575 lines
23 KiB
Python
575 lines
23 KiB
Python
|
#SPDX-License-Identifier: GPL-2.0
|
||
|
|
import re
|
||
|
|
import csv
|
||
|
|
import json
|
||
|
|
import argparse
|
||
|
|
from pathlib import Path
|
||
|
|
import subprocess
|
||
|
|
|
||
|
|
class Validator:
|
||
|
|
def __init__(self, rulefname, reportfname='', t=5, debug=False, datafname='', fullrulefname='', workload='true', metrics=''):
|
||
|
|
self.rulefname = rulefname
|
||
|
|
self.reportfname = reportfname
|
||
|
|
self.rules = None
|
||
|
|
self.collectlist:str = metrics
|
||
|
|
self.metrics = self.__set_metrics(metrics)
|
||
|
|
self.skiplist = set()
|
||
|
|
self.tolerance = t
|
||
|
|
|
||
|
|
self.workloads = [x for x in workload.split(",") if x]
|
||
|
|
self.wlidx = 0 # idx of current workloads
|
||
|
|
self.allresults = dict() # metric results of all workload
|
||
|
|
self.allignoremetrics = dict() # metrics with no results or negative results
|
||
|
|
self.allfailtests = dict()
|
||
|
|
self.alltotalcnt = dict()
|
||
|
|
self.allpassedcnt = dict()
|
||
|
|
self.allerrlist = dict()
|
||
|
|
|
||
|
|
self.results = dict() # metric results of current workload
|
||
|
|
# vars for test pass/failure statistics
|
||
|
|
self.ignoremetrics= set() # metrics with no results or negative results, neg result counts as a failed test
|
||
|
|
self.failtests = dict()
|
||
|
|
self.totalcnt = 0
|
||
|
|
self.passedcnt = 0
|
||
|
|
# vars for errors
|
||
|
|
self.errlist = list()
|
||
|
|
|
||
|
|
# vars for Rule Generator
|
||
|
|
self.pctgmetrics = set() # Percentage rule
|
||
|
|
|
||
|
|
# vars for debug
|
||
|
|
self.datafname = datafname
|
||
|
|
self.debug = debug
|
||
|
|
self.fullrulefname = fullrulefname
|
||
|
|
|
||
|
|
def __set_metrics(self, metrics=''):
|
||
|
|
if metrics != '':
|
||
|
|
return set(metrics.split(","))
|
||
|
|
else:
|
||
|
|
return set()
|
||
|
|
|
||
|
|
def read_json(self, filename: str) -> dict:
|
||
|
|
try:
|
||
|
|
with open(Path(filename).resolve(), "r") as f:
|
||
|
|
data = json.loads(f.read())
|
||
|
|
except OSError as e:
|
||
|
|
print(f"Error when reading file {e}")
|
||
|
|
sys.exit()
|
||
|
|
|
||
|
|
return data
|
||
|
|
|
||
|
|
def json_dump(self, data, output_file):
|
||
|
|
parent = Path(output_file).parent
|
||
|
|
if not parent.exists():
|
||
|
|
parent.mkdir(parents=True)
|
||
|
|
|
||
|
|
with open(output_file, "w+") as output_file:
|
||
|
|
json.dump(data,
|
||
|
|
output_file,
|
||
|
|
ensure_ascii=True,
|
||
|
|
indent=4)
|
||
|
|
|
||
|
|
def get_results(self, idx:int = 0):
|
||
|
|
return self.results[idx]
|
||
|
|
|
||
|
|
def get_bounds(self, lb, ub, error, alias={}, ridx:int = 0) -> list:
|
||
|
|
"""
|
||
|
|
Get bounds and tolerance from lb, ub, and error.
|
||
|
|
If missing lb, use 0.0; missing ub, use float('inf); missing error, use self.tolerance.
|
||
|
|
|
||
|
|
@param lb: str/float, lower bound
|
||
|
|
@param ub: str/float, upper bound
|
||
|
|
@param error: float/str, error tolerance
|
||
|
|
@returns: lower bound, return inf if the lower bound is a metric value and is not collected
|
||
|
|
upper bound, return -1 if the upper bound is a metric value and is not collected
|
||
|
|
tolerance, denormalized base on upper bound value
|
||
|
|
"""
|
||
|
|
# init ubv and lbv to invalid values
|
||
|
|
def get_bound_value (bound, initval, ridx):
|
||
|
|
val = initval
|
||
|
|
if isinstance(bound, int) or isinstance(bound, float):
|
||
|
|
val = bound
|
||
|
|
elif isinstance(bound, str):
|
||
|
|
if bound == '':
|
||
|
|
val = float("inf")
|
||
|
|
elif bound in alias:
|
||
|
|
vall = self.get_value(alias[ub], ridx)
|
||
|
|
if vall:
|
||
|
|
val = vall[0]
|
||
|
|
elif bound.replace('.', '1').isdigit():
|
||
|
|
val = float(bound)
|
||
|
|
else:
|
||
|
|
print("Wrong bound: {0}".format(bound))
|
||
|
|
else:
|
||
|
|
print("Wrong bound: {0}".format(bound))
|
||
|
|
return val
|
||
|
|
|
||
|
|
ubv = get_bound_value(ub, -1, ridx)
|
||
|
|
lbv = get_bound_value(lb, float('inf'), ridx)
|
||
|
|
t = get_bound_value(error, self.tolerance, ridx)
|
||
|
|
|
||
|
|
# denormalize error threshold
|
||
|
|
denormerr = t * ubv / 100 if ubv != 100 and ubv > 0 else t
|
||
|
|
|
||
|
|
return lbv, ubv, denormerr
|
||
|
|
|
||
|
|
def get_value(self, name:str, ridx:int = 0) -> list:
|
||
|
|
"""
|
||
|
|
Get value of the metric from self.results.
|
||
|
|
If result of this metric is not provided, the metric name will be added into self.ignoremetics and self.errlist.
|
||
|
|
All future test(s) on this metric will fail.
|
||
|
|
|
||
|
|
@param name: name of the metric
|
||
|
|
@returns: list with value found in self.results; list is empty when value is not found.
|
||
|
|
"""
|
||
|
|
results = []
|
||
|
|
data = self.results[ridx] if ridx in self.results else self.results[0]
|
||
|
|
if name not in self.ignoremetrics:
|
||
|
|
if name in data:
|
||
|
|
results.append(data[name])
|
||
|
|
elif name.replace('.', '1').isdigit():
|
||
|
|
results.append(float(name))
|
||
|
|
else:
|
||
|
|
self.ignoremetrics.add(name)
|
||
|
|
return results
|
||
|
|
|
||
|
|
def check_bound(self, val, lb, ub, err):
|
||
|
|
return True if val <= ub + err and val >= lb - err else False
|
||
|
|
|
||
|
|
# Positive Value Sanity check
|
||
|
|
def pos_val_test(self):
|
||
|
|
"""
|
||
|
|
Check if metrics value are non-negative.
|
||
|
|
One metric is counted as one test.
|
||
|
|
Failure: when metric value is negative or not provided.
|
||
|
|
Metrics with negative value will be added into the self.failtests['PositiveValueTest'] and self.ignoremetrics.
|
||
|
|
"""
|
||
|
|
negmetric = dict()
|
||
|
|
pcnt = 0
|
||
|
|
tcnt = 0
|
||
|
|
rerun = list()
|
||
|
|
for name, val in self.get_results().items():
|
||
|
|
if val < 0:
|
||
|
|
negmetric[name] = val
|
||
|
|
rerun.append(name)
|
||
|
|
else:
|
||
|
|
pcnt += 1
|
||
|
|
tcnt += 1
|
||
|
|
if len(rerun) > 0 and len(rerun) < 20:
|
||
|
|
second_results = dict()
|
||
|
|
self.second_test(rerun, second_results)
|
||
|
|
for name, val in second_results.items():
|
||
|
|
if name not in negmetric: continue
|
||
|
|
if val >= 0:
|
||
|
|
del negmetric[name]
|
||
|
|
pcnt += 1
|
||
|
|
|
||
|
|
self.failtests['PositiveValueTest']['Total Tests'] = tcnt
|
||
|
|
self.failtests['PositiveValueTest']['Passed Tests'] = pcnt
|
||
|
|
if len(negmetric.keys()):
|
||
|
|
self.ignoremetrics.update(negmetric.keys())
|
||
|
|
negmessage = ["{0}(={1:.4f})".format(name, val) for name, val in negmetric.items()]
|
||
|
|
self.failtests['PositiveValueTest']['Failed Tests'].append({'NegativeValue': negmessage})
|
||
|
|
|
||
|
|
return
|
||
|
|
|
||
|
|
def evaluate_formula(self, formula:str, alias:dict, ridx:int = 0):
|
||
|
|
"""
|
||
|
|
Evaluate the value of formula.
|
||
|
|
|
||
|
|
@param formula: the formula to be evaluated
|
||
|
|
@param alias: the dict has alias to metric name mapping
|
||
|
|
@returns: value of the formula is success; -1 if the one or more metric value not provided
|
||
|
|
"""
|
||
|
|
stack = []
|
||
|
|
b = 0
|
||
|
|
errs = []
|
||
|
|
sign = "+"
|
||
|
|
f = str()
|
||
|
|
|
||
|
|
#TODO: support parenthesis?
|
||
|
|
for i in range(len(formula)):
|
||
|
|
if i+1 == len(formula) or formula[i] in ('+', '-', '*', '/'):
|
||
|
|
s = alias[formula[b:i]] if i+1 < len(formula) else alias[formula[b:]]
|
||
|
|
v = self.get_value(s, ridx)
|
||
|
|
if not v:
|
||
|
|
errs.append(s)
|
||
|
|
else:
|
||
|
|
f = f + "{0}(={1:.4f})".format(s, v[0])
|
||
|
|
if sign == "*":
|
||
|
|
stack[-1] = stack[-1] * v
|
||
|
|
elif sign == "/":
|
||
|
|
stack[-1] = stack[-1] / v
|
||
|
|
elif sign == '-':
|
||
|
|
stack.append(-v[0])
|
||
|
|
else:
|
||
|
|
stack.append(v[0])
|
||
|
|
if i + 1 < len(formula):
|
||
|
|
sign = formula[i]
|
||
|
|
f += sign
|
||
|
|
b = i + 1
|
||
|
|
|
||
|
|
if len(errs) > 0:
|
||
|
|
return -1, "Metric value missing: "+','.join(errs)
|
||
|
|
|
||
|
|
val = sum(stack)
|
||
|
|
return val, f
|
||
|
|
|
||
|
|
# Relationships Tests
|
||
|
|
def relationship_test(self, rule: dict):
|
||
|
|
"""
|
||
|
|
Validate if the metrics follow the required relationship in the rule.
|
||
|
|
eg. lower_bound <= eval(formula)<= upper_bound
|
||
|
|
One rule is counted as ont test.
|
||
|
|
Failure: when one or more metric result(s) not provided, or when formula evaluated outside of upper/lower bounds.
|
||
|
|
|
||
|
|
@param rule: dict with metric name(+alias), formula, and required upper and lower bounds.
|
||
|
|
"""
|
||
|
|
alias = dict()
|
||
|
|
for m in rule['Metrics']:
|
||
|
|
alias[m['Alias']] = m['Name']
|
||
|
|
lbv, ubv, t = self.get_bounds(rule['RangeLower'], rule['RangeUpper'], rule['ErrorThreshold'], alias, ridx=rule['RuleIndex'])
|
||
|
|
val, f = self.evaluate_formula(rule['Formula'], alias, ridx=rule['RuleIndex'])
|
||
|
|
if val == -1:
|
||
|
|
self.failtests['RelationshipTest']['Failed Tests'].append({'RuleIndex': rule['RuleIndex'], 'Description':f})
|
||
|
|
elif not self.check_bound(val, lbv, ubv, t):
|
||
|
|
lb = rule['RangeLower']
|
||
|
|
ub = rule['RangeUpper']
|
||
|
|
if isinstance(lb, str):
|
||
|
|
if lb in alias:
|
||
|
|
lb = alias[lb]
|
||
|
|
if isinstance(ub, str):
|
||
|
|
if ub in alias:
|
||
|
|
ub = alias[ub]
|
||
|
|
self.failtests['RelationshipTest']['Failed Tests'].append({'RuleIndex': rule['RuleIndex'], 'Formula':f,
|
||
|
|
'RangeLower': lb, 'LowerBoundValue': self.get_value(lb),
|
||
|
|
'RangeUpper': ub, 'UpperBoundValue':self.get_value(ub),
|
||
|
|
'ErrorThreshold': t, 'CollectedValue': val})
|
||
|
|
else:
|
||
|
|
self.passedcnt += 1
|
||
|
|
self.failtests['RelationshipTest']['Passed Tests'] += 1
|
||
|
|
self.totalcnt += 1
|
||
|
|
self.failtests['RelationshipTest']['Total Tests'] += 1
|
||
|
|
|
||
|
|
return
|
||
|
|
|
||
|
|
|
||
|
|
# Single Metric Test
|
||
|
|
def single_test(self, rule:dict):
|
||
|
|
"""
|
||
|
|
Validate if the metrics are in the required value range.
|
||
|
|
eg. lower_bound <= metrics_value <= upper_bound
|
||
|
|
One metric is counted as one test in this type of test.
|
||
|
|
One rule may include one or more metrics.
|
||
|
|
Failure: when the metric value not provided or the value is outside the bounds.
|
||
|
|
This test updates self.total_cnt and records failed tests in self.failtest['SingleMetricTest'].
|
||
|
|
|
||
|
|
@param rule: dict with metrics to validate and the value range requirement
|
||
|
|
"""
|
||
|
|
lbv, ubv, t = self.get_bounds(rule['RangeLower'], rule['RangeUpper'], rule['ErrorThreshold'])
|
||
|
|
metrics = rule['Metrics']
|
||
|
|
passcnt = 0
|
||
|
|
totalcnt = 0
|
||
|
|
faillist = list()
|
||
|
|
failures = dict()
|
||
|
|
rerun = list()
|
||
|
|
for m in metrics:
|
||
|
|
totalcnt += 1
|
||
|
|
result = self.get_value(m['Name'])
|
||
|
|
if len(result) > 0 and self.check_bound(result[0], lbv, ubv, t) or m['Name'] in self.skiplist:
|
||
|
|
passcnt += 1
|
||
|
|
else:
|
||
|
|
failures[m['Name']] = result
|
||
|
|
rerun.append(m['Name'])
|
||
|
|
|
||
|
|
if len(rerun) > 0 and len(rerun) < 20:
|
||
|
|
second_results = dict()
|
||
|
|
self.second_test(rerun, second_results)
|
||
|
|
for name, val in second_results.items():
|
||
|
|
if name not in failures: continue
|
||
|
|
if self.check_bound(val, lbv, ubv, t):
|
||
|
|
passcnt += 1
|
||
|
|
del failures[name]
|
||
|
|
else:
|
||
|
|
failures[name] = val
|
||
|
|
self.results[0][name] = val
|
||
|
|
|
||
|
|
self.totalcnt += totalcnt
|
||
|
|
self.passedcnt += passcnt
|
||
|
|
self.failtests['SingleMetricTest']['Total Tests'] += totalcnt
|
||
|
|
self.failtests['SingleMetricTest']['Passed Tests'] += passcnt
|
||
|
|
if len(failures.keys()) != 0:
|
||
|
|
faillist = [{'MetricName':name, 'CollectedValue':val} for name, val in failures.items()]
|
||
|
|
self.failtests['SingleMetricTest']['Failed Tests'].append({'RuleIndex':rule['RuleIndex'],
|
||
|
|
'RangeLower': rule['RangeLower'],
|
||
|
|
'RangeUpper': rule['RangeUpper'],
|
||
|
|
'ErrorThreshold':rule['ErrorThreshold'],
|
||
|
|
'Failure':faillist})
|
||
|
|
|
||
|
|
return
|
||
|
|
|
||
|
|
def create_report(self):
|
||
|
|
"""
|
||
|
|
Create final report and write into a JSON file.
|
||
|
|
"""
|
||
|
|
alldata = list()
|
||
|
|
for i in range(0, len(self.workloads)):
|
||
|
|
reportstas = {"Total Rule Count": self.alltotalcnt[i], "Passed Rule Count": self.allpassedcnt[i]}
|
||
|
|
data = {"Metric Validation Statistics": reportstas, "Tests in Category": self.allfailtests[i],
|
||
|
|
"Errors":self.allerrlist[i]}
|
||
|
|
alldata.append({"Workload": self.workloads[i], "Report": data})
|
||
|
|
|
||
|
|
json_str = json.dumps(alldata, indent=4)
|
||
|
|
print("Test validation finished. Final report: ")
|
||
|
|
print(json_str)
|
||
|
|
|
||
|
|
if self.debug:
|
||
|
|
allres = [{"Workload": self.workloads[i], "Results": self.allresults[i]} for i in range(0, len(self.workloads))]
|
||
|
|
self.json_dump(allres, self.datafname)
|
||
|
|
|
||
|
|
def check_rule(self, testtype, metric_list):
|
||
|
|
"""
|
||
|
|
Check if the rule uses metric(s) that not exist in current platform.
|
||
|
|
|
||
|
|
@param metric_list: list of metrics from the rule.
|
||
|
|
@return: False when find one metric out in Metric file. (This rule should not skipped.)
|
||
|
|
True when all metrics used in the rule are found in Metric file.
|
||
|
|
"""
|
||
|
|
if testtype == "RelationshipTest":
|
||
|
|
for m in metric_list:
|
||
|
|
if m['Name'] not in self.metrics:
|
||
|
|
return False
|
||
|
|
return True
|
||
|
|
|
||
|
|
# Start of Collector and Converter
|
||
|
|
def convert(self, data: list, metricvalues:dict):
|
||
|
|
"""
|
||
|
|
Convert collected metric data from the -j output to dict of {metric_name:value}.
|
||
|
|
"""
|
||
|
|
for json_string in data:
|
||
|
|
try:
|
||
|
|
result =json.loads(json_string)
|
||
|
|
if "metric-unit" in result and result["metric-unit"] != "(null)" and result["metric-unit"] != "":
|
||
|
|
name = result["metric-unit"].split(" ")[1] if len(result["metric-unit"].split(" ")) > 1 \
|
||
|
|
else result["metric-unit"]
|
||
|
|
metricvalues[name.lower()] = float(result["metric-value"])
|
||
|
|
except ValueError as error:
|
||
|
|
continue
|
||
|
|
return
|
||
|
|
|
||
|
|
def _run_perf(self, metric, workload: str):
|
||
|
|
tool = 'perf'
|
||
|
|
command = [tool, 'stat', '-j', '-M', f"{metric}", "-a"]
|
||
|
|
wl = workload.split()
|
||
|
|
command.extend(wl)
|
||
|
|
print(" ".join(command))
|
||
|
|
cmd = subprocess.run(command, stderr=subprocess.PIPE, encoding='utf-8')
|
||
|
|
data = [x+'}' for x in cmd.stderr.split('}\n') if x]
|
||
|
|
return data
|
||
|
|
|
||
|
|
|
||
|
|
def collect_perf(self, workload: str):
|
||
|
|
"""
|
||
|
|
Collect metric data with "perf stat -M" on given workload with -a and -j.
|
||
|
|
"""
|
||
|
|
self.results = dict()
|
||
|
|
print(f"Starting perf collection")
|
||
|
|
print(f"Long workload: {workload}")
|
||
|
|
collectlist = dict()
|
||
|
|
if self.collectlist != "":
|
||
|
|
collectlist[0] = {x for x in self.collectlist.split(",")}
|
||
|
|
else:
|
||
|
|
collectlist[0] = set(list(self.metrics))
|
||
|
|
# Create metric set for relationship rules
|
||
|
|
for rule in self.rules:
|
||
|
|
if rule["TestType"] == "RelationshipTest":
|
||
|
|
metrics = [m["Name"] for m in rule["Metrics"]]
|
||
|
|
if not any(m not in collectlist[0] for m in metrics):
|
||
|
|
collectlist[rule["RuleIndex"]] = [",".join(list(set(metrics)))]
|
||
|
|
|
||
|
|
for idx, metrics in collectlist.items():
|
||
|
|
if idx == 0: wl = "true"
|
||
|
|
else: wl = workload
|
||
|
|
for metric in metrics:
|
||
|
|
data = self._run_perf(metric, wl)
|
||
|
|
if idx not in self.results: self.results[idx] = dict()
|
||
|
|
self.convert(data, self.results[idx])
|
||
|
|
return
|
||
|
|
|
||
|
|
def second_test(self, collectlist, second_results):
|
||
|
|
workload = self.workloads[self.wlidx]
|
||
|
|
for metric in collectlist:
|
||
|
|
data = self._run_perf(metric, workload)
|
||
|
|
self.convert(data, second_results)
|
||
|
|
|
||
|
|
# End of Collector and Converter
|
||
|
|
|
||
|
|
# Start of Rule Generator
|
||
|
|
def parse_perf_metrics(self):
|
||
|
|
"""
|
||
|
|
Read and parse perf metric file:
|
||
|
|
1) find metrics with '1%' or '100%' as ScaleUnit for Percent check
|
||
|
|
2) create metric name list
|
||
|
|
"""
|
||
|
|
command = ['perf', 'list', '-j', '--details', 'metrics']
|
||
|
|
cmd = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf-8')
|
||
|
|
try:
|
||
|
|
data = json.loads(cmd.stdout)
|
||
|
|
for m in data:
|
||
|
|
if 'MetricName' not in m:
|
||
|
|
print("Warning: no metric name")
|
||
|
|
continue
|
||
|
|
name = m['MetricName'].lower()
|
||
|
|
self.metrics.add(name)
|
||
|
|
if 'ScaleUnit' in m and (m['ScaleUnit'] == '1%' or m['ScaleUnit'] == '100%'):
|
||
|
|
self.pctgmetrics.add(name.lower())
|
||
|
|
except ValueError as error:
|
||
|
|
print(f"Error when parsing metric data")
|
||
|
|
sys.exit()
|
||
|
|
|
||
|
|
return
|
||
|
|
|
||
|
|
def remove_unsupported_rules(self, rules):
|
||
|
|
new_rules = []
|
||
|
|
for rule in rules:
|
||
|
|
add_rule = True
|
||
|
|
for m in rule["Metrics"]:
|
||
|
|
if m["Name"] in self.skiplist or m["Name"] not in self.metrics:
|
||
|
|
add_rule = False
|
||
|
|
break
|
||
|
|
if add_rule:
|
||
|
|
new_rules.append(rule)
|
||
|
|
return new_rules
|
||
|
|
|
||
|
|
def create_rules(self):
|
||
|
|
"""
|
||
|
|
Create full rules which includes:
|
||
|
|
1) All the rules from the "relationshi_rules" file
|
||
|
|
2) SingleMetric rule for all the 'percent' metrics
|
||
|
|
|
||
|
|
Reindex all the rules to avoid repeated RuleIndex
|
||
|
|
"""
|
||
|
|
data = self.read_json(self.rulefname)
|
||
|
|
rules = data['RelationshipRules']
|
||
|
|
self.skiplist = set([name.lower() for name in data['SkipList']])
|
||
|
|
self.rules = self.remove_unsupported_rules(rules)
|
||
|
|
pctgrule = {'RuleIndex':0,
|
||
|
|
'TestType':'SingleMetricTest',
|
||
|
|
'RangeLower':'0',
|
||
|
|
'RangeUpper': '100',
|
||
|
|
'ErrorThreshold': self.tolerance,
|
||
|
|
'Description':'Metrics in percent unit have value with in [0, 100]',
|
||
|
|
'Metrics': [{'Name': m.lower()} for m in self.pctgmetrics]}
|
||
|
|
self.rules.append(pctgrule)
|
||
|
|
|
||
|
|
# Re-index all rules to avoid repeated RuleIndex
|
||
|
|
idx = 1
|
||
|
|
for r in self.rules:
|
||
|
|
r['RuleIndex'] = idx
|
||
|
|
idx += 1
|
||
|
|
|
||
|
|
if self.debug:
|
||
|
|
#TODO: need to test and generate file name correctly
|
||
|
|
data = {'RelationshipRules':self.rules, 'SupportedMetrics': [{"MetricName": name} for name in self.metrics]}
|
||
|
|
self.json_dump(data, self.fullrulefname)
|
||
|
|
|
||
|
|
return
|
||
|
|
# End of Rule Generator
|
||
|
|
|
||
|
|
def _storewldata(self, key):
|
||
|
|
'''
|
||
|
|
Store all the data of one workload into the corresponding data structure for all workloads.
|
||
|
|
@param key: key to the dictionaries (index of self.workloads).
|
||
|
|
'''
|
||
|
|
self.allresults[key] = self.results
|
||
|
|
self.allignoremetrics[key] = self.ignoremetrics
|
||
|
|
self.allfailtests[key] = self.failtests
|
||
|
|
self.alltotalcnt[key] = self.totalcnt
|
||
|
|
self.allpassedcnt[key] = self.passedcnt
|
||
|
|
self.allerrlist[key] = self.errlist
|
||
|
|
|
||
|
|
#Initialize data structures before data validation of each workload
|
||
|
|
def _init_data(self):
|
||
|
|
|
||
|
|
testtypes = ['PositiveValueTest', 'RelationshipTest', 'SingleMetricTest']
|
||
|
|
self.results = dict()
|
||
|
|
self.ignoremetrics= set()
|
||
|
|
self.errlist = list()
|
||
|
|
self.failtests = {k:{'Total Tests':0, 'Passed Tests':0, 'Failed Tests':[]} for k in testtypes}
|
||
|
|
self.totalcnt = 0
|
||
|
|
self.passedcnt = 0
|
||
|
|
|
||
|
|
def test(self):
|
||
|
|
'''
|
||
|
|
The real entry point of the test framework.
|
||
|
|
This function loads the validation rule JSON file and Standard Metric file to create rules for
|
||
|
|
testing and namemap dictionaries.
|
||
|
|
It also reads in result JSON file for testing.
|
||
|
|
|
||
|
|
In the test process, it passes through each rule and launch correct test function bases on the
|
||
|
|
'TestType' field of the rule.
|
||
|
|
|
||
|
|
The final report is written into a JSON file.
|
||
|
|
'''
|
||
|
|
if not self.collectlist:
|
||
|
|
self.parse_perf_metrics()
|
||
|
|
self.create_rules()
|
||
|
|
for i in range(0, len(self.workloads)):
|
||
|
|
self.wlidx = i
|
||
|
|
self._init_data()
|
||
|
|
self.collect_perf(self.workloads[i])
|
||
|
|
# Run positive value test
|
||
|
|
self.pos_val_test()
|
||
|
|
for r in self.rules:
|
||
|
|
# skip rules that uses metrics not exist in this platform
|
||
|
|
testtype = r['TestType']
|
||
|
|
if not self.check_rule(testtype, r['Metrics']):
|
||
|
|
continue
|
||
|
|
if testtype == 'RelationshipTest':
|
||
|
|
self.relationship_test(r)
|
||
|
|
elif testtype == 'SingleMetricTest':
|
||
|
|
self.single_test(r)
|
||
|
|
else:
|
||
|
|
print("Unsupported Test Type: ", testtype)
|
||
|
|
self.errlist.append("Unsupported Test Type from rule: " + r['RuleIndex'])
|
||
|
|
self._storewldata(i)
|
||
|
|
print("Workload: ", self.workloads[i])
|
||
|
|
print("Total metrics collected: ", self.failtests['PositiveValueTest']['Total Tests'])
|
||
|
|
print("Non-negative metric count: ", self.failtests['PositiveValueTest']['Passed Tests'])
|
||
|
|
print("Total Test Count: ", self.totalcnt)
|
||
|
|
print("Passed Test Count: ", self.passedcnt)
|
||
|
|
|
||
|
|
self.create_report()
|
||
|
|
return sum(self.alltotalcnt.values()) != sum(self.allpassedcnt.values())
|
||
|
|
# End of Class Validator
|
||
|
|
|
||
|
|
|
||
|
|
def main() -> None:
|
||
|
|
parser = argparse.ArgumentParser(description="Launch metric value validation")
|
||
|
|
|
||
|
|
parser.add_argument("-rule", help="Base validation rule file", required=True)
|
||
|
|
parser.add_argument("-output_dir", help="Path for validator output file, report file", required=True)
|
||
|
|
parser.add_argument("-debug", help="Debug run, save intermediate data to files", action="store_true", default=False)
|
||
|
|
parser.add_argument("-wl", help="Workload to run while data collection", default="true")
|
||
|
|
parser.add_argument("-m", help="Metric list to validate", default="")
|
||
|
|
args = parser.parse_args()
|
||
|
|
outpath = Path(args.output_dir)
|
||
|
|
reportf = Path.joinpath(outpath, 'perf_report.json')
|
||
|
|
fullrule = Path.joinpath(outpath, 'full_rule.json')
|
||
|
|
datafile = Path.joinpath(outpath, 'perf_data.json')
|
||
|
|
|
||
|
|
validator = Validator(args.rule, reportf, debug=args.debug,
|
||
|
|
datafname=datafile, fullrulefname=fullrule, workload=args.wl,
|
||
|
|
metrics=args.m)
|
||
|
|
ret = validator.test()
|
||
|
|
|
||
|
|
return ret
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
import sys
|
||
|
|
sys.exit(main())
|
||
|
|
|
||
|
|
|
||
|
|
|