import base64
import datetime
import itertools
import json
from xml.sax.saxutils import quoteattr as quo
import lit.Test
def by_suite_and_test_path(test):
return (test.suite.name, id(test.suite), test.path_in_suite)
class JsonReport(object):
def __init__(self, output_file):
self.output_file = output_file
def write_results(self, tests, elapsed):
unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED}
tests = [t for t in tests if t.result.code not in unexecuted_codes]
data = {}
data['__version__'] = lit.__versioninfo__
data['elapsed'] = elapsed
data['tests'] = tests_data = []
for test in tests:
test_data = {
'name': test.getFullName(),
'code': test.result.code.name,
'output': test.result.output,
'elapsed': test.result.elapsed}
if test.result.metrics:
test_data['metrics'] = metrics_data = {}
for key, value in test.result.metrics.items():
metrics_data[key] = value.todata()
if test.result.microResults:
for key, micro_test in test.result.microResults.items():
parent_name = test.getFullName()
micro_full_name = parent_name + ':' + key
micro_test_data = {
'name': micro_full_name,
'code': micro_test.code.name,
'output': micro_test.output,
'elapsed': micro_test.elapsed}
if micro_test.metrics:
micro_test_data['metrics'] = micro_metrics_data = {}
for key, value in micro_test.metrics.items():
micro_metrics_data[key] = value.todata()
tests_data.append(micro_test_data)
tests_data.append(test_data)
with open(self.output_file, 'w') as file:
json.dump(data, file, indent=2, sort_keys=True)
file.write('\n')
_invalid_xml_chars_dict = {c: None for c in range(32) if chr(c) not in ('\t', '\n', '\r')}
def remove_invalid_xml_chars(s):
return s.translate(_invalid_xml_chars_dict)
class XunitReport(object):
def __init__(self, output_file):
self.output_file = output_file
self.skipped_codes = {lit.Test.EXCLUDED,
lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
def write_results(self, tests, elapsed):
tests.sort(key=by_suite_and_test_path)
tests_by_suite = itertools.groupby(tests, lambda t: t.suite)
with open(self.output_file, 'w') as file:
file.write('<?xml version="1.0" encoding="UTF-8"?>\n')
file.write('<testsuites time="{time:.2f}">\n'.format(time=elapsed))
for suite, test_iter in tests_by_suite:
self._write_testsuite(file, suite, list(test_iter))
file.write('</testsuites>\n')
def _write_testsuite(self, file, suite, tests):
skipped = sum(1 for t in tests if t.result.code in self.skipped_codes)
failures = sum(1 for t in tests if t.isFailure())
name = suite.config.name.replace('.', '-')
file.write(f'<testsuite name={quo(name)} tests="{len(tests)}" failures="{failures}" skipped="{skipped}">\n')
for test in tests:
self._write_test(file, test, name)
file.write('</testsuite>\n')
def _write_test(self, file, test, suite_name):
path = '/'.join(test.path_in_suite[:-1]).replace('.', '_')
class_name = f'{suite_name}.{path or suite_name}'
name = test.path_in_suite[-1]
time = test.result.elapsed or 0.0
file.write(f'<testcase classname={quo(class_name)} name={quo(name)} time="{time:.2f}"')
if test.isFailure():
file.write('>\n <failure><![CDATA[')
output = test.result.output.replace(']]>', ']]]]><![CDATA[>')
if isinstance(output, bytes):
output = output.decode("utf-8", 'ignore')
output = remove_invalid_xml_chars(output)
file.write(output)
file.write(']]></failure>\n</testcase>\n')
elif test.result.code in self.skipped_codes:
reason = self._get_skip_reason(test)
file.write(f'>\n <skipped message={quo(reason)}/>\n</testcase>\n')
else:
file.write('/>\n')
def _get_skip_reason(self, test):
code = test.result.code
if code == lit.Test.EXCLUDED:
return 'Test not selected (--filter, --max-tests)'
if code == lit.Test.SKIPPED:
return 'User interrupt'
assert code == lit.Test.UNSUPPORTED
features = test.getMissingRequiredFeatures()
if features:
return 'Missing required feature(s): ' + ', '.join(features)
return 'Unsupported configuration'
def gen_resultdb_test_entry(
test_name, start_time, elapsed_time, test_output, result_code, is_expected
):
test_data = {
'testId': test_name,
'start_time': datetime.datetime.fromtimestamp(start_time).isoformat() + 'Z',
'duration': '%.9fs' % elapsed_time,
'summary_html': '<p><text-artifact artifact-id="artifact-content-in-request"></p>',
'artifacts': {
'artifact-content-in-request': {
'contents': base64.b64encode(test_output.encode('utf-8')).decode(
'utf-8'
),
},
},
'expected': is_expected,
}
if (
result_code == lit.Test.PASS
or result_code == lit.Test.XPASS
or result_code == lit.Test.FLAKYPASS
):
test_data['status'] = 'PASS'
elif result_code == lit.Test.FAIL or result_code == lit.Test.XFAIL:
test_data['status'] = 'FAIL'
elif (
result_code == lit.Test.UNSUPPORTED
or result_code == lit.Test.SKIPPED
or result_code == lit.Test.EXCLUDED
):
test_data['status'] = 'SKIP'
elif result_code == lit.Test.UNRESOLVED or result_code == lit.Test.TIMEOUT:
test_data['status'] = 'ABORT'
return test_data
class ResultDBReport(object):
def __init__(self, output_file):
self.output_file = output_file
def write_results(self, tests, elapsed):
unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED}
tests = [t for t in tests if t.result.code not in unexecuted_codes]
data = {}
data['__version__'] = lit.__versioninfo__
data['elapsed'] = elapsed
data['tests'] = tests_data = []
for test in tests:
tests_data.append(
gen_resultdb_test_entry(
test_name=test.getFullName(),
start_time=test.result.start,
elapsed_time=test.result.elapsed,
test_output=test.result.output,
result_code=test.result.code,
is_expected=not test.result.code.isFailure,
)
)
if test.result.microResults:
for key, micro_test in test.result.microResults.items():
parent_name = test.getFullName()
micro_full_name = parent_name + ':' + key + 'microres'
tests_data.append(
gen_resultdb_test_entry(
test_name=micro_full_name,
start_time=micro_test.start
if micro_test.start
else test.result.start,
elapsed_time=micro_test.elapsed
if micro_test.elapsed
else test.result.elapsed,
test_output=micro_test.output,
result_code=micro_test.code,
is_expected=not micro_test.code.isFailure,
)
)
with open(self.output_file, 'w') as file:
json.dump(data, file, indent=2, sort_keys=True)
file.write('\n')
class TimeTraceReport(object):
def __init__(self, output_file):
self.output_file = output_file
self.skipped_codes = {lit.Test.EXCLUDED,
lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
def write_results(self, tests, elapsed):
first_start_time = min([t.result.start for t in tests])
events = [self._get_test_event(
x, first_start_time) for x in tests if x.result.code not in self.skipped_codes]
json_data = {'traceEvents': events}
with open(self.output_file, "w") as time_trace_file:
json.dump(json_data, time_trace_file, indent=2, sort_keys=True)
def _get_test_event(self, test, first_start_time):
test_name = test.getFullName()
elapsed_time = test.result.elapsed or 0.0
start_time = test.result.start - first_start_time if test.result.start else 0.0
pid = test.result.pid or 0
return {
'pid': pid,
'tid': 1,
'ph': 'X',
'ts': int(start_time * 1000000.),
'dur': int(elapsed_time * 1000000.),
'name': test_name,
}