Merge pull request #4062 from adelez/mako_template

Use mako template for HTML report.
reviewable/pr4035/r4^2
adelez 9 years ago
commit 7b63a6c8ea
  1. 141
      reports/interop_html_report.template
  2. 168
      tools/run_tests/report_utils.py
  3. 4
      tools/run_tests/run_interop_tests.py
  4. 2
      tools/run_tests/run_tests.py

@ -0,0 +1,141 @@
<!DOCTYPE html>
<html lang="en">
<head><title>Interop Test Result</title></head>
<body>
<%def name="fill_one_test_result(shortname, resultset)">
% if shortname in resultset:
## Because interop tests does not have runs_per_test flag, each test is
## run once. So there should only be one element for each result.
<% result = resultset[shortname][0] %>
% if result.state == 'PASSED':
<td bgcolor="green">PASS</td>
% else:
<%
tooltip = ''
if result.returncode > 0 or result.message:
if result.returncode > 0:
tooltip = 'returncode: %d ' % result.returncode
if result.message:
tooltip = '%smessage: %s' % (tooltip, result.message)
%>
% if result.state == 'FAILED':
<td bgcolor="red">
% if tooltip:
<a href="#" data-toggle="tooltip" data-placement="auto" title="${tooltip | h}">FAIL</a></td>
% else:
FAIL</td>
% endif
% elif result.state == 'TIMEOUT':
<td bgcolor="yellow">
% if tooltip:
<a href="#" data-toggle="tooltip" data-placement="auto" title="${tooltip | h}">TIMEOUT</a></td>
% else:
TIMEOUT</td>
% endif
% endif
% endif
% else:
<td bgcolor="magenta">Not implemented</td>
% endif
</%def>
% if num_failures > 1:
<p><h2><font color="red">${num_failures} tests failed!</font></h2></p>
% elif num_failures:
<p><h2><font color="red">${num_failures} test failed!</font></h2></p>
% else:
<p><h2><font color="green">All tests passed!</font></h2></p>
% endif
% if cloud_to_prod:
## Each column header is the client language.
<h2>Cloud to Prod</h2>
<table style="width:100%" border="1">
<tr bgcolor="#00BFFF">
<th>Client languages &#9658;<br/>Test Cases &#9660;</th>
% for client_lang in client_langs:
<th>${client_lang}</th>
% endfor
</tr>
% for test_case in test_cases + auth_test_cases:
<tr><td><b>${test_case}</b></td>
% for client_lang in client_langs:
<%
if test_case in auth_test_cases:
shortname = 'cloud_to_prod_auth:%s:%s' % (client_lang, test_case)
else:
shortname = 'cloud_to_prod:%s:%s' % (client_lang, test_case)
%>
${fill_one_test_result(shortname, resultset)}
% endfor
</tr>
% endfor
</table>
% endif
% if http2_interop:
## Each column header is the server language.
<h2>HTTP/2 Interop</h2>
<table style="width:100%" border="1">
<tr bgcolor="#00BFFF">
<th>Servers &#9658;<br/>Test Cases &#9660;</th>
% for server_lang in server_langs:
<th>${server_lang}</th>
% endfor
% if cloud_to_prod:
<th>prod</th>
% endif
</tr>
% for test_case in http2_cases:
<tr><td><b>${test_case}</b></td>
## Fill up the cells with test result.
% for server_lang in server_langs:
<%
shortname = 'cloud_to_cloud:http2:%s_server:%s' % (
server_lang, test_case)
%>
${fill_one_test_result(shortname, resultset)}
% endfor
% if cloud_to_prod:
<% shortname = 'cloud_to_prod:http2:%s' % test_case %>
${fill_one_test_result(shortname, resultset)}
% endif
</tr>
% endfor
</table>
% endif
% if server_langs:
% for test_case in test_cases:
## Each column header is the client language.
<h2>${test_case}</h2>
<table style="width:100%" border="1">
<tr bgcolor="#00BFFF">
<th>Client languages &#9658;<br/>Server languages &#9660;</th>
% for client_lang in client_langs:
<th>${client_lang}</th>
% endfor
</tr>
## Each row head is the server language.
% for server_lang in server_langs:
<tr>
<td><b>${server_lang}</b></td>
% for client_lang in client_langs:
<%
shortname = 'cloud_to_cloud:%s:%s_server:%s' % (
client_lang, server_lang, test_case)
%>
${fill_one_test_result(shortname, resultset)}
% endfor
</tr>
% endfor
</table>
% endfor
% endif
<script>
$(document).ready(function(){$('[data-toggle="tooltip"]').tooltip();});
</script>
</body>
</html>

@ -29,6 +29,11 @@
"""Generate XML and HTML test reports."""
try:
from mako.runtime import Context
from mako.template import Template
except (ImportError):
pass # Mako not installed but it is ok.
import os
import string
import xml.etree.cElementTree as ET
@ -49,7 +54,7 @@ def _filter_msg(msg, output_format):
return msg
def render_xml_report(resultset, xml_report):
def render_junit_xml_report(resultset, xml_report):
"""Generate JUnit-like XML report."""
root = ET.Element('testsuites')
testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc',
@ -69,147 +74,36 @@ def render_xml_report(resultset, xml_report):
tree.write(xml_report, encoding='UTF-8')
# TODO(adelez): Use mako template.
def fill_one_test_result(shortname, resultset, html_str):
if shortname in resultset:
# Because interop tests does not have runs_per_test flag, each test is run
# once. So there should only be one element for each result.
result = resultset[shortname][0]
if result.state == 'PASSED':
html_str = '%s<td bgcolor=\"green\">PASS</td>\n' % html_str
else:
tooltip = ''
if result.returncode > 0 or result.message:
if result.returncode > 0:
tooltip = 'returncode: %d ' % result.returncode
if result.message:
escaped_msg = _filter_msg(result.message, 'HTML')
tooltip = '%smessage: %s' % (tooltip, escaped_msg)
if result.state == 'FAILED':
html_str = '%s<td bgcolor=\"red\">' % html_str
if tooltip:
html_str = ('%s<a href=\"#\" data-toggle=\"tooltip\" '
'data-placement=\"auto\" title=\"%s\">FAIL</a></td>\n' %
(html_str, tooltip))
else:
html_str = '%sFAIL</td>\n' % html_str
elif result.state == 'TIMEOUT':
html_str = '%s<td bgcolor=\"yellow\">' % html_str
if tooltip:
html_str = ('%s<a href=\"#\" data-toggle=\"tooltip\" '
'data-placement=\"auto\" title=\"%s\">TIMEOUT</a></td>\n'
% (html_str, tooltip))
else:
html_str = '%sTIMEOUT</td>\n' % html_str
else:
html_str = '%s<td bgcolor=\"magenta\">Not implemented</td>\n' % html_str
return html_str
def render_interop_html_report(
client_langs, server_langs, test_cases, auth_test_cases, http2_cases,
resultset, num_failures, cloud_to_prod, http2_interop):
"""Generate HTML report for interop tests."""
html_report_dir = 'reports'
template_file = os.path.join(html_report_dir, 'interop_html_report.template')
try:
mytemplate = Template(filename=template_file, format_exceptions=True)
except NameError:
print 'Mako template is not installed. Skipping HTML report generation.'
return
except IOError as e:
print 'Failed to find the template %s: %s' % (template_file, e)
return
def render_html_report(client_langs, server_langs, test_cases, auth_test_cases,
http2_cases, resultset, num_failures, cloud_to_prod,
http2_interop):
"""Generate html report."""
sorted_test_cases = sorted(test_cases)
sorted_auth_test_cases = sorted(auth_test_cases)
sorted_http2_cases = sorted(http2_cases)
sorted_client_langs = sorted(client_langs)
sorted_server_langs = sorted(server_langs)
html_str = ('<!DOCTYPE html>\n'
'<html lang=\"en\">\n'
'<head><title>Interop Test Result</title></head>\n'
'<body>\n')
if num_failures > 1:
html_str = (
'%s<p><h2><font color=\"red\">%d tests failed!</font></h2></p>\n' %
(html_str, num_failures))
elif num_failures:
html_str = (
'%s<p><h2><font color=\"red\">%d test failed!</font></h2></p>\n' %
(html_str, num_failures))
else:
html_str = (
'%s<p><h2><font color=\"green\">All tests passed!</font></h2></p>\n' %
html_str)
if cloud_to_prod:
# Each column header is the client language.
html_str = ('%s<h2>Cloud to Prod</h2>\n'
'<table style=\"width:100%%\" border=\"1\">\n'
'<tr bgcolor=\"#00BFFF\">\n'
'<th>Client languages &#9658;</th>\n') % html_str
for client_lang in sorted_client_langs:
html_str = '%s<th>%s\n' % (html_str, client_lang)
html_str = '%s</tr>\n' % html_str
for test_case in sorted_test_cases + sorted_auth_test_cases:
html_str = '%s<tr><td><b>%s</b></td>\n' % (html_str, test_case)
for client_lang in sorted_client_langs:
if not test_case in sorted_auth_test_cases:
shortname = 'cloud_to_prod:%s:%s' % (client_lang, test_case)
else:
shortname = 'cloud_to_prod_auth:%s:%s' % (client_lang, test_case)
html_str = fill_one_test_result(shortname, resultset, html_str)
html_str = '%s</tr>\n' % html_str
html_str = '%s</table>\n' % html_str
if http2_interop:
# Each column header is the server language.
html_str = ('%s<h2>HTTP/2 Interop</h2>\n'
'<table style=\"width:100%%\" border=\"1\">\n'
'<tr bgcolor=\"#00BFFF\">\n'
'<th>Servers &#9658;<br/>'
'Test Cases &#9660;</th>\n') % html_str
for server_lang in sorted_server_langs:
html_str = '%s<th>%s\n' % (html_str, server_lang)
if cloud_to_prod:
html_str = '%s<th>%s\n' % (html_str, "prod")
html_str = '%s</tr>\n' % html_str
for test_case in sorted_http2_cases:
html_str = '%s<tr><td><b>%s</b></td>\n' % (html_str, test_case)
# Fill up the cells with test result.
for server_lang in sorted_server_langs:
shortname = 'cloud_to_cloud:%s:%s_server:%s' % (
"http2", server_lang, test_case)
html_str = fill_one_test_result(shortname, resultset, html_str)
if cloud_to_prod:
shortname = 'cloud_to_prod:%s:%s' % ("http2", test_case)
html_str = fill_one_test_result(shortname, resultset, html_str)
html_str = '%s</tr>\n' % html_str
html_str = '%s</table>\n' % html_str
if server_langs:
for test_case in sorted_test_cases:
# Each column header is the client language.
html_str = ('%s<h2>%s</h2>\n'
'<table style=\"width:100%%\" border=\"1\">\n'
'<tr bgcolor=\"#00BFFF\">\n'
'<th>Client languages &#9658;<br/>'
'Server languages &#9660;</th>\n') % (html_str, test_case)
for client_lang in sorted_client_langs:
html_str = '%s<th>%s\n' % (html_str, client_lang)
html_str = '%s</tr>\n' % html_str
# Each row head is the server language.
for server_lang in sorted_server_langs:
html_str = '%s<tr><td><b>%s</b></td>\n' % (html_str, server_lang)
# Fill up the cells with test result.
for client_lang in sorted_client_langs:
shortname = 'cloud_to_cloud:%s:%s_server:%s' % (
client_lang, server_lang, test_case)
html_str = fill_one_test_result(shortname, resultset, html_str)
html_str = '%s</tr>\n' % html_str
html_str = '%s</table>\n' % html_str
html_str = ('%s\n'
'<script>\n'
'$(document).ready(function(){'
'$(\'[data-toggle=\"tooltip\"]\').tooltip();\n'
'});\n'
'</script>\n'
'</body>\n'
'</html>') % html_str
# Write to reports/index.html as set up in Jenkins plugin.
html_report_dir = 'reports'
if not os.path.exists(html_report_dir):
os.mkdir(html_report_dir)
args = {'client_langs': sorted_client_langs,
'server_langs': sorted_server_langs,
'test_cases': sorted_test_cases,
'auth_test_cases': sorted_auth_test_cases,
'http2_cases': sorted_http2_cases,
'resultset': resultset,
'num_failures': num_failures,
'cloud_to_prod': cloud_to_prod,
'http2_interop': http2_interop}
html_file_path = os.path.join(html_report_dir, 'index.html')
with open(html_file_path, 'w') as f:
f.write(html_str)
with open(html_file_path, 'w') as output_file:
mytemplate.render_context(Context(output_file, **args))

@ -686,9 +686,9 @@ try:
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)
report_utils.render_xml_report(resultset, 'report.xml')
report_utils.render_junit_xml_report(resultset, 'report.xml')
report_utils.render_html_report(
report_utils.render_interop_html_report(
set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES,
_HTTP2_TEST_CASES, resultset, num_failures,
args.cloud_to_prod_auth or args.cloud_to_prod, args.http2_interop)

@ -902,7 +902,7 @@ def _build_and_run(
for antagonist in antagonists:
antagonist.kill()
if xml_report and resultset:
report_utils.render_xml_report(resultset, xml_report)
report_utils.render_junit_xml_report(resultset, xml_report)
number_failures, _ = jobset.run(
post_tests_steps, maxjobs=1, stop_on_failure=True,

Loading…
Cancel
Save