Chiebot-Mirror
/
googletest
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

gtest: Output a canned test case for test suite setup / teardown failures in XML/JSON

Browse Source 
This surfaces useful information about the environment failure in a structured form.

As we can see from the updated test, previously unsurfaced information is now present.

PiperOrigin-RevId: 709892315
Change-Id: I2656294d50c33f995bef5c96195a66cff3c4b907
pull/4677/merge
Abseil Team 5 months ago committed by Copybara-Service
parent e54519b094
commit 7d76a231b0
4 changed files with 198 additions and 28 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 50
      googletest/src/gtest.cc
  2. 95
      googletest/test/googletest-json-output-unittest.py
  3. 65
      googletest/test/gtest_xml_output_unittest.py
  4. 16
      googletest/test/gtest_xml_output_unittest_.cc

50
googletest/src/gtest.cc
Unescape View File

@ -3989,6 +3989,12 @@ class XmlUnitTestResultPrinter : public EmptyTestEventListener {
static void OutputXmlTestSuiteForTestResult(::std::ostream* stream,
const TestResult& result);
// Streams a test case XML stanza containing the given test result.
//
// Requires: result.Failed()
static void OutputXmlTestCaseForTestResult(::std::ostream* stream,
const TestResult& result);
// Streams an XML representation of a TestResult object.
static void OutputXmlTestResult(::std::ostream* stream,
const TestResult& result);
@ -4236,6 +4242,15 @@ void XmlUnitTestResultPrinter::OutputXmlTestSuiteForTestResult(
FormatEpochTimeInMillisAsIso8601(result.start_timestamp()));
*stream << ">";
OutputXmlTestCaseForTestResult(stream, result);
// Complete the test suite.
*stream << " </testsuite>\n";
}
// Streams a test case XML stanza containing the given test result.
void XmlUnitTestResultPrinter::OutputXmlTestCaseForTestResult(
::std::ostream* stream, const TestResult& result) {
// Output the boilerplate for a minimal test case with a single test.
*stream << " <testcase";
OutputXmlAttribute(stream, "testcase", "name", "");
@ -4250,9 +4265,6 @@ void XmlUnitTestResultPrinter::OutputXmlTestSuiteForTestResult(
// Output the actual test result.
OutputXmlTestResult(stream, result);
// Complete the test suite.
*stream << " </testsuite>\n";
}
// Prints an XML representation of a TestInfo object.
@ -4379,6 +4391,10 @@ void XmlUnitTestResultPrinter::PrintXmlTestSuite(std::ostream* stream,
if (test_suite.GetTestInfo(i)->is_reportable())
OutputXmlTestInfo(stream, test_suite.name(), *test_suite.GetTestInfo(i));
}
if (test_suite.ad_hoc_test_result().Failed()) {
OutputXmlTestCaseForTestResult(stream, test_suite.ad_hoc_test_result());
}
*stream << " </" << kTestsuite << ">\n";
}
@ -4518,6 +4534,12 @@ class JsonUnitTestResultPrinter : public EmptyTestEventListener {
static void OutputJsonTestSuiteForTestResult(::std::ostream* stream,
const TestResult& result);
// Streams a test case JSON stanza containing the given test result.
//
// Requires: result.Failed()
static void OutputJsonTestCaseForTestResult(::std::ostream* stream,
const TestResult& result);
// Streams a JSON representation of a TestResult object.
static void OutputJsonTestResult(::std::ostream* stream,
const TestResult& result);
@ -4688,6 +4710,15 @@ void JsonUnitTestResultPrinter::OutputJsonTestSuiteForTestResult(
}
*stream << Indent(6) << "\"testsuite\": [\n";
OutputJsonTestCaseForTestResult(stream, result);
// Finish the test suite.
*stream << "\n" << Indent(6) << "]\n" << Indent(4) << "}";
}
// Streams a test case JSON stanza containing the given test result.
void JsonUnitTestResultPrinter::OutputJsonTestCaseForTestResult(
::std::ostream* stream, const TestResult& result) {
// Output the boilerplate for a new test case.
*stream << Indent(8) << "{\n";
OutputJsonKey(stream, "testcase", "name", "", Indent(10));
@ -4704,9 +4735,6 @@ void JsonUnitTestResultPrinter::OutputJsonTestSuiteForTestResult(
// Output the actual test result.
OutputJsonTestResult(stream, result);
// Finish the test suite.
*stream << "\n" << Indent(6) << "]\n" << Indent(4) << "}";
}
// Prints a JSON representation of a TestInfo object.
@ -4851,6 +4879,16 @@ void JsonUnitTestResultPrinter::PrintJsonTestSuite(
OutputJsonTestInfo(stream, test_suite.name(), *test_suite.GetTestInfo(i));
}
}
// If there was a failure in the test suite setup or teardown include that in
// the output.
if (test_suite.ad_hoc_test_result().Failed()) {
if (comma) {
*stream << ",\n";
}
OutputJsonTestCaseForTestResult(stream, test_suite.ad_hoc_test_result());
}
*stream << "\n" << kIndent << "]\n" << Indent(4) << "}";
}

95
googletest/test/googletest-json-output-unittest.py
Unescape View File

@ -57,7 +57,7 @@ else:
STACK_TRACE_TEMPLATE = '\n'
EXPECTED_NON_EMPTY = {
'tests': 26,
'tests': 28,
'failures': 5,
'disabled': 2,
'errors': 0,
@ -419,6 +419,83 @@ EXPECTED_NON_EMPTY = {
},
],
},
{
'name': 'SetupFailTest',
'tests': 1,
'failures': 0,
'disabled': 0,
'errors': 0,
'time': '*',
'timestamp': '*',
'testsuite': [
{
'name': 'NoopPassingTest',
'file': 'gtest_xml_output_unittest_.cc',
'line': 168,
'status': 'RUN',
'result': 'SKIPPED',
'timestamp': '*',
'time': '*',
'classname': 'SetupFailTest',
'skipped': [
{'message': 'gtest_xml_output_unittest_.cc:*\n'}
],
},
{
'name': '',
'status': 'RUN',
'result': 'COMPLETED',
'timestamp': '*',
'time': '*',
'classname': '',
'failures': [{
'failure': (
'gtest_xml_output_unittest_.cc:*\nExpected equality'
' of these values:\n 1\n 2'
+ STACK_TRACE_TEMPLATE
),
'type': '',
}],
},
],
},
{
'name': 'TearDownFailTest',
'tests': 1,
'failures': 0,
'disabled': 0,
'errors': 0,
'timestamp': '*',
'time': '*',
'testsuite': [
{
'name': 'NoopPassingTest',
'file': 'gtest_xml_output_unittest_.cc',
'line': 175,
'status': 'RUN',
'result': 'COMPLETED',
'timestamp': '*',
'time': '*',
'classname': 'TearDownFailTest',
},
{
'name': '',
'status': 'RUN',
'result': 'COMPLETED',
'timestamp': '*',
'time': '*',
'classname': '',
'failures': [{
'failure': (
'gtest_xml_output_unittest_.cc:*\nExpected equality'
' of these values:\n 1\n 2'
+ STACK_TRACE_TEMPLATE
),
'type': '',
}],
},
],
},
{
'name': 'TypedTest/0',
'tests': 1,
@ -431,7 +508,7 @@ EXPECTED_NON_EMPTY = {
'name': 'HasTypeParamAttribute',
'type_param': 'int',
'file': 'gtest_xml_output_unittest_.cc',
'line': 173,
'line': 189,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
@ -451,7 +528,7 @@ EXPECTED_NON_EMPTY = {
'name': 'HasTypeParamAttribute',
'type_param': 'long',
'file': 'gtest_xml_output_unittest_.cc',
'line': 173,
'line': 189,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
@ -471,7 +548,7 @@ EXPECTED_NON_EMPTY = {
'name': 'HasTypeParamAttribute',
'type_param': 'int',
'file': 'gtest_xml_output_unittest_.cc',
'line': 180,
'line': 196,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
@ -491,7 +568,7 @@ EXPECTED_NON_EMPTY = {
'name': 'HasTypeParamAttribute',
'type_param': 'long',
'file': 'gtest_xml_output_unittest_.cc',
'line': 180,
'line': 196,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
@ -512,7 +589,7 @@ EXPECTED_NON_EMPTY = {
'name': 'HasValueParamAttribute/0',
'value_param': '33',
'file': 'gtest_xml_output_unittest_.cc',
'line': 164,
'line': 180,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
@ -523,7 +600,7 @@ EXPECTED_NON_EMPTY = {
'name': 'HasValueParamAttribute/1',
'value_param': '42',
'file': 'gtest_xml_output_unittest_.cc',
'line': 164,
'line': 180,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
@ -534,7 +611,7 @@ EXPECTED_NON_EMPTY = {
'name': 'AnotherTestThatHasValueParamAttribute/0',
'value_param': '33',
'file': 'gtest_xml_output_unittest_.cc',
'line': 165,
'line': 181,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
@ -545,7 +622,7 @@ EXPECTED_NON_EMPTY = {
'name': 'AnotherTestThatHasValueParamAttribute/1',
'value_param': '42',
'file': 'gtest_xml_output_unittest_.cc',
'line': 165,
'line': 181,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',

65
googletest/test/gtest_xml_output_unittest.py
Unescape View File

@ -29,14 +29,14 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Unit test for the gtest_xml_output module"""
"""Unit test for the gtest_xml_output module."""
import datetime
import errno
import os
import re
import sys
from xml.dom import minidom, Node
from xml.dom import minidom
from googletest.test import gtest_test_utils
from googletest.test import gtest_xml_test_utils
@ -67,7 +67,7 @@ else:
sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG)
EXPECTED_NON_EMPTY_XML = """<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="26" failures="5" disabled="2" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
<testsuites tests="28" failures="5" disabled="2" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="Succeeds" file="gtest_xml_output_unittest_.cc" line="53" status="run" result="completed" time="*" timestamp="*" classname="SuccessfulTest"/>
</testsuite>
@ -173,23 +173,44 @@ It is good practice to tell why you skip a test.
</properties>
</testcase>
</testsuite>
<testsuite name="SetupFailTest" tests="1" failures="0" disabled="0" skipped="1" errors="0" time="*" timestamp="*">
<testcase name="NoopPassingTest" file="gtest_xml_output_unittest_.cc" line="168" status="run" result="skipped" time="*" timestamp="*" classname="SetupFailTest">
<skipped message="gtest_xml_output_unittest_.cc:*&#x0A;"><![CDATA[gtest_xml_output_unittest_.cc:*
]]></skipped>
</testcase>
<testcase name="" status="run" result="completed" classname="" time="*" timestamp="*">
<failure message="gtest_xml_output_unittest_.cc:*&#x0A;Expected equality of these values:&#x0A; 1&#x0A; 2%(stack_entity)s" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
Expected equality of these values:
1
2%(stack)s]]></failure>
</testcase>
</testsuite>
<testsuite name="TearDownFailTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="NoopPassingTest" file="gtest_xml_output_unittest_.cc" line="175" status="run" result="completed" time="*" timestamp="*" classname="TearDownFailTest"/>
<testcase name="" status="run" result="completed" classname="" time="*" timestamp="*">
<failure message="gtest_xml_output_unittest_.cc:*&#x0A;Expected equality of these values:&#x0A; 1&#x0A; 2%(stack_entity)s" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
Expected equality of these values:
1
2%(stack)s]]></failure>
</testcase>
</testsuite>
<testsuite name="Single/ValueParamTest" tests="4" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="164" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="HasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="164" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="AnotherTestThatHasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="165" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="AnotherTestThatHasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="165" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="HasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="180" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="HasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="180" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="AnotherTestThatHasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="181" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="AnotherTestThatHasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="181" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
</testsuite>
<testsuite name="TypedTest/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="173" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/0" />
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="189" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/0" />
</testsuite>
<testsuite name="TypedTest/1" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="173" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/1" />
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="189" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/1" />
</testsuite>
<testsuite name="Single/TypeParameterizedTestSuite/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="180" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0" />
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="196" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0" />
</testsuite>
<testsuite name="Single/TypeParameterizedTestSuite/1" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="180" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/1" />
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="196" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/1" />
</testsuite>
</testsuites>""" % {
'stack': STACK_TRACE_TEMPLATE,
@ -205,6 +226,24 @@ EXPECTED_FILTERED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?>
</testsuite>
</testsuites>"""
ACTUAL_OUTPUT = """<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="3" failures="0" disabled="0" errors="0" time="*" timestamp="*" ad_hoc_property="42" name="AllTests">
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="Succeeds" file="gtest_xml_output_unittest_.cc" line="53" status="run" result="completed" time="*" timestamp="*" classname="SuccessfulTest"/>
</testsuite>
<testsuite name="PropertyRecordingTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*" SetUpTestSuite="yes" TearDownTestSuite="aye">
<testcase name="IntValuedProperty" file="gtest_xml_output_unittest_.cc" line="125" status="run" result="completed" time="*" timestamp="*" classname="PropertyRecordingTest">
<properties>
<property name="key_int" value="1"/>
</properties>
</testcase>
</testsuite>
<testsuite name="Single/TypeParameterizedTestSuite/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" type_param="int" file="gtest_xml_output_unittest_.cc" line="196" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0"/>
</testsuite>
</testsuites>
"""
EXPECTED_SHARDED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="3" failures="0" disabled="0" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
@ -217,8 +256,8 @@ EXPECTED_SHARDED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?>
</properties>
</testcase>
</testsuite>
<testsuite name="Single/ValueParamTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="164" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testsuite name="Single/TypeParameterizedTestSuite/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" type_param="*" file="gtest_xml_output_unittest_.cc" line="196" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0" />
</testsuite>
</testsuites>"""

16
googletest/test/gtest_xml_output_unittest_.cc
Unescape View File

@ -158,6 +158,22 @@ TEST(NoFixtureTest, ExternalUtilityThatCallsRecordStringValuedProperty) {
ExternalUtilityThatCallsRecordProperty("key_for_utility_string", "1");
}
// Ensures that SetUpTestSuite and TearDownTestSuite failures are reported in
// the XML output.
class SetupFailTest : public ::testing::Test {
protected:
static void SetUpTestSuite() { ASSERT_EQ(1, 2); }
};
TEST_F(SetupFailTest, NoopPassingTest) {}
class TearDownFailTest : public ::testing::Test {
protected:
static void TearDownTestSuite() { ASSERT_EQ(1, 2); }
};
TEST_F(TearDownFailTest, NoopPassingTest) {}
// Verifies that the test parameter value is output in the 'value_param'
// XML attribute for value-parameterized tests.
class ValueParamTest : public TestWithParam<int> {};

Write Preview
Loading…
Cancel
Save