diff options
| author | Lexi Winter <lexi@le-fay.org> | 2025-06-29 19:25:29 +0100 |
|---|---|---|
| committer | Lexi Winter <lexi@le-fay.org> | 2025-06-29 19:25:29 +0100 |
| commit | bc524d70253a4ab2fe40c3ca3e5666e267c0a4d1 (patch) | |
| tree | 1e629e7b46b1d9972a973bc93fd100bcebd395be /src/catch2/reporters | |
| download | nihil-bc524d70253a4ab2fe40c3ca3e5666e267c0a4d1.tar.gz nihil-bc524d70253a4ab2fe40c3ca3e5666e267c0a4d1.tar.bz2 | |
import catch2 3.8.1vendor/catch2/3.8.1vendor/catch2
Diffstat (limited to 'src/catch2/reporters')
33 files changed, 4616 insertions, 0 deletions
diff --git a/src/catch2/reporters/catch_reporter_automake.cpp b/src/catch2/reporters/catch_reporter_automake.cpp new file mode 100644 index 0000000..5e506a6 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_automake.cpp @@ -0,0 +1,37 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#include <catch2/reporters/catch_reporter_automake.hpp> +#include <catch2/catch_test_case_info.hpp> + +#include <ostream> + +namespace Catch { + + AutomakeReporter::~AutomakeReporter() = default; + + void AutomakeReporter::testCaseEnded(TestCaseStats const& _testCaseStats) { + // Possible values to emit are PASS, XFAIL, SKIP, FAIL, XPASS and ERROR. + m_stream << ":test-result: "; + if ( _testCaseStats.totals.testCases.skipped > 0 ) { + m_stream << "SKIP"; + } else if (_testCaseStats.totals.assertions.allPassed()) { + m_stream << "PASS"; + } else if (_testCaseStats.totals.assertions.allOk()) { + m_stream << "XFAIL"; + } else { + m_stream << "FAIL"; + } + m_stream << ' ' << _testCaseStats.testInfo->name << '\n'; + StreamingReporterBase::testCaseEnded(_testCaseStats); + } + + void AutomakeReporter::skipTest(TestCaseInfo const& testInfo) { + m_stream << ":test-result: SKIP " << testInfo.name << '\n'; + } + +} // end namespace Catch diff --git a/src/catch2/reporters/catch_reporter_automake.hpp b/src/catch2/reporters/catch_reporter_automake.hpp new file mode 100644 index 0000000..a639428 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_automake.hpp @@ -0,0 +1,38 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef CATCH_REPORTER_AUTOMAKE_HPP_INCLUDED +#define CATCH_REPORTER_AUTOMAKE_HPP_INCLUDED + +#include <catch2/reporters/catch_reporter_streaming_base.hpp> +#include <catch2/internal/catch_move_and_forward.hpp> + +#include <string> + +namespace Catch { + + class AutomakeReporter final : public StreamingReporterBase { + public: + // GCC5 compat: we cannot use inherited constructor, because it + // doesn't implement backport of P0136 + AutomakeReporter(ReporterConfig&& _config): + StreamingReporterBase(CATCH_MOVE(_config)) + {} + ~AutomakeReporter() override; + + static std::string getDescription() { + using namespace std::string_literals; + return "Reports test results in the format of Automake .trs files"s; + } + + void testCaseEnded(TestCaseStats const& _testCaseStats) override; + void skipTest(TestCaseInfo const& testInfo) override; + }; + +} // end namespace Catch + +#endif // CATCH_REPORTER_AUTOMAKE_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporter_common_base.cpp b/src/catch2/reporters/catch_reporter_common_base.cpp new file mode 100644 index 0000000..a1ca76a --- /dev/null +++ b/src/catch2/reporters/catch_reporter_common_base.cpp @@ -0,0 +1,49 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +#include <catch2/reporters/catch_reporter_common_base.hpp> + +#include <catch2/reporters/catch_reporter_helpers.hpp> +#include <catch2/internal/catch_console_colour.hpp> +#include <catch2/internal/catch_istream.hpp> + + +namespace Catch { + ReporterBase::ReporterBase( ReporterConfig&& config ): + IEventListener( config.fullConfig() ), + m_wrapped_stream( CATCH_MOVE(config).takeStream() ), + m_stream( m_wrapped_stream->stream() ), + m_colour( makeColourImpl( config.colourMode(), m_wrapped_stream.get() ) ), + m_customOptions( config.customOptions() ) + {} + + ReporterBase::~ReporterBase() = default; + + void ReporterBase::listReporters( + std::vector<ReporterDescription> const& descriptions ) { + defaultListReporters(m_stream, descriptions, m_config->verbosity()); + } + + void ReporterBase::listListeners( + std::vector<ListenerDescription> const& descriptions ) { + defaultListListeners( m_stream, descriptions ); + } + + void ReporterBase::listTests(std::vector<TestCaseHandle> const& tests) { + defaultListTests(m_stream, + m_colour.get(), + tests, + m_config->hasTestFilters(), + m_config->verbosity()); + } + + void ReporterBase::listTags(std::vector<TagInfo> const& tags) { + defaultListTags( m_stream, tags, m_config->hasTestFilters() ); + } + +} // namespace Catch diff --git a/src/catch2/reporters/catch_reporter_common_base.hpp b/src/catch2/reporters/catch_reporter_common_base.hpp new file mode 100644 index 0000000..b4f0a9f --- /dev/null +++ b/src/catch2/reporters/catch_reporter_common_base.hpp @@ -0,0 +1,79 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef CATCH_REPORTER_COMMON_BASE_HPP_INCLUDED +#define CATCH_REPORTER_COMMON_BASE_HPP_INCLUDED + +#include <catch2/interfaces/catch_interfaces_reporter.hpp> + +#include <map> +#include <string> + +namespace Catch { + class ColourImpl; + + /** + * This is the base class for all reporters. + * + * If are writing a reporter, you must derive from this type, or one + * of the helper reporter bases that are derived from this type. + * + * ReporterBase centralizes handling of various common tasks in reporters, + * like storing the right stream for the reporters to write to, and + * providing the default implementation of the different listing events. + */ + class ReporterBase : public IEventListener { + protected: + //! The stream wrapper as passed to us by outside code + Detail::unique_ptr<IStream> m_wrapped_stream; + //! Cached output stream from `m_wrapped_stream` to reduce + //! number of indirect calls needed to write output. + std::ostream& m_stream; + //! Colour implementation this reporter was configured for + Detail::unique_ptr<ColourImpl> m_colour; + //! The custom reporter options user passed down to the reporter + std::map<std::string, std::string> m_customOptions; + + public: + ReporterBase( ReporterConfig&& config ); + ~ReporterBase() override; // = default; + + /** + * Provides a simple default listing of reporters. + * + * Should look roughly like the reporter listing in v2 and earlier + * versions of Catch2. + */ + void listReporters( + std::vector<ReporterDescription> const& descriptions ) override; + /** + * Provides a simple default listing of listeners + * + * Looks similarly to listing of reporters, but with listener type + * instead of reporter name. + */ + void listListeners( + std::vector<ListenerDescription> const& descriptions ) override; + /** + * Provides a simple default listing of tests. + * + * Should look roughly like the test listing in v2 and earlier versions + * of Catch2. Especially supports low-verbosity listing that mimics the + * old `--list-test-names-only` output. + */ + void listTests( std::vector<TestCaseHandle> const& tests ) override; + /** + * Provides a simple default listing of tags. + * + * Should look roughly like the tag listing in v2 and earlier versions + * of Catch2. + */ + void listTags( std::vector<TagInfo> const& tags ) override; + }; +} // namespace Catch + +#endif // CATCH_REPORTER_COMMON_BASE_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporter_compact.cpp b/src/catch2/reporters/catch_reporter_compact.cpp new file mode 100644 index 0000000..0f85594 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_compact.cpp @@ -0,0 +1,254 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#include <catch2/reporters/catch_reporter_compact.hpp> + +#include <catch2/catch_get_random_seed.hpp> +#include <catch2/catch_test_spec.hpp> +#include <catch2/reporters/catch_reporter_helpers.hpp> +#include <catch2/interfaces/catch_interfaces_config.hpp> +#include <catch2/internal/catch_platform.hpp> +#include <catch2/internal/catch_console_colour.hpp> +#include <catch2/internal/catch_string_manip.hpp> +#include <catch2/internal/catch_stringref.hpp> + +#include <ostream> + +namespace Catch { +namespace { + + // Colour::LightGrey + static constexpr Colour::Code compactDimColour = Colour::FileName; + +#ifdef CATCH_PLATFORM_MAC + static constexpr Catch::StringRef compactFailedString = "FAILED"_sr; + static constexpr Catch::StringRef compactPassedString = "PASSED"_sr; +#else + static constexpr Catch::StringRef compactFailedString = "failed"_sr; + static constexpr Catch::StringRef compactPassedString = "passed"_sr; +#endif + +// Implementation of CompactReporter formatting +class AssertionPrinter { +public: + AssertionPrinter& operator= (AssertionPrinter const&) = delete; + AssertionPrinter(AssertionPrinter const&) = delete; + AssertionPrinter(std::ostream& _stream, AssertionStats const& _stats, bool _printInfoMessages, ColourImpl* colourImpl_) + : stream(_stream) + , result(_stats.assertionResult) + , messages(_stats.infoMessages) + , itMessage(_stats.infoMessages.begin()) + , printInfoMessages(_printInfoMessages) + , colourImpl(colourImpl_) + {} + + void print() { + printSourceInfo(); + + itMessage = messages.begin(); + + switch (result.getResultType()) { + case ResultWas::Ok: + printResultType(Colour::ResultSuccess, compactPassedString); + printOriginalExpression(); + printReconstructedExpression(); + if (!result.hasExpression()) + printRemainingMessages(Colour::None); + else + printRemainingMessages(); + break; + case ResultWas::ExpressionFailed: + if (result.isOk()) + printResultType(Colour::ResultSuccess, compactFailedString + " - but was ok"_sr); + else + printResultType(Colour::Error, compactFailedString); + printOriginalExpression(); + printReconstructedExpression(); + printRemainingMessages(); + break; + case ResultWas::ThrewException: + printResultType(Colour::Error, compactFailedString); + printIssue("unexpected exception with message:"); + printMessage(); + printExpressionWas(); + printRemainingMessages(); + break; + case ResultWas::FatalErrorCondition: + printResultType(Colour::Error, compactFailedString); + printIssue("fatal error condition with message:"); + printMessage(); + printExpressionWas(); + printRemainingMessages(); + break; + case ResultWas::DidntThrowException: + printResultType(Colour::Error, compactFailedString); + printIssue("expected exception, got none"); + printExpressionWas(); + printRemainingMessages(); + break; + case ResultWas::Info: + printResultType(Colour::None, "info"_sr); + printMessage(); + printRemainingMessages(); + break; + case ResultWas::Warning: + printResultType(Colour::None, "warning"_sr); + printMessage(); + printRemainingMessages(); + break; + case ResultWas::ExplicitFailure: + printResultType(Colour::Error, compactFailedString); + printIssue("explicitly"); + printRemainingMessages(Colour::None); + break; + case ResultWas::ExplicitSkip: + printResultType(Colour::Skip, "skipped"_sr); + printMessage(); + printRemainingMessages(); + break; + // These cases are here to prevent compiler warnings + case ResultWas::Unknown: + case ResultWas::FailureBit: + case ResultWas::Exception: + printResultType(Colour::Error, "** internal error **"); + break; + } + } + +private: + void printSourceInfo() const { + stream << colourImpl->guardColour( Colour::FileName ) + << result.getSourceInfo() << ':'; + } + + void printResultType(Colour::Code colour, StringRef passOrFail) const { + if (!passOrFail.empty()) { + stream << colourImpl->guardColour(colour) << ' ' << passOrFail; + stream << ':'; + } + } + + void printIssue(char const* issue) const { + stream << ' ' << issue; + } + + void printExpressionWas() { + if (result.hasExpression()) { + stream << ';'; + { + stream << colourImpl->guardColour(compactDimColour) << " expression was:"; + } + printOriginalExpression(); + } + } + + void printOriginalExpression() const { + if (result.hasExpression()) { + stream << ' ' << result.getExpression(); + } + } + + void printReconstructedExpression() const { + if (result.hasExpandedExpression()) { + stream << colourImpl->guardColour(compactDimColour) << " for: "; + stream << result.getExpandedExpression(); + } + } + + void printMessage() { + if (itMessage != messages.end()) { + stream << " '" << itMessage->message << '\''; + ++itMessage; + } + } + + void printRemainingMessages(Colour::Code colour = compactDimColour) { + if (itMessage == messages.end()) + return; + + const auto itEnd = messages.cend(); + const auto N = static_cast<std::size_t>(itEnd - itMessage); + + stream << colourImpl->guardColour( colour ) << " with " + << pluralise( N, "message"_sr ) << ':'; + + while (itMessage != itEnd) { + // If this assertion is a warning ignore any INFO messages + if (printInfoMessages || itMessage->type != ResultWas::Info) { + printMessage(); + if (itMessage != itEnd) { + stream << colourImpl->guardColour(compactDimColour) << " and"; + } + continue; + } + ++itMessage; + } + } + +private: + std::ostream& stream; + AssertionResult const& result; + std::vector<MessageInfo> const& messages; + std::vector<MessageInfo>::const_iterator itMessage; + bool printInfoMessages; + ColourImpl* colourImpl; +}; + +} // anon namespace + + std::string CompactReporter::getDescription() { + return "Reports test results on a single line, suitable for IDEs"; + } + + void CompactReporter::noMatchingTestCases( StringRef unmatchedSpec ) { + m_stream << "No test cases matched '" << unmatchedSpec << "'\n"; + } + + void CompactReporter::testRunStarting( TestRunInfo const& ) { + if ( m_config->testSpec().hasFilters() ) { + m_stream << m_colour->guardColour( Colour::BrightYellow ) + << "Filters: " + << m_config->testSpec() + << '\n'; + } + m_stream << "RNG seed: " << getSeed() << '\n'; + } + + void CompactReporter::assertionEnded( AssertionStats const& _assertionStats ) { + AssertionResult const& result = _assertionStats.assertionResult; + + bool printInfoMessages = true; + + // Drop out if result was successful and we're not printing those + if( !m_config->includeSuccessfulResults() && result.isOk() ) { + if( result.getResultType() != ResultWas::Warning && result.getResultType() != ResultWas::ExplicitSkip ) + return; + printInfoMessages = false; + } + + AssertionPrinter printer( m_stream, _assertionStats, printInfoMessages, m_colour.get() ); + printer.print(); + + m_stream << '\n' << std::flush; + } + + void CompactReporter::sectionEnded(SectionStats const& _sectionStats) { + double dur = _sectionStats.durationInSeconds; + if ( shouldShowDuration( *m_config, dur ) ) { + m_stream << getFormattedDuration( dur ) << " s: " << _sectionStats.sectionInfo.name << '\n' << std::flush; + } + } + + void CompactReporter::testRunEnded( TestRunStats const& _testRunStats ) { + printTestRunTotals( m_stream, *m_colour, _testRunStats.totals ); + m_stream << "\n\n" << std::flush; + StreamingReporterBase::testRunEnded( _testRunStats ); + } + + CompactReporter::~CompactReporter() = default; + +} // end namespace Catch diff --git a/src/catch2/reporters/catch_reporter_compact.hpp b/src/catch2/reporters/catch_reporter_compact.hpp new file mode 100644 index 0000000..d95bbff --- /dev/null +++ b/src/catch2/reporters/catch_reporter_compact.hpp @@ -0,0 +1,39 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef CATCH_REPORTER_COMPACT_HPP_INCLUDED +#define CATCH_REPORTER_COMPACT_HPP_INCLUDED + + +#include <catch2/reporters/catch_reporter_streaming_base.hpp> + + +namespace Catch { + + class CompactReporter final : public StreamingReporterBase { + public: + using StreamingReporterBase::StreamingReporterBase; + + ~CompactReporter() override; + + static std::string getDescription(); + + void noMatchingTestCases( StringRef unmatchedSpec ) override; + + void testRunStarting( TestRunInfo const& _testInfo ) override; + + void assertionEnded(AssertionStats const& _assertionStats) override; + + void sectionEnded(SectionStats const& _sectionStats) override; + + void testRunEnded(TestRunStats const& _testRunStats) override; + + }; + +} // end namespace Catch + +#endif // CATCH_REPORTER_COMPACT_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporter_console.cpp b/src/catch2/reporters/catch_reporter_console.cpp new file mode 100644 index 0000000..9529f39 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_console.cpp @@ -0,0 +1,668 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#include <catch2/reporters/catch_reporter_console.hpp> + +#include <catch2/interfaces/catch_interfaces_config.hpp> +#include <catch2/catch_test_spec.hpp> +#include <catch2/internal/catch_console_colour.hpp> +#include <catch2/internal/catch_string_manip.hpp> +#include <catch2/catch_version.hpp> +#include <catch2/internal/catch_textflow.hpp> +#include <catch2/internal/catch_reusable_string_stream.hpp> +#include <catch2/internal/catch_stringref.hpp> +#include <catch2/catch_test_case_info.hpp> +#include <catch2/internal/catch_console_width.hpp> +#include <catch2/reporters/catch_reporter_helpers.hpp> +#include <catch2/internal/catch_move_and_forward.hpp> +#include <catch2/catch_get_random_seed.hpp> + +#include <cstdio> + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable:4061) // Not all labels are EXPLICITLY handled in switch + // Note that 4062 (not all labels are handled and default is missing) is enabled +#endif + +#if defined(__clang__) +# pragma clang diagnostic push +// For simplicity, benchmarking-only helpers are always enabled +# pragma clang diagnostic ignored "-Wunused-function" +#endif + + + +namespace Catch { + +namespace { + +// Formatter impl for ConsoleReporter +class ConsoleAssertionPrinter { +public: + ConsoleAssertionPrinter& operator= (ConsoleAssertionPrinter const&) = delete; + ConsoleAssertionPrinter(ConsoleAssertionPrinter const&) = delete; + ConsoleAssertionPrinter(std::ostream& _stream, AssertionStats const& _stats, ColourImpl* colourImpl_, bool _printInfoMessages) + : stream(_stream), + stats(_stats), + result(_stats.assertionResult), + colour(Colour::None), + messages(_stats.infoMessages), + colourImpl(colourImpl_), + printInfoMessages(_printInfoMessages) { + switch (result.getResultType()) { + case ResultWas::Ok: + colour = Colour::Success; + passOrFail = "PASSED"_sr; + //if( result.hasMessage() ) + if (messages.size() == 1) + messageLabel = "with message"_sr; + if (messages.size() > 1) + messageLabel = "with messages"_sr; + break; + case ResultWas::ExpressionFailed: + if (result.isOk()) { + colour = Colour::Success; + passOrFail = "FAILED - but was ok"_sr; + } else { + colour = Colour::Error; + passOrFail = "FAILED"_sr; + } + if (messages.size() == 1) + messageLabel = "with message"_sr; + if (messages.size() > 1) + messageLabel = "with messages"_sr; + break; + case ResultWas::ThrewException: + colour = Colour::Error; + passOrFail = "FAILED"_sr; + // todo switch + switch (messages.size()) { case 0: + messageLabel = "due to unexpected exception with "_sr; + break; + case 1: + messageLabel = "due to unexpected exception with message"_sr; + break; + default: + messageLabel = "due to unexpected exception with messages"_sr; + break; + } + break; + case ResultWas::FatalErrorCondition: + colour = Colour::Error; + passOrFail = "FAILED"_sr; + messageLabel = "due to a fatal error condition"_sr; + break; + case ResultWas::DidntThrowException: + colour = Colour::Error; + passOrFail = "FAILED"_sr; + messageLabel = "because no exception was thrown where one was expected"_sr; + break; + case ResultWas::Info: + messageLabel = "info"_sr; + break; + case ResultWas::Warning: + messageLabel = "warning"_sr; + break; + case ResultWas::ExplicitFailure: + passOrFail = "FAILED"_sr; + colour = Colour::Error; + if (messages.size() == 1) + messageLabel = "explicitly with message"_sr; + if (messages.size() > 1) + messageLabel = "explicitly with messages"_sr; + break; + case ResultWas::ExplicitSkip: + colour = Colour::Skip; + passOrFail = "SKIPPED"_sr; + if (messages.size() == 1) + messageLabel = "explicitly with message"_sr; + if (messages.size() > 1) + messageLabel = "explicitly with messages"_sr; + break; + // These cases are here to prevent compiler warnings + case ResultWas::Unknown: + case ResultWas::FailureBit: + case ResultWas::Exception: + passOrFail = "** internal error **"_sr; + colour = Colour::Error; + break; + } + } + + void print() const { + printSourceInfo(); + if (stats.totals.assertions.total() > 0) { + printResultType(); + printOriginalExpression(); + printReconstructedExpression(); + } else { + stream << '\n'; + } + printMessage(); + } + +private: + void printResultType() const { + if (!passOrFail.empty()) { + stream << colourImpl->guardColour(colour) << passOrFail << ":\n"; + } + } + void printOriginalExpression() const { + if (result.hasExpression()) { + stream << colourImpl->guardColour( Colour::OriginalExpression ) + << " " << result.getExpressionInMacro() << '\n'; + } + } + void printReconstructedExpression() const { + if (result.hasExpandedExpression()) { + stream << "with expansion:\n"; + stream << colourImpl->guardColour( Colour::ReconstructedExpression ) + << TextFlow::Column( result.getExpandedExpression() ) + .indent( 2 ) + << '\n'; + } + } + void printMessage() const { + if (!messageLabel.empty()) + stream << messageLabel << ':' << '\n'; + for (auto const& msg : messages) { + // If this assertion is a warning ignore any INFO messages + if (printInfoMessages || msg.type != ResultWas::Info) + stream << TextFlow::Column(msg.message).indent(2) << '\n'; + } + } + void printSourceInfo() const { + stream << colourImpl->guardColour( Colour::FileName ) + << result.getSourceInfo() << ": "; + } + + std::ostream& stream; + AssertionStats const& stats; + AssertionResult const& result; + Colour::Code colour; + StringRef passOrFail; + StringRef messageLabel; + std::vector<MessageInfo> const& messages; + ColourImpl* colourImpl; + bool printInfoMessages; +}; + +std::size_t makeRatio( std::uint64_t number, std::uint64_t total ) { + const auto ratio = total > 0 ? CATCH_CONFIG_CONSOLE_WIDTH * number / total : 0; + return (ratio == 0 && number > 0) ? 1 : static_cast<std::size_t>(ratio); +} + +std::size_t& +findMax( std::size_t& i, std::size_t& j, std::size_t& k, std::size_t& l ) { + if (i > j && i > k && i > l) + return i; + else if (j > k && j > l) + return j; + else if (k > l) + return k; + else + return l; +} + +struct ColumnBreak {}; +struct RowBreak {}; +struct OutputFlush {}; + +class Duration { + enum class Unit : uint8_t { + Auto, + Nanoseconds, + Microseconds, + Milliseconds, + Seconds, + Minutes + }; + static const uint64_t s_nanosecondsInAMicrosecond = 1000; + static const uint64_t s_nanosecondsInAMillisecond = 1000 * s_nanosecondsInAMicrosecond; + static const uint64_t s_nanosecondsInASecond = 1000 * s_nanosecondsInAMillisecond; + static const uint64_t s_nanosecondsInAMinute = 60 * s_nanosecondsInASecond; + + double m_inNanoseconds; + Unit m_units; + +public: + explicit Duration(double inNanoseconds, Unit units = Unit::Auto) + : m_inNanoseconds(inNanoseconds), + m_units(units) { + if (m_units == Unit::Auto) { + if (m_inNanoseconds < s_nanosecondsInAMicrosecond) + m_units = Unit::Nanoseconds; + else if (m_inNanoseconds < s_nanosecondsInAMillisecond) + m_units = Unit::Microseconds; + else if (m_inNanoseconds < s_nanosecondsInASecond) + m_units = Unit::Milliseconds; + else if (m_inNanoseconds < s_nanosecondsInAMinute) + m_units = Unit::Seconds; + else + m_units = Unit::Minutes; + } + + } + + auto value() const -> double { + switch (m_units) { + case Unit::Microseconds: + return m_inNanoseconds / static_cast<double>(s_nanosecondsInAMicrosecond); + case Unit::Milliseconds: + return m_inNanoseconds / static_cast<double>(s_nanosecondsInAMillisecond); + case Unit::Seconds: + return m_inNanoseconds / static_cast<double>(s_nanosecondsInASecond); + case Unit::Minutes: + return m_inNanoseconds / static_cast<double>(s_nanosecondsInAMinute); + default: + return m_inNanoseconds; + } + } + StringRef unitsAsString() const { + switch (m_units) { + case Unit::Nanoseconds: + return "ns"_sr; + case Unit::Microseconds: + return "us"_sr; + case Unit::Milliseconds: + return "ms"_sr; + case Unit::Seconds: + return "s"_sr; + case Unit::Minutes: + return "m"_sr; + default: + return "** internal error **"_sr; + } + + } + friend auto operator << (std::ostream& os, Duration const& duration) -> std::ostream& { + return os << duration.value() << ' ' << duration.unitsAsString(); + } +}; +} // end anon namespace + +enum class Justification : uint8_t { + Left, + Right +}; + +struct ColumnInfo { + std::string name; + std::size_t width; + Justification justification; +}; + +class TablePrinter { + std::ostream& m_os; + std::vector<ColumnInfo> m_columnInfos; + ReusableStringStream m_oss; + int m_currentColumn = -1; + bool m_isOpen = false; + +public: + TablePrinter( std::ostream& os, std::vector<ColumnInfo> columnInfos ) + : m_os( os ), + m_columnInfos( CATCH_MOVE( columnInfos ) ) {} + + auto columnInfos() const -> std::vector<ColumnInfo> const& { + return m_columnInfos; + } + + void open() { + if (!m_isOpen) { + m_isOpen = true; + *this << RowBreak(); + + TextFlow::Columns headerCols; + for (auto const& info : m_columnInfos) { + assert(info.width > 2); + headerCols += TextFlow::Column(info.name).width(info.width - 2); + headerCols += TextFlow::Spacer( 2 ); + } + m_os << headerCols << '\n'; + + m_os << lineOfChars('-') << '\n'; + } + } + void close() { + if (m_isOpen) { + *this << RowBreak(); + m_os << '\n' << std::flush; + m_isOpen = false; + } + } + + template<typename T> + friend TablePrinter& operator<< (TablePrinter& tp, T const& value) { + tp.m_oss << value; + return tp; + } + + friend TablePrinter& operator<< (TablePrinter& tp, ColumnBreak) { + auto colStr = tp.m_oss.str(); + const auto strSize = colStr.size(); + tp.m_oss.str(""); + tp.open(); + if (tp.m_currentColumn == static_cast<int>(tp.m_columnInfos.size() - 1)) { + tp.m_currentColumn = -1; + tp.m_os << '\n'; + } + tp.m_currentColumn++; + + auto colInfo = tp.m_columnInfos[tp.m_currentColumn]; + auto padding = (strSize + 1 < colInfo.width) + ? std::string(colInfo.width - (strSize + 1), ' ') + : std::string(); + if (colInfo.justification == Justification::Left) + tp.m_os << colStr << padding << ' '; + else + tp.m_os << padding << colStr << ' '; + return tp; + } + + friend TablePrinter& operator<< (TablePrinter& tp, RowBreak) { + if (tp.m_currentColumn > 0) { + tp.m_os << '\n'; + tp.m_currentColumn = -1; + } + return tp; + } + + friend TablePrinter& operator<<(TablePrinter& tp, OutputFlush) { + tp.m_os << std::flush; + return tp; + } +}; + +ConsoleReporter::ConsoleReporter(ReporterConfig&& config): + StreamingReporterBase( CATCH_MOVE( config ) ), + m_tablePrinter(Detail::make_unique<TablePrinter>(m_stream, + [&config]() -> std::vector<ColumnInfo> { + if (config.fullConfig()->benchmarkNoAnalysis()) + { + return{ + { "benchmark name", CATCH_CONFIG_CONSOLE_WIDTH - 43, Justification::Left }, + { " samples", 14, Justification::Right }, + { " iterations", 14, Justification::Right }, + { " mean", 14, Justification::Right } + }; + } + else + { + return{ + { "benchmark name", CATCH_CONFIG_CONSOLE_WIDTH - 43, Justification::Left }, + { "samples mean std dev", 14, Justification::Right }, + { "iterations low mean low std dev", 14, Justification::Right }, + { "est run time high mean high std dev", 14, Justification::Right } + }; + } + }())) {} +ConsoleReporter::~ConsoleReporter() = default; + +std::string ConsoleReporter::getDescription() { + return "Reports test results as plain lines of text"; +} + +void ConsoleReporter::noMatchingTestCases( StringRef unmatchedSpec ) { + m_stream << "No test cases matched '" << unmatchedSpec << "'\n"; +} + +void ConsoleReporter::reportInvalidTestSpec( StringRef arg ) { + m_stream << "Invalid Filter: " << arg << '\n'; +} + +void ConsoleReporter::assertionStarting(AssertionInfo const&) {} + +void ConsoleReporter::assertionEnded(AssertionStats const& _assertionStats) { + AssertionResult const& result = _assertionStats.assertionResult; + + bool includeResults = m_config->includeSuccessfulResults() || !result.isOk(); + + // Drop out if result was successful but we're not printing them. + // TODO: Make configurable whether skips should be printed + if (!includeResults && result.getResultType() != ResultWas::Warning && result.getResultType() != ResultWas::ExplicitSkip) + return; + + lazyPrint(); + + ConsoleAssertionPrinter printer(m_stream, _assertionStats, m_colour.get(), includeResults); + printer.print(); + m_stream << '\n' << std::flush; +} + +void ConsoleReporter::sectionStarting(SectionInfo const& _sectionInfo) { + m_tablePrinter->close(); + m_headerPrinted = false; + StreamingReporterBase::sectionStarting(_sectionInfo); +} +void ConsoleReporter::sectionEnded(SectionStats const& _sectionStats) { + m_tablePrinter->close(); + if (_sectionStats.missingAssertions) { + lazyPrint(); + auto guard = + m_colour->guardColour( Colour::ResultError ).engage( m_stream ); + if (m_sectionStack.size() > 1) + m_stream << "\nNo assertions in section"; + else + m_stream << "\nNo assertions in test case"; + m_stream << " '" << _sectionStats.sectionInfo.name << "'\n\n" << std::flush; + } + double dur = _sectionStats.durationInSeconds; + if (shouldShowDuration(*m_config, dur)) { + m_stream << getFormattedDuration(dur) << " s: " << _sectionStats.sectionInfo.name << '\n' << std::flush; + } + if (m_headerPrinted) { + m_headerPrinted = false; + } + StreamingReporterBase::sectionEnded(_sectionStats); +} + +void ConsoleReporter::benchmarkPreparing( StringRef name ) { + lazyPrintWithoutClosingBenchmarkTable(); + + auto nameCol = TextFlow::Column( static_cast<std::string>( name ) ) + .width( m_tablePrinter->columnInfos()[0].width - 2 ); + + bool firstLine = true; + for (auto line : nameCol) { + if (!firstLine) + (*m_tablePrinter) << ColumnBreak() << ColumnBreak() << ColumnBreak(); + else + firstLine = false; + + (*m_tablePrinter) << line << ColumnBreak(); + } +} + +void ConsoleReporter::benchmarkStarting(BenchmarkInfo const& info) { + (*m_tablePrinter) << info.samples << ColumnBreak() + << info.iterations << ColumnBreak(); + if ( !m_config->benchmarkNoAnalysis() ) { + ( *m_tablePrinter ) + << Duration( info.estimatedDuration ) << ColumnBreak(); + } + ( *m_tablePrinter ) << OutputFlush{}; +} +void ConsoleReporter::benchmarkEnded(BenchmarkStats<> const& stats) { + if (m_config->benchmarkNoAnalysis()) + { + (*m_tablePrinter) << Duration(stats.mean.point.count()) << ColumnBreak(); + } + else + { + (*m_tablePrinter) << ColumnBreak() + << Duration(stats.mean.point.count()) << ColumnBreak() + << Duration(stats.mean.lower_bound.count()) << ColumnBreak() + << Duration(stats.mean.upper_bound.count()) << ColumnBreak() << ColumnBreak() + << Duration(stats.standardDeviation.point.count()) << ColumnBreak() + << Duration(stats.standardDeviation.lower_bound.count()) << ColumnBreak() + << Duration(stats.standardDeviation.upper_bound.count()) << ColumnBreak() << ColumnBreak() << ColumnBreak() << ColumnBreak() << ColumnBreak(); + } +} + +void ConsoleReporter::benchmarkFailed( StringRef error ) { + auto guard = m_colour->guardColour( Colour::Red ).engage( m_stream ); + (*m_tablePrinter) + << "Benchmark failed (" << error << ')' + << ColumnBreak() << RowBreak(); +} + +void ConsoleReporter::testCaseEnded(TestCaseStats const& _testCaseStats) { + m_tablePrinter->close(); + StreamingReporterBase::testCaseEnded(_testCaseStats); + m_headerPrinted = false; +} +void ConsoleReporter::testRunEnded(TestRunStats const& _testRunStats) { + printTotalsDivider(_testRunStats.totals); + printTestRunTotals( m_stream, *m_colour, _testRunStats.totals ); + m_stream << '\n' << std::flush; + StreamingReporterBase::testRunEnded(_testRunStats); +} +void ConsoleReporter::testRunStarting(TestRunInfo const& _testRunInfo) { + StreamingReporterBase::testRunStarting(_testRunInfo); + if ( m_config->testSpec().hasFilters() ) { + m_stream << m_colour->guardColour( Colour::BrightYellow ) << "Filters: " + << m_config->testSpec() << '\n'; + } + m_stream << "Randomness seeded to: " << getSeed() << '\n'; +} + +void ConsoleReporter::lazyPrint() { + + m_tablePrinter->close(); + lazyPrintWithoutClosingBenchmarkTable(); +} + +void ConsoleReporter::lazyPrintWithoutClosingBenchmarkTable() { + + if ( !m_testRunInfoPrinted ) { + lazyPrintRunInfo(); + } + if (!m_headerPrinted) { + printTestCaseAndSectionHeader(); + m_headerPrinted = true; + } +} +void ConsoleReporter::lazyPrintRunInfo() { + m_stream << '\n' + << lineOfChars( '~' ) << '\n' + << m_colour->guardColour( Colour::SecondaryText ) + << currentTestRunInfo.name << " is a Catch2 v" << libraryVersion() + << " host application.\n" + << "Run with -? for options\n\n"; + + m_testRunInfoPrinted = true; +} +void ConsoleReporter::printTestCaseAndSectionHeader() { + assert(!m_sectionStack.empty()); + printOpenHeader(currentTestCaseInfo->name); + + if (m_sectionStack.size() > 1) { + auto guard = m_colour->guardColour( Colour::Headers ).engage( m_stream ); + + auto + it = m_sectionStack.begin() + 1, // Skip first section (test case) + itEnd = m_sectionStack.end(); + for (; it != itEnd; ++it) + printHeaderString(it->name, 2); + } + + SourceLineInfo lineInfo = m_sectionStack.back().lineInfo; + + + m_stream << lineOfChars( '-' ) << '\n' + << m_colour->guardColour( Colour::FileName ) << lineInfo << '\n' + << lineOfChars( '.' ) << "\n\n" + << std::flush; +} + +void ConsoleReporter::printClosedHeader(std::string const& _name) { + printOpenHeader(_name); + m_stream << lineOfChars('.') << '\n'; +} +void ConsoleReporter::printOpenHeader(std::string const& _name) { + m_stream << lineOfChars('-') << '\n'; + { + auto guard = m_colour->guardColour( Colour::Headers ).engage( m_stream ); + printHeaderString(_name); + } +} + +void ConsoleReporter::printHeaderString(std::string const& _string, std::size_t indent) { + // We want to get a bit fancy with line breaking here, so that subsequent + // lines start after ":" if one is present, e.g. + // ``` + // blablabla: Fancy + // linebreaking + // ``` + // but we also want to avoid problems with overly long indentation causing + // the text to take up too many lines, e.g. + // ``` + // blablabla: F + // a + // n + // c + // y + // . + // . + // . + // ``` + // So we limit the prefix indentation check to first quarter of the possible + // width + std::size_t idx = _string.find( ": " ); + if ( idx != std::string::npos && idx < CATCH_CONFIG_CONSOLE_WIDTH / 4 ) { + idx += 2; + } else { + idx = 0; + } + m_stream << TextFlow::Column( _string ) + .indent( indent + idx ) + .initialIndent( indent ) + << '\n'; +} + +void ConsoleReporter::printTotalsDivider(Totals const& totals) { + if (totals.testCases.total() > 0) { + std::size_t failedRatio = makeRatio(totals.testCases.failed, totals.testCases.total()); + std::size_t failedButOkRatio = makeRatio(totals.testCases.failedButOk, totals.testCases.total()); + std::size_t passedRatio = makeRatio(totals.testCases.passed, totals.testCases.total()); + std::size_t skippedRatio = makeRatio(totals.testCases.skipped, totals.testCases.total()); + while (failedRatio + failedButOkRatio + passedRatio + skippedRatio < CATCH_CONFIG_CONSOLE_WIDTH - 1) + findMax(failedRatio, failedButOkRatio, passedRatio, skippedRatio)++; + while (failedRatio + failedButOkRatio + passedRatio > CATCH_CONFIG_CONSOLE_WIDTH - 1) + findMax(failedRatio, failedButOkRatio, passedRatio, skippedRatio)--; + + m_stream << m_colour->guardColour( Colour::Error ) + << std::string( failedRatio, '=' ) + << m_colour->guardColour( Colour::ResultExpectedFailure ) + << std::string( failedButOkRatio, '=' ); + if ( totals.testCases.allPassed() ) { + m_stream << m_colour->guardColour( Colour::ResultSuccess ) + << std::string( passedRatio, '=' ); + } else { + m_stream << m_colour->guardColour( Colour::Success ) + << std::string( passedRatio, '=' ); + } + m_stream << m_colour->guardColour( Colour::Skip ) + << std::string( skippedRatio, '=' ); + } else { + m_stream << m_colour->guardColour( Colour::Warning ) + << std::string( CATCH_CONFIG_CONSOLE_WIDTH - 1, '=' ); + } + m_stream << '\n'; +} + +} // end namespace Catch + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +#if defined(__clang__) +# pragma clang diagnostic pop +#endif diff --git a/src/catch2/reporters/catch_reporter_console.hpp b/src/catch2/reporters/catch_reporter_console.hpp new file mode 100644 index 0000000..2437726 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_console.hpp @@ -0,0 +1,67 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef CATCH_REPORTER_CONSOLE_HPP_INCLUDED +#define CATCH_REPORTER_CONSOLE_HPP_INCLUDED + +#include <catch2/reporters/catch_reporter_streaming_base.hpp> +#include <catch2/internal/catch_unique_ptr.hpp> + +namespace Catch { + // Fwd decls + class TablePrinter; + + class ConsoleReporter final : public StreamingReporterBase { + Detail::unique_ptr<TablePrinter> m_tablePrinter; + + public: + ConsoleReporter(ReporterConfig&& config); + ~ConsoleReporter() override; + static std::string getDescription(); + + void noMatchingTestCases( StringRef unmatchedSpec ) override; + void reportInvalidTestSpec( StringRef arg ) override; + + void assertionStarting(AssertionInfo const&) override; + + void assertionEnded(AssertionStats const& _assertionStats) override; + + void sectionStarting(SectionInfo const& _sectionInfo) override; + void sectionEnded(SectionStats const& _sectionStats) override; + + void benchmarkPreparing( StringRef name ) override; + void benchmarkStarting(BenchmarkInfo const& info) override; + void benchmarkEnded(BenchmarkStats<> const& stats) override; + void benchmarkFailed( StringRef error ) override; + + void testCaseEnded(TestCaseStats const& _testCaseStats) override; + void testRunEnded(TestRunStats const& _testRunStats) override; + void testRunStarting(TestRunInfo const& _testRunInfo) override; + + private: + void lazyPrint(); + + void lazyPrintWithoutClosingBenchmarkTable(); + void lazyPrintRunInfo(); + void printTestCaseAndSectionHeader(); + + void printClosedHeader(std::string const& _name); + void printOpenHeader(std::string const& _name); + + // if string has a : in first line will set indent to follow it on + // subsequent lines + void printHeaderString(std::string const& _string, std::size_t indent = 0); + + void printTotalsDivider(Totals const& totals); + + bool m_headerPrinted = false; + bool m_testRunInfoPrinted = false; + }; + +} // end namespace Catch + +#endif // CATCH_REPORTER_CONSOLE_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporter_cumulative_base.cpp b/src/catch2/reporters/catch_reporter_cumulative_base.cpp new file mode 100644 index 0000000..0916963 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_cumulative_base.cpp @@ -0,0 +1,158 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#include <catch2/reporters/catch_reporter_cumulative_base.hpp> + +#include <catch2/internal/catch_move_and_forward.hpp> + +#include <algorithm> +#include <cassert> + +namespace Catch { + namespace { + struct BySectionInfo { + BySectionInfo( SectionInfo const& other ): m_other( other ) {} + BySectionInfo( BySectionInfo const& other ) = default; + bool operator()( + Detail::unique_ptr<CumulativeReporterBase::SectionNode> const& + node ) const { + return ( + ( node->stats.sectionInfo.name == m_other.name ) && + ( node->stats.sectionInfo.lineInfo == m_other.lineInfo ) ); + } + void operator=( BySectionInfo const& ) = delete; + + private: + SectionInfo const& m_other; + }; + + } // namespace + + namespace Detail { + AssertionOrBenchmarkResult::AssertionOrBenchmarkResult( + AssertionStats const& assertion ): + m_assertion( assertion ) {} + + AssertionOrBenchmarkResult::AssertionOrBenchmarkResult( + BenchmarkStats<> const& benchmark ): + m_benchmark( benchmark ) {} + + bool AssertionOrBenchmarkResult::isAssertion() const { + return m_assertion.some(); + } + bool AssertionOrBenchmarkResult::isBenchmark() const { + return m_benchmark.some(); + } + + AssertionStats const& AssertionOrBenchmarkResult::asAssertion() const { + assert(m_assertion.some()); + + return *m_assertion; + } + BenchmarkStats<> const& AssertionOrBenchmarkResult::asBenchmark() const { + assert(m_benchmark.some()); + + return *m_benchmark; + } + + } + + CumulativeReporterBase::~CumulativeReporterBase() = default; + + void CumulativeReporterBase::benchmarkEnded(BenchmarkStats<> const& benchmarkStats) { + m_sectionStack.back()->assertionsAndBenchmarks.emplace_back(benchmarkStats); + } + + void + CumulativeReporterBase::sectionStarting( SectionInfo const& sectionInfo ) { + // We need a copy, because SectionStats expect to take ownership + SectionStats incompleteStats( SectionInfo(sectionInfo), Counts(), 0, false ); + SectionNode* node; + if ( m_sectionStack.empty() ) { + if ( !m_rootSection ) { + m_rootSection = + Detail::make_unique<SectionNode>( incompleteStats ); + } + node = m_rootSection.get(); + } else { + SectionNode& parentNode = *m_sectionStack.back(); + auto it = std::find_if( parentNode.childSections.begin(), + parentNode.childSections.end(), + BySectionInfo( sectionInfo ) ); + if ( it == parentNode.childSections.end() ) { + auto newNode = + Detail::make_unique<SectionNode>( incompleteStats ); + node = newNode.get(); + parentNode.childSections.push_back( CATCH_MOVE( newNode ) ); + } else { + node = it->get(); + } + } + + m_deepestSection = node; + m_sectionStack.push_back( node ); + } + + void CumulativeReporterBase::assertionEnded( + AssertionStats const& assertionStats ) { + assert( !m_sectionStack.empty() ); + // AssertionResult holds a pointer to a temporary DecomposedExpression, + // which getExpandedExpression() calls to build the expression string. + // Our section stack copy of the assertionResult will likely outlive the + // temporary, so it must be expanded or discarded now to avoid calling + // a destroyed object later. + if ( m_shouldStoreFailedAssertions && + !assertionStats.assertionResult.isOk() ) { + static_cast<void>( + assertionStats.assertionResult.getExpandedExpression() ); + } + if ( m_shouldStoreSuccesfulAssertions && + assertionStats.assertionResult.isOk() ) { + static_cast<void>( + assertionStats.assertionResult.getExpandedExpression() ); + } + SectionNode& sectionNode = *m_sectionStack.back(); + sectionNode.assertionsAndBenchmarks.emplace_back( assertionStats ); + } + + void CumulativeReporterBase::sectionEnded( SectionStats const& sectionStats ) { + assert( !m_sectionStack.empty() ); + SectionNode& node = *m_sectionStack.back(); + node.stats = sectionStats; + m_sectionStack.pop_back(); + } + + void CumulativeReporterBase::testCaseEnded( + TestCaseStats const& testCaseStats ) { + auto node = Detail::make_unique<TestCaseNode>( testCaseStats ); + assert( m_sectionStack.size() == 0 ); + node->children.push_back( CATCH_MOVE(m_rootSection) ); + m_testCases.push_back( CATCH_MOVE(node) ); + + assert( m_deepestSection ); + m_deepestSection->stdOut = testCaseStats.stdOut; + m_deepestSection->stdErr = testCaseStats.stdErr; + } + + + void CumulativeReporterBase::testRunEnded( TestRunStats const& testRunStats ) { + assert(!m_testRun && "CumulativeReporterBase assumes there can only be one test run"); + m_testRun = Detail::make_unique<TestRunNode>( testRunStats ); + m_testRun->children.swap( m_testCases ); + testRunEndedCumulative(); + } + + bool CumulativeReporterBase::SectionNode::hasAnyAssertions() const { + return std::any_of( + assertionsAndBenchmarks.begin(), + assertionsAndBenchmarks.end(), + []( Detail::AssertionOrBenchmarkResult const& res ) { + return res.isAssertion(); + } ); + } + +} // end namespace Catch diff --git a/src/catch2/reporters/catch_reporter_cumulative_base.hpp b/src/catch2/reporters/catch_reporter_cumulative_base.hpp new file mode 100644 index 0000000..267b39f --- /dev/null +++ b/src/catch2/reporters/catch_reporter_cumulative_base.hpp @@ -0,0 +1,151 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef CATCH_REPORTER_CUMULATIVE_BASE_HPP_INCLUDED +#define CATCH_REPORTER_CUMULATIVE_BASE_HPP_INCLUDED + +#include <catch2/reporters/catch_reporter_common_base.hpp> +#include <catch2/internal/catch_move_and_forward.hpp> +#include <catch2/internal/catch_unique_ptr.hpp> +#include <catch2/internal/catch_optional.hpp> + +#include <string> +#include <vector> + +namespace Catch { + + namespace Detail { + + //! Represents either an assertion or a benchmark result to be handled by cumulative reporter later + class AssertionOrBenchmarkResult { + // This should really be a variant, but this is much faster + // to write and the data layout here is already terrible + // enough that we do not have to care about the object size. + Optional<AssertionStats> m_assertion; + Optional<BenchmarkStats<>> m_benchmark; + public: + AssertionOrBenchmarkResult(AssertionStats const& assertion); + AssertionOrBenchmarkResult(BenchmarkStats<> const& benchmark); + + bool isAssertion() const; + bool isBenchmark() const; + + AssertionStats const& asAssertion() const; + BenchmarkStats<> const& asBenchmark() const; + }; + } + + /** + * Utility base for reporters that need to handle all results at once + * + * It stores tree of all test cases, sections and assertions, and after the + * test run is finished, calls into `testRunEndedCumulative` to pass the + * control to the deriving class. + * + * If you are deriving from this class and override any testing related + * member functions, you should first call into the base's implementation to + * avoid breaking the tree construction. + * + * Due to the way this base functions, it has to expand assertions up-front, + * even if they are later unused (e.g. because the deriving reporter does + * not report successful assertions, or because the deriving reporter does + * not use assertion expansion at all). Derived classes can use two + * customization points, `m_shouldStoreSuccesfulAssertions` and + * `m_shouldStoreFailedAssertions`, to disable the expansion and gain extra + * performance. **Accessing the assertion expansions if it wasn't stored is + * UB.** + */ + class CumulativeReporterBase : public ReporterBase { + public: + template<typename T, typename ChildNodeT> + struct Node { + explicit Node( T const& _value ) : value( _value ) {} + + using ChildNodes = std::vector<Detail::unique_ptr<ChildNodeT>>; + T value; + ChildNodes children; + }; + struct SectionNode { + explicit SectionNode(SectionStats const& _stats) : stats(_stats) {} + + bool operator == (SectionNode const& other) const { + return stats.sectionInfo.lineInfo == other.stats.sectionInfo.lineInfo; + } + + bool hasAnyAssertions() const; + + SectionStats stats; + std::vector<Detail::unique_ptr<SectionNode>> childSections; + std::vector<Detail::AssertionOrBenchmarkResult> assertionsAndBenchmarks; + std::string stdOut; + std::string stdErr; + }; + + + using TestCaseNode = Node<TestCaseStats, SectionNode>; + using TestRunNode = Node<TestRunStats, TestCaseNode>; + + // GCC5 compat: we cannot use inherited constructor, because it + // doesn't implement backport of P0136 + CumulativeReporterBase(ReporterConfig&& _config): + ReporterBase(CATCH_MOVE(_config)) + {} + ~CumulativeReporterBase() override; + + void benchmarkPreparing( StringRef ) override {} + void benchmarkStarting( BenchmarkInfo const& ) override {} + void benchmarkEnded( BenchmarkStats<> const& benchmarkStats ) override; + void benchmarkFailed( StringRef ) override {} + + void noMatchingTestCases( StringRef ) override {} + void reportInvalidTestSpec( StringRef ) override {} + void fatalErrorEncountered( StringRef /*error*/ ) override {} + + void testRunStarting( TestRunInfo const& ) override {} + + void testCaseStarting( TestCaseInfo const& ) override {} + void testCasePartialStarting( TestCaseInfo const&, uint64_t ) override {} + void sectionStarting( SectionInfo const& sectionInfo ) override; + + void assertionStarting( AssertionInfo const& ) override {} + + void assertionEnded( AssertionStats const& assertionStats ) override; + void sectionEnded( SectionStats const& sectionStats ) override; + void testCasePartialEnded( TestCaseStats const&, uint64_t ) override {} + void testCaseEnded( TestCaseStats const& testCaseStats ) override; + void testRunEnded( TestRunStats const& testRunStats ) override; + //! Customization point: called after last test finishes (testRunEnded has been handled) + virtual void testRunEndedCumulative() = 0; + + void skipTest(TestCaseInfo const&) override {} + + protected: + //! Should the cumulative base store the assertion expansion for successful assertions? + bool m_shouldStoreSuccesfulAssertions = true; + //! Should the cumulative base store the assertion expansion for failed assertions? + bool m_shouldStoreFailedAssertions = true; + + // We need lazy construction here. We should probably refactor it + // later, after the events are redone. + //! The root node of the test run tree. + Detail::unique_ptr<TestRunNode> m_testRun; + + private: + // Note: We rely on pointer identity being stable, which is why + // we store pointers to the nodes rather than the values. + std::vector<Detail::unique_ptr<TestCaseNode>> m_testCases; + // Root section of the _current_ test case + Detail::unique_ptr<SectionNode> m_rootSection; + // Deepest section of the _current_ test case + SectionNode* m_deepestSection = nullptr; + // Stack of _active_ sections in the _current_ test case + std::vector<SectionNode*> m_sectionStack; + }; + +} // end namespace Catch + +#endif // CATCH_REPORTER_CUMULATIVE_BASE_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporter_event_listener.cpp b/src/catch2/reporters/catch_reporter_event_listener.cpp new file mode 100644 index 0000000..e94063b --- /dev/null +++ b/src/catch2/reporters/catch_reporter_event_listener.cpp @@ -0,0 +1,40 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +#include <catch2/reporters/catch_reporter_event_listener.hpp> + +namespace Catch { + + void EventListenerBase::fatalErrorEncountered( StringRef ) {} + + void EventListenerBase::benchmarkPreparing( StringRef ) {} + void EventListenerBase::benchmarkStarting( BenchmarkInfo const& ) {} + void EventListenerBase::benchmarkEnded( BenchmarkStats<> const& ) {} + void EventListenerBase::benchmarkFailed( StringRef ) {} + + void EventListenerBase::assertionStarting( AssertionInfo const& ) {} + + void EventListenerBase::assertionEnded( AssertionStats const& ) {} + void EventListenerBase::listReporters( + std::vector<ReporterDescription> const& ) {} + void EventListenerBase::listListeners( + std::vector<ListenerDescription> const& ) {} + void EventListenerBase::listTests( std::vector<TestCaseHandle> const& ) {} + void EventListenerBase::listTags( std::vector<TagInfo> const& ) {} + void EventListenerBase::noMatchingTestCases( StringRef ) {} + void EventListenerBase::reportInvalidTestSpec( StringRef ) {} + void EventListenerBase::testRunStarting( TestRunInfo const& ) {} + void EventListenerBase::testCaseStarting( TestCaseInfo const& ) {} + void EventListenerBase::testCasePartialStarting(TestCaseInfo const&, uint64_t) {} + void EventListenerBase::sectionStarting( SectionInfo const& ) {} + void EventListenerBase::sectionEnded( SectionStats const& ) {} + void EventListenerBase::testCasePartialEnded(TestCaseStats const&, uint64_t) {} + void EventListenerBase::testCaseEnded( TestCaseStats const& ) {} + void EventListenerBase::testRunEnded( TestRunStats const& ) {} + void EventListenerBase::skipTest( TestCaseInfo const& ) {} +} // namespace Catch diff --git a/src/catch2/reporters/catch_reporter_event_listener.hpp b/src/catch2/reporters/catch_reporter_event_listener.hpp new file mode 100644 index 0000000..346263e --- /dev/null +++ b/src/catch2/reporters/catch_reporter_event_listener.hpp @@ -0,0 +1,60 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef CATCH_REPORTER_EVENT_LISTENER_HPP_INCLUDED +#define CATCH_REPORTER_EVENT_LISTENER_HPP_INCLUDED + +#include <catch2/interfaces/catch_interfaces_reporter.hpp> + +namespace Catch { + + /** + * Base class to simplify implementing listeners. + * + * Provides empty default implementation for all IEventListener member + * functions, so that a listener implementation can pick which + * member functions it actually cares about. + */ + class EventListenerBase : public IEventListener { + public: + using IEventListener::IEventListener; + + void reportInvalidTestSpec( StringRef unmatchedSpec ) override; + void fatalErrorEncountered( StringRef error ) override; + + void benchmarkPreparing( StringRef name ) override; + void benchmarkStarting( BenchmarkInfo const& benchmarkInfo ) override; + void benchmarkEnded( BenchmarkStats<> const& benchmarkStats ) override; + void benchmarkFailed( StringRef error ) override; + + void assertionStarting( AssertionInfo const& assertionInfo ) override; + void assertionEnded( AssertionStats const& assertionStats ) override; + + void listReporters( + std::vector<ReporterDescription> const& descriptions ) override; + void listListeners( + std::vector<ListenerDescription> const& descriptions ) override; + void listTests( std::vector<TestCaseHandle> const& tests ) override; + void listTags( std::vector<TagInfo> const& tagInfos ) override; + + void noMatchingTestCases( StringRef unmatchedSpec ) override; + void testRunStarting( TestRunInfo const& testRunInfo ) override; + void testCaseStarting( TestCaseInfo const& testInfo ) override; + void testCasePartialStarting( TestCaseInfo const& testInfo, + uint64_t partNumber ) override; + void sectionStarting( SectionInfo const& sectionInfo ) override; + void sectionEnded( SectionStats const& sectionStats ) override; + void testCasePartialEnded( TestCaseStats const& testCaseStats, + uint64_t partNumber ) override; + void testCaseEnded( TestCaseStats const& testCaseStats ) override; + void testRunEnded( TestRunStats const& testRunStats ) override; + void skipTest( TestCaseInfo const& testInfo ) override; + }; + +} // end namespace Catch + +#endif // CATCH_REPORTER_EVENT_LISTENER_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporter_helpers.cpp b/src/catch2/reporters/catch_reporter_helpers.cpp new file mode 100644 index 0000000..ffb32ff --- /dev/null +++ b/src/catch2/reporters/catch_reporter_helpers.cpp @@ -0,0 +1,343 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +#include <catch2/reporters/catch_reporter_helpers.hpp> +#include <catch2/interfaces/catch_interfaces_config.hpp> +#include <catch2/internal/catch_console_width.hpp> +#include <catch2/internal/catch_errno_guard.hpp> +#include <catch2/internal/catch_textflow.hpp> +#include <catch2/internal/catch_reusable_string_stream.hpp> +#include <catch2/internal/catch_string_manip.hpp> +#include <catch2/internal/catch_console_colour.hpp> +#include <catch2/catch_tostring.hpp> +#include <catch2/catch_test_case_info.hpp> + +#include <algorithm> +#include <cfloat> +#include <cstdio> +#include <ostream> +#include <iomanip> + +namespace Catch { + + namespace { + void listTestNamesOnly(std::ostream& out, + std::vector<TestCaseHandle> const& tests) { + for (auto const& test : tests) { + auto const& testCaseInfo = test.getTestCaseInfo(); + + if (startsWith(testCaseInfo.name, '#')) { + out << '"' << testCaseInfo.name << '"'; + } else { + out << testCaseInfo.name; + } + + out << '\n'; + } + out << std::flush; + } + } // end unnamed namespace + + + // Because formatting using c++ streams is stateful, drop down to C is + // required Alternatively we could use stringstream, but its performance + // is... not good. + std::string getFormattedDuration( double duration ) { + // Max exponent + 1 is required to represent the whole part + // + 1 for decimal point + // + 3 for the 3 decimal places + // + 1 for null terminator + const std::size_t maxDoubleSize = DBL_MAX_10_EXP + 1 + 1 + 3 + 1; + char buffer[maxDoubleSize]; + + // Save previous errno, to prevent sprintf from overwriting it + ErrnoGuard guard; +#ifdef _MSC_VER + size_t printedLength = static_cast<size_t>( + sprintf_s( buffer, "%.3f", duration ) ); +#else + size_t printedLength = static_cast<size_t>( + std::snprintf( buffer, maxDoubleSize, "%.3f", duration ) ); +#endif + return std::string( buffer, printedLength ); + } + + bool shouldShowDuration( IConfig const& config, double duration ) { + if ( config.showDurations() == ShowDurations::Always ) { + return true; + } + if ( config.showDurations() == ShowDurations::Never ) { + return false; + } + const double min = config.minDuration(); + return min >= 0 && duration >= min; + } + + std::string serializeFilters( std::vector<std::string> const& filters ) { + // We add a ' ' separator between each filter + size_t serialized_size = filters.size() - 1; + for (auto const& filter : filters) { + serialized_size += filter.size(); + } + + std::string serialized; + serialized.reserve(serialized_size); + bool first = true; + + for (auto const& filter : filters) { + if (!first) { + serialized.push_back(' '); + } + first = false; + serialized.append(filter); + } + + return serialized; + } + + std::ostream& operator<<( std::ostream& out, lineOfChars value ) { + for ( size_t idx = 0; idx < CATCH_CONFIG_CONSOLE_WIDTH - 1; ++idx ) { + out.put( value.c ); + } + return out; + } + + void + defaultListReporters( std::ostream& out, + std::vector<ReporterDescription> const& descriptions, + Verbosity verbosity ) { + out << "Available reporters:\n"; + const auto maxNameLen = + std::max_element( descriptions.begin(), + descriptions.end(), + []( ReporterDescription const& lhs, + ReporterDescription const& rhs ) { + return lhs.name.size() < rhs.name.size(); + } ) + ->name.size(); + + for ( auto const& desc : descriptions ) { + if ( verbosity == Verbosity::Quiet ) { + out << TextFlow::Column( desc.name ) + .indent( 2 ) + .width( 5 + maxNameLen ) + << '\n'; + } else { + out << TextFlow::Column( desc.name + ':' ) + .indent( 2 ) + .width( 5 + maxNameLen ) + + TextFlow::Column( desc.description ) + .initialIndent( 0 ) + .indent( 2 ) + .width( CATCH_CONFIG_CONSOLE_WIDTH - maxNameLen - 8 ) + << '\n'; + } + } + out << '\n' << std::flush; + } + + void defaultListListeners( std::ostream& out, + std::vector<ListenerDescription> const& descriptions ) { + out << "Registered listeners:\n"; + + if(descriptions.empty()) { + return; + } + + const auto maxNameLen = + std::max_element( descriptions.begin(), + descriptions.end(), + []( ListenerDescription const& lhs, + ListenerDescription const& rhs ) { + return lhs.name.size() < rhs.name.size(); + } ) + ->name.size(); + + for ( auto const& desc : descriptions ) { + out << TextFlow::Column( static_cast<std::string>( desc.name ) + + ':' ) + .indent( 2 ) + .width( maxNameLen + 5 ) + + TextFlow::Column( desc.description ) + .initialIndent( 0 ) + .indent( 2 ) + .width( CATCH_CONFIG_CONSOLE_WIDTH - maxNameLen - 8 ) + << '\n'; + } + + out << '\n' << std::flush; + } + + void defaultListTags( std::ostream& out, + std::vector<TagInfo> const& tags, + bool isFiltered ) { + if ( isFiltered ) { + out << "Tags for matching test cases:\n"; + } else { + out << "All available tags:\n"; + } + + for ( auto const& tagCount : tags ) { + ReusableStringStream rss; + rss << " " << std::setw( 2 ) << tagCount.count << " "; + auto str = rss.str(); + auto wrapper = TextFlow::Column( tagCount.all() ) + .initialIndent( 0 ) + .indent( str.size() ) + .width( CATCH_CONFIG_CONSOLE_WIDTH - 10 ); + out << str << wrapper << '\n'; + } + out << pluralise(tags.size(), "tag"_sr) << "\n\n" << std::flush; + } + + void defaultListTests(std::ostream& out, ColourImpl* streamColour, std::vector<TestCaseHandle> const& tests, bool isFiltered, Verbosity verbosity) { + // We special case this to provide the equivalent of old + // `--list-test-names-only`, which could then be used by the + // `--input-file` option. + if (verbosity == Verbosity::Quiet) { + listTestNamesOnly(out, tests); + return; + } + + if (isFiltered) { + out << "Matching test cases:\n"; + } else { + out << "All available test cases:\n"; + } + + for (auto const& test : tests) { + auto const& testCaseInfo = test.getTestCaseInfo(); + Colour::Code colour = testCaseInfo.isHidden() + ? Colour::SecondaryText + : Colour::None; + auto colourGuard = streamColour->guardColour( colour ).engage( out ); + + out << TextFlow::Column(testCaseInfo.name).indent(2) << '\n'; + if (verbosity >= Verbosity::High) { + out << TextFlow::Column(Catch::Detail::stringify(testCaseInfo.lineInfo)).indent(4) << '\n'; + } + if (!testCaseInfo.tags.empty() && + verbosity > Verbosity::Quiet) { + out << TextFlow::Column(testCaseInfo.tagsAsString()).indent(6) << '\n'; + } + } + + if (isFiltered) { + out << pluralise(tests.size(), "matching test case"_sr); + } else { + out << pluralise(tests.size(), "test case"_sr); + } + out << "\n\n" << std::flush; + } + + namespace { + class SummaryColumn { + public: + SummaryColumn( std::string suffix, Colour::Code colour ): + m_suffix( CATCH_MOVE( suffix ) ), m_colour( colour ) {} + + SummaryColumn&& addRow( std::uint64_t count ) && { + std::string row = std::to_string(count); + auto const new_width = std::max( m_width, row.size() ); + if ( new_width > m_width ) { + for ( auto& oldRow : m_rows ) { + oldRow.insert( 0, new_width - m_width, ' ' ); + } + } else { + row.insert( 0, m_width - row.size(), ' ' ); + } + m_width = new_width; + m_rows.push_back( row ); + return std::move( *this ); + } + + std::string const& getSuffix() const { return m_suffix; } + Colour::Code getColour() const { return m_colour; } + std::string const& getRow( std::size_t index ) const { + return m_rows[index]; + } + + private: + std::string m_suffix; + Colour::Code m_colour; + std::size_t m_width = 0; + std::vector<std::string> m_rows; + }; + + void printSummaryRow( std::ostream& stream, + ColourImpl& colour, + StringRef label, + std::vector<SummaryColumn> const& cols, + std::size_t row ) { + for ( auto const& col : cols ) { + auto const& value = col.getRow( row ); + auto const& suffix = col.getSuffix(); + if ( suffix.empty() ) { + stream << label << ": "; + if ( value != "0" ) { + stream << value; + } else { + stream << colour.guardColour( Colour::Warning ) + << "- none -"; + } + } else if ( value != "0" ) { + stream << colour.guardColour( Colour::LightGrey ) << " | " + << colour.guardColour( col.getColour() ) << value + << ' ' << suffix; + } + } + stream << '\n'; + } + } // namespace + + void printTestRunTotals( std::ostream& stream, + ColourImpl& streamColour, + Totals const& totals ) { + if ( totals.testCases.total() == 0 ) { + stream << streamColour.guardColour( Colour::Warning ) + << "No tests ran\n"; + return; + } + + if ( totals.assertions.total() > 0 && totals.testCases.allPassed() ) { + stream << streamColour.guardColour( Colour::ResultSuccess ) + << "All tests passed"; + stream << " (" + << pluralise( totals.assertions.passed, "assertion"_sr ) + << " in " + << pluralise( totals.testCases.passed, "test case"_sr ) + << ')' << '\n'; + return; + } + + std::vector<SummaryColumn> columns; + // Don't include "skipped assertions" in total count + const auto totalAssertionCount = + totals.assertions.total() - totals.assertions.skipped; + columns.push_back( SummaryColumn( "", Colour::None ) + .addRow( totals.testCases.total() ) + .addRow( totalAssertionCount ) ); + columns.push_back( SummaryColumn( "passed", Colour::Success ) + .addRow( totals.testCases.passed ) + .addRow( totals.assertions.passed ) ); + columns.push_back( SummaryColumn( "failed", Colour::ResultError ) + .addRow( totals.testCases.failed ) + .addRow( totals.assertions.failed ) ); + columns.push_back( SummaryColumn( "skipped", Colour::Skip ) + .addRow( totals.testCases.skipped ) + // Don't print "skipped assertions" + .addRow( 0 ) ); + columns.push_back( + SummaryColumn( "failed as expected", Colour::ResultExpectedFailure ) + .addRow( totals.testCases.failedButOk ) + .addRow( totals.assertions.failedButOk ) ); + printSummaryRow( stream, streamColour, "test cases"_sr, columns, 0 ); + printSummaryRow( stream, streamColour, "assertions"_sr, columns, 1 ); + } + +} // namespace Catch diff --git a/src/catch2/reporters/catch_reporter_helpers.hpp b/src/catch2/reporters/catch_reporter_helpers.hpp new file mode 100644 index 0000000..316cb40 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_helpers.hpp @@ -0,0 +1,95 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef CATCH_REPORTER_HELPERS_HPP_INCLUDED +#define CATCH_REPORTER_HELPERS_HPP_INCLUDED + +#include <iosfwd> +#include <string> +#include <vector> + +#include <catch2/internal/catch_list.hpp> +#include <catch2/interfaces/catch_interfaces_config.hpp> +#include <catch2/catch_totals.hpp> + +namespace Catch { + + class IConfig; + class TestCaseHandle; + class ColourImpl; + + // Returns double formatted as %.3f (format expected on output) + std::string getFormattedDuration( double duration ); + + //! Should the reporter show duration of test given current configuration? + bool shouldShowDuration( IConfig const& config, double duration ); + + std::string serializeFilters( std::vector<std::string> const& filters ); + + struct lineOfChars { + char c; + constexpr lineOfChars( char c_ ): c( c_ ) {} + + friend std::ostream& operator<<( std::ostream& out, lineOfChars value ); + }; + + /** + * Lists reporter descriptions to the provided stream in user-friendly + * format + * + * Used as the default listing implementation by the first party reporter + * bases. The output should be backwards compatible with the output of + * Catch2 v2 binaries. + */ + void + defaultListReporters( std::ostream& out, + std::vector<ReporterDescription> const& descriptions, + Verbosity verbosity ); + + /** + * Lists listeners descriptions to the provided stream in user-friendly + * format + */ + void defaultListListeners( std::ostream& out, + std::vector<ListenerDescription> const& descriptions ); + + /** + * Lists tag information to the provided stream in user-friendly format + * + * Used as the default listing implementation by the first party reporter + * bases. The output should be backwards compatible with the output of + * Catch2 v2 binaries. + */ + void defaultListTags( std::ostream& out, std::vector<TagInfo> const& tags, bool isFiltered ); + + /** + * Lists test case information to the provided stream in user-friendly + * format + * + * Used as the default listing implementation by the first party reporter + * bases. The output is backwards compatible with the output of Catch2 + * v2 binaries, and also supports the format specific to the old + * `--list-test-names-only` option, for people who used it in integrations. + */ + void defaultListTests( std::ostream& out, + ColourImpl* streamColour, + std::vector<TestCaseHandle> const& tests, + bool isFiltered, + Verbosity verbosity ); + + /** + * Prints test run totals to the provided stream in user-friendly format + * + * Used by the console and compact reporters. + */ + void printTestRunTotals( std::ostream& stream, + ColourImpl& streamColour, + Totals const& totals ); + +} // end namespace Catch + +#endif // CATCH_REPORTER_HELPERS_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporter_json.cpp b/src/catch2/reporters/catch_reporter_json.cpp new file mode 100644 index 0000000..6a8e655 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_json.cpp @@ -0,0 +1,372 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +// +#include <catch2/catch_test_case_info.hpp> +#include <catch2/catch_test_spec.hpp> +#include <catch2/catch_version.hpp> +#include <catch2/interfaces/catch_interfaces_config.hpp> +#include <catch2/internal/catch_list.hpp> +#include <catch2/internal/catch_string_manip.hpp> +#include <catch2/reporters/catch_reporter_json.hpp> + +namespace Catch { + namespace { + void writeSourceInfo( JsonObjectWriter& writer, + SourceLineInfo const& sourceInfo ) { + auto source_location_writer = + writer.write( "source-location"_sr ).writeObject(); + source_location_writer.write( "filename"_sr ) + .write( sourceInfo.file ); + source_location_writer.write( "line"_sr ).write( sourceInfo.line ); + } + + void writeTags( JsonArrayWriter writer, std::vector<Tag> const& tags ) { + for ( auto const& tag : tags ) { + writer.write( tag.original ); + } + } + + void writeProperties( JsonArrayWriter writer, + TestCaseInfo const& info ) { + if ( info.isHidden() ) { writer.write( "is-hidden"_sr ); } + if ( info.okToFail() ) { writer.write( "ok-to-fail"_sr ); } + if ( info.expectedToFail() ) { + writer.write( "expected-to-fail"_sr ); + } + if ( info.throws() ) { writer.write( "throws"_sr ); } + } + + } // namespace + + JsonReporter::JsonReporter( ReporterConfig&& config ): + StreamingReporterBase{ CATCH_MOVE( config ) } { + + m_preferences.shouldRedirectStdOut = true; + // TBD: Do we want to report all assertions? XML reporter does + // not, but for machine-parseable reporters I think the answer + // should be yes. + m_preferences.shouldReportAllAssertions = true; + + m_objectWriters.emplace( m_stream ); + m_writers.emplace( Writer::Object ); + auto& writer = m_objectWriters.top(); + + writer.write( "version"_sr ).write( 1 ); + + { + auto metadata_writer = writer.write( "metadata"_sr ).writeObject(); + metadata_writer.write( "name"_sr ).write( m_config->name() ); + metadata_writer.write( "rng-seed"_sr ).write( m_config->rngSeed() ); + metadata_writer.write( "catch2-version"_sr ) + .write( libraryVersion() ); + if ( m_config->testSpec().hasFilters() ) { + metadata_writer.write( "filters"_sr ) + .write( m_config->testSpec() ); + } + } + } + + JsonReporter::~JsonReporter() { + endListing(); + // TODO: Ensure this closes the top level object, add asserts + assert( m_writers.size() == 1 && "Only the top level object should be open" ); + assert( m_writers.top() == Writer::Object ); + endObject(); + m_stream << '\n' << std::flush; + assert( m_writers.empty() ); + } + + JsonArrayWriter& JsonReporter::startArray() { + m_arrayWriters.emplace( m_arrayWriters.top().writeArray() ); + m_writers.emplace( Writer::Array ); + return m_arrayWriters.top(); + } + JsonArrayWriter& JsonReporter::startArray( StringRef key ) { + m_arrayWriters.emplace( + m_objectWriters.top().write( key ).writeArray() ); + m_writers.emplace( Writer::Array ); + return m_arrayWriters.top(); + } + + JsonObjectWriter& JsonReporter::startObject() { + m_objectWriters.emplace( m_arrayWriters.top().writeObject() ); + m_writers.emplace( Writer::Object ); + return m_objectWriters.top(); + } + JsonObjectWriter& JsonReporter::startObject( StringRef key ) { + m_objectWriters.emplace( + m_objectWriters.top().write( key ).writeObject() ); + m_writers.emplace( Writer::Object ); + return m_objectWriters.top(); + } + + void JsonReporter::endObject() { + assert( isInside( Writer::Object ) ); + m_objectWriters.pop(); + m_writers.pop(); + } + void JsonReporter::endArray() { + assert( isInside( Writer::Array ) ); + m_arrayWriters.pop(); + m_writers.pop(); + } + + bool JsonReporter::isInside( Writer writer ) { + return !m_writers.empty() && m_writers.top() == writer; + } + + void JsonReporter::startListing() { + if ( !m_startedListing ) { startObject( "listings"_sr ); } + m_startedListing = true; + } + void JsonReporter::endListing() { + if ( m_startedListing ) { endObject(); } + m_startedListing = false; + } + + std::string JsonReporter::getDescription() { + return "Outputs listings as JSON. Test listing is Work-in-Progress!"; + } + + void JsonReporter::testRunStarting( TestRunInfo const& runInfo ) { + StreamingReporterBase::testRunStarting( runInfo ); + endListing(); + + assert( isInside( Writer::Object ) ); + startObject( "test-run"_sr ); + startArray( "test-cases"_sr ); + } + + static void writeCounts( JsonObjectWriter&& writer, Counts const& counts ) { + writer.write( "passed"_sr ).write( counts.passed ); + writer.write( "failed"_sr ).write( counts.failed ); + writer.write( "fail-but-ok"_sr ).write( counts.failedButOk ); + writer.write( "skipped"_sr ).write( counts.skipped ); + } + + void JsonReporter::testRunEnded(TestRunStats const& runStats) { + assert( isInside( Writer::Array ) ); + // End "test-cases" + endArray(); + + { + auto totals = + m_objectWriters.top().write( "totals"_sr ).writeObject(); + writeCounts( totals.write( "assertions"_sr ).writeObject(), + runStats.totals.assertions ); + writeCounts( totals.write( "test-cases"_sr ).writeObject(), + runStats.totals.testCases ); + } + + // End the "test-run" object + endObject(); + } + + void JsonReporter::testCaseStarting( TestCaseInfo const& tcInfo ) { + StreamingReporterBase::testCaseStarting( tcInfo ); + + assert( isInside( Writer::Array ) && + "We should be in the 'test-cases' array" ); + startObject(); + // "test-info" prelude + { + auto testInfo = + m_objectWriters.top().write( "test-info"_sr ).writeObject(); + // TODO: handle testName vs className!! + testInfo.write( "name"_sr ).write( tcInfo.name ); + writeSourceInfo(testInfo, tcInfo.lineInfo); + writeTags( testInfo.write( "tags"_sr ).writeArray(), tcInfo.tags ); + writeProperties( testInfo.write( "properties"_sr ).writeArray(), + tcInfo ); + } + + + // Start the array for individual test runs (testCasePartial pairs) + startArray( "runs"_sr ); + } + + void JsonReporter::testCaseEnded( TestCaseStats const& tcStats ) { + StreamingReporterBase::testCaseEnded( tcStats ); + + // We need to close the 'runs' array before finishing the test case + assert( isInside( Writer::Array ) ); + endArray(); + + { + auto totals = + m_objectWriters.top().write( "totals"_sr ).writeObject(); + writeCounts( totals.write( "assertions"_sr ).writeObject(), + tcStats.totals.assertions ); + // We do not write the test case totals, because there will always be just one test case here. + // TODO: overall "result" -> success, skip, fail here? Or in partial result? + } + // We do not write out stderr/stdout, because we instead wrote those out in partial runs + + // TODO: aborting? + + // And we also close this test case's object + assert( isInside( Writer::Object ) ); + endObject(); + } + + void JsonReporter::testCasePartialStarting( TestCaseInfo const& /*tcInfo*/, + uint64_t index ) { + startObject(); + m_objectWriters.top().write( "run-idx"_sr ).write( index ); + startArray( "path"_sr ); + // TODO: we want to delay most of the printing to the 'root' section + // TODO: childSection key name? + } + + void JsonReporter::testCasePartialEnded( TestCaseStats const& tcStats, + uint64_t /*index*/ ) { + // Fixme: the top level section handles this. + //// path object + endArray(); + if ( !tcStats.stdOut.empty() ) { + m_objectWriters.top() + .write( "captured-stdout"_sr ) + .write( tcStats.stdOut ); + } + if ( !tcStats.stdErr.empty() ) { + m_objectWriters.top() + .write( "captured-stderr"_sr ) + .write( tcStats.stdErr ); + } + { + auto totals = + m_objectWriters.top().write( "totals"_sr ).writeObject(); + writeCounts( totals.write( "assertions"_sr ).writeObject(), + tcStats.totals.assertions ); + // We do not write the test case totals, because there will + // always be just one test case here. + // TODO: overall "result" -> success, skip, fail here? Or in + // partial result? + } + // TODO: aborting? + // run object + endObject(); + } + + void JsonReporter::sectionStarting( SectionInfo const& sectionInfo ) { + assert( isInside( Writer::Array ) && + "Section should always start inside an object" ); + // We want to nest top level sections, even though it shares name + // and source loc with the TEST_CASE + auto& sectionObject = startObject(); + sectionObject.write( "kind"_sr ).write( "section"_sr ); + sectionObject.write( "name"_sr ).write( sectionInfo.name ); + writeSourceInfo( m_objectWriters.top(), sectionInfo.lineInfo ); + + + // TBD: Do we want to create this event lazily? It would become + // rather complex, but we could do it, and it would look + // better for empty sections. OTOH, empty sections should + // be rare. + startArray( "path"_sr ); + } + void JsonReporter::sectionEnded( SectionStats const& /*sectionStats */) { + // End the subpath array + endArray(); + // TODO: metadata + // TODO: what info do we have here? + + // End the section object + endObject(); + } + + void JsonReporter::assertionStarting( AssertionInfo const& /*assertionInfo*/ ) {} + void JsonReporter::assertionEnded( AssertionStats const& assertionStats ) { + // TODO: There is lot of different things to handle here, but + // we can fill it in later, after we show that the basic + // outline and streaming reporter impl works well enough. + //if ( !m_config->includeSuccessfulResults() + // && assertionStats.assertionResult.isOk() ) { + // return; + //} + assert( isInside( Writer::Array ) ); + auto assertionObject = m_arrayWriters.top().writeObject(); + + assertionObject.write( "kind"_sr ).write( "assertion"_sr ); + writeSourceInfo( assertionObject, + assertionStats.assertionResult.getSourceInfo() ); + assertionObject.write( "status"_sr ) + .write( assertionStats.assertionResult.isOk() ); + // TODO: handling of result. + // TODO: messages + // TODO: totals? + } + + + void JsonReporter::benchmarkPreparing( StringRef name ) { (void)name; } + void JsonReporter::benchmarkStarting( BenchmarkInfo const& ) {} + void JsonReporter::benchmarkEnded( BenchmarkStats<> const& ) {} + void JsonReporter::benchmarkFailed( StringRef error ) { (void)error; } + + void JsonReporter::listReporters( + std::vector<ReporterDescription> const& descriptions ) { + startListing(); + + auto writer = + m_objectWriters.top().write( "reporters"_sr ).writeArray(); + for ( auto const& desc : descriptions ) { + auto desc_writer = writer.writeObject(); + desc_writer.write( "name"_sr ).write( desc.name ); + desc_writer.write( "description"_sr ).write( desc.description ); + } + } + void JsonReporter::listListeners( + std::vector<ListenerDescription> const& descriptions ) { + startListing(); + + auto writer = + m_objectWriters.top().write( "listeners"_sr ).writeArray(); + + for ( auto const& desc : descriptions ) { + auto desc_writer = writer.writeObject(); + desc_writer.write( "name"_sr ).write( desc.name ); + desc_writer.write( "description"_sr ).write( desc.description ); + } + } + void JsonReporter::listTests( std::vector<TestCaseHandle> const& tests ) { + startListing(); + + auto writer = m_objectWriters.top().write( "tests"_sr ).writeArray(); + + for ( auto const& test : tests ) { + auto desc_writer = writer.writeObject(); + auto const& info = test.getTestCaseInfo(); + + desc_writer.write( "name"_sr ).write( info.name ); + desc_writer.write( "class-name"_sr ).write( info.className ); + { + auto tag_writer = desc_writer.write( "tags"_sr ).writeArray(); + for ( auto const& tag : info.tags ) { + tag_writer.write( tag.original ); + } + } + writeSourceInfo( desc_writer, info.lineInfo ); + } + } + void JsonReporter::listTags( std::vector<TagInfo> const& tags ) { + startListing(); + + auto writer = m_objectWriters.top().write( "tags"_sr ).writeArray(); + for ( auto const& tag : tags ) { + auto tag_writer = writer.writeObject(); + { + auto aliases_writer = + tag_writer.write( "aliases"_sr ).writeArray(); + for ( auto alias : tag.spellings ) { + aliases_writer.write( alias ); + } + } + tag_writer.write( "count"_sr ).write( tag.count ); + } + } +} // namespace Catch diff --git a/src/catch2/reporters/catch_reporter_json.hpp b/src/catch2/reporters/catch_reporter_json.hpp new file mode 100644 index 0000000..c938ca3 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_json.hpp @@ -0,0 +1,95 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +#ifndef CATCH_REPORTER_JSON_HPP_INCLUDED +#define CATCH_REPORTER_JSON_HPP_INCLUDED + +#include <catch2/catch_timer.hpp> +#include <catch2/internal/catch_jsonwriter.hpp> +#include <catch2/reporters/catch_reporter_streaming_base.hpp> + +#include <stack> + +namespace Catch { + class JsonReporter : public StreamingReporterBase { + public: + JsonReporter( ReporterConfig&& config ); + + ~JsonReporter() override; + + static std::string getDescription(); + + public: // StreamingReporterBase + void testRunStarting( TestRunInfo const& runInfo ) override; + void testRunEnded( TestRunStats const& runStats ) override; + + void testCaseStarting( TestCaseInfo const& tcInfo ) override; + void testCaseEnded( TestCaseStats const& tcStats ) override; + + void testCasePartialStarting( TestCaseInfo const& tcInfo, + uint64_t index ) override; + void testCasePartialEnded( TestCaseStats const& tcStats, + uint64_t index ) override; + + void sectionStarting( SectionInfo const& sectionInfo ) override; + void sectionEnded( SectionStats const& sectionStats ) override; + + void assertionStarting( AssertionInfo const& assertionInfo ) override; + void assertionEnded( AssertionStats const& assertionStats ) override; + + //void testRunEndedCumulative() override; + + void benchmarkPreparing( StringRef name ) override; + void benchmarkStarting( BenchmarkInfo const& ) override; + void benchmarkEnded( BenchmarkStats<> const& ) override; + void benchmarkFailed( StringRef error ) override; + + void listReporters( + std::vector<ReporterDescription> const& descriptions ) override; + void listListeners( + std::vector<ListenerDescription> const& descriptions ) override; + void listTests( std::vector<TestCaseHandle> const& tests ) override; + void listTags( std::vector<TagInfo> const& tags ) override; + + private: + Timer m_testCaseTimer; + enum class Writer { + Object, + Array + }; + + JsonArrayWriter& startArray(); + JsonArrayWriter& startArray( StringRef key ); + + JsonObjectWriter& startObject(); + JsonObjectWriter& startObject( StringRef key ); + + void endObject(); + void endArray(); + + bool isInside( Writer writer ); + + void startListing(); + void endListing(); + + // Invariant: + // When m_writers is not empty and its top element is + // - Writer::Object, then m_objectWriters is not be empty + // - Writer::Array, then m_arrayWriters shall not be empty + std::stack<JsonObjectWriter> m_objectWriters{}; + std::stack<JsonArrayWriter> m_arrayWriters{}; + std::stack<Writer> m_writers{}; + + bool m_startedListing = false; + + // std::size_t m_sectionDepth = 0; + // std::size_t m_sectionStarted = 0; + }; +} // namespace Catch + +#endif // CATCH_REPORTER_JSON_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporter_junit.cpp b/src/catch2/reporters/catch_reporter_junit.cpp new file mode 100644 index 0000000..27bdfe2 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_junit.cpp @@ -0,0 +1,309 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#include <catch2/reporters/catch_reporter_junit.hpp> + +#include <catch2/reporters/catch_reporter_helpers.hpp> +#include <catch2/catch_tostring.hpp> +#include <catch2/internal/catch_string_manip.hpp> +#include <catch2/internal/catch_textflow.hpp> +#include <catch2/interfaces/catch_interfaces_config.hpp> +#include <catch2/catch_test_case_info.hpp> +#include <catch2/catch_test_spec.hpp> +#include <catch2/internal/catch_move_and_forward.hpp> + +#include <cassert> +#include <ctime> +#include <algorithm> +#include <iomanip> + +namespace Catch { + + namespace { + std::string getCurrentTimestamp() { + time_t rawtime; + std::time(&rawtime); + + std::tm timeInfo = {}; +#if defined (_MSC_VER) || defined (__MINGW32__) + gmtime_s(&timeInfo, &rawtime); +#elif defined (CATCH_PLATFORM_PLAYSTATION) + gmtime_s(&rawtime, &timeInfo); +#elif defined (__IAR_SYSTEMS_ICC__) + timeInfo = *std::gmtime(&rawtime); +#else + gmtime_r(&rawtime, &timeInfo); +#endif + + auto const timeStampSize = sizeof("2017-01-16T17:06:45Z"); + char timeStamp[timeStampSize]; + const char * const fmt = "%Y-%m-%dT%H:%M:%SZ"; + + std::strftime(timeStamp, timeStampSize, fmt, &timeInfo); + + return std::string(timeStamp, timeStampSize - 1); + } + + std::string fileNameTag(std::vector<Tag> const& tags) { + auto it = std::find_if(begin(tags), + end(tags), + [] (Tag const& tag) { + return tag.original.size() > 0 + && tag.original[0] == '#'; }); + if (it != tags.end()) { + return static_cast<std::string>( + it->original.substr(1, it->original.size() - 1) + ); + } + return std::string(); + } + + // Formats the duration in seconds to 3 decimal places. + // This is done because some genius defined Maven Surefire schema + // in a way that only accepts 3 decimal places, and tools like + // Jenkins use that schema for validation JUnit reporter output. + std::string formatDuration( double seconds ) { + ReusableStringStream rss; + rss << std::fixed << std::setprecision( 3 ) << seconds; + return rss.str(); + } + + static void normalizeNamespaceMarkers(std::string& str) { + std::size_t pos = str.find( "::" ); + while ( pos != std::string::npos ) { + str.replace( pos, 2, "." ); + pos += 1; + pos = str.find( "::", pos ); + } + } + + } // anonymous namespace + + JunitReporter::JunitReporter( ReporterConfig&& _config ) + : CumulativeReporterBase( CATCH_MOVE(_config) ), + xml( m_stream ) + { + m_preferences.shouldRedirectStdOut = true; + m_preferences.shouldReportAllAssertions = false; + m_shouldStoreSuccesfulAssertions = false; + } + + std::string JunitReporter::getDescription() { + return "Reports test results in an XML format that looks like Ant's junitreport target"; + } + + void JunitReporter::testRunStarting( TestRunInfo const& runInfo ) { + CumulativeReporterBase::testRunStarting( runInfo ); + xml.startElement( "testsuites" ); + suiteTimer.start(); + stdOutForSuite.clear(); + stdErrForSuite.clear(); + unexpectedExceptions = 0; + } + + void JunitReporter::testCaseStarting( TestCaseInfo const& testCaseInfo ) { + m_okToFail = testCaseInfo.okToFail(); + } + + void JunitReporter::assertionEnded( AssertionStats const& assertionStats ) { + if( assertionStats.assertionResult.getResultType() == ResultWas::ThrewException && !m_okToFail ) + unexpectedExceptions++; + CumulativeReporterBase::assertionEnded( assertionStats ); + } + + void JunitReporter::testCaseEnded( TestCaseStats const& testCaseStats ) { + stdOutForSuite += testCaseStats.stdOut; + stdErrForSuite += testCaseStats.stdErr; + CumulativeReporterBase::testCaseEnded( testCaseStats ); + } + + void JunitReporter::testRunEndedCumulative() { + const auto suiteTime = suiteTimer.getElapsedSeconds(); + writeRun( *m_testRun, suiteTime ); + xml.endElement(); + } + + void JunitReporter::writeRun( TestRunNode const& testRunNode, double suiteTime ) { + XmlWriter::ScopedElement e = xml.scopedElement( "testsuite" ); + + TestRunStats const& stats = testRunNode.value; + xml.writeAttribute( "name"_sr, stats.runInfo.name ); + xml.writeAttribute( "errors"_sr, unexpectedExceptions ); + xml.writeAttribute( "failures"_sr, stats.totals.assertions.failed-unexpectedExceptions ); + xml.writeAttribute( "skipped"_sr, stats.totals.assertions.skipped ); + xml.writeAttribute( "tests"_sr, stats.totals.assertions.total() ); + xml.writeAttribute( "hostname"_sr, "tbd"_sr ); // !TBD + if( m_config->showDurations() == ShowDurations::Never ) + xml.writeAttribute( "time"_sr, ""_sr ); + else + xml.writeAttribute( "time"_sr, formatDuration( suiteTime ) ); + xml.writeAttribute( "timestamp"_sr, getCurrentTimestamp() ); + + // Write properties + { + auto properties = xml.scopedElement("properties"); + xml.scopedElement("property") + .writeAttribute("name"_sr, "random-seed"_sr) + .writeAttribute("value"_sr, m_config->rngSeed()); + if (m_config->testSpec().hasFilters()) { + xml.scopedElement("property") + .writeAttribute("name"_sr, "filters"_sr) + .writeAttribute("value"_sr, m_config->testSpec()); + } + } + + // Write test cases + for( auto const& child : testRunNode.children ) + writeTestCase( *child ); + + xml.scopedElement( "system-out" ).writeText( trim( stdOutForSuite ), XmlFormatting::Newline ); + xml.scopedElement( "system-err" ).writeText( trim( stdErrForSuite ), XmlFormatting::Newline ); + } + + void JunitReporter::writeTestCase( TestCaseNode const& testCaseNode ) { + TestCaseStats const& stats = testCaseNode.value; + + // All test cases have exactly one section - which represents the + // test case itself. That section may have 0-n nested sections + assert( testCaseNode.children.size() == 1 ); + SectionNode const& rootSection = *testCaseNode.children.front(); + + std::string className = + static_cast<std::string>( stats.testInfo->className ); + + if( className.empty() ) { + className = fileNameTag(stats.testInfo->tags); + if ( className.empty() ) { + className = "global"; + } + } + + if ( !m_config->name().empty() ) + className = static_cast<std::string>(m_config->name()) + '.' + className; + + normalizeNamespaceMarkers(className); + + writeSection( className, "", rootSection, stats.testInfo->okToFail() ); + } + + void JunitReporter::writeSection( std::string const& className, + std::string const& rootName, + SectionNode const& sectionNode, + bool testOkToFail) { + std::string name = trim( sectionNode.stats.sectionInfo.name ); + if( !rootName.empty() ) + name = rootName + '/' + name; + + if ( sectionNode.stats.assertions.total() > 0 + || !sectionNode.stdOut.empty() + || !sectionNode.stdErr.empty() ) { + XmlWriter::ScopedElement e = xml.scopedElement( "testcase" ); + if( className.empty() ) { + xml.writeAttribute( "classname"_sr, name ); + xml.writeAttribute( "name"_sr, "root"_sr ); + } + else { + xml.writeAttribute( "classname"_sr, className ); + xml.writeAttribute( "name"_sr, name ); + } + xml.writeAttribute( "time"_sr, formatDuration( sectionNode.stats.durationInSeconds ) ); + // This is not ideal, but it should be enough to mimic gtest's + // junit output. + // Ideally the JUnit reporter would also handle `skipTest` + // events and write those out appropriately. + xml.writeAttribute( "status"_sr, "run"_sr ); + + if (sectionNode.stats.assertions.failedButOk) { + xml.scopedElement("skipped") + .writeAttribute("message", "TEST_CASE tagged with !mayfail"); + } + + writeAssertions( sectionNode ); + + + if( !sectionNode.stdOut.empty() ) + xml.scopedElement( "system-out" ).writeText( trim( sectionNode.stdOut ), XmlFormatting::Newline ); + if( !sectionNode.stdErr.empty() ) + xml.scopedElement( "system-err" ).writeText( trim( sectionNode.stdErr ), XmlFormatting::Newline ); + } + for( auto const& childNode : sectionNode.childSections ) + if( className.empty() ) + writeSection( name, "", *childNode, testOkToFail ); + else + writeSection( className, name, *childNode, testOkToFail ); + } + + void JunitReporter::writeAssertions( SectionNode const& sectionNode ) { + for (auto const& assertionOrBenchmark : sectionNode.assertionsAndBenchmarks) { + if (assertionOrBenchmark.isAssertion()) { + writeAssertion(assertionOrBenchmark.asAssertion()); + } + } + } + + void JunitReporter::writeAssertion( AssertionStats const& stats ) { + AssertionResult const& result = stats.assertionResult; + if ( !result.isOk() || + result.getResultType() == ResultWas::ExplicitSkip ) { + std::string elementName; + switch( result.getResultType() ) { + case ResultWas::ThrewException: + case ResultWas::FatalErrorCondition: + elementName = "error"; + break; + case ResultWas::ExplicitFailure: + case ResultWas::ExpressionFailed: + case ResultWas::DidntThrowException: + elementName = "failure"; + break; + case ResultWas::ExplicitSkip: + elementName = "skipped"; + break; + // We should never see these here: + case ResultWas::Info: + case ResultWas::Warning: + case ResultWas::Ok: + case ResultWas::Unknown: + case ResultWas::FailureBit: + case ResultWas::Exception: + elementName = "internalError"; + break; + } + + XmlWriter::ScopedElement e = xml.scopedElement( elementName ); + + xml.writeAttribute( "message"_sr, result.getExpression() ); + xml.writeAttribute( "type"_sr, result.getTestMacroName() ); + + ReusableStringStream rss; + if ( result.getResultType() == ResultWas::ExplicitSkip ) { + rss << "SKIPPED\n"; + } else { + rss << "FAILED" << ":\n"; + if (result.hasExpression()) { + rss << " "; + rss << result.getExpressionInMacro(); + rss << '\n'; + } + if (result.hasExpandedExpression()) { + rss << "with expansion:\n"; + rss << TextFlow::Column(result.getExpandedExpression()).indent(2) << '\n'; + } + } + + if( result.hasMessage() ) + rss << result.getMessage() << '\n'; + for( auto const& msg : stats.infoMessages ) + if( msg.type == ResultWas::Info ) + rss << msg.message << '\n'; + + rss << "at " << result.getSourceInfo(); + xml.writeText( rss.str(), XmlFormatting::Newline ); + } + } + +} // end namespace Catch diff --git a/src/catch2/reporters/catch_reporter_junit.hpp b/src/catch2/reporters/catch_reporter_junit.hpp new file mode 100644 index 0000000..7cb53c2 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_junit.hpp @@ -0,0 +1,56 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef CATCH_REPORTER_JUNIT_HPP_INCLUDED +#define CATCH_REPORTER_JUNIT_HPP_INCLUDED + + +#include <catch2/reporters/catch_reporter_cumulative_base.hpp> +#include <catch2/internal/catch_xmlwriter.hpp> +#include <catch2/catch_timer.hpp> + +namespace Catch { + + class JunitReporter final : public CumulativeReporterBase { + public: + JunitReporter(ReporterConfig&& _config); + + static std::string getDescription(); + + void testRunStarting(TestRunInfo const& runInfo) override; + + void testCaseStarting(TestCaseInfo const& testCaseInfo) override; + void assertionEnded(AssertionStats const& assertionStats) override; + + void testCaseEnded(TestCaseStats const& testCaseStats) override; + + void testRunEndedCumulative() override; + + private: + void writeRun(TestRunNode const& testRunNode, double suiteTime); + + void writeTestCase(TestCaseNode const& testCaseNode); + + void writeSection( std::string const& className, + std::string const& rootName, + SectionNode const& sectionNode, + bool testOkToFail ); + + void writeAssertions(SectionNode const& sectionNode); + void writeAssertion(AssertionStats const& stats); + + XmlWriter xml; + Timer suiteTimer; + std::string stdOutForSuite; + std::string stdErrForSuite; + unsigned int unexpectedExceptions = 0; + bool m_okToFail = false; + }; + +} // end namespace Catch + +#endif // CATCH_REPORTER_JUNIT_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporter_multi.cpp b/src/catch2/reporters/catch_reporter_multi.cpp new file mode 100644 index 0000000..531902b --- /dev/null +++ b/src/catch2/reporters/catch_reporter_multi.cpp @@ -0,0 +1,197 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#include <catch2/reporters/catch_reporter_multi.hpp> + +#include <catch2/catch_config.hpp> +#include <catch2/internal/catch_move_and_forward.hpp> +#include <catch2/internal/catch_stdstreams.hpp> + +#include <ostream> + +namespace Catch { + void MultiReporter::updatePreferences(IEventListener const& reporterish) { + m_preferences.shouldRedirectStdOut |= + reporterish.getPreferences().shouldRedirectStdOut; + m_preferences.shouldReportAllAssertions |= + reporterish.getPreferences().shouldReportAllAssertions; + } + + void MultiReporter::addListener( IEventListenerPtr&& listener ) { + updatePreferences(*listener); + m_reporterLikes.insert(m_reporterLikes.begin() + m_insertedListeners, CATCH_MOVE(listener) ); + ++m_insertedListeners; + } + + void MultiReporter::addReporter( IEventListenerPtr&& reporter ) { + updatePreferences(*reporter); + + // We will need to output the captured stdout if there are reporters + // that do not want it captured. + // We do not consider listeners, because it is generally assumed that + // listeners are output-transparent, even though they can ask for stdout + // capture to do something with it. + m_haveNoncapturingReporters |= !reporter->getPreferences().shouldRedirectStdOut; + + // Reporters can always be placed to the back without breaking the + // reporting order + m_reporterLikes.push_back( CATCH_MOVE( reporter ) ); + } + + void MultiReporter::noMatchingTestCases( StringRef unmatchedSpec ) { + for ( auto& reporterish : m_reporterLikes ) { + reporterish->noMatchingTestCases( unmatchedSpec ); + } + } + + void MultiReporter::fatalErrorEncountered( StringRef error ) { + for ( auto& reporterish : m_reporterLikes ) { + reporterish->fatalErrorEncountered( error ); + } + } + + void MultiReporter::reportInvalidTestSpec( StringRef arg ) { + for ( auto& reporterish : m_reporterLikes ) { + reporterish->reportInvalidTestSpec( arg ); + } + } + + void MultiReporter::benchmarkPreparing( StringRef name ) { + for (auto& reporterish : m_reporterLikes) { + reporterish->benchmarkPreparing(name); + } + } + void MultiReporter::benchmarkStarting( BenchmarkInfo const& benchmarkInfo ) { + for ( auto& reporterish : m_reporterLikes ) { + reporterish->benchmarkStarting( benchmarkInfo ); + } + } + void MultiReporter::benchmarkEnded( BenchmarkStats<> const& benchmarkStats ) { + for ( auto& reporterish : m_reporterLikes ) { + reporterish->benchmarkEnded( benchmarkStats ); + } + } + + void MultiReporter::benchmarkFailed( StringRef error ) { + for (auto& reporterish : m_reporterLikes) { + reporterish->benchmarkFailed(error); + } + } + + void MultiReporter::testRunStarting( TestRunInfo const& testRunInfo ) { + for ( auto& reporterish : m_reporterLikes ) { + reporterish->testRunStarting( testRunInfo ); + } + } + + void MultiReporter::testCaseStarting( TestCaseInfo const& testInfo ) { + for ( auto& reporterish : m_reporterLikes ) { + reporterish->testCaseStarting( testInfo ); + } + } + + void + MultiReporter::testCasePartialStarting( TestCaseInfo const& testInfo, + uint64_t partNumber ) { + for ( auto& reporterish : m_reporterLikes ) { + reporterish->testCasePartialStarting( testInfo, partNumber ); + } + } + + void MultiReporter::sectionStarting( SectionInfo const& sectionInfo ) { + for ( auto& reporterish : m_reporterLikes ) { + reporterish->sectionStarting( sectionInfo ); + } + } + + void MultiReporter::assertionStarting( AssertionInfo const& assertionInfo ) { + for ( auto& reporterish : m_reporterLikes ) { + reporterish->assertionStarting( assertionInfo ); + } + } + + void MultiReporter::assertionEnded( AssertionStats const& assertionStats ) { + const bool reportByDefault = + assertionStats.assertionResult.getResultType() != ResultWas::Ok || + m_config->includeSuccessfulResults(); + + for ( auto & reporterish : m_reporterLikes ) { + if ( reportByDefault || + reporterish->getPreferences().shouldReportAllAssertions ) { + reporterish->assertionEnded( assertionStats ); + } + } + } + + void MultiReporter::sectionEnded( SectionStats const& sectionStats ) { + for ( auto& reporterish : m_reporterLikes ) { + reporterish->sectionEnded( sectionStats ); + } + } + + void MultiReporter::testCasePartialEnded( TestCaseStats const& testStats, + uint64_t partNumber ) { + if ( m_preferences.shouldRedirectStdOut && + m_haveNoncapturingReporters ) { + if ( !testStats.stdOut.empty() ) { + Catch::cout() << testStats.stdOut << std::flush; + } + if ( !testStats.stdErr.empty() ) { + Catch::cerr() << testStats.stdErr << std::flush; + } + } + + for ( auto& reporterish : m_reporterLikes ) { + reporterish->testCasePartialEnded( testStats, partNumber ); + } + } + + void MultiReporter::testCaseEnded( TestCaseStats const& testCaseStats ) { + for ( auto& reporterish : m_reporterLikes ) { + reporterish->testCaseEnded( testCaseStats ); + } + } + + void MultiReporter::testRunEnded( TestRunStats const& testRunStats ) { + for ( auto& reporterish : m_reporterLikes ) { + reporterish->testRunEnded( testRunStats ); + } + } + + + void MultiReporter::skipTest( TestCaseInfo const& testInfo ) { + for ( auto& reporterish : m_reporterLikes ) { + reporterish->skipTest( testInfo ); + } + } + + void MultiReporter::listReporters(std::vector<ReporterDescription> const& descriptions) { + for (auto& reporterish : m_reporterLikes) { + reporterish->listReporters(descriptions); + } + } + + void MultiReporter::listListeners( + std::vector<ListenerDescription> const& descriptions ) { + for ( auto& reporterish : m_reporterLikes ) { + reporterish->listListeners( descriptions ); + } + } + + void MultiReporter::listTests(std::vector<TestCaseHandle> const& tests) { + for (auto& reporterish : m_reporterLikes) { + reporterish->listTests(tests); + } + } + + void MultiReporter::listTags(std::vector<TagInfo> const& tags) { + for (auto& reporterish : m_reporterLikes) { + reporterish->listTags(tags); + } + } + +} // end namespace Catch diff --git a/src/catch2/reporters/catch_reporter_multi.hpp b/src/catch2/reporters/catch_reporter_multi.hpp new file mode 100644 index 0000000..6611383 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_multi.hpp @@ -0,0 +1,72 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef CATCH_REPORTER_MULTI_HPP_INCLUDED +#define CATCH_REPORTER_MULTI_HPP_INCLUDED + +#include <catch2/interfaces/catch_interfaces_reporter.hpp> + +namespace Catch { + + class MultiReporter final : public IEventListener { + /* + * Stores all added reporters and listeners + * + * All Listeners are stored before all reporters, and individual + * listeners/reporters are stored in order of insertion. + */ + std::vector<IEventListenerPtr> m_reporterLikes; + bool m_haveNoncapturingReporters = false; + + // Keep track of how many listeners we have already inserted, + // so that we can insert them into the main vector at the right place + size_t m_insertedListeners = 0; + + void updatePreferences(IEventListener const& reporterish); + + public: + using IEventListener::IEventListener; + + void addListener( IEventListenerPtr&& listener ); + void addReporter( IEventListenerPtr&& reporter ); + + public: // IEventListener + + void noMatchingTestCases( StringRef unmatchedSpec ) override; + void fatalErrorEncountered( StringRef error ) override; + void reportInvalidTestSpec( StringRef arg ) override; + + void benchmarkPreparing( StringRef name ) override; + void benchmarkStarting( BenchmarkInfo const& benchmarkInfo ) override; + void benchmarkEnded( BenchmarkStats<> const& benchmarkStats ) override; + void benchmarkFailed( StringRef error ) override; + + void testRunStarting( TestRunInfo const& testRunInfo ) override; + void testCaseStarting( TestCaseInfo const& testInfo ) override; + void testCasePartialStarting(TestCaseInfo const& testInfo, uint64_t partNumber) override; + void sectionStarting( SectionInfo const& sectionInfo ) override; + void assertionStarting( AssertionInfo const& assertionInfo ) override; + + void assertionEnded( AssertionStats const& assertionStats ) override; + void sectionEnded( SectionStats const& sectionStats ) override; + void testCasePartialEnded(TestCaseStats const& testStats, uint64_t partNumber) override; + void testCaseEnded( TestCaseStats const& testCaseStats ) override; + void testRunEnded( TestRunStats const& testRunStats ) override; + + void skipTest( TestCaseInfo const& testInfo ) override; + + void listReporters(std::vector<ReporterDescription> const& descriptions) override; + void listListeners(std::vector<ListenerDescription> const& descriptions) override; + void listTests(std::vector<TestCaseHandle> const& tests) override; + void listTags(std::vector<TagInfo> const& tags) override; + + + }; + +} // end namespace Catch + +#endif // CATCH_REPORTER_MULTI_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporter_registrars.cpp b/src/catch2/reporters/catch_reporter_registrars.cpp new file mode 100644 index 0000000..2a3ac95 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_registrars.cpp @@ -0,0 +1,36 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +#include <catch2/reporters/catch_reporter_registrars.hpp> + +#include <catch2/interfaces/catch_interfaces_registry_hub.hpp> +#include <catch2/internal/catch_compiler_capabilities.hpp> + +namespace Catch { + namespace Detail { + + void registerReporterImpl( std::string const& name, + IReporterFactoryPtr reporterPtr ) { + CATCH_TRY { + getMutableRegistryHub().registerReporter( + name, CATCH_MOVE( reporterPtr ) ); + } + CATCH_CATCH_ALL { + // Do not throw when constructing global objects, instead + // register the exception to be processed later + getMutableRegistryHub().registerStartupException(); + } + } + + void registerListenerImpl( Detail::unique_ptr<EventListenerFactory> listenerFactory ) { + getMutableRegistryHub().registerListener( CATCH_MOVE(listenerFactory) ); + } + + + } // namespace Detail +} // namespace Catch diff --git a/src/catch2/reporters/catch_reporter_registrars.hpp b/src/catch2/reporters/catch_reporter_registrars.hpp new file mode 100644 index 0000000..a93963f --- /dev/null +++ b/src/catch2/reporters/catch_reporter_registrars.hpp @@ -0,0 +1,131 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef CATCH_REPORTER_REGISTRARS_HPP_INCLUDED +#define CATCH_REPORTER_REGISTRARS_HPP_INCLUDED + +#include <catch2/interfaces/catch_interfaces_reporter_factory.hpp> +#include <catch2/internal/catch_compiler_capabilities.hpp> +#include <catch2/internal/catch_unique_name.hpp> +#include <catch2/internal/catch_unique_ptr.hpp> +#include <catch2/internal/catch_move_and_forward.hpp> +#include <catch2/internal/catch_void_type.hpp> + +#include <type_traits> + +namespace Catch { + + namespace Detail { + + template <typename T, typename = void> + struct has_description : std::false_type {}; + + template <typename T> + struct has_description< + T, + void_t<decltype( T::getDescription() )>> + : std::true_type {}; + + //! Indirection for reporter registration, so that the error handling is + //! independent on the reporter's concrete type + void registerReporterImpl( std::string const& name, + IReporterFactoryPtr reporterPtr ); + //! Actually registers the factory, independent on listener's concrete type + void registerListenerImpl( Detail::unique_ptr<EventListenerFactory> listenerFactory ); + } // namespace Detail + + class IEventListener; + using IEventListenerPtr = Detail::unique_ptr<IEventListener>; + + template <typename T> + class ReporterFactory : public IReporterFactory { + + IEventListenerPtr create( ReporterConfig&& config ) const override { + return Detail::make_unique<T>( CATCH_MOVE(config) ); + } + + std::string getDescription() const override { + return T::getDescription(); + } + }; + + + template<typename T> + class ReporterRegistrar { + public: + explicit ReporterRegistrar( std::string const& name ) { + registerReporterImpl( name, + Detail::make_unique<ReporterFactory<T>>() ); + } + }; + + template<typename T> + class ListenerRegistrar { + + class TypedListenerFactory : public EventListenerFactory { + StringRef m_listenerName; + + std::string getDescriptionImpl( std::true_type ) const { + return T::getDescription(); + } + + std::string getDescriptionImpl( std::false_type ) const { + return "(No description provided)"; + } + + public: + TypedListenerFactory( StringRef listenerName ): + m_listenerName( listenerName ) {} + + IEventListenerPtr create( IConfig const* config ) const override { + return Detail::make_unique<T>( config ); + } + + StringRef getName() const override { + return m_listenerName; + } + + std::string getDescription() const override { + return getDescriptionImpl( Detail::has_description<T>{} ); + } + }; + + public: + ListenerRegistrar(StringRef listenerName) { + registerListenerImpl( Detail::make_unique<TypedListenerFactory>(listenerName) ); + } + }; +} + +#if !defined(CATCH_CONFIG_DISABLE) + +# define CATCH_REGISTER_REPORTER( name, reporterType ) \ + CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \ + CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + namespace { \ + Catch::ReporterRegistrar<reporterType> INTERNAL_CATCH_UNIQUE_NAME( \ + catch_internal_RegistrarFor )( name ); \ + } \ + CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION + +# define CATCH_REGISTER_LISTENER( listenerType ) \ + CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \ + CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + namespace { \ + Catch::ListenerRegistrar<listenerType> INTERNAL_CATCH_UNIQUE_NAME( \ + catch_internal_RegistrarFor )( #listenerType##_catch_sr ); \ + } \ + CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION + +#else // CATCH_CONFIG_DISABLE + +#define CATCH_REGISTER_REPORTER(name, reporterType) +#define CATCH_REGISTER_LISTENER(listenerType) + +#endif // CATCH_CONFIG_DISABLE + +#endif // CATCH_REPORTER_REGISTRARS_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporter_sonarqube.cpp b/src/catch2/reporters/catch_reporter_sonarqube.cpp new file mode 100644 index 0000000..2c3eb1c --- /dev/null +++ b/src/catch2/reporters/catch_reporter_sonarqube.cpp @@ -0,0 +1,162 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#include <catch2/reporters/catch_reporter_sonarqube.hpp> + +#include <catch2/internal/catch_string_manip.hpp> +#include <catch2/catch_test_case_info.hpp> +#include <catch2/internal/catch_reusable_string_stream.hpp> +#include <catch2/interfaces/catch_interfaces_config.hpp> +#include <catch2/catch_test_spec.hpp> +#include <catch2/reporters/catch_reporter_helpers.hpp> + +#include <map> + +namespace Catch { + + namespace { + std::string createMetadataString(IConfig const& config) { + ReusableStringStream sstr; + if ( config.testSpec().hasFilters() ) { + sstr << "filters='" + << config.testSpec() + << "' "; + } + sstr << "rng-seed=" << config.rngSeed(); + return sstr.str(); + } + } + + void SonarQubeReporter::testRunStarting(TestRunInfo const& testRunInfo) { + CumulativeReporterBase::testRunStarting(testRunInfo); + + xml.writeComment( createMetadataString( *m_config ) ); + xml.startElement("testExecutions"); + xml.writeAttribute("version"_sr, '1'); + } + + void SonarQubeReporter::writeRun( TestRunNode const& runNode ) { + std::map<StringRef, std::vector<TestCaseNode const*>> testsPerFile; + + for ( auto const& child : runNode.children ) { + testsPerFile[child->value.testInfo->lineInfo.file].push_back( + child.get() ); + } + + for ( auto const& kv : testsPerFile ) { + writeTestFile( kv.first, kv.second ); + } + } + + void SonarQubeReporter::writeTestFile(StringRef filename, std::vector<TestCaseNode const*> const& testCaseNodes) { + XmlWriter::ScopedElement e = xml.scopedElement("file"); + xml.writeAttribute("path"_sr, filename); + + for (auto const& child : testCaseNodes) + writeTestCase(*child); + } + + void SonarQubeReporter::writeTestCase(TestCaseNode const& testCaseNode) { + // All test cases have exactly one section - which represents the + // test case itself. That section may have 0-n nested sections + assert(testCaseNode.children.size() == 1); + SectionNode const& rootSection = *testCaseNode.children.front(); + writeSection("", rootSection, testCaseNode.value.testInfo->okToFail()); + } + + void SonarQubeReporter::writeSection(std::string const& rootName, SectionNode const& sectionNode, bool okToFail) { + std::string name = trim(sectionNode.stats.sectionInfo.name); + if (!rootName.empty()) + name = rootName + '/' + name; + + if ( sectionNode.stats.assertions.total() > 0 + || !sectionNode.stdOut.empty() + || !sectionNode.stdErr.empty() ) { + XmlWriter::ScopedElement e = xml.scopedElement("testCase"); + xml.writeAttribute("name"_sr, name); + xml.writeAttribute("duration"_sr, static_cast<long>(sectionNode.stats.durationInSeconds * 1000)); + + writeAssertions(sectionNode, okToFail); + } + + for (auto const& childNode : sectionNode.childSections) + writeSection(name, *childNode, okToFail); + } + + void SonarQubeReporter::writeAssertions(SectionNode const& sectionNode, bool okToFail) { + for (auto const& assertionOrBenchmark : sectionNode.assertionsAndBenchmarks) { + if (assertionOrBenchmark.isAssertion()) { + writeAssertion(assertionOrBenchmark.asAssertion(), okToFail); + } + } + } + + void SonarQubeReporter::writeAssertion(AssertionStats const& stats, bool okToFail) { + AssertionResult const& result = stats.assertionResult; + if ( !result.isOk() || + result.getResultType() == ResultWas::ExplicitSkip ) { + std::string elementName; + if (okToFail) { + elementName = "skipped"; + } else { + switch (result.getResultType()) { + case ResultWas::ThrewException: + case ResultWas::FatalErrorCondition: + elementName = "error"; + break; + case ResultWas::ExplicitFailure: + case ResultWas::ExpressionFailed: + case ResultWas::DidntThrowException: + elementName = "failure"; + break; + case ResultWas::ExplicitSkip: + elementName = "skipped"; + break; + // We should never see these here: + case ResultWas::Info: + case ResultWas::Warning: + case ResultWas::Ok: + case ResultWas::Unknown: + case ResultWas::FailureBit: + case ResultWas::Exception: + elementName = "internalError"; + break; + } + } + + XmlWriter::ScopedElement e = xml.scopedElement(elementName); + + ReusableStringStream messageRss; + messageRss << result.getTestMacroName() << '(' << result.getExpression() << ')'; + xml.writeAttribute("message"_sr, messageRss.str()); + + ReusableStringStream textRss; + if ( result.getResultType() == ResultWas::ExplicitSkip ) { + textRss << "SKIPPED\n"; + } else { + textRss << "FAILED:\n"; + if (result.hasExpression()) { + textRss << '\t' << result.getExpressionInMacro() << '\n'; + } + if (result.hasExpandedExpression()) { + textRss << "with expansion:\n\t" << result.getExpandedExpression() << '\n'; + } + } + + if (result.hasMessage()) + textRss << result.getMessage() << '\n'; + + for (auto const& msg : stats.infoMessages) + if (msg.type == ResultWas::Info) + textRss << msg.message << '\n'; + + textRss << "at " << result.getSourceInfo(); + xml.writeText(textRss.str(), XmlFormatting::Newline); + } + } + +} // end namespace Catch diff --git a/src/catch2/reporters/catch_reporter_sonarqube.hpp b/src/catch2/reporters/catch_reporter_sonarqube.hpp new file mode 100644 index 0000000..509f411 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_sonarqube.hpp @@ -0,0 +1,59 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef CATCH_REPORTER_SONARQUBE_HPP_INCLUDED +#define CATCH_REPORTER_SONARQUBE_HPP_INCLUDED + +#include <catch2/reporters/catch_reporter_cumulative_base.hpp> + +#include <catch2/internal/catch_xmlwriter.hpp> +#include <catch2/internal/catch_move_and_forward.hpp> + +namespace Catch { + + class SonarQubeReporter final : public CumulativeReporterBase { + public: + SonarQubeReporter(ReporterConfig&& config) + : CumulativeReporterBase(CATCH_MOVE(config)) + , xml(m_stream) { + m_preferences.shouldRedirectStdOut = true; + m_preferences.shouldReportAllAssertions = false; + m_shouldStoreSuccesfulAssertions = false; + } + + static std::string getDescription() { + using namespace std::string_literals; + return "Reports test results in the Generic Test Data SonarQube XML format"s; + } + + void testRunStarting( TestRunInfo const& testRunInfo ) override; + + void testRunEndedCumulative() override { + writeRun( *m_testRun ); + xml.endElement(); + } + + void writeRun( TestRunNode const& runNode ); + + void writeTestFile(StringRef filename, std::vector<TestCaseNode const*> const& testCaseNodes); + + void writeTestCase(TestCaseNode const& testCaseNode); + + void writeSection(std::string const& rootName, SectionNode const& sectionNode, bool okToFail); + + void writeAssertions(SectionNode const& sectionNode, bool okToFail); + + void writeAssertion(AssertionStats const& stats, bool okToFail); + + private: + XmlWriter xml; + }; + + +} // end namespace Catch + +#endif // CATCH_REPORTER_SONARQUBE_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporter_streaming_base.cpp b/src/catch2/reporters/catch_reporter_streaming_base.cpp new file mode 100644 index 0000000..f1cc425 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_streaming_base.cpp @@ -0,0 +1,23 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#include <catch2/reporters/catch_reporter_streaming_base.hpp> + +namespace Catch { + + StreamingReporterBase::~StreamingReporterBase() = default; + + void + StreamingReporterBase::testRunStarting( TestRunInfo const& _testRunInfo ) { + currentTestRunInfo = _testRunInfo; + } + + void StreamingReporterBase::testRunEnded( TestRunStats const& ) { + currentTestCaseInfo = nullptr; + } + +} // end namespace Catch diff --git a/src/catch2/reporters/catch_reporter_streaming_base.hpp b/src/catch2/reporters/catch_reporter_streaming_base.hpp new file mode 100644 index 0000000..5448000 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_streaming_base.hpp @@ -0,0 +1,73 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef CATCH_REPORTER_STREAMING_BASE_HPP_INCLUDED +#define CATCH_REPORTER_STREAMING_BASE_HPP_INCLUDED + +#include <catch2/reporters/catch_reporter_common_base.hpp> +#include <catch2/internal/catch_move_and_forward.hpp> + +#include <vector> + +namespace Catch { + + class StreamingReporterBase : public ReporterBase { + public: + // GCC5 compat: we cannot use inherited constructor, because it + // doesn't implement backport of P0136 + StreamingReporterBase(ReporterConfig&& _config): + ReporterBase(CATCH_MOVE(_config)) + {} + ~StreamingReporterBase() override; + + void benchmarkPreparing( StringRef ) override {} + void benchmarkStarting( BenchmarkInfo const& ) override {} + void benchmarkEnded( BenchmarkStats<> const& ) override {} + void benchmarkFailed( StringRef ) override {} + + void fatalErrorEncountered( StringRef /*error*/ ) override {} + void noMatchingTestCases( StringRef /*unmatchedSpec*/ ) override {} + void reportInvalidTestSpec( StringRef /*invalidArgument*/ ) override {} + + void testRunStarting( TestRunInfo const& _testRunInfo ) override; + + void testCaseStarting(TestCaseInfo const& _testInfo) override { + currentTestCaseInfo = &_testInfo; + } + void testCasePartialStarting( TestCaseInfo const&, uint64_t ) override {} + void sectionStarting(SectionInfo const& _sectionInfo) override { + m_sectionStack.push_back(_sectionInfo); + } + + void assertionStarting( AssertionInfo const& ) override {} + void assertionEnded( AssertionStats const& ) override {} + + void sectionEnded(SectionStats const& /* _sectionStats */) override { + m_sectionStack.pop_back(); + } + void testCasePartialEnded( TestCaseStats const&, uint64_t ) override {} + void testCaseEnded(TestCaseStats const& /* _testCaseStats */) override { + currentTestCaseInfo = nullptr; + } + void testRunEnded( TestRunStats const& /* _testRunStats */ ) override; + + void skipTest(TestCaseInfo const&) override { + // Don't do anything with this by default. + // It can optionally be overridden in the derived class. + } + + protected: + TestRunInfo currentTestRunInfo{ "test run has not started yet"_sr }; + TestCaseInfo const* currentTestCaseInfo = nullptr; + + //! Stack of all _active_ sections in the _current_ test case + std::vector<SectionInfo> m_sectionStack; + }; + +} // end namespace Catch + +#endif // CATCH_REPORTER_STREAMING_BASE_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporter_tap.cpp b/src/catch2/reporters/catch_reporter_tap.cpp new file mode 100644 index 0000000..67d406f --- /dev/null +++ b/src/catch2/reporters/catch_reporter_tap.cpp @@ -0,0 +1,228 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#include <catch2/reporters/catch_reporter_tap.hpp> +#include <catch2/internal/catch_console_colour.hpp> +#include <catch2/internal/catch_string_manip.hpp> +#include <catch2/catch_test_case_info.hpp> +#include <catch2/interfaces/catch_interfaces_config.hpp> +#include <catch2/catch_test_spec.hpp> +#include <catch2/reporters/catch_reporter_helpers.hpp> + +#include <algorithm> +#include <ostream> + +namespace Catch { + + namespace { + // Yes, this has to be outside the class and namespaced by naming. + // Making older compiler happy is hard. + static constexpr StringRef tapFailedString = "not ok"_sr; + static constexpr StringRef tapPassedString = "ok"_sr; + static constexpr Colour::Code tapDimColour = Colour::FileName; + + class TapAssertionPrinter { + public: + TapAssertionPrinter& operator= (TapAssertionPrinter const&) = delete; + TapAssertionPrinter(TapAssertionPrinter const&) = delete; + TapAssertionPrinter(std::ostream& _stream, AssertionStats const& _stats, std::size_t _counter, ColourImpl* colour_) + : stream(_stream) + , result(_stats.assertionResult) + , messages(_stats.infoMessages) + , itMessage(_stats.infoMessages.begin()) + , printInfoMessages(true) + , counter(_counter) + , colourImpl( colour_ ) {} + + void print() { + itMessage = messages.begin(); + + switch (result.getResultType()) { + case ResultWas::Ok: + printResultType(tapPassedString); + printOriginalExpression(); + printReconstructedExpression(); + if (!result.hasExpression()) + printRemainingMessages(Colour::None); + else + printRemainingMessages(); + break; + case ResultWas::ExpressionFailed: + if (result.isOk()) { + printResultType(tapPassedString); + } else { + printResultType(tapFailedString); + } + printOriginalExpression(); + printReconstructedExpression(); + if (result.isOk()) { + printIssue(" # TODO"); + } + printRemainingMessages(); + break; + case ResultWas::ThrewException: + printResultType(tapFailedString); + printIssue("unexpected exception with message:"_sr); + printMessage(); + printExpressionWas(); + printRemainingMessages(); + break; + case ResultWas::FatalErrorCondition: + printResultType(tapFailedString); + printIssue("fatal error condition with message:"_sr); + printMessage(); + printExpressionWas(); + printRemainingMessages(); + break; + case ResultWas::DidntThrowException: + printResultType(tapFailedString); + printIssue("expected exception, got none"_sr); + printExpressionWas(); + printRemainingMessages(); + break; + case ResultWas::Info: + printResultType("info"_sr); + printMessage(); + printRemainingMessages(); + break; + case ResultWas::Warning: + printResultType("warning"_sr); + printMessage(); + printRemainingMessages(); + break; + case ResultWas::ExplicitFailure: + printResultType(tapFailedString); + printIssue("explicitly"_sr); + printRemainingMessages(Colour::None); + break; + case ResultWas::ExplicitSkip: + printResultType(tapPassedString); + printIssue(" # SKIP"_sr); + printMessage(); + printRemainingMessages(); + break; + // These cases are here to prevent compiler warnings + case ResultWas::Unknown: + case ResultWas::FailureBit: + case ResultWas::Exception: + printResultType("** internal error **"_sr); + break; + } + } + + private: + void printResultType(StringRef passOrFail) const { + if (!passOrFail.empty()) { + stream << passOrFail << ' ' << counter << " -"; + } + } + + void printIssue(StringRef issue) const { + stream << ' ' << issue; + } + + void printExpressionWas() { + if (result.hasExpression()) { + stream << ';'; + stream << colourImpl->guardColour( tapDimColour ) + << " expression was:"; + printOriginalExpression(); + } + } + + void printOriginalExpression() const { + if (result.hasExpression()) { + stream << ' ' << result.getExpression(); + } + } + + void printReconstructedExpression() const { + if (result.hasExpandedExpression()) { + stream << colourImpl->guardColour( tapDimColour ) << " for: "; + + std::string expr = result.getExpandedExpression(); + std::replace(expr.begin(), expr.end(), '\n', ' '); + stream << expr; + } + } + + void printMessage() { + if (itMessage != messages.end()) { + stream << " '" << itMessage->message << '\''; + ++itMessage; + } + } + + void printRemainingMessages(Colour::Code colour = tapDimColour) { + if (itMessage == messages.end()) { + return; + } + + // using messages.end() directly (or auto) yields compilation error: + std::vector<MessageInfo>::const_iterator itEnd = messages.end(); + const std::size_t N = static_cast<std::size_t>(itEnd - itMessage); + + stream << colourImpl->guardColour( colour ) << " with " + << pluralise( N, "message"_sr ) << ':'; + + for (; itMessage != itEnd; ) { + // If this assertion is a warning ignore any INFO messages + if (printInfoMessages || itMessage->type != ResultWas::Info) { + stream << " '" << itMessage->message << '\''; + if (++itMessage != itEnd) { + stream << colourImpl->guardColour(tapDimColour) << " and"; + } + } + } + } + + private: + std::ostream& stream; + AssertionResult const& result; + std::vector<MessageInfo> const& messages; + std::vector<MessageInfo>::const_iterator itMessage; + bool printInfoMessages; + std::size_t counter; + ColourImpl* colourImpl; + }; + + } // End anonymous namespace + + void TAPReporter::testRunStarting( TestRunInfo const& ) { + if ( m_config->testSpec().hasFilters() ) { + m_stream << "# filters: " << m_config->testSpec() << '\n'; + } + m_stream << "# rng-seed: " << m_config->rngSeed() << '\n'; + } + + void TAPReporter::noMatchingTestCases( StringRef unmatchedSpec ) { + m_stream << "# No test cases matched '" << unmatchedSpec << "'\n"; + } + + void TAPReporter::assertionEnded(AssertionStats const& _assertionStats) { + ++counter; + + m_stream << "# " << currentTestCaseInfo->name << '\n'; + TapAssertionPrinter printer(m_stream, _assertionStats, counter, m_colour.get()); + printer.print(); + + m_stream << '\n' << std::flush; + } + + void TAPReporter::testRunEnded(TestRunStats const& _testRunStats) { + m_stream << "1.." << _testRunStats.totals.assertions.total(); + if (_testRunStats.totals.testCases.total() == 0) { + m_stream << " # Skipped: No tests ran."; + } + m_stream << "\n\n" << std::flush; + StreamingReporterBase::testRunEnded(_testRunStats); + } + + + + +} // end namespace Catch diff --git a/src/catch2/reporters/catch_reporter_tap.hpp b/src/catch2/reporters/catch_reporter_tap.hpp new file mode 100644 index 0000000..e6889bb --- /dev/null +++ b/src/catch2/reporters/catch_reporter_tap.hpp @@ -0,0 +1,42 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef CATCH_REPORTER_TAP_HPP_INCLUDED +#define CATCH_REPORTER_TAP_HPP_INCLUDED + +#include <catch2/reporters/catch_reporter_streaming_base.hpp> +#include <catch2/internal/catch_move_and_forward.hpp> + +namespace Catch { + + class TAPReporter final : public StreamingReporterBase { + public: + TAPReporter( ReporterConfig&& config ): + StreamingReporterBase( CATCH_MOVE(config) ) { + m_preferences.shouldReportAllAssertions = true; + } + + static std::string getDescription() { + using namespace std::string_literals; + return "Reports test results in TAP format, suitable for test harnesses"s; + } + + void testRunStarting( TestRunInfo const& testInfo ) override; + + void noMatchingTestCases( StringRef unmatchedSpec ) override; + + void assertionEnded(AssertionStats const& _assertionStats) override; + + void testRunEnded(TestRunStats const& _testRunStats) override; + + private: + std::size_t counter = 0; + }; + +} // end namespace Catch + +#endif // CATCH_REPORTER_TAP_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporter_teamcity.cpp b/src/catch2/reporters/catch_reporter_teamcity.cpp new file mode 100644 index 0000000..38aa55a --- /dev/null +++ b/src/catch2/reporters/catch_reporter_teamcity.cpp @@ -0,0 +1,177 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#include <catch2/reporters/catch_reporter_teamcity.hpp> + +#include <catch2/reporters/catch_reporter_helpers.hpp> +#include <catch2/internal/catch_string_manip.hpp> +#include <catch2/internal/catch_enforce.hpp> +#include <catch2/internal/catch_textflow.hpp> +#include <catch2/catch_test_case_info.hpp> + +#include <cassert> +#include <ostream> + +namespace Catch { + + namespace { + // if string has a : in first line will set indent to follow it on + // subsequent lines + void printHeaderString(std::ostream& os, std::string const& _string, std::size_t indent = 0) { + std::size_t i = _string.find(": "); + if (i != std::string::npos) + i += 2; + else + i = 0; + os << TextFlow::Column(_string) + .indent(indent + i) + .initialIndent(indent) << '\n'; + } + + std::string escape(StringRef str) { + std::string escaped = static_cast<std::string>(str); + replaceInPlace(escaped, "|", "||"); + replaceInPlace(escaped, "'", "|'"); + replaceInPlace(escaped, "\n", "|n"); + replaceInPlace(escaped, "\r", "|r"); + replaceInPlace(escaped, "[", "|["); + replaceInPlace(escaped, "]", "|]"); + return escaped; + } + } // end anonymous namespace + + + TeamCityReporter::~TeamCityReporter() = default; + + void TeamCityReporter::testRunStarting( TestRunInfo const& runInfo ) { + m_stream << "##teamcity[testSuiteStarted name='" << escape( runInfo.name ) + << "']\n"; + } + + void TeamCityReporter::testRunEnded( TestRunStats const& runStats ) { + m_stream << "##teamcity[testSuiteFinished name='" + << escape( runStats.runInfo.name ) << "']\n"; + } + + void TeamCityReporter::assertionEnded(AssertionStats const& assertionStats) { + AssertionResult const& result = assertionStats.assertionResult; + if ( !result.isOk() || + result.getResultType() == ResultWas::ExplicitSkip ) { + + ReusableStringStream msg; + if (!m_headerPrintedForThisSection) + printSectionHeader(msg.get()); + m_headerPrintedForThisSection = true; + + msg << result.getSourceInfo() << '\n'; + + switch (result.getResultType()) { + case ResultWas::ExpressionFailed: + msg << "expression failed"; + break; + case ResultWas::ThrewException: + msg << "unexpected exception"; + break; + case ResultWas::FatalErrorCondition: + msg << "fatal error condition"; + break; + case ResultWas::DidntThrowException: + msg << "no exception was thrown where one was expected"; + break; + case ResultWas::ExplicitFailure: + msg << "explicit failure"; + break; + case ResultWas::ExplicitSkip: + msg << "explicit skip"; + break; + + // We shouldn't get here because of the isOk() test + case ResultWas::Ok: + case ResultWas::Info: + case ResultWas::Warning: + CATCH_ERROR("Internal error in TeamCity reporter"); + // These cases are here to prevent compiler warnings + case ResultWas::Unknown: + case ResultWas::FailureBit: + case ResultWas::Exception: + CATCH_ERROR("Not implemented"); + } + if (assertionStats.infoMessages.size() == 1) + msg << " with message:"; + if (assertionStats.infoMessages.size() > 1) + msg << " with messages:"; + for (auto const& messageInfo : assertionStats.infoMessages) + msg << "\n \"" << messageInfo.message << '"'; + + + if (result.hasExpression()) { + msg << + "\n " << result.getExpressionInMacro() << "\n" + "with expansion:\n" + " " << result.getExpandedExpression() << '\n'; + } + + if ( result.getResultType() == ResultWas::ExplicitSkip ) { + m_stream << "##teamcity[testIgnored"; + } else if ( currentTestCaseInfo->okToFail() ) { + msg << "- failure ignore as test marked as 'ok to fail'\n"; + m_stream << "##teamcity[testIgnored"; + } else { + m_stream << "##teamcity[testFailed"; + } + m_stream << " name='" << escape( currentTestCaseInfo->name ) << '\'' + << " message='" << escape( msg.str() ) << '\'' << "]\n"; + } + m_stream.flush(); + } + + void TeamCityReporter::testCaseStarting(TestCaseInfo const& testInfo) { + m_testTimer.start(); + StreamingReporterBase::testCaseStarting(testInfo); + m_stream << "##teamcity[testStarted name='" + << escape(testInfo.name) << "']\n"; + m_stream.flush(); + } + + void TeamCityReporter::testCaseEnded(TestCaseStats const& testCaseStats) { + StreamingReporterBase::testCaseEnded(testCaseStats); + auto const& testCaseInfo = *testCaseStats.testInfo; + if (!testCaseStats.stdOut.empty()) + m_stream << "##teamcity[testStdOut name='" + << escape(testCaseInfo.name) + << "' out='" << escape(testCaseStats.stdOut) << "']\n"; + if (!testCaseStats.stdErr.empty()) + m_stream << "##teamcity[testStdErr name='" + << escape(testCaseInfo.name) + << "' out='" << escape(testCaseStats.stdErr) << "']\n"; + m_stream << "##teamcity[testFinished name='" + << escape(testCaseInfo.name) << "' duration='" + << m_testTimer.getElapsedMilliseconds() << "']\n"; + m_stream.flush(); + } + + void TeamCityReporter::printSectionHeader(std::ostream& os) { + assert(!m_sectionStack.empty()); + + if (m_sectionStack.size() > 1) { + os << lineOfChars('-') << '\n'; + + std::vector<SectionInfo>::const_iterator + it = m_sectionStack.begin() + 1, // Skip first section (test case) + itEnd = m_sectionStack.end(); + for (; it != itEnd; ++it) + printHeaderString(os, it->name); + os << lineOfChars('-') << '\n'; + } + + SourceLineInfo lineInfo = m_sectionStack.front().lineInfo; + + os << lineInfo << '\n'; + os << lineOfChars('.') << "\n\n"; + } + +} // end namespace Catch diff --git a/src/catch2/reporters/catch_reporter_teamcity.hpp b/src/catch2/reporters/catch_reporter_teamcity.hpp new file mode 100644 index 0000000..662e989 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_teamcity.hpp @@ -0,0 +1,66 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef CATCH_REPORTER_TEAMCITY_HPP_INCLUDED +#define CATCH_REPORTER_TEAMCITY_HPP_INCLUDED + +#include <catch2/reporters/catch_reporter_streaming_base.hpp> +#include <catch2/catch_timer.hpp> + +#include <cstring> + +#ifdef __clang__ +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wpadded" +#endif + +namespace Catch { + + class TeamCityReporter final : public StreamingReporterBase { + public: + TeamCityReporter( ReporterConfig&& _config ) + : StreamingReporterBase( CATCH_MOVE(_config) ) + { + m_preferences.shouldRedirectStdOut = true; + } + + ~TeamCityReporter() override; + + static std::string getDescription() { + using namespace std::string_literals; + return "Reports test results as TeamCity service messages"s; + } + + void testRunStarting( TestRunInfo const& runInfo ) override; + void testRunEnded( TestRunStats const& runStats ) override; + + + void assertionEnded(AssertionStats const& assertionStats) override; + + void sectionStarting(SectionInfo const& sectionInfo) override { + m_headerPrintedForThisSection = false; + StreamingReporterBase::sectionStarting( sectionInfo ); + } + + void testCaseStarting(TestCaseInfo const& testInfo) override; + + void testCaseEnded(TestCaseStats const& testCaseStats) override; + + private: + void printSectionHeader(std::ostream& os); + + bool m_headerPrintedForThisSection = false; + Timer m_testTimer; + }; + +} // end namespace Catch + +#ifdef __clang__ +# pragma clang diagnostic pop +#endif + +#endif // CATCH_REPORTER_TEAMCITY_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporter_xml.cpp b/src/catch2/reporters/catch_reporter_xml.cpp new file mode 100644 index 0000000..35a3028 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_xml.cpp @@ -0,0 +1,333 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#include <catch2/reporters/catch_reporter_xml.hpp> + +#include <catch2/reporters/catch_reporter_helpers.hpp> +#include <catch2/interfaces/catch_interfaces_config.hpp> +#include <catch2/catch_test_spec.hpp> +#include <catch2/internal/catch_string_manip.hpp> +#include <catch2/internal/catch_list.hpp> +#include <catch2/catch_test_case_info.hpp> +#include <catch2/internal/catch_move_and_forward.hpp> +#include <catch2/catch_version.hpp> + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable:4061) // Not all labels are EXPLICITLY handled in switch + // Note that 4062 (not all labels are handled + // and default is missing) is enabled +#endif + +namespace Catch { + XmlReporter::XmlReporter( ReporterConfig&& _config ) + : StreamingReporterBase( CATCH_MOVE(_config) ), + m_xml(m_stream) + { + m_preferences.shouldRedirectStdOut = true; + m_preferences.shouldReportAllAssertions = true; + } + + XmlReporter::~XmlReporter() = default; + + std::string XmlReporter::getDescription() { + return "Reports test results as an XML document"; + } + + std::string XmlReporter::getStylesheetRef() const { + return std::string(); + } + + void XmlReporter::writeSourceInfo( SourceLineInfo const& sourceInfo ) { + m_xml + .writeAttribute( "filename"_sr, sourceInfo.file ) + .writeAttribute( "line"_sr, sourceInfo.line ); + } + + void XmlReporter::testRunStarting( TestRunInfo const& testInfo ) { + StreamingReporterBase::testRunStarting( testInfo ); + std::string stylesheetRef = getStylesheetRef(); + if( !stylesheetRef.empty() ) + m_xml.writeStylesheetRef( stylesheetRef ); + m_xml.startElement("Catch2TestRun") + .writeAttribute("name"_sr, m_config->name()) + .writeAttribute("rng-seed"_sr, m_config->rngSeed()) + .writeAttribute("xml-format-version"_sr, 3) + .writeAttribute("catch2-version"_sr, libraryVersion()); + if ( m_config->testSpec().hasFilters() ) { + m_xml.writeAttribute( "filters"_sr, m_config->testSpec() ); + } + } + + void XmlReporter::testCaseStarting( TestCaseInfo const& testInfo ) { + StreamingReporterBase::testCaseStarting(testInfo); + m_xml.startElement( "TestCase" ) + .writeAttribute( "name"_sr, trim( StringRef(testInfo.name) ) ) + .writeAttribute( "tags"_sr, testInfo.tagsAsString() ); + + writeSourceInfo( testInfo.lineInfo ); + + if ( m_config->showDurations() == ShowDurations::Always ) + m_testCaseTimer.start(); + m_xml.ensureTagClosed(); + } + + void XmlReporter::sectionStarting( SectionInfo const& sectionInfo ) { + StreamingReporterBase::sectionStarting( sectionInfo ); + if( m_sectionDepth++ > 0 ) { + m_xml.startElement( "Section" ) + .writeAttribute( "name"_sr, trim( StringRef(sectionInfo.name) ) ); + writeSourceInfo( sectionInfo.lineInfo ); + m_xml.ensureTagClosed(); + } + } + + void XmlReporter::assertionStarting( AssertionInfo const& ) { } + + void XmlReporter::assertionEnded( AssertionStats const& assertionStats ) { + + AssertionResult const& result = assertionStats.assertionResult; + + bool includeResults = m_config->includeSuccessfulResults() || !result.isOk(); + + if( includeResults || result.getResultType() == ResultWas::Warning ) { + // Print any info messages in <Info> tags. + for( auto const& msg : assertionStats.infoMessages ) { + if( msg.type == ResultWas::Info && includeResults ) { + auto t = m_xml.scopedElement( "Info" ); + writeSourceInfo( msg.lineInfo ); + t.writeText( msg.message ); + } else if ( msg.type == ResultWas::Warning ) { + auto t = m_xml.scopedElement( "Warning" ); + writeSourceInfo( msg.lineInfo ); + t.writeText( msg.message ); + } + } + } + + // Drop out if result was successful but we're not printing them. + if ( !includeResults && result.getResultType() != ResultWas::Warning && + result.getResultType() != ResultWas::ExplicitSkip ) { + return; + } + + // Print the expression if there is one. + if( result.hasExpression() ) { + m_xml.startElement( "Expression" ) + .writeAttribute( "success"_sr, result.succeeded() ) + .writeAttribute( "type"_sr, result.getTestMacroName() ); + + writeSourceInfo( result.getSourceInfo() ); + + m_xml.scopedElement( "Original" ) + .writeText( result.getExpression() ); + m_xml.scopedElement( "Expanded" ) + .writeText( result.getExpandedExpression() ); + } + + // And... Print a result applicable to each result type. + switch( result.getResultType() ) { + case ResultWas::ThrewException: + m_xml.startElement( "Exception" ); + writeSourceInfo( result.getSourceInfo() ); + m_xml.writeText( result.getMessage() ); + m_xml.endElement(); + break; + case ResultWas::FatalErrorCondition: + m_xml.startElement( "FatalErrorCondition" ); + writeSourceInfo( result.getSourceInfo() ); + m_xml.writeText( result.getMessage() ); + m_xml.endElement(); + break; + case ResultWas::Info: + m_xml.scopedElement( "Info" ) + .writeText( result.getMessage() ); + break; + case ResultWas::Warning: + // Warning will already have been written + break; + case ResultWas::ExplicitFailure: + m_xml.startElement( "Failure" ); + writeSourceInfo( result.getSourceInfo() ); + m_xml.writeText( result.getMessage() ); + m_xml.endElement(); + break; + case ResultWas::ExplicitSkip: + m_xml.startElement( "Skip" ); + writeSourceInfo( result.getSourceInfo() ); + m_xml.writeText( result.getMessage() ); + m_xml.endElement(); + break; + default: + break; + } + + if( result.hasExpression() ) + m_xml.endElement(); + } + + void XmlReporter::sectionEnded( SectionStats const& sectionStats ) { + StreamingReporterBase::sectionEnded( sectionStats ); + if ( --m_sectionDepth > 0 ) { + { + XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResults" ); + e.writeAttribute( "successes"_sr, sectionStats.assertions.passed ); + e.writeAttribute( "failures"_sr, sectionStats.assertions.failed ); + e.writeAttribute( "expectedFailures"_sr, sectionStats.assertions.failedButOk ); + e.writeAttribute( "skipped"_sr, sectionStats.assertions.skipped > 0 ); + + if ( m_config->showDurations() == ShowDurations::Always ) + e.writeAttribute( "durationInSeconds"_sr, sectionStats.durationInSeconds ); + } + // Ends assertion tag + m_xml.endElement(); + } + } + + void XmlReporter::testCaseEnded( TestCaseStats const& testCaseStats ) { + StreamingReporterBase::testCaseEnded( testCaseStats ); + XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResult" ); + e.writeAttribute( "success"_sr, testCaseStats.totals.assertions.allOk() ); + e.writeAttribute( "skips"_sr, testCaseStats.totals.assertions.skipped ); + + if ( m_config->showDurations() == ShowDurations::Always ) + e.writeAttribute( "durationInSeconds"_sr, m_testCaseTimer.getElapsedSeconds() ); + if( !testCaseStats.stdOut.empty() ) + m_xml.scopedElement( "StdOut" ).writeText( trim( StringRef(testCaseStats.stdOut) ), XmlFormatting::Newline ); + if( !testCaseStats.stdErr.empty() ) + m_xml.scopedElement( "StdErr" ).writeText( trim( StringRef(testCaseStats.stdErr) ), XmlFormatting::Newline ); + + m_xml.endElement(); + } + + void XmlReporter::testRunEnded( TestRunStats const& testRunStats ) { + StreamingReporterBase::testRunEnded( testRunStats ); + m_xml.scopedElement( "OverallResults" ) + .writeAttribute( "successes"_sr, testRunStats.totals.assertions.passed ) + .writeAttribute( "failures"_sr, testRunStats.totals.assertions.failed ) + .writeAttribute( "expectedFailures"_sr, testRunStats.totals.assertions.failedButOk ) + .writeAttribute( "skips"_sr, testRunStats.totals.assertions.skipped ); + m_xml.scopedElement( "OverallResultsCases") + .writeAttribute( "successes"_sr, testRunStats.totals.testCases.passed ) + .writeAttribute( "failures"_sr, testRunStats.totals.testCases.failed ) + .writeAttribute( "expectedFailures"_sr, testRunStats.totals.testCases.failedButOk ) + .writeAttribute( "skips"_sr, testRunStats.totals.testCases.skipped ); + m_xml.endElement(); + } + + void XmlReporter::benchmarkPreparing( StringRef name ) { + m_xml.startElement("BenchmarkResults") + .writeAttribute("name"_sr, name); + } + + void XmlReporter::benchmarkStarting(BenchmarkInfo const &info) { + m_xml.writeAttribute("samples"_sr, info.samples) + .writeAttribute("resamples"_sr, info.resamples) + .writeAttribute("iterations"_sr, info.iterations) + .writeAttribute("clockResolution"_sr, info.clockResolution) + .writeAttribute("estimatedDuration"_sr, info.estimatedDuration) + .writeComment("All values in nano seconds"_sr); + } + + void XmlReporter::benchmarkEnded(BenchmarkStats<> const& benchmarkStats) { + m_xml.scopedElement("mean") + .writeAttribute("value"_sr, benchmarkStats.mean.point.count()) + .writeAttribute("lowerBound"_sr, benchmarkStats.mean.lower_bound.count()) + .writeAttribute("upperBound"_sr, benchmarkStats.mean.upper_bound.count()) + .writeAttribute("ci"_sr, benchmarkStats.mean.confidence_interval); + m_xml.scopedElement("standardDeviation") + .writeAttribute("value"_sr, benchmarkStats.standardDeviation.point.count()) + .writeAttribute("lowerBound"_sr, benchmarkStats.standardDeviation.lower_bound.count()) + .writeAttribute("upperBound"_sr, benchmarkStats.standardDeviation.upper_bound.count()) + .writeAttribute("ci"_sr, benchmarkStats.standardDeviation.confidence_interval); + m_xml.scopedElement("outliers") + .writeAttribute("variance"_sr, benchmarkStats.outlierVariance) + .writeAttribute("lowMild"_sr, benchmarkStats.outliers.low_mild) + .writeAttribute("lowSevere"_sr, benchmarkStats.outliers.low_severe) + .writeAttribute("highMild"_sr, benchmarkStats.outliers.high_mild) + .writeAttribute("highSevere"_sr, benchmarkStats.outliers.high_severe); + m_xml.endElement(); + } + + void XmlReporter::benchmarkFailed(StringRef error) { + m_xml.scopedElement("failed"). + writeAttribute("message"_sr, error); + m_xml.endElement(); + } + + void XmlReporter::listReporters(std::vector<ReporterDescription> const& descriptions) { + auto outerTag = m_xml.scopedElement("AvailableReporters"); + for (auto const& reporter : descriptions) { + auto inner = m_xml.scopedElement("Reporter"); + m_xml.startElement("Name", XmlFormatting::Indent) + .writeText(reporter.name, XmlFormatting::None) + .endElement(XmlFormatting::Newline); + m_xml.startElement("Description", XmlFormatting::Indent) + .writeText(reporter.description, XmlFormatting::None) + .endElement(XmlFormatting::Newline); + } + } + + void XmlReporter::listListeners(std::vector<ListenerDescription> const& descriptions) { + auto outerTag = m_xml.scopedElement( "RegisteredListeners" ); + for ( auto const& listener : descriptions ) { + auto inner = m_xml.scopedElement( "Listener" ); + m_xml.startElement( "Name", XmlFormatting::Indent ) + .writeText( listener.name, XmlFormatting::None ) + .endElement( XmlFormatting::Newline ); + m_xml.startElement( "Description", XmlFormatting::Indent ) + .writeText( listener.description, XmlFormatting::None ) + .endElement( XmlFormatting::Newline ); + } + } + + void XmlReporter::listTests(std::vector<TestCaseHandle> const& tests) { + auto outerTag = m_xml.scopedElement("MatchingTests"); + for (auto const& test : tests) { + auto innerTag = m_xml.scopedElement("TestCase"); + auto const& testInfo = test.getTestCaseInfo(); + m_xml.startElement("Name", XmlFormatting::Indent) + .writeText(testInfo.name, XmlFormatting::None) + .endElement(XmlFormatting::Newline); + m_xml.startElement("ClassName", XmlFormatting::Indent) + .writeText(testInfo.className, XmlFormatting::None) + .endElement(XmlFormatting::Newline); + m_xml.startElement("Tags", XmlFormatting::Indent) + .writeText(testInfo.tagsAsString(), XmlFormatting::None) + .endElement(XmlFormatting::Newline); + + auto sourceTag = m_xml.scopedElement("SourceInfo"); + m_xml.startElement("File", XmlFormatting::Indent) + .writeText(testInfo.lineInfo.file, XmlFormatting::None) + .endElement(XmlFormatting::Newline); + m_xml.startElement("Line", XmlFormatting::Indent) + .writeText(std::to_string(testInfo.lineInfo.line), XmlFormatting::None) + .endElement(XmlFormatting::Newline); + } + } + + void XmlReporter::listTags(std::vector<TagInfo> const& tags) { + auto outerTag = m_xml.scopedElement("TagsFromMatchingTests"); + for (auto const& tag : tags) { + auto innerTag = m_xml.scopedElement("Tag"); + m_xml.startElement("Count", XmlFormatting::Indent) + .writeText(std::to_string(tag.count), XmlFormatting::None) + .endElement(XmlFormatting::Newline); + auto aliasTag = m_xml.scopedElement("Aliases"); + for (auto const& alias : tag.spellings) { + m_xml.startElement("Alias", XmlFormatting::Indent) + .writeText(alias, XmlFormatting::None) + .endElement(XmlFormatting::Newline); + } + } + } + +} // end namespace Catch + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif diff --git a/src/catch2/reporters/catch_reporter_xml.hpp b/src/catch2/reporters/catch_reporter_xml.hpp new file mode 100644 index 0000000..bead7a8 --- /dev/null +++ b/src/catch2/reporters/catch_reporter_xml.hpp @@ -0,0 +1,66 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef CATCH_REPORTER_XML_HPP_INCLUDED +#define CATCH_REPORTER_XML_HPP_INCLUDED + +#include <catch2/reporters/catch_reporter_streaming_base.hpp> + +#include <catch2/internal/catch_xmlwriter.hpp> +#include <catch2/catch_timer.hpp> + + +namespace Catch { + class XmlReporter : public StreamingReporterBase { + public: + XmlReporter(ReporterConfig&& _config); + + ~XmlReporter() override; + + static std::string getDescription(); + + virtual std::string getStylesheetRef() const; + + void writeSourceInfo(SourceLineInfo const& sourceInfo); + + public: // StreamingReporterBase + + void testRunStarting(TestRunInfo const& testInfo) override; + + void testCaseStarting(TestCaseInfo const& testInfo) override; + + void sectionStarting(SectionInfo const& sectionInfo) override; + + void assertionStarting(AssertionInfo const&) override; + + void assertionEnded(AssertionStats const& assertionStats) override; + + void sectionEnded(SectionStats const& sectionStats) override; + + void testCaseEnded(TestCaseStats const& testCaseStats) override; + + void testRunEnded(TestRunStats const& testRunStats) override; + + void benchmarkPreparing( StringRef name ) override; + void benchmarkStarting(BenchmarkInfo const&) override; + void benchmarkEnded(BenchmarkStats<> const&) override; + void benchmarkFailed( StringRef error ) override; + + void listReporters(std::vector<ReporterDescription> const& descriptions) override; + void listListeners(std::vector<ListenerDescription> const& descriptions) override; + void listTests(std::vector<TestCaseHandle> const& tests) override; + void listTags(std::vector<TagInfo> const& tags) override; + + private: + Timer m_testCaseTimer; + XmlWriter m_xml; + int m_sectionDepth = 0; + }; + +} // end namespace Catch + +#endif // CATCH_REPORTER_XML_HPP_INCLUDED diff --git a/src/catch2/reporters/catch_reporters_all.hpp b/src/catch2/reporters/catch_reporters_all.hpp new file mode 100644 index 0000000..5c713fe --- /dev/null +++ b/src/catch2/reporters/catch_reporters_all.hpp @@ -0,0 +1,41 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +/** \file + * This is a convenience header for Catch2's Reporter support. It includes + * **all** of Catch2 headers related to reporters, including all reporters. + * + * Generally the Catch2 users should use specific includes they need, + * but this header can be used instead for ease-of-experimentation, or + * just plain convenience, at the cost of (significantly) increased + * compilation times. + * + * When a new header (reporter) is added to either the `reporter` folder, + * or to the corresponding internal subfolder, it should be added here. + */ + +#ifndef CATCH_REPORTERS_ALL_HPP_INCLUDED +#define CATCH_REPORTERS_ALL_HPP_INCLUDED + +#include <catch2/reporters/catch_reporter_automake.hpp> +#include <catch2/reporters/catch_reporter_common_base.hpp> +#include <catch2/reporters/catch_reporter_compact.hpp> +#include <catch2/reporters/catch_reporter_console.hpp> +#include <catch2/reporters/catch_reporter_cumulative_base.hpp> +#include <catch2/reporters/catch_reporter_event_listener.hpp> +#include <catch2/reporters/catch_reporter_helpers.hpp> +#include <catch2/reporters/catch_reporter_json.hpp> +#include <catch2/reporters/catch_reporter_junit.hpp> +#include <catch2/reporters/catch_reporter_multi.hpp> +#include <catch2/reporters/catch_reporter_registrars.hpp> +#include <catch2/reporters/catch_reporter_sonarqube.hpp> +#include <catch2/reporters/catch_reporter_streaming_base.hpp> +#include <catch2/reporters/catch_reporter_tap.hpp> +#include <catch2/reporters/catch_reporter_teamcity.hpp> +#include <catch2/reporters/catch_reporter_xml.hpp> + +#endif // CATCH_REPORTERS_ALL_HPP_INCLUDED |
