libwreport  3.38
utils/tests.h
1 #ifndef WREPORT_TESTS_H
2 #define WREPORT_TESTS_H
3 
12 #include <string>
13 #include <sstream>
14 #include <exception>
15 #include <filesystem>
16 #include <functional>
17 #include <vector>
18 #include <cstdint>
19 
20 namespace wreport {
21 namespace tests {
22 struct LocationInfo;
23 }
24 }
25 
26 /*
27  * These global arguments will be shadowed by local variables in functions that
28  * implement tests.
29  *
30  * They are here to act as default root nodes to fulfill method signatures when
31  * tests are called from outside other tests.
32  */
33 extern const wreport::tests::LocationInfo wreport_test_location_info;
34 
35 namespace wreport {
36 namespace tests {
37 
55 struct LocationInfo : public std::stringstream
56 {
57  LocationInfo() {}
58 
63  std::ostream& operator()();
64 };
65 
68 {
69  const char* file;
70  int line;
71  const char* call;
72  std::string local_info;
73 
74  TestStackFrame(const char* file_, int line_, const char* call_)
75  : file(file_), line(line_), call(call_)
76  {
77  }
78 
79  TestStackFrame(const char* file_, int line_, const char* call_, const LocationInfo& local_info_)
80  : file(file_), line(line_), call(call_), local_info(local_info_.str())
81  {
82  }
83 
84  std::string format() const;
85 
86  void format(std::ostream& out) const;
87 };
88 
89 struct TestStack : public std::vector<TestStackFrame>
90 {
91  using vector::vector;
92 
94  std::string backtrace() const;
95 
97  void backtrace(std::ostream& out) const;
98 };
99 
104 struct TestFailed : public std::exception
105 {
106  std::string message;
107  TestStack stack;
108 
109  explicit TestFailed(const std::exception& e);
110 
111  template<typename ...Args>
112  TestFailed(const std::exception& e, Args&&... args)
113  : TestFailed(e)
114  {
115  add_stack_info(std::forward<Args>(args)...);
116  }
117 
118  explicit TestFailed(const std::string& message_) : message(message_) {}
119 
120  template<typename ...Args>
121  TestFailed(const std::string& message_, Args&&... args)
122  : TestFailed(message_)
123  {
124  add_stack_info(std::forward<Args>(args)...);
125  }
126 
127  const char* what() const noexcept override { return message.c_str(); }
128 
129  template<typename ...Args>
130  void add_stack_info(Args&&... args) { stack.emplace_back(std::forward<Args>(args)...); }
131 };
132 
136 struct TestSkipped : public std::exception
137 {
138  std::string reason;
139 
140  TestSkipped();
141  explicit TestSkipped(const std::string& reason);
142 };
143 
148 #define WREPORT_TEST_INFO(name) \
149  wreport::tests::LocationInfo wreport_test_location_info; \
150  wreport::tests::LocationInfo& name = wreport_test_location_info
151 
152 
160 template<typename A>
162 void assert_true(const A& actual)
163 {
164  if (actual) return;
165  std::stringstream ss;
166  ss << "actual value " << actual << " is not true";
167  throw TestFailed(ss.str());
168 }
169 
170 [[noreturn]] void assert_true(std::nullptr_t actual);
171 
173 template<typename A>
174 void assert_false(const A& actual)
175 {
176  if (!actual) return;
177  std::stringstream ss;
178  ss << "actual value " << actual << " is not false";
179  throw TestFailed(ss.str());
180 }
181 
182 void assert_false(std::nullptr_t actual);
183 
184 template<typename LIST>
185 static inline void _format_list(std::ostream& o, const LIST& list) {
186  bool first = true;
187  o << "[";
188  for (const auto& v: list)
189  {
190  if (first)
191  first = false;
192  else
193  o << ", ";
194  o << v;
195  }
196  o << "]";
197 }
198 
199 template<typename T>
200 void assert_equal(const std::vector<T>& actual, const std::vector<T>& expected)
201 {
202  if (actual == expected) return;
203  std::stringstream ss;
204  ss << "value ";
205  _format_list(ss, actual);
206  ss << " is different than the expected ";
207  _format_list(ss, expected);
208  throw TestFailed(ss.str());
209 }
210 
211 template<typename T>
212 void assert_equal(const std::vector<T>& actual, const std::initializer_list<T>& expected)
213 {
214  if (actual == expected) return;
215  std::stringstream ss;
216  ss << "value ";
217  _format_list(ss, actual);
218  ss << " is different than the expected ";
219  _format_list(ss, expected);
220  throw TestFailed(ss.str());
221 }
222 
227 template<typename A, typename E>
228 void assert_equal(const A& actual, const E& expected)
229 {
230  if (actual == expected) return;
231  std::stringstream ss;
232  ss << "value '" << actual << "' is different than the expected '" << expected << "'";
233  throw TestFailed(ss.str());
234 }
235 
240 template<typename A, typename E>
241 void assert_not_equal(const A& actual, const E& expected)
242 {
243  if (actual != expected) return;
244  std::stringstream ss;
245  ss << "value '" << actual << "' is not different than the expected '" << expected << "'";
246  throw TestFailed(ss.str());
247 }
248 
250 template<typename A, typename E>
251 void assert_less(const A& actual, const E& expected)
252 {
253  if (actual < expected) return;
254  std::stringstream ss;
255  ss << "value '" << actual << "' is not less than the expected '" << expected << "'";
256  throw TestFailed(ss.str());
257 }
258 
260 template<typename A, typename E>
261 void assert_less_equal(const A& actual, const E& expected)
262 {
263  if (actual <= expected) return;
264  std::stringstream ss;
265  ss << "value '" << actual << "' is not less than or equals to the expected '" << expected << "'";
266  throw TestFailed(ss.str());
267 }
268 
270 template<typename A, typename E>
271 void assert_greater(const A& actual, const E& expected)
272 {
273  if (actual > expected) return;
274  std::stringstream ss;
275  ss << "value '" << actual << "' is not greater than the expected '" << expected << "'";
276  throw TestFailed(ss.str());
277 }
278 
280 template<typename A, typename E>
281 void assert_greater_equal(const A& actual, const E& expected)
282 {
283  if (actual >= expected) return;
284  std::stringstream ss;
285  ss << "value '" << actual << "' is not greater than or equals to the expected '" << expected << "'";
286  throw TestFailed(ss.str());
287 }
288 
290 void assert_startswith(const std::string& actual, const std::string& expected);
291 
293 void assert_endswith(const std::string& actual, const std::string& expected);
294 
296 void assert_contains(const std::string& actual, const std::string& expected);
297 
299 void assert_not_contains(const std::string& actual, const std::string& expected);
300 
307 void assert_re_matches(const std::string& actual, const std::string& expected);
308 
315 void assert_not_re_matches(const std::string& actual, const std::string& expected);
316 
317 
318 template<class A>
319 struct Actual
320 {
321  A _actual;
322  Actual(const A& actual) : _actual(actual) {}
323  Actual(const Actual&) = default;
324  Actual(Actual&&) = default;
325  ~Actual() = default;
326  Actual& operator=(const Actual&) = delete;
327  Actual& operator=(Actual&&) = delete;
328 
329  void istrue() const { assert_true(_actual); }
330  void isfalse() const { assert_false(_actual); }
331  template<typename E> void operator==(const E& expected) const { assert_equal(_actual, expected); }
332  template<typename E> void operator!=(const E& expected) const { assert_not_equal(_actual, expected); }
333  template<typename E> void operator<(const E& expected) const { return assert_less(_actual, expected); }
334  template<typename E> void operator<=(const E& expected) const { return assert_less_equal(_actual, expected); }
335  template<typename E> void operator>(const E& expected) const { return assert_greater(_actual, expected); }
336  template<typename E> void operator>=(const E& expected) const { return assert_greater_equal(_actual, expected); }
337 };
338 
340 {
341  const char* _actual;
342  ActualCString(const char* s) : _actual(s) {}
343 
344  void istrue() const { return assert_true(_actual); }
345  void isfalse() const { return assert_false(_actual); }
346  void operator==(const char* expected) const;
347  void operator==(const std::string& expected) const;
348  void operator!=(const char* expected) const;
349  void operator!=(const std::string& expected) const;
350  void operator<(const std::string& expected) const;
351  void operator<=(const std::string& expected) const;
352  void operator>(const std::string& expected) const;
353  void operator>=(const std::string& expected) const;
354  void startswith(const std::string& expected) const;
355  void endswith(const std::string& expected) const;
356  void contains(const std::string& expected) const;
357  void not_contains(const std::string& expected) const;
358  void matches(const std::string& re) const;
359  void not_matches(const std::string& re) const;
360 };
361 
362 struct ActualStdString : public Actual<std::string>
363 {
364  explicit ActualStdString(const std::string& s) : Actual<std::string>(s) {}
365 
367  void operator==(const std::vector<uint8_t>& expected) const;
369  void operator!=(const std::vector<uint8_t>& expected) const;
370  void startswith(const std::string& expected) const;
371  void endswith(const std::string& expected) const;
372  void contains(const std::string& expected) const;
373  void not_contains(const std::string& expected) const;
374  void matches(const std::string& re) const;
375  void not_matches(const std::string& re) const;
376 };
377 
378 struct ActualPath : public Actual<std::filesystem::path>
379 {
380  explicit ActualPath(const std::filesystem::path& p) : Actual<std::filesystem::path>(p) {}
381 
384 
385  // Check if the normalized paths match
386  void is(const std::filesystem::path& expected) const;
387  [[deprecated("Use path_startswith")]] void startswith(const std::string& data) const;
388 
389  void path_startswith(const std::filesystem::path& expected) const;
390  void path_endswith(const std::filesystem::path& expected) const;
391  void path_contains(const std::filesystem::path& expected) const;
392  void path_not_contains(const std::filesystem::path& expected) const;
393 
394  void exists() const;
395  void not_exists() const;
396  void empty() const;
397  void not_empty() const;
398 
399  void contents_startwith(const std::string& data) const;
400  void contents_equal(const std::string& data) const;
401  void contents_equal(const std::vector<uint8_t>& data) const;
402  void contents_equal(const std::initializer_list<std::string>& lines) const;
403  void contents_match(const std::string& data_re) const;
404  void contents_match(const std::initializer_list<std::string>& lines_re) const;
405 };
406 
407 struct ActualDouble : public Actual<double>
408 {
409  using Actual::Actual;
410 
411  void almost_equal(double expected, unsigned places) const;
412  void not_almost_equal(double expected, unsigned places) const;
413 };
414 
415 template<typename A>
416 inline Actual<A> actual(const A& actual) { return Actual<A>(actual); }
417 inline ActualCString actual(const char* actual) { return ActualCString(actual); }
418 inline ActualCString actual(char* actual) { return ActualCString(actual); }
419 inline ActualStdString actual(const std::string& actual) { return ActualStdString(actual); }
420 inline ActualStdString actual(const std::vector<uint8_t>& actual) { return ActualStdString(std::string(actual.begin(), actual.end())); }
421 inline ActualPath actual(const std::filesystem::path& actual) { return ActualPath(actual); }
422 inline ActualDouble actual(double actual) { return ActualDouble(actual); }
423 
424 struct ActualFunction : public Actual<std::function<void()>>
425 {
426  using Actual::Actual;
427 
428  void throws(const std::string& what_match) const;
429 };
430 
431 inline ActualFunction actual_function(std::function<void()> actual) { return ActualFunction(actual); }
432 
433 inline ActualPath actual_path(const char* pathname) { return ActualPath(pathname); }
434 inline ActualPath actual_path(const std::string& pathname) { return ActualPath(pathname); }
435 inline ActualPath actual_file(const char* pathname) { return ActualPath(pathname); }
436 inline ActualPath actual_file(const std::string& pathname) { return ActualPath(pathname); }
437 
445 #define wassert(...) \
446  do { try { \
447  __VA_ARGS__ ; \
448  } catch (wreport::tests::TestFailed& e1) { \
449  e1.add_stack_info(__FILE__, __LINE__, #__VA_ARGS__, wreport_test_location_info); \
450  throw; \
451  } catch (std::exception& e2) { \
452  throw wreport::tests::TestFailed(e2, __FILE__, __LINE__, #__VA_ARGS__, wreport_test_location_info); \
453  } } while(0)
454 
456 #define wassert_true(...) wassert(actual(__VA_ARGS__).istrue())
457 
459 #define wassert_false(...) wassert(actual(__VA_ARGS__).isfalse())
460 
466 #define wassert_throws(exc, ...) \
467  [&]() { try { \
468  __VA_ARGS__ ; \
469  wfail_test(#__VA_ARGS__ " did not throw " #exc); \
470  } catch (TestFailed& e1) { \
471  throw; \
472  } catch (exc& e2) { \
473  return e2; \
474  } catch (std::exception& e3) { \
475  std::string msg(#__VA_ARGS__ " did not throw " #exc " but threw "); \
476  msg += typeid(e3).name(); \
477  msg += " instead"; \
478  wfail_test(msg); \
479  } }()
480 
488 #define wcallchecked(func) \
489  [&]() { try { \
490  return func; \
491  } catch (wreport::tests::TestFailed& e) { \
492  e.add_stack_info(__FILE__, __LINE__, #func, wreport_test_location_info); \
493  throw; \
494  } catch (std::exception& e) { \
495  throw wreport::tests::TestFailed(e, __FILE__, __LINE__, #func, wreport_test_location_info); \
496  } }()
497 
501 #define wfail_test(msg) wassert(throw wreport::tests::TestFailed((msg)))
502 
503 struct TestCase;
504 struct TestController;
505 struct TestRegistry;
506 struct TestCaseResult;
507 struct TestMethod;
508 struct TestMethodResult;
509 
510 
515 {
517  std::string name;
518 
520  std::string doc;
521 
527  std::function<void()> test_function;
528 
529  TestMethod(const std::string& name_)
530  : name(name_), test_function() {}
531 
532  TestMethod(const std::string& name_, std::function<void()> test_function_)
533  : name(name_), test_function(test_function_) {}
534 };
535 
536 
541 struct TestCase
542 {
544  std::string name;
545 
547  std::vector<TestMethod> methods;
548 
550  bool tests_registered = false;
551 
552 
553  TestCase(const std::string& name);
554  virtual ~TestCase() {}
555 
559  void register_tests_once();
560 
568  virtual void register_tests() = 0;
569 
573  virtual void setup() {}
574 
578  virtual void teardown() {}
579 
583  virtual void method_setup(TestMethodResult&) {}
584 
589 
597  virtual TestCaseResult run_tests(TestController& controller);
598 
611  virtual TestMethodResult run_test(TestController& controller, TestMethod& method);
612 
617  TestMethod& add_method(const std::string& name_)
618  {
619  methods.emplace_back(name_);
620  return methods.back();
621  }
622 
626  template<typename ...Args>
627  TestMethod& add_method(const std::string& name_, std::function<void()> test_function)
628  {
629  methods.emplace_back(name_, test_function);
630  return methods.back();
631  }
632 
636  template<typename ...Args>
637  TestMethod& add_method(const std::string& name_, const std::string& doc, std::function<void()> test_function)
638  {
639  methods.emplace_back(name_, test_function);
640  methods.back().doc = doc;
641  return methods.back();
642  }
643 };
644 
645 
656 struct Fixture
657 {
658  // Called before each test
659  void test_setup() {}
660 
661  // Called after each test
662  void test_teardown() {}
663 };
664 
665 template<typename Fixture, typename... Args>
666 static inline Fixture* fixture_factory(Args... args)
667 {
668  return new Fixture(args...);
669 }
670 
674 template<typename FIXTURE>
675 class FixtureTestCase : public TestCase
676 {
677 public:
678  typedef FIXTURE Fixture;
679 
680  Fixture* fixture = nullptr;
681  std::function<Fixture*()> make_fixture;
682 
683  template<typename... Args>
684  FixtureTestCase(const std::string& name_, Args... args)
685  : TestCase(name_)
686  {
687  make_fixture = std::bind(fixture_factory<FIXTURE, Args...>, args...);
688  }
689  FixtureTestCase(const FixtureTestCase&) = delete;
690  FixtureTestCase(FixtureTestCase&&) = delete;
691  FixtureTestCase& operator=(const FixtureTestCase&) = delete;
692  FixtureTestCase& operator=(FixtureTestCase&) = delete;
693 
694  void setup() override
695  {
696  TestCase::setup();
697  fixture = make_fixture();
698  }
699 
700  void teardown() override
701  {
702  delete fixture;
703  fixture = nullptr;
705  }
706 
707  void method_setup(TestMethodResult& mr) override
708  {
710  if (fixture) fixture->test_setup();
711  }
712 
714  {
715  if (fixture) fixture->test_teardown();
717  }
718 
723  template<typename ...Args>
724  TestMethod& add_method(const std::string& name_, std::function<void(FIXTURE&)> test_function)
725  {
726  return TestCase::add_method(name_, [=]() { test_function(*fixture); });
727  }
728 
733  template<typename ...Args>
734  TestMethod& add_method(const std::string& name_, const std::string& doc, std::function<void(FIXTURE&)> test_function)
735  {
736  return TestCase::add_method(name_, doc, [=]() { test_function(*fixture); });
737  }
738 };
739 
740 }
741 }
742 #endif
virtual TestCaseResult run_tests(TestController &controller)
Call setup(), run all the tests that have been registered, then call teardown().
Result of running a whole test case.
Definition: testrunner.h:96
Test case collecting several test methods, and self-registering with the singleton instance of TestRe...
Definition: utils/tests.h:541
std::string name
Name of the test case.
Definition: utils/tests.h:544
Information about one stack frame in the test execution stack.
Definition: utils/tests.h:67
Add information to the test backtrace for the tests run in the current scope.
Definition: utils/tests.h:55
Exception thrown when a test or a test case needs to be skipped.
Definition: utils/tests.h:136
virtual void method_setup(TestMethodResult &)
Set up before the test method is run.
Definition: utils/tests.h:583
void register_tests_once()
Idempotent wrapper for register_tests()
bool tests_registered
Set to true the first time register_tests_once is run.
Definition: utils/tests.h:550
Abstract interface for the objects that supervise test execution.
Definition: testrunner.h:158
Definition: utils/tests.h:319
virtual TestMethodResult run_test(TestController &controller, TestMethod &method)
Run a test method.
void teardown() override
Clean up after the test case is run.
Definition: utils/tests.h:700
Exception thrown when a test assertion fails, normally by Location::fail_test.
Definition: utils/tests.h:104
Definition: utils/tests.h:378
void method_teardown(TestMethodResult &mr) override
Clean up after the test method is run.
Definition: utils/tests.h:713
std::vector< TestMethod > methods
All registered test methods.
Definition: utils/tests.h:547
std::string backtrace() const
Return the formatted backtrace for this location.
TestMethod & add_method(const std::string &name_, const std::string &doc, std::function< void()> test_function)
Register a new test method, including documentation.
Definition: utils/tests.h:637
Definition: utils/tests.h:89
virtual void setup()
Set up the test case before it is run.
Definition: utils/tests.h:573
Definition: utils/tests.h:339
std::string doc
Documentation attached to this test method.
Definition: utils/tests.h:520
Definition: utils/tests.h:424
Definition: utils/tests.h:407
Result of running a test method.
Definition: testrunner.h:26
TestMethod & add_method(const std::string &name_, const std::string &doc, std::function< void(FIXTURE &)> test_function)
Register a new test method that takes a reference to the fixture as argument, including documentation...
Definition: utils/tests.h:734
void setup() override
Set up the test case before it is run.
Definition: utils/tests.h:694
std::function< void()> test_function
Main body of the test method.
Definition: utils/tests.h:527
void method_setup(TestMethodResult &mr) override
Set up before the test method is run.
Definition: utils/tests.h:707
Definition: utils/tests.h:362
virtual void register_tests()=0
This will be called before running the test case, to populate it with its test methods.
TestMethod & add_method(const std::string &name_)
Register a new test method, with the actual test function to be added later.
Definition: utils/tests.h:617
String functions.
Definition: benchmark.h:13
Test case that includes a fixture.
Definition: utils/tests.h:675
TestMethod & add_method(const std::string &name_, std::function< void()> test_function)
Register a new test method.
Definition: utils/tests.h:627
Test method information.
Definition: utils/tests.h:514
virtual void method_teardown(TestMethodResult &)
Clean up after the test method is run.
Definition: utils/tests.h:588
virtual void teardown()
Clean up after the test case is run.
Definition: utils/tests.h:578
TestMethod & add_method(const std::string &name_, std::function< void(FIXTURE &)> test_function)
Register a new test method that takes a reference to the fixture as argument.
Definition: utils/tests.h:724
std::string name
Name of the test method.
Definition: utils/tests.h:517
Base class for test fixtures.
Definition: utils/tests.h:656
std::ostream & operator()()
Clear the current information and return the output stream to which new information can be sent...