libwreport  3.10
utils/tests.h
1 #ifndef WREPORT_TESTS_H
2 #define WREPORT_TESTS_H
3 
12 #include <string>
13 #include <sstream>
14 #include <exception>
15 #include <functional>
16 #include <vector>
17 #include <typeinfo>
18 
19 namespace wreport {
20 namespace tests {
21 struct LocationInfo;
22 }
23 }
24 
25 /*
26  * These global arguments will be shadowed by local variables in functions that
27  * implement tests.
28  *
29  * They are here to act as default root nodes to fulfill method signatures when
30  * tests are called from outside other tests.
31  */
32 extern const wreport::tests::LocationInfo wreport_test_location_info;
33 
34 namespace wreport {
35 namespace tests {
36 
54 struct LocationInfo : public std::stringstream
55 {
56  LocationInfo() {}
57 
62  std::ostream& operator()();
63 };
64 
67 {
68  const char* file;
69  int line;
70  const char* call;
71  std::string local_info;
72 
73  TestStackFrame(const char* file, int line, const char* call)
74  : file(file), line(line), call(call)
75  {
76  }
77 
78  TestStackFrame(const char* file, int line, const char* call, const LocationInfo& local_info)
79  : file(file), line(line), call(call), local_info(local_info.str())
80  {
81  }
82 
83  std::string format() const;
84 
85  void format(std::ostream& out) const;
86 };
87 
88 struct TestStack : public std::vector<TestStackFrame>
89 {
90  using vector::vector;
91 
93  std::string backtrace() const;
94 
96  void backtrace(std::ostream& out) const;
97 };
98 
103 struct TestFailed : public std::exception
104 {
105  std::string message;
106  TestStack stack;
107 
108  TestFailed(const std::exception& e);
109 
110  template<typename ...Args>
111  TestFailed(const std::exception& e, Args&&... args)
112  : TestFailed(e)
113  {
114  add_stack_info(std::forward<Args>(args)...);
115  }
116 
117  TestFailed(const std::string& message) : message(message) {}
118 
119  template<typename ...Args>
120  TestFailed(const std::string& message, Args&&... args)
121  : TestFailed(message)
122  {
123  add_stack_info(std::forward<Args>(args)...);
124  }
125 
126  const char* what() const noexcept override { return message.c_str(); }
127 
128  template<typename ...Args>
129  void add_stack_info(Args&&... args) { stack.emplace_back(std::forward<Args>(args)...); }
130 };
131 
135 struct TestSkipped : public std::exception
136 {
137 };
138 
143 #define WREPORT_TEST_INFO(name) \
144  wreport::tests::LocationInfo wreport_test_location_info; \
145  wreport::tests::LocationInfo& name = wreport_test_location_info
146 
147 
155 template<typename A>
157 void assert_true(const A& actual)
158 {
159  if (actual) return;
160  std::stringstream ss;
161  ss << "actual value " << actual << " is not true";
162  throw TestFailed(ss.str());
163 };
164 
165 void assert_true(std::nullptr_t actual);
166 
168 template<typename A>
169 void assert_false(const A& actual)
170 {
171  if (!actual) return;
172  std::stringstream ss;
173  ss << "actual value " << actual << " is not false";
174  throw TestFailed(ss.str());
175 };
176 
177 void assert_false(std::nullptr_t actual);
178 
183 template<typename A, typename E>
184 void assert_equal(const A& actual, const E& expected)
185 {
186  if (actual == expected) return;
187  std::stringstream ss;
188  ss << "value '" << actual << "' is different than the expected '" << expected << "'";
189  throw TestFailed(ss.str());
190 }
191 
196 template<typename A, typename E>
197 void assert_not_equal(const A& actual, const E& expected)
198 {
199  if (actual != expected) return;
200  std::stringstream ss;
201  ss << "value '" << actual << "' is not different than the expected '" << expected << "'";
202  throw TestFailed(ss.str());
203 }
204 
206 template<typename A, typename E>
207 void assert_less(const A& actual, const E& expected)
208 {
209  if (actual < expected) return;
210  std::stringstream ss;
211  ss << "value '" << actual << "' is not less than the expected '" << expected << "'";
212  throw TestFailed(ss.str());
213 }
214 
216 template<typename A, typename E>
217 void assert_less_equal(const A& actual, const E& expected)
218 {
219  if (actual <= expected) return;
220  std::stringstream ss;
221  ss << "value '" << actual << "' is not less than or equals to the expected '" << expected << "'";
222  throw TestFailed(ss.str());
223 }
224 
226 template<typename A, typename E>
227 void assert_greater(const A& actual, const E& expected)
228 {
229  if (actual > expected) return;
230  std::stringstream ss;
231  ss << "value '" << actual << "' is not greater than the expected '" << expected << "'";
232  throw TestFailed(ss.str());
233 }
234 
236 template<typename A, typename E>
237 void assert_greater_equal(const A& actual, const E& expected)
238 {
239  if (actual >= expected) return;
240  std::stringstream ss;
241  ss << "value '" << actual << "' is not greater than or equals to the expected '" << expected << "'";
242  throw TestFailed(ss.str());
243 }
244 
246 void assert_startswith(const std::string& actual, const std::string& expected);
247 
249 void assert_endswith(const std::string& actual, const std::string& expected);
250 
252 void assert_contains(const std::string& actual, const std::string& expected);
253 
255 void assert_not_contains(const std::string& actual, const std::string& expected);
256 
263 void assert_re_matches(const std::string& actual, const std::string& expected);
264 
271 void assert_not_re_matches(const std::string& actual, const std::string& expected);
272 
273 
274 template<class A>
275 struct Actual
276 {
277  A _actual;
278  Actual(const A& actual) : _actual(actual) {}
279  ~Actual() {}
280 
281  void istrue() const { assert_true(_actual); }
282  void isfalse() const { assert_false(_actual); }
283  template<typename E> void operator==(const E& expected) const { assert_equal(_actual, expected); }
284  template<typename E> void operator!=(const E& expected) const { assert_not_equal(_actual, expected); }
285  template<typename E> void operator<(const E& expected) const { return assert_less(_actual, expected); }
286  template<typename E> void operator<=(const E& expected) const { return assert_less_equal(_actual, expected); }
287  template<typename E> void operator>(const E& expected) const { return assert_greater(_actual, expected); }
288  template<typename E> void operator>=(const E& expected) const { return assert_greater_equal(_actual, expected); }
289 };
290 
292 {
293  const char* _actual;
294  ActualCString(const char* s) : _actual(s) {}
295 
296  void istrue() const { return assert_true(_actual); }
297  void isfalse() const { return assert_false(_actual); }
298  void operator==(const char* expected) const;
299  void operator==(const std::string& expected) const;
300  void operator!=(const char* expected) const;
301  void operator!=(const std::string& expected) const;
302  void operator<(const std::string& expected) const;
303  void operator<=(const std::string& expected) const;
304  void operator>(const std::string& expected) const;
305  void operator>=(const std::string& expected) const;
306  void startswith(const std::string& expected) const;
307  void endswith(const std::string& expected) const;
308  void contains(const std::string& expected) const;
309  void not_contains(const std::string& expected) const;
310  void matches(const std::string& re) const;
311  void not_matches(const std::string& re) const;
312 };
313 
314 struct ActualStdString : public Actual<std::string>
315 {
316  ActualStdString(const std::string& s) : Actual<std::string>(s) {}
317 
318  void startswith(const std::string& expected) const;
319  void endswith(const std::string& expected) const;
320  void contains(const std::string& expected) const;
321  void not_contains(const std::string& expected) const;
322  void matches(const std::string& re) const;
323  void not_matches(const std::string& re) const;
324 };
325 
326 struct ActualDouble : public Actual<double>
327 {
328  using Actual::Actual;
329 
330  void almost_equal(double expected, unsigned places) const;
331  void not_almost_equal(double expected, unsigned places) const;
332 };
333 
334 template<typename A>
335 inline Actual<A> actual(const A& actual) { return Actual<A>(actual); }
336 inline ActualCString actual(const char* actual) { return ActualCString(actual); }
337 inline ActualCString actual(char* actual) { return ActualCString(actual); }
338 inline ActualStdString actual(const std::string& actual) { return ActualStdString(actual); }
339 inline ActualDouble actual(double actual) { return ActualDouble(actual); }
340 
341 struct ActualFunction : public Actual<std::function<void()>>
342 {
343  using Actual::Actual;
344 
345  void throws(const std::string& what_match) const;
346 };
347 
348 inline ActualFunction actual_function(std::function<void()> actual) { return ActualFunction(actual); }
349 
350 struct ActualFile : public Actual<std::string>
351 {
352  using Actual::Actual;
353 
354  void exists() const;
355  void not_exists() const;
356  void startswith(const std::string& data) const;
357 };
358 
359 inline ActualFile actual_file(const std::string& pathname) { return ActualFile(pathname); }
360 
368 #define wassert(...) \
369  do { try { \
370  __VA_ARGS__ ; \
371  } catch (wreport::tests::TestFailed& e) { \
372  e.add_stack_info(__FILE__, __LINE__, #__VA_ARGS__, wreport_test_location_info); \
373  throw; \
374  } catch (std::exception& e) { \
375  throw wreport::tests::TestFailed(e, __FILE__, __LINE__, #__VA_ARGS__, wreport_test_location_info); \
376  } } while(0)
377 
379 #define wassert_true(...) wassert(actual(__VA_ARGS__).istrue())
380 
382 #define wassert_false(...) wassert(actual(__VA_ARGS__).isfalse())
383 
389 #define wassert_throws(exc, ...) \
390  [&]() { try { \
391  __VA_ARGS__ ; \
392  wfail_test(#__VA_ARGS__ " did not throw " #exc); \
393  } catch (exc& e) { \
394  return e; \
395  } catch (std::exception& e) { \
396  std::string msg(#__VA_ARGS__ " did not throw " #exc " but threw "); \
397  msg += typeid(e).name(); \
398  msg += " instead"; \
399  wfail_test(msg); \
400  } }()
401 
409 #define wcallchecked(func) \
410  [&]() { try { \
411  return func; \
412  } catch (wreport::tests::TestFailed& e) { \
413  e.add_stack_info(__FILE__, __LINE__, #func, wreport_test_location_info); \
414  throw; \
415  } catch (std::exception& e) { \
416  throw wreport::tests::TestFailed(e, __FILE__, __LINE__, #func, wreport_test_location_info); \
417  } }()
418 
422 #define wfail_test(msg) wassert(throw wreport::tests::TestFailed((msg)))
423 
424 struct TestCase;
425 
430 {
432  std::string test_case;
433 
435  std::string test_method;
436 
438  std::string error_message;
439 
442 
444  std::string exception_typeid;
445 
447  bool skipped = false;
448 
449 
450  TestMethodResult(const std::string& test_case, const std::string& test_method)
451  : test_case(test_case), test_method(test_method) {}
452 
453  void set_failed(TestFailed& e)
454  {
455  error_message = e.what();
456  error_stack = e.stack;
457  if (error_message.empty())
458  error_message = "test failed with an empty error message";
459  }
460 
461  void set_exception(std::exception& e)
462  {
463  error_message = e.what();
464  if (error_message.empty())
465  error_message = "test threw an exception with an empty error message";
466  exception_typeid = typeid(e).name();
467  }
468 
469  void set_unknown_exception()
470  {
471  error_message = "unknown exception caught";
472  }
473 
474  void set_setup_exception(std::exception& e)
475  {
476  error_message = "[setup failed: ";
477  error_message += e.what();
478  error_message += "]";
479  }
480 
481  void set_teardown_exception(std::exception& e)
482  {
483  error_message = "[teardown failed: ";
484  error_message += e.what();
485  error_message += "]";
486  }
487 
488  bool is_success() const
489  {
490  return error_message.empty();
491  }
492 };
493 
498 {
500  std::string test_case;
502  std::vector<TestMethodResult> methods;
504  std::string fail_setup;
507  std::string fail_teardown;
509  bool skipped = false;
510 
511  TestCaseResult(const std::string& test_case) : test_case(test_case) {}
512 
513  void set_setup_failed()
514  {
515  fail_setup = "test case setup method threw an unknown exception";
516  }
517 
518  void set_setup_failed(std::exception& e)
519  {
520  fail_setup = "test case setup method threw an exception: ";
521  fail_setup += e.what();
522  }
523 
524  void set_teardown_failed()
525  {
526  fail_teardown = "test case teardown method threw an unknown exception";
527  }
528 
529  void set_teardown_failed(std::exception& e)
530  {
531  fail_teardown = "test case teardown method threw an exception: ";
532  fail_teardown += e.what();
533  }
534 
535  void add_test_method(TestMethodResult&& e)
536  {
537  methods.emplace_back(std::move(e));
538  }
539 
540  bool is_success() const
541  {
542  if (!fail_setup.empty() || !fail_teardown.empty()) return false;
543  for (const auto& m: methods)
544  if (!m.is_success())
545  return false;
546  return true;
547  }
548 };
549 
550 struct TestCase;
551 struct TestCaseResult;
552 struct TestMethod;
553 struct TestMethodResult;
554 
562 {
563  virtual ~TestController() {}
564 
570  virtual bool test_case_begin(const TestCase& test_case, const TestCaseResult& test_case_result) { return true; }
571 
575  virtual void test_case_end(const TestCase& test_case, const TestCaseResult& test_case_result) {}
576 
582  virtual bool test_method_begin(const TestMethod& test_method, const TestMethodResult& test_method_result) { return true; }
583 
587  virtual void test_method_end(const TestMethod& test_method, const TestMethodResult& test_method_result) {}
588 };
589 
597 {
599  std::string whitelist;
600 
602  std::string blacklist;
603 
604  bool test_case_begin(const TestCase& test_case, const TestCaseResult& test_case_result) override;
605  void test_case_end(const TestCase& test_case, const TestCaseResult& test_case_result) override;
606  bool test_method_begin(const TestMethod& test_method, const TestMethodResult& test_method_result) override;
607  void test_method_end(const TestMethod& test_method, const TestMethodResult& test_method_result) override;
608 
609  bool test_method_should_run(const std::string& fullname) const;
610 };
611 
612 
620 {
622  std::vector<TestCase*> entries;
623 
630  void register_test_case(TestCase& test_case);
631 
638  void iterate_test_methods(std::function<void(const TestCase&, const TestMethod&)>);
639 
643  std::vector<TestCaseResult> run_tests(TestController& controller);
644 
646  static TestRegistry& get();
647 };
648 
653 {
655  std::string name;
656 
658  std::string doc;
659 
665  std::function<void()> test_function;
666 
667  TestMethod(const std::string& name)
668  : name(name) {}
669 
670  TestMethod(const std::string& name, std::function<void()> test_function)
671  : name(name), test_function(test_function) {}
672 };
673 
674 
679 struct TestCase
680 {
682  std::string name;
683 
685  std::vector<TestMethod> methods;
686 
688  bool tests_registered = false;
689 
690 
691  TestCase(const std::string& name)
692  : name(name)
693  {
695  }
696  virtual ~TestCase() {}
697 
701  void register_tests_once();
702 
710  virtual void register_tests() = 0;
711 
715  virtual void setup() {}
716 
720  virtual void teardown() {}
721 
725  virtual void method_setup(TestMethodResult&) {}
726 
731 
739  virtual TestCaseResult run_tests(TestController& controller);
740 
753  virtual TestMethodResult run_test(TestController& controller, TestMethod& method);
754 
759  TestMethod& add_method(const std::string& name)
760  {
761  methods.emplace_back(name);
762  return methods.back();
763  }
764 
768  template<typename ...Args>
769  TestMethod& add_method(const std::string& name, std::function<void()> test_function)
770  {
771  methods.emplace_back(name, test_function);
772  return methods.back();
773  }
774 
778  template<typename ...Args>
779  TestMethod& add_method(const std::string& name, const std::string& doc, std::function<void()> test_function)
780  {
781  methods.emplace_back(name, test_function);
782  methods.back().doc = doc;
783  return methods.back();
784  }
785 };
786 
787 
798 struct Fixture
799 {
800  // Called before each test
801  void test_setup() {}
802 
803  // Called after each test
804  void test_teardown() {}
805 };
806 
807 template<typename Fixture, typename... Args>
808 static inline Fixture* fixture_factory(Args... args)
809 {
810  return new Fixture(args...);
811 }
812 
816 template<typename FIXTURE>
817 class FixtureTestCase : public TestCase
818 {
819 public:
820  typedef FIXTURE Fixture;
821 
822  Fixture* fixture = nullptr;
823  std::function<Fixture*()> make_fixture;
824 
825  template<typename... Args>
826  FixtureTestCase(const std::string& name, Args... args)
827  : TestCase(name)
828  {
829  make_fixture = std::bind(fixture_factory<FIXTURE, Args...>, args...);
830  }
831 
832  void setup() override
833  {
834  TestCase::setup();
835  fixture = make_fixture();
836  }
837 
838  void teardown() override
839  {
840  delete fixture;
841  fixture = 0;
843  }
844 
845  void method_setup(TestMethodResult& mr) override
846  {
848  if (fixture) fixture->test_setup();
849  }
850 
852  {
853  if (fixture) fixture->test_teardown();
855  }
856 
861  template<typename ...Args>
862  TestMethod& add_method(const std::string& name, std::function<void(FIXTURE&)> test_function)
863  {
864  return TestCase::add_method(name, [=]() { test_function(*fixture); });
865  }
866 
871  template<typename ...Args>
872  TestMethod& add_method(const std::string& name, const std::string& doc, std::function<void(FIXTURE&)> test_function)
873  {
874  return TestCase::add_method(name, doc, [=]() { test_function(*fixture); });
875  }
876 };
877 
878 }
879 }
880 #endif
Test registry.
Definition: utils/tests.h:619
virtual void test_method_end(const TestMethod &test_method, const TestMethodResult &test_method_result)
Called after running a test method.
Definition: utils/tests.h:587
virtual TestCaseResult run_tests(TestController &controller)
Call setup(), run all the tests that have been registered, then call teardown().
std::string exception_typeid
If non-empty, the test threw an exception and this is its type ID.
Definition: utils/tests.h:444
void test_method_end(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called after running a test method.
Result of running a whole test case.
Definition: utils/tests.h:497
TestMethod & add_method(const std::string &name, std::function< void(FIXTURE &)> test_function)
Register a new test method that takes a reference to the fixture as argument.
Definition: utils/tests.h:862
Test case collecting several test methods, and self-registering with the singleton instance of TestRe...
Definition: utils/tests.h:679
std::string name
Name of the test case.
Definition: utils/tests.h:682
Information about one stack frame in the test execution stack.
Definition: utils/tests.h:66
Add information to the test backtrace for the tests run in the current scope.
Definition: utils/tests.h:54
std::string test_method
Name of the test method.
Definition: utils/tests.h:435
Exception thrown when a test or a test case needs to be skipped.
Definition: utils/tests.h:135
virtual bool test_method_begin(const TestMethod &test_method, const TestMethodResult &test_method_result)
Called before running a test method.
Definition: utils/tests.h:582
virtual void method_setup(TestMethodResult &)
Set up before the test method is run.
Definition: utils/tests.h:725
std::vector< TestMethodResult > methods
Outcome of all the methods that have been run.
Definition: utils/tests.h:502
void register_tests_once()
Idempotent wrapper for register_tests()
Simple default implementation of TestController.
Definition: utils/tests.h:596
TestStack error_stack
Stack frame of where the error happened.
Definition: utils/tests.h:441
bool tests_registered
Set to true the first time register_tests_once is run.
Definition: utils/tests.h:688
Abstract interface for the objects that supervise test execution.
Definition: utils/tests.h:561
std::string fail_teardown
Set to a non-empty string if the teardown method of the test case failed.
Definition: utils/tests.h:507
Definition: utils/tests.h:275
bool skipped
Set to true if this test case has been skipped.
Definition: utils/tests.h:509
virtual TestMethodResult run_test(TestController &controller, TestMethod &method)
Run a test method.
void teardown() override
Clean up after the test case is run.
Definition: utils/tests.h:838
std::vector< TestCaseResult > run_tests(TestController &controller)
Run all the registered tests using the given controller.
Exception thrown when a test assertion fails, normally by Location::fail_test.
Definition: utils/tests.h:103
TestMethod & add_method(const std::string &name)
Register a new test method, with the actual test function to be added later.
Definition: utils/tests.h:759
void method_teardown(TestMethodResult &mr) override
Clean up after the test method is run.
Definition: utils/tests.h:851
void register_test_case(TestCase &test_case)
Register a new test case.
std::string backtrace() const
Return the formatted backtrace for this location.
std::vector< TestMethod > methods
All registered test methods.
Definition: utils/tests.h:685
TestMethod & add_method(const std::string &name, const std::string &doc, std::function< void(FIXTURE &)> test_function)
Register a new test method that takes a reference to the fixture as argument, including documentation...
Definition: utils/tests.h:872
TestMethod & add_method(const std::string &name, std::function< void()> test_function)
Register a new test method.
Definition: utils/tests.h:769
Definition: utils/tests.h:88
static TestRegistry & get()
Get the singleton instance of TestRegistry.
virtual void setup()
Set up the test case before it is run.
Definition: utils/tests.h:715
std::vector< TestCase * > entries
All known test cases.
Definition: utils/tests.h:622
Definition: utils/tests.h:291
std::string doc
Documentation attached to this test method.
Definition: utils/tests.h:658
virtual void test_case_end(const TestCase &test_case, const TestCaseResult &test_case_result)
Called after running a test case.
Definition: utils/tests.h:575
std::string test_case
Name of the test case.
Definition: utils/tests.h:500
Definition: utils/tests.h:341
Definition: utils/tests.h:326
bool skipped
True if the test has been skipped.
Definition: utils/tests.h:447
std::string blacklist
Any method matching this glob expression will not be run.
Definition: utils/tests.h:602
Result of running a test method.
Definition: utils/tests.h:429
virtual bool test_case_begin(const TestCase &test_case, const TestCaseResult &test_case_result)
Called before running a test case.
Definition: utils/tests.h:570
void iterate_test_methods(std::function< void(const TestCase &, const TestMethod &)>)
Iterate on all test methods known by this registry.
Definition: utils/tests.h:350
void setup() override
Set up the test case before it is run.
Definition: utils/tests.h:832
std::function< void()> test_function
Main body of the test method.
Definition: utils/tests.h:665
void method_setup(TestMethodResult &mr) override
Set up before the test method is run.
Definition: utils/tests.h:845
Definition: utils/tests.h:314
std::string whitelist
Any method not matching this glob expression will not be run.
Definition: utils/tests.h:599
virtual void register_tests()=0
This will be called before running the test case, to populate it with its test methods.
std::string error_message
If non-empty, the test failed with this error.
Definition: utils/tests.h:438
Test case that includes a fixture.
Definition: utils/tests.h:817
Test method information.
Definition: utils/tests.h:652
virtual void method_teardown(TestMethodResult &)
Clean up after the test method is run.
Definition: utils/tests.h:730
virtual void teardown()
Clean up after the test case is run.
Definition: utils/tests.h:720
std::string name
Name of the test method.
Definition: utils/tests.h:655
Base class for test fixtures.
Definition: utils/tests.h:798
std::string fail_setup
Set to a non-empty string if the setup method of the test case failed.
Definition: utils/tests.h:504
std::string test_case
Name of the test case.
Definition: utils/tests.h:432
bool test_method_begin(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called before running a test method.
std::ostream & operator()()
Clear the current information and return the output stream to which new information can be sent...
bool test_case_begin(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called before running a test case.
TestMethod & add_method(const std::string &name, const std::string &doc, std::function< void()> test_function)
Register a new test method, including documentation.
Definition: utils/tests.h:779
void test_case_end(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called after running a test case.