Add "FLAKY_TEST" test directive to support re-running flaky tests.

Some of the mutex tests fail on machines with high load. This patch implements
the test directive "// FLAKY_TEST" which allows a test to be run 3 times
before it's considered a failure.


git-svn-id: https://llvm.org/svn/llvm-project/libcxx/trunk@280050 91177308-0d34-0410-b5e6-96231b3b80d8

test/libcxx/test/format.py

@@ -114,6 +114,9 @@ class LibcxxTestFormat(object):
     def _evaluate_pass_test(self, test, tmpBase, lit_config):
         execDir = os.path.dirname(test.getExecPath())
         source_path = test.getSourcePath()
+        with open(source_path, 'r') as f:
+            contents = f.read()
+        is_flaky = 'FLAKY_TEST' in contents
         exec_path = tmpBase + '.exe'
         object_path = tmpBase + '.o'
         # Create the output directory if it does not already exist.
@@ -139,14 +142,21 @@ class LibcxxTestFormat(object):
             # should add a `// FILE-DEP: foo.dat` to each test to track this.
             data_files = [os.path.join(local_cwd, f)
                           for f in os.listdir(local_cwd) if f.endswith('.dat')]
-            cmd, out, err, rc = self.executor.run(exec_path, [exec_path],
-                                                  local_cwd, data_files, env)
-            if rc != 0:
-                report = libcxx.util.makeReport(cmd, out, err, rc)
-                report = "Compiled With: %s\n%s" % (compile_cmd, report)
-                report += "Compiled test failed unexpectedly!"
-                return lit.Test.FAIL, report
-            return lit.Test.PASS, ''
+            max_retry = 3 if is_flaky else 1
+            for retry_count in range(max_retry):
+                cmd, out, err, rc = self.executor.run(exec_path, [exec_path],
+                                                      local_cwd, data_files,
+                                                      env)
+                if rc == 0:
+                    res = lit.Test.PASS if retry_count == 0 else lit.Test.FLAKYPASS
+                    return res, ''
+                elif rc != 0 and retry_count + 1 == max_retry:
+                    report = libcxx.util.makeReport(cmd, out, err, rc)
+                    report = "Compiled With: %s\n%s" % (compile_cmd, report)
+                    report += "Compiled test failed unexpectedly!"
+                    return lit.Test.FAIL, report
+
+            assert False # Unreachable
         finally:
             # Note that cleanup of exec_file happens in `_clean()`. If you
             # override this, cleanup is your reponsibility.

test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex.pass.cpp

@@ -10,6 +10,8 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03, c++11
 
+// FLAKY_TEST
+
 // <shared_mutex>
 
 // template <class Mutex> class shared_lock;

test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_duration.pass.cpp

@@ -10,6 +10,8 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03, c++11
 
+// FLAKY_TEST
+
 // <shared_mutex>
 
 // class timed_mutex;

test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_time_point.pass.cpp

@@ -10,6 +10,8 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03, c++11
 
+// FLAKY_TEST
+
 // <shared_mutex>
 
 // class shared_timed_mutex;

test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_duration.pass.cpp

@@ -9,6 +9,8 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 
+// FLAKY_TEST
+
 // <mutex>
 
 // class timed_mutex;

test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_time_point.pass.cpp

@@ -9,6 +9,8 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 
+// FLAKY_TEST
+
 // <mutex>
 
 // class timed_mutex;

test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock.pass.cpp

@@ -10,6 +10,8 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++03, c++98, c++11, c++14
 
+// FLAKY_TEST
+
 // <shared_mutex>
 
 // class shared_mutex;

test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock_shared.pass.cpp

@@ -10,6 +10,8 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++03, c++98, c++11, c++14
 
+// FLAKY_TEST
+
 // <shared_mutex>
 
 // class shared_mutex;

test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/try_lock_shared.pass.cpp

@@ -10,6 +10,8 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++03, c++98, c++11, c++14
 
+// FLAKY_TEST
+
 // <shared_mutex>
 
 // class shared_mutex;

test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock.pass.cpp

@@ -10,6 +10,8 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++03, c++98, c++11
 
+// FLAKY_TEST
+
 // <shared_mutex>
 
 // class shared_timed_mutex;

test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock_shared.pass.cpp

@@ -10,6 +10,8 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++03, c++98, c++11
 
+// FLAKY_TEST
+
 // <shared_mutex>
 
 // class shared_timed_mutex;

test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_for.pass.cpp

@@ -10,6 +10,8 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++03, c++98, c++11
 
+// FLAKY_TEST
+
 // <shared_mutex>
 
 // class shared_timed_mutex;

test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared.pass.cpp

@@ -10,6 +10,8 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++03, c++98, c++11
 
+// FLAKY_TEST
+
 // <shared_mutex>
 
 // class shared_timed_mutex;

test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_for.pass.cpp

@@ -10,6 +10,8 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++03, c++98, c++11
 
+// FLAKY_TEST
+
 // <shared_mutex>
 
 // class shared_timed_mutex;

test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_until.pass.cpp

@@ -10,6 +10,8 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++03, c++98, c++11
 
+// FLAKY_TEST
+
 // <shared_mutex>
 
 // class shared_timed_mutex;

test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_until.pass.cpp

@@ -10,6 +10,8 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++03, c++98, c++11
 
+// FLAKY_TEST
+
 // <shared_mutex>
 
 // class shared_timed_mutex;