Introduce Custom Test Frameworks

src/test/incremental/issue-49595/issue_49595.rs

@@ -15,12 +15,11 @@
 #![feature(rustc_attrs)]
 #![crate_type = "rlib"]
 
-#![rustc_partition_codegened(module="issue_49595-__test", cfg="cfail2")]
+#![rustc_partition_codegened(module="issue_49595-tests", cfg="cfail2")]
 #![rustc_partition_codegened(module="issue_49595-lit_test", cfg="cfail3")]
 
 mod tests {
-    #[cfg_attr(not(cfail1), ignore)]
-    #[test]
+    #[cfg_attr(not(cfail1), test)]
     fn test() {
     }
 }

src/test/ui/cfg-non-opt-expr.rs

@@ -9,6 +9,7 @@
 // except according to those terms.
 
 #![feature(stmt_expr_attributes)]
+#![feature(custom_test_frameworks)]
 
 fn main() {
     let _ = #[cfg(unset)] ();
@@ -17,6 +18,6 @@ fn main() {
     //~^ ERROR removing an expression is not supported in this position
     let _ = [1, 2, 3][#[cfg(unset)] 1];
     //~^ ERROR removing an expression is not supported in this position
-    let _ = #[test] ();
+    let _ = #[test_case] ();
     //~^ ERROR removing an expression is not supported in this position
 }

src/test/ui/cfg-non-opt-expr.stderr

@@ -1,26 +1,26 @@
 error: removing an expression is not supported in this position
-  --> $DIR/cfg-non-opt-expr.rs:14:13
+  --> $DIR/cfg-non-opt-expr.rs:15:13
    |
 LL |     let _ = #[cfg(unset)] ();
    |             ^^^^^^^^^^^^^
 
 error: removing an expression is not supported in this position
-  --> $DIR/cfg-non-opt-expr.rs:16:21
+  --> $DIR/cfg-non-opt-expr.rs:17:21
    |
 LL |     let _ = 1 + 2 + #[cfg(unset)] 3;
    |                     ^^^^^^^^^^^^^
 
 error: removing an expression is not supported in this position
-  --> $DIR/cfg-non-opt-expr.rs:18:23
+  --> $DIR/cfg-non-opt-expr.rs:19:23
    |
 LL |     let _ = [1, 2, 3][#[cfg(unset)] 1];
    |                       ^^^^^^^^^^^^^
 
 error: removing an expression is not supported in this position
-  --> $DIR/cfg-non-opt-expr.rs:20:13
+  --> $DIR/cfg-non-opt-expr.rs:21:13
    |
-LL |     let _ = #[test] ();
-   |             ^^^^^^^
+LL |     let _ = #[test_case] ();
+   |             ^^^^^^^^^^^^
 
 error: aborting due to 4 previous errors
 

src/test/ui/custom-test-frameworks-simple.rs

@@ -0,0 +1,32 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// compile-flags: --test
+// run-pass
+
+#![feature(custom_test_frameworks)]
+#![test_runner(crate::foo_runner)]
+
+#[cfg(test)]
+fn foo_runner(ts: &[&Fn(usize)->()]) {
+    for (i, t) in ts.iter().enumerate() {
+        t(i);
+    }
+}
+
+#[test_case]
+fn test1(i: usize) {
+    println!("Hi #{}", i);
+}
+
+#[test_case]
+fn test2(i: usize) {
+    println!("Hey #{}", i);
+}

src/test/ui/custom_test_frameworks/auxiliary/dynamic_runner.rs

@@ -0,0 +1,45 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use std::process::exit;
+
+pub trait Testable {
+    // Name of the test
+    fn name(&self) -> String;
+
+    // Tests pass by default
+    fn run(&self) -> bool {
+        true
+    }
+
+    // A test can generate subtests
+    fn subtests(&self) -> Vec<Box<dyn Testable>> {
+        vec![]
+    }
+}
+
+fn run_test(t: &dyn Testable) -> bool {
+    let success = t.subtests().into_iter().all(|sub_t| run_test(&*sub_t)) && t.run();
+    println!("{}...{}", t.name(), if success { "SUCCESS" } else { "FAIL" });
+    success
+}
+
+pub fn runner(tests: &[&dyn Testable]) {
+    let mut failed = false;
+    for t in tests {
+        if !run_test(*t) {
+            failed = true;
+        }
+    }
+
+    if failed {
+        exit(1);
+    }
+}

src/test/ui/custom_test_frameworks/auxiliary/example_runner.rs

@@ -0,0 +1,20 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+pub trait Testable {
+    fn name(&self) -> String;
+    fn run(&self) -> Option<String>; // None will be success, Some is the error message
+}
+
+pub fn runner(tests: &[&dyn Testable]) {
+    for t in tests {
+        print!("{}........{}", t.name(), t.run().unwrap_or_else(|| "SUCCESS".to_string()));
+    }
+}

src/test/ui/custom_test_frameworks/dynamic.rs

@@ -0,0 +1,45 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// run-pass
+// aux-build:dynamic_runner.rs
+// compile-flags:--test
+#![feature(custom_test_frameworks)]
+#![test_runner(dynamic_runner::runner)]
+
+extern crate dynamic_runner;
+
+pub struct AllFoo(&'static str);
+struct IsFoo(String);
+
+impl dynamic_runner::Testable for AllFoo {
+    fn name(&self) -> String {
+        String::from(self.0)
+    }
+
+    fn subtests(&self) -> Vec<Box<dyn dynamic_runner::Testable>> {
+        self.0.split(" ").map(|word|
+            Box::new(IsFoo(word.into())) as Box<dyn dynamic_runner::Testable>
+        ).collect()
+    }
+}
+
+impl dynamic_runner::Testable for IsFoo {
+    fn name(&self) -> String {
+        self.0.clone()
+    }
+
+    fn run(&self) -> bool {
+        self.0 == "foo"
+    }
+}
+
+#[test_case]
+const TEST_2: AllFoo = AllFoo("foo foo");

src/test/ui/custom_test_frameworks/full.rs

@@ -0,0 +1,38 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// run-pass
+// aux-build:example_runner.rs
+// compile-flags:--test
+
+#![feature(custom_test_frameworks)]
+#![test_runner(example_runner::runner)]
+extern crate example_runner;
+
+pub struct IsFoo(&'static str);
+
+impl example_runner::Testable for IsFoo {
+    fn name(&self) -> String {
+        self.0.to_string()
+    }
+
+    fn run(&self) -> Option<String> {
+        if self.0 != "foo" {
+            return Some(format!("{} != foo", self.0));
+        }
+        None
+    }
+}
+
+#[test_case]
+const TEST_1: IsFoo = IsFoo("hello");
+
+#[test_case]
+const TEST_2: IsFoo = IsFoo("foo");

src/test/ui/custom_test_frameworks/mismatch.rs

@@ -0,0 +1,19 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:example_runner.rs
+// compile-flags:--test
+#![feature(custom_test_frameworks)]
+#![test_runner(example_runner::runner)]
+
+extern crate example_runner;
+
+#[test]
+fn wrong_kind(){}

src/test/ui/custom_test_frameworks/mismatch.stderr

@@ -0,0 +1,11 @@
+error[E0277]: the trait bound `test::TestDescAndFn: example_runner::Testable` is not satisfied
+  --> $DIR/mismatch.rs:19:1
+   |
+LL | fn wrong_kind(){}
+   | ^^^^^^^^^^^^^^^^^ the trait `example_runner::Testable` is not implemented for `test::TestDescAndFn`
+   |
+   = note: required for the cast to the object type `dyn example_runner::Testable`
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0277`.

src/test/ui/feature-gate-custom_test_frameworks.rs

@@ -0,0 +1,13 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![test_runner(main)] //~ ERROR Custom Test Frameworks is an unstable feature
+
+fn main() {}

src/test/ui/feature-gate-custom_test_frameworks.stderr

@@ -0,0 +1,11 @@
+error[E0658]: Custom Test Frameworks is an unstable feature (see issue #50297)
+  --> $DIR/feature-gate-custom_test_frameworks.rs:11:1
+   |
+LL | #![test_runner(main)] //~ ERROR Custom Test Frameworks is an unstable feature
+   | ^^^^^^^^^^^^^^^^^^^^^
+   |
+   = help: add #![feature(custom_test_frameworks)] to the crate attributes to enable
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0658`.

src/test/ui/inaccessible-test-modules.stderr

@@ -2,7 +2,7 @@ error[E0432]: unresolved import `__test`
   --> $DIR/inaccessible-test-modules.rs:15:5
    |
 LL | use __test as x; //~ ERROR unresolved import `__test`
-   |     ^^^^^^^^^^^ no `__test` in the root. Did you mean to use `__test`?
+   |     ^^^^^^^^^^^ no `__test` in the root. Did you mean to use `test`?
 
 error[E0432]: unresolved import `__test_reexports`
   --> $DIR/inaccessible-test-modules.rs:16:5

src/test/ui/issues/issue-11692-2.rs

@@ -10,5 +10,5 @@
 
 fn main() {
     concat!(test!());
-    //~^ ERROR cannot find macro `test!` in this scope
+    //~^ error: `test` can only be used in attributes
 }

src/test/ui/issues/issue-11692-2.stderr

@@ -1,4 +1,4 @@
-error: cannot find macro `test!` in this scope
+error: `test` can only be used in attributes
   --> $DIR/issue-11692-2.rs:12:13
    |
 LL |     concat!(test!());

src/test/ui/issues/issue-12997-2.stderr

@@ -5,7 +5,7 @@ LL | fn bar(x: isize) { }
    | ^^^^^^^^^^^^^^^^^^^^ expected isize, found mutable reference
    |
    = note: expected type `isize`
-              found type `&mut __test::test::Bencher`
+              found type `&mut test::Bencher`
 
 error: aborting due to previous error
 

src/test/ui/lint/test-inner-fn.rs

@@ -8,11 +8,11 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-// compile-flags: --test -D unnameable_test_functions
+// compile-flags: --test -D unnameable_test_items
 
 #[test]
 fn foo() {
-    #[test] //~ ERROR cannot test inner function [unnameable_test_functions]
+    #[test] //~ ERROR cannot test inner items [unnameable_test_items]
     fn bar() {}
     bar();
 }
@@ -20,7 +20,7 @@ fn foo() {
 mod x {
     #[test]
     fn foo() {
-        #[test] //~ ERROR cannot test inner function [unnameable_test_functions]
+        #[test] //~ ERROR cannot test inner items [unnameable_test_items]
         fn bar() {}
         bar();
     }

src/test/ui/lint/test-inner-fn.stderr

@@ -1,15 +1,15 @@
-error: cannot test inner function
+error: cannot test inner items
   --> $DIR/test-inner-fn.rs:15:5
    |
-LL |     #[test] //~ ERROR cannot test inner function [unnameable_test_functions]
+LL |     #[test] //~ ERROR cannot test inner items [unnameable_test_items]
    |     ^^^^^^^
    |
-   = note: requested on the command line with `-D unnameable-test-functions`
+   = note: requested on the command line with `-D unnameable-test-items`
 
-error: cannot test inner function
+error: cannot test inner items
   --> $DIR/test-inner-fn.rs:23:9
    |
-LL |         #[test] //~ ERROR cannot test inner function [unnameable_test_functions]
+LL |         #[test] //~ ERROR cannot test inner items [unnameable_test_items]
    |         ^^^^^^^
 
 error: aborting due to 2 previous errors

src/test/ui/rfc-1937-termination-trait/termination-trait-test-wrong-type.stderr

@@ -7,7 +7,7 @@ LL | | }
    | |_^ `main` can only return types that implement `std::process::Termination`
    |
    = help: the trait `std::process::Termination` is not implemented for `std::result::Result<f32, std::num::ParseIntError>`
-   = note: required by `__test::test::assert_test_result`
+   = note: required by `test::assert_test_result`
 
 error: aborting due to previous error
 

src/test/ui/test-on-macro.rs

@@ -0,0 +1,23 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// compile-pass
+// compile-flags:--test
+
+#![deny(warnings)]
+
+macro_rules! foo {
+    () => (fn foo(){})
+}
+
+#[test]
+foo!();
+
+fn main(){}

src/test/ui/test-on-macro.stderr

@@ -0,0 +1,6 @@
+warning: #[test] attribute should not be used on macros. Use #[cfg(test)] instead.
+  --> $DIR/test-on-macro.rs:21:1
+   |
+LL | foo!();
+   | ^^^^^^^
+