From 5a84d641cd72adf202c29b9ce998d27bbd8ca8ac Mon Sep 17 00:00:00 2001
From: Eric Huss <eric@huss.org>
Date: Tue, 22 Apr 2025 10:56:05 -0700
Subject: [PATCH] Migrate pass/fail `mdbook test` to BookTest

---
 tests/cli/mod.rs                              |  1 -
 tests/cli/test.rs                             | 34 ------------
 tests/testing.rs                              | 21 -------
 tests/testsuite/main.rs                       |  1 +
 tests/testsuite/test.rs                       | 55 +++++++++++++++++++
 tests/testsuite/test/failing_tests/book.toml  |  2 +
 .../test/failing_tests/src/SUMMARY.md         |  4 ++
 .../test/failing_tests/src/failing.md         |  5 ++
 .../test/failing_tests/src/failing_include.md |  5 ++
 .../testsuite/test/failing_tests/src/test1.rs | 11 ++++
 tests/testsuite/test/passing_tests/book.toml  |  2 +
 .../test/passing_tests/src/SUMMARY.md         |  6 ++
 .../test/passing_tests/src/passing1.md        | 29 ++++++++++
 .../test/passing_tests/src/passing2.md        |  5 ++
 .../testsuite/test/passing_tests/src/test1.rs |  1 +
 .../testsuite/test/passing_tests/src/test2.rs | 11 ++++
 .../testsuite/test/passing_tests/src/test3.rs |  1 +
 17 files changed, 138 insertions(+), 56 deletions(-)
 delete mode 100644 tests/cli/test.rs
 create mode 100644 tests/testsuite/test.rs
 create mode 100644 tests/testsuite/test/failing_tests/book.toml
 create mode 100644 tests/testsuite/test/failing_tests/src/SUMMARY.md
 create mode 100644 tests/testsuite/test/failing_tests/src/failing.md
 create mode 100644 tests/testsuite/test/failing_tests/src/failing_include.md
 create mode 100644 tests/testsuite/test/failing_tests/src/test1.rs
 create mode 100644 tests/testsuite/test/passing_tests/book.toml
 create mode 100644 tests/testsuite/test/passing_tests/src/SUMMARY.md
 create mode 100644 tests/testsuite/test/passing_tests/src/passing1.md
 create mode 100644 tests/testsuite/test/passing_tests/src/passing2.md
 create mode 100644 tests/testsuite/test/passing_tests/src/test1.rs
 create mode 100644 tests/testsuite/test/passing_tests/src/test2.rs
 create mode 100644 tests/testsuite/test/passing_tests/src/test3.rs

diff --git a/tests/cli/mod.rs b/tests/cli/mod.rs
index 989f443f..4c2d1892 100644
--- a/tests/cli/mod.rs
+++ b/tests/cli/mod.rs
@@ -1,3 +1,2 @@
 mod build;
 mod cmd;
-mod test;
diff --git a/tests/cli/test.rs b/tests/cli/test.rs
deleted file mode 100644
index 63114d3a..00000000
--- a/tests/cli/test.rs
+++ /dev/null
@@ -1,34 +0,0 @@
-use crate::cli::cmd::mdbook_cmd;
-use crate::dummy_book::DummyBook;
-
-use predicates::boolean::PredicateBooleanExt;
-
-#[test]
-fn mdbook_cli_can_correctly_test_a_passing_book() {
-    let temp = DummyBook::new().with_passing_test(true).build().unwrap();
-
-    let mut cmd = mdbook_cmd();
-    cmd.arg("test").current_dir(temp.path());
-    cmd.assert().success()
-      .stderr(predicates::str::is_match(r##"Testing chapter [^:]*: "README.md""##).unwrap())
-      .stderr(predicates::str::is_match(r##"Testing chapter [^:]*: "intro.md""##).unwrap())
-      .stderr(predicates::str::is_match(r##"Testing chapter [^:]*: "first[\\/]index.md""##).unwrap())
-      .stderr(predicates::str::is_match(r##"Testing chapter [^:]*: "first[\\/]nested.md""##).unwrap())
-      .stderr(predicates::str::is_match(r##"returned an error:\n\n"##).unwrap().not())
-      .stderr(predicates::str::is_match(r##"Nested_Chapter::Rustdoc_include_works_with_anchors_too \(line \d+\) ... FAILED"##).unwrap().not());
-}
-
-#[test]
-fn mdbook_cli_detects_book_with_failing_tests() {
-    let temp = DummyBook::new().with_passing_test(false).build().unwrap();
-
-    let mut cmd = mdbook_cmd();
-    cmd.arg("test").current_dir(temp.path());
-    cmd.assert().failure()
-      .stderr(predicates::str::is_match(r##"Testing chapter [^:]*: "README.md""##).unwrap())
-      .stderr(predicates::str::is_match(r##"Testing chapter [^:]*: "intro.md""##).unwrap())
-      .stderr(predicates::str::is_match(r##"Testing chapter [^:]*: "first[\\/]index.md""##).unwrap())
-      .stderr(predicates::str::is_match(r##"Testing chapter [^:]*: "first[\\/]nested.md""##).unwrap())
-      .stderr(predicates::str::is_match(r##"returned an error:\n\n"##).unwrap())
-      .stderr(predicates::str::is_match(r##"Nested_Chapter::Rustdoc_include_works_with_anchors_too \(line \d+\) ... FAILED"##).unwrap());
-}
diff --git a/tests/testing.rs b/tests/testing.rs
index 3030c5cb..6f5a299a 100644
--- a/tests/testing.rs
+++ b/tests/testing.rs
@@ -4,27 +4,6 @@ use crate::dummy_book::DummyBook;
 
 use mdbook::MDBook;
 
-#[test]
-fn mdbook_can_correctly_test_a_passing_book() {
-    let temp = DummyBook::new().with_passing_test(true).build().unwrap();
-    let mut md = MDBook::load(temp.path()).unwrap();
-
-    let result = md.test(vec![]);
-    assert!(
-        result.is_ok(),
-        "Tests failed with {}",
-        result.err().unwrap()
-    );
-}
-
-#[test]
-fn mdbook_detects_book_with_failing_tests() {
-    let temp = DummyBook::new().with_passing_test(false).build().unwrap();
-    let mut md = MDBook::load(temp.path()).unwrap();
-
-    assert!(md.test(vec![]).is_err());
-}
-
 #[test]
 fn mdbook_test_chapter() {
     let temp = DummyBook::new().with_passing_test(true).build().unwrap();
diff --git a/tests/testsuite/main.rs b/tests/testsuite/main.rs
index 95756d51..e622fe0f 100644
--- a/tests/testsuite/main.rs
+++ b/tests/testsuite/main.rs
@@ -17,6 +17,7 @@ mod renderer;
 mod rendering;
 #[cfg(feature = "search")]
 mod search;
+mod test;
 
 mod prelude {
     pub use crate::book_test::BookTest;
diff --git a/tests/testsuite/test.rs b/tests/testsuite/test.rs
new file mode 100644
index 00000000..b39b183c
--- /dev/null
+++ b/tests/testsuite/test.rs
@@ -0,0 +1,55 @@
+//! Tests for the `mdbook test` command.
+
+use crate::prelude::*;
+
+// Simple test for passing tests.
+#[test]
+fn passing_tests() {
+    BookTest::from_dir("test/passing_tests").run("test", |cmd| {
+        cmd.expect_stdout(str![[""]]).expect_stderr(str![[r#"
+[TIMESTAMP] [INFO] (mdbook::book): Testing chapter 'Intro': "intro.md"
+[TIMESTAMP] [INFO] (mdbook::book): Testing chapter 'Passing 1': "passing1.md"
+[TIMESTAMP] [INFO] (mdbook::book): Testing chapter 'Passing 2': "passing2.md"
+
+"#]]);
+    });
+}
+
+// Test for a test failure
+#[test]
+fn failing_tests() {
+    BookTest::from_dir("test/failing_tests").run("test", |cmd| {
+        cmd.expect_code(101)
+            .expect_stdout(str![[""]])
+            // This redacts a large number of lines that come from rustdoc and
+            // libtest. If the output from those ever changes, then it would not
+            // make it possible to test against different versions of Rust. This
+            // still includes a little bit of output, so if that is a problem,
+            // add more redactions.
+            .expect_stderr(str![[r#"
+[TIMESTAMP] [INFO] (mdbook::book): Testing chapter 'Failing Tests': "failing.md"
+[TIMESTAMP] [ERROR] (mdbook::book): rustdoc returned an error:
+
+--- stdout
+
+...
+test failing.md - Failing_Tests (line 3) ... FAILED
+...
+thread 'main' panicked at failing.md:3:1:
+fail
+...
+[TIMESTAMP] [INFO] (mdbook::book): Testing chapter 'Failing Include': "failing_include.md"
+[TIMESTAMP] [ERROR] (mdbook::book): rustdoc returned an error:
+
+--- stdout
+...
+test failing_include.md - Failing_Include (line 3) ... FAILED
+...
+thread 'main' panicked at failing_include.md:3:1:
+failing!
+...
+[TIMESTAMP] [ERROR] (mdbook::utils): Error: One or more tests failed
+
+"#]]);
+    });
+}
diff --git a/tests/testsuite/test/failing_tests/book.toml b/tests/testsuite/test/failing_tests/book.toml
new file mode 100644
index 00000000..37c9bcf4
--- /dev/null
+++ b/tests/testsuite/test/failing_tests/book.toml
@@ -0,0 +1,2 @@
+[book]
+title = "failing_tests"
diff --git a/tests/testsuite/test/failing_tests/src/SUMMARY.md b/tests/testsuite/test/failing_tests/src/SUMMARY.md
new file mode 100644
index 00000000..b141f28d
--- /dev/null
+++ b/tests/testsuite/test/failing_tests/src/SUMMARY.md
@@ -0,0 +1,4 @@
+# Summary
+
+- [Failing Tests](./failing.md)
+- [Failing Include](./failing_include.md)
diff --git a/tests/testsuite/test/failing_tests/src/failing.md b/tests/testsuite/test/failing_tests/src/failing.md
new file mode 100644
index 00000000..635f8b51
--- /dev/null
+++ b/tests/testsuite/test/failing_tests/src/failing.md
@@ -0,0 +1,5 @@
+# Failing Tests
+
+```rust
+panic!("fail");
+```
diff --git a/tests/testsuite/test/failing_tests/src/failing_include.md b/tests/testsuite/test/failing_tests/src/failing_include.md
new file mode 100644
index 00000000..7c369200
--- /dev/null
+++ b/tests/testsuite/test/failing_tests/src/failing_include.md
@@ -0,0 +1,5 @@
+# Failing Include
+
+```rust
+{{#include test1.rs:FAILING}}
+```
diff --git a/tests/testsuite/test/failing_tests/src/test1.rs b/tests/testsuite/test/failing_tests/src/test1.rs
new file mode 100644
index 00000000..b3f19502
--- /dev/null
+++ b/tests/testsuite/test/failing_tests/src/test1.rs
@@ -0,0 +1,11 @@
+fn test2() {
+    println!("test2");
+}
+
+// ANCHOR: PASSING
+println!("passing!");
+// ANCHOR_END: PASSING
+
+// ANCHOR: FAILING
+panic!("failing!");
+// ANCHOR_END: FAILING
diff --git a/tests/testsuite/test/passing_tests/book.toml b/tests/testsuite/test/passing_tests/book.toml
new file mode 100644
index 00000000..959cd17c
--- /dev/null
+++ b/tests/testsuite/test/passing_tests/book.toml
@@ -0,0 +1,2 @@
+[book]
+title = "passing_tests"
diff --git a/tests/testsuite/test/passing_tests/src/SUMMARY.md b/tests/testsuite/test/passing_tests/src/SUMMARY.md
new file mode 100644
index 00000000..fb2a2b53
--- /dev/null
+++ b/tests/testsuite/test/passing_tests/src/SUMMARY.md
@@ -0,0 +1,6 @@
+# Summary
+
+[Intro](./intro.md)
+
+- [Passing 1](./passing1.md)
+- [Passing 2](./passing2.md)
diff --git a/tests/testsuite/test/passing_tests/src/passing1.md b/tests/testsuite/test/passing_tests/src/passing1.md
new file mode 100644
index 00000000..9f1b91ba
--- /dev/null
+++ b/tests/testsuite/test/passing_tests/src/passing1.md
@@ -0,0 +1,29 @@
+# Passing Tests 1
+
+```rust
+assert!(true);
+```
+
+```rust
+println!("hello!");
+```
+
+## Also check includes
+
+```rust
+{{#include test1.rs}}
+```
+
+```rust
+{{#include test2.rs:2}}
+```
+
+```rust
+{{#include test2.rs:PASSING}}
+```
+
+```rust
+{{#rustdoc_include test3.rs:2}}
+```
+
+{{#playground test1.rs}}
diff --git a/tests/testsuite/test/passing_tests/src/passing2.md b/tests/testsuite/test/passing_tests/src/passing2.md
new file mode 100644
index 00000000..dc290970
--- /dev/null
+++ b/tests/testsuite/test/passing_tests/src/passing2.md
@@ -0,0 +1,5 @@
+# Passing Tests 2
+
+```rust
+println!("also passing");
+```
diff --git a/tests/testsuite/test/passing_tests/src/test1.rs b/tests/testsuite/test/passing_tests/src/test1.rs
new file mode 100644
index 00000000..226bd6af
--- /dev/null
+++ b/tests/testsuite/test/passing_tests/src/test1.rs
@@ -0,0 +1 @@
+println!("test1");
diff --git a/tests/testsuite/test/passing_tests/src/test2.rs b/tests/testsuite/test/passing_tests/src/test2.rs
new file mode 100644
index 00000000..b3f19502
--- /dev/null
+++ b/tests/testsuite/test/passing_tests/src/test2.rs
@@ -0,0 +1,11 @@
+fn test2() {
+    println!("test2");
+}
+
+// ANCHOR: PASSING
+println!("passing!");
+// ANCHOR_END: PASSING
+
+// ANCHOR: FAILING
+panic!("failing!");
+// ANCHOR_END: FAILING
diff --git a/tests/testsuite/test/passing_tests/src/test3.rs b/tests/testsuite/test/passing_tests/src/test3.rs
new file mode 100644
index 00000000..1832ea31
--- /dev/null
+++ b/tests/testsuite/test/passing_tests/src/test3.rs
@@ -0,0 +1 @@
+println!("test3");