jesus/mdbook
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Introduce the new BookTest-based testsuite

Browse source 
This is a new testsuite intended to replace the other tests, which
provides an easy facility to update tests, validate output, and more.
This commit is contained in:
Eric Huss 2025-04-21 18:22:08 -07:00
parent 566a42c4f7
commit f10d23e893
5 changed files with 592 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

96
Cargo.lock generated
View file

@ -75,6 +75,15 @@ version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
[[package]]
name = "anstyle-lossy"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "934ff8719effd2023a48cf63e69536c1c3ced9d3895068f6f5cc9a4ff845e59b"
dependencies = [
"anstyle",
]
[[package]]
name = "anstyle-parse"
version = "0.2.6"
@ -93,6 +102,19 @@ dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "anstyle-svg"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3607949e9f6de49ea4bafe12f5e4fd73613ebf24795e48587302a8cc0e4bb35"
dependencies = [
"anstream",
"anstyle",
"anstyle-lossy",
"html-escape",
"unicode-width",
]
[[package]]
name = "anstyle-wincon"
version = "3.0.7"
@ -288,6 +310,15 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
[[package]]
name = "content_inspector"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7bda66e858c683005a53a9a60c69a4aca7eeaa45d124526e389f7aec8e62f38"
dependencies = [
"memchr",
]
[[package]]
name = "core-foundation-sys"
version = "0.8.7"
@ -460,6 +491,12 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
[[package]]
name = "dunce"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
[[package]]
name = "elasticlunr-rs"
version = "3.0.2"
@ -737,6 +774,15 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "html-escape"
version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476"
dependencies = [
"utf8-width",
]
[[package]]
name = "html5ever"
version = "0.26.0"
@ -1236,6 +1282,7 @@ dependencies = [
"serde_json",
"sha2",
"shlex",
"snapbox",
"tempfile",
"tokio",
"toml",
@ -1920,6 +1967,12 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "similar"
version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
[[package]]
name = "siphasher"
version = "0.3.11"
@ -1947,6 +2000,37 @@ version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
[[package]]
name = "snapbox"
version = "0.6.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96dcfc4581e3355d70ac2ee14cfdf81dce3d85c85f1ed9e2c1d3013f53b3436b"
dependencies = [
"anstream",
"anstyle",
"anstyle-svg",
"content_inspector",
"dunce",
"filetime",
"normalize-line-endings",
"regex",
"serde",
"serde_json",
"similar",
"snapbox-macros",
"tempfile",
"walkdir",
]
[[package]]
name = "snapbox-macros"
version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16569f53ca23a41bb6f62e0a5084aa1661f4814a67fa33696a79073e03a664af"
dependencies = [
"anstream",
]
[[package]]
name = "socket2"
version = "0.5.8"
@ -2261,6 +2345,12 @@ version = "1.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034"
[[package]]
name = "unicode-width"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
[[package]]
name = "url"
version = "2.5.4"
@ -2284,6 +2374,12 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
[[package]]
name = "utf8-width"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3"
[[package]]
name = "utf8_iter"
version = "1.0.4"

1
Cargo.toml
View file

@ -68,6 +68,7 @@ assert_cmd = "2.0.11"
predicates = "3.0.3"
select = "0.6.0"
semver = "1.0.17"
snapbox = { version = "0.6.21", features = ["diff", "dir", "term-svg", "regex", "json"] }
pretty_assertions = "1.3.0"
walkdir = "2.3.3"

43
tests/testsuite/README.md Normal file
View file

@ -0,0 +1,43 @@
# Testsuite
## Introduction
This is the main testsuite for exercising all functionality of mdBook.
Tests should be organized into modules based around major features. Tests should use `BookTest` to drive the test. `BookTest` will set up a temp directory, and provides a variety of methods to help create a build books.
## Basic structure of a test
Using `BookTest`, you typically use it to copy a directory into a temp directory, and then run mdbook commands in that temp directory. You can run the `mdbook` executable, or use the mdbook API to perform whatever tasks you need. Running the executable has the benefit of being able to validate the console output.
See `build::basic_build` for a simple test example. I recommend reviewing the methods on `BookTest` to learn more, and reviewing some of the existing tests to get a feel for how they are structured.
For example, let's say you are creating a new theme test. In the `testsuite/theme` directory, create a new directory with the book source that you want to exercise. At a minimum, this needs a `src/SUMMARY.md`, but often you'll also want `book.toml`. Then, in `testsuite/theme.rs`, add a test with `BookTest::from_dir("theme/mytest")`, and then use the methods to perform whatever actions you want.
`BookTest` is designed to be able to chain a series of actions. For example, you can do something like:
```rust
BookTest::from_dir("theme/mytest")
.build()
.check_main_file("book/index.html", str![["file contents"]])
.change_file("src/index.md", "new contents")
.build()
.check_main_file("book/index.html", str![["new contents"]]);
```
## Snapbox
The testsuite uses [`snapbox`] to drive most of the tests. This library provides the ability to compare strings using a variety of methods. These strings are written in the source code using either the [`str!`] or [`file!`] macros.
The magic is that you can set the `SNAPSHOTS=overwrite` environment variable, and snapbox will automatically update the strings contents of `str!`, or the file contents of `file!`. This makes it easier to update tests. Snapbox provides nice diffing output, and quite a few other features.
Expected contents can have wildcards like `...` (matches any lines) or `[..]` (matches any characters on a line). See [snapbox filters] for more info and other filters.
Typically when writing a test, I'll just start with an empty `str!` or `file!`, and let snapbox fill it in. Then I review the contents to make sure they are what I expect.
Note that there is some normalization applied to the strings. See `book_test::assert` for how some of these normalizations happen.
[`snapbox`]: https://docs.rs/snapbox/latest/snapbox/
[`str!`]: https://docs.rs/snapbox/latest/snapbox/macro.str.html
[`file!`]: https://docs.rs/snapbox/latest/snapbox/macro.file.html
[snapbox filters]: https://docs.rs/snapbox/latest/snapbox/assert/struct.Assert.html#method.eq

442
tests/testsuite/book_test.rs Normal file
View file

@ -0,0 +1,442 @@
//! Utility for building and running tests against mdbook.
use mdbook::book::BookBuilder;
use mdbook::MDBook;
use snapbox::IntoData;
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::atomic::{AtomicU32, Ordering};
/// Test number used for generating unique temp directory names.
static NEXT_TEST_ID: AtomicU32 = AtomicU32::new(0);
#[derive(Clone, Copy, Eq, PartialEq)]
enum StatusCode {
Success,
Failure,
Code(i32),
}
/// Main helper for driving mdbook tests.
pub struct BookTest {
/// The temp directory where the test should perform its work.
pub dir: PathBuf,
assert: snapbox::Assert,
/// This indicates whether or not the book has been built.
built: bool,
}
impl BookTest {
/// Creates a new test, copying the contents from the given directory into
/// a temp directory.
pub fn from_dir(dir: &str) -> BookTest {
// Copy this test book to a temp directory.
let dir = Path::new("tests/testsuite").join(dir);
assert!(dir.exists(), "{dir:?} should exist");
let tmp = Self::new_tmp();
mdbook::utils::fs::copy_files_except_ext(
&dir,
&tmp,
true,
Some(&PathBuf::from("book")),
&[],
)
.unwrap_or_else(|e| panic!("failed to copy test book {dir:?} to {tmp:?}: {e:?}"));
Self::new(tmp)
}
/// Creates a new test with an empty temp directory.
pub fn empty() -> BookTest {
Self::new(Self::new_tmp())
}
/// Creates a new test with the given function to initialize a new book.
///
/// The book itself is not built.
pub fn init(f: impl Fn(&mut BookBuilder)) -> BookTest {
let tmp = Self::new_tmp();
let mut bb = MDBook::init(&tmp);
f(&mut bb);
bb.build()
.unwrap_or_else(|e| panic!("failed to initialize book at {tmp:?}: {e:?}"));
Self::new(tmp)
}
fn new_tmp() -> PathBuf {
let id = NEXT_TEST_ID.fetch_add(1, Ordering::SeqCst);
let tmp = Path::new(env!("CARGO_TARGET_TMPDIR"))
.join("ts")
.join(format!("t{id}"));
if tmp.exists() {
std::fs::remove_dir_all(&tmp)
.unwrap_or_else(|e| panic!("failed to remove {tmp:?}: {e:?}"));
}
std::fs::create_dir_all(&tmp).unwrap_or_else(|e| panic!("failed to create {tmp:?}: {e:?}"));
tmp
}
fn new(dir: PathBuf) -> BookTest {
let assert = assert(&dir);
BookTest {
dir,
assert,
built: false,
}
}
/// Checks the contents of an HTML file that it has the given contents
/// between the `<main>` tag.
///
/// Normally the contents outside of the `<main>` tag aren't interesting,
/// and they add a significant amount of noise.
pub fn check_main_file(&mut self, path: &str, expected: impl IntoData) -> &mut Self {
if !self.built {
self.build();
}
let full_path = self.dir.join(path);
let actual = read_to_string(&full_path);
let start = actual
.find("<main>")
.unwrap_or_else(|| panic!("didn't find <main> in:\n{actual}"));
let end = actual.find("</main>").unwrap();
let contents = actual[start + 6..end - 7].trim();
self.assert.eq(contents, expected);
self
}
/// Checks the summary contents of `toc.js` against the expected value.
pub fn check_toc_js(&mut self, expected: impl IntoData) -> &mut Self {
if !self.built {
self.build();
}
let inner = self.toc_js_html();
// Would be nice if this were prettified, but a primitive wrapping will do for now.
let inner = inner.replace("><", ">\n<");
self.assert.eq(inner, expected);
self
}
/// Returns the summary contents from `toc.js`.
pub fn toc_js_html(&self) -> String {
let full_path = self.dir.join("book/toc.js");
let actual = read_to_string(&full_path);
let inner = actual
.lines()
.filter_map(|line| {
let line = line.trim().strip_prefix("this.innerHTML = '")?;
let line = line.strip_suffix("';")?;
Some(line)
})
.next()
.expect("should have innerHTML");
inner.to_string()
}
/// Checks that the contents of the given file matches the expected value.
pub fn check_file(&mut self, path: &str, expected: impl IntoData) -> &mut Self {
if !self.built {
self.build();
}
let path = self.dir.join(path);
let actual = read_to_string(&path);
self.assert.eq(actual, expected);
self
}
/// Checks that the given file contains the given string somewhere.
pub fn check_file_contains(&mut self, path: &str, expected: &str) -> &mut Self {
if !self.built {
self.build();
}
let path = self.dir.join(path);
let actual = read_to_string(&path);
assert!(
actual.contains(expected),
"Did not find {expected:?} in {path:?}\n\n{actual}",
);
self
}
/// Checks that the given file does not contain the given string anywhere.
///
/// Beware that using this is fragile, as it may be unable to catch
/// regressions (it can't tell the difference between success, or the
/// string being looked for changed).
pub fn check_file_doesnt_contain(&mut self, path: &str, string: &str) -> &mut Self {
if !self.built {
self.build();
}
let path = self.dir.join(path);
let actual = read_to_string(&path);
assert!(
!actual.contains(string),
"Unexpectedly found {string:?} in {path:?}\n\n{actual}",
);
self
}
/// Checks that the list of files at the given path matches the given value.
pub fn check_file_list(&mut self, path: &str, expected: impl IntoData) -> &mut Self {
let mut all_paths: Vec<_> = walkdir::WalkDir::new(&self.dir.join(path))
.into_iter()
// Skip the outer directory.
.skip(1)
.map(|e| {
e.unwrap()
.into_path()
.strip_prefix(&self.dir)
.unwrap()
.to_str()
.unwrap()
.replace('\\', "/")
})
.collect();
all_paths.sort();
let actual = all_paths.join("\n");
self.assert.eq(actual, expected);
self
}
/// Loads an [`MDBook`] from the temp directory.
pub fn load_book(&self) -> MDBook {
MDBook::load(&self.dir).unwrap_or_else(|e| panic!("book failed to load: {e:?}"))
}
/// Builds the book in the temp directory.
pub fn build(&mut self) -> &mut Self {
let book = self.load_book();
book.build()
.unwrap_or_else(|e| panic!("book failed to build: {e:?}"));
self.built = true;
self
}
/// Runs the `mdbook` binary in the temp directory.
///
/// This runs `mdbook` with the given args. The args are split on spaces
/// (if you need args with spaces, use the `args` method). The given
/// callback receives a [`BookCommand`] for you to customize how the
/// executable is run.
pub fn run(&mut self, args: &str, f: impl Fn(&mut BookCommand)) -> &mut Self {
let mut cmd = BookCommand {
assert: self.assert.clone(),
dir: self.dir.clone(),
args: split_args(args),
env: BTreeMap::new(),
expect_status: StatusCode::Success,
expect_stderr_data: None,
expect_stdout_data: None,
};
f(&mut cmd);
cmd.run();
self
}
/// Change a file's contents in the given path.
pub fn change_file(&mut self, path: impl AsRef<Path>, body: &str) -> &mut Self {
let path = self.dir.join(path);
std::fs::write(&path, body).unwrap_or_else(|e| panic!("failed to write {path:?}: {e:?}"));
self
}
/// Builds a Rust program with the given src.
///
/// The given path should be the path where to output the executable in
/// the temp directory.
pub fn rust_program(&mut self, path: &str, src: &str) -> &mut Self {
let rs = self.dir.join(path).with_extension("rs");
let parent = rs.parent().unwrap();
if !parent.exists() {
std::fs::create_dir_all(&parent).unwrap();
}
std::fs::write(&rs, src).unwrap_or_else(|e| panic!("failed to write {rs:?}: {e:?}"));
let status = std::process::Command::new("rustc")
.arg(&rs)
.current_dir(&parent)
.status()
.expect("rustc should run");
assert!(status.success());
self
}
}
/// A builder for preparing to run the `mdbook` executable.
///
/// By default, it expects the process to succeed.
pub struct BookCommand {
pub dir: PathBuf,
assert: snapbox::Assert,
args: Vec<String>,
env: BTreeMap<String, Option<String>>,
expect_status: StatusCode,
expect_stderr_data: Option<snapbox::Data>,
expect_stdout_data: Option<snapbox::Data>,
}
impl BookCommand {
/// Indicates that the process should fail.
pub fn expect_failure(&mut self) -> &mut Self {
self.expect_status = StatusCode::Failure;
self
}
/// Indicates the process should fail with the given exit code.
pub fn expect_code(&mut self, code: i32) -> &mut Self {
self.expect_status = StatusCode::Code(code);
self
}
/// Verifies that stderr matches the given value.
pub fn expect_stderr(&mut self, expected: impl snapbox::IntoData) -> &mut Self {
self.expect_stderr_data = Some(expected.into_data());
self
}
/// Verifies that stdout matches the given value.
pub fn expect_stdout(&mut self, expected: impl snapbox::IntoData) -> &mut Self {
self.expect_stdout_data = Some(expected.into_data());
self
}
/// Adds arguments to the command to run.
pub fn args(&mut self, args: &[&str]) -> &mut Self {
self.args.extend(args.into_iter().map(|t| t.to_string()));
self
}
/// Specifies an environment variable to set on the executable.
pub fn env<T: Into<String>>(&mut self, key: &str, value: T) -> &mut Self {
self.env.insert(key.to_string(), Some(value.into()));
self
}
/// Runs the command, and verifies the output.
fn run(&mut self) {
let mut cmd = Command::new(env!("CARGO_BIN_EXE_mdbook"));
cmd.current_dir(&self.dir)
.args(&self.args)
.env_remove("RUST_LOG")
// Don't read the system git config which is out of our control.
.env("GIT_CONFIG_NOSYSTEM", "1")
.env("GIT_CONFIG_GLOBAL", &self.dir)
.env("GIT_CONFIG_SYSTEM", &self.dir)
.env_remove("GIT_AUTHOR_EMAIL")
.env_remove("GIT_AUTHOR_NAME")
.env_remove("GIT_COMMITTER_EMAIL")
.env_remove("GIT_COMMITTER_NAME");
for (k, v) in &self.env {
match v {
Some(v) => cmd.env(k, v),
None => cmd.env_remove(k),
};
}
let output = cmd.output().expect("mdbook should be runnable");
let stdout = std::str::from_utf8(&output.stdout).expect("stdout is not utf8");
let stderr = std::str::from_utf8(&output.stderr).expect("stderr is not utf8");
let render_output = || format!("\n--- stdout\n{stdout}\n--- stderr\n{stderr}");
match (self.expect_status, output.status.success()) {
(StatusCode::Success, false) => {
panic!("mdbook failed, but expected success{}", render_output())
}
(StatusCode::Failure, true) => {
panic!("mdbook succeeded, but expected failure{}", render_output())
}
(StatusCode::Code(expected), _) => match output.status.code() {
Some(actual) => assert_eq!(
actual, expected,
"process exit code did not match as expected"
),
None => panic!("process exited via signal {:?}", output.status),
},
_ => {}
}
self.expect_status = StatusCode::Success; // Reset to default.
if let Some(expect_stderr_data) = &self.expect_stderr_data {
if let Err(e) = self.assert.try_eq(
Some(&"stderr"),
stderr.into_data(),
expect_stderr_data.clone(),
) {
panic!("{e}");
}
}
if let Some(expect_stdout_data) = &self.expect_stdout_data {
if let Err(e) = self.assert.try_eq(
Some(&"stdout"),
stdout.into_data(),
expect_stdout_data.clone(),
) {
panic!("{e}");
}
}
}
}
fn split_args(s: &str) -> Vec<String> {
s.split_whitespace()
.map(|arg| {
if arg.contains(&['"', '\''][..]) {
panic!("shell-style argument parsing is not supported");
}
String::from(arg)
})
.collect()
}
static LITERAL_REDACTIONS: &[(&str, &str)] = &[
// Unix message for an entity was not found
("[NOT_FOUND]", "No such file or directory (os error 2)"),
// Windows message for an entity was not found
(
"[NOT_FOUND]",
"The system cannot find the file specified. (os error 2)",
),
(
"[NOT_FOUND]",
"The system cannot find the path specified. (os error 3)",
),
("[NOT_FOUND]", "program not found"),
// Unix message for exit status
("[EXIT_STATUS]", "exit status"),
// Windows message for exit status
("[EXIT_STATUS]", "exit code"),
("[TAB]", "\t"),
("[EXE]", std::env::consts::EXE_SUFFIX),
];
/// This makes it easier to write regex replacements that are guaranteed to only
/// get compiled once
macro_rules! regex {
($re:literal $(,)?) => {{
static RE: std::sync::OnceLock<regex::Regex> = std::sync::OnceLock::new();
RE.get_or_init(|| regex::Regex::new($re).unwrap())
}};
}
fn assert(root: &Path) -> snapbox::Assert {
let mut subs = snapbox::Redactions::new();
subs.insert("[ROOT]", root.to_path_buf()).unwrap();
subs.insert(
"[TIMESTAMP]",
regex!(r"(?m)(?<redacted>20\d\d-\d{2}-\d{2} \d{2}:\d{2}:\d{2})"),
)
.unwrap();
subs.insert("[VERSION]", mdbook::MDBOOK_VERSION).unwrap();
subs.extend(LITERAL_REDACTIONS.into_iter().cloned())
.unwrap();
snapbox::Assert::new()
.action_env(snapbox::assert::DEFAULT_ACTION_ENV)
.redact_with(subs)
}
/// Helper to read a string from the filesystem.
#[track_caller]
pub fn read_to_string<P: AsRef<Path>>(path: P) -> String {
let path = path.as_ref();
std::fs::read_to_string(path).unwrap_or_else(|e| panic!("could not read file {path:?}: {e:?}"))
}

10
tests/testsuite/main.rs Normal file
View file

@ -0,0 +1,10 @@
//! Main testsuite for exercising all functionality of mdBook.
//!
//! See README.md for documentation.
mod book_test;
mod prelude {
pub use crate::book_test::BookTest;
pub use snapbox::str;
}