diff --git a/Cargo.lock b/Cargo.lock index 082543f4..ae1b6f75 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,6 +75,15 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +[[package]] +name = "anstyle-lossy" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "934ff8719effd2023a48cf63e69536c1c3ced9d3895068f6f5cc9a4ff845e59b" +dependencies = [ + "anstyle", +] + [[package]] name = "anstyle-parse" version = "0.2.6" @@ -93,6 +102,19 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "anstyle-svg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3607949e9f6de49ea4bafe12f5e4fd73613ebf24795e48587302a8cc0e4bb35" +dependencies = [ + "anstream", + "anstyle", + "anstyle-lossy", + "html-escape", + "unicode-width", +] + [[package]] name = "anstyle-wincon" version = "3.0.7" @@ -288,6 +310,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "content_inspector" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7bda66e858c683005a53a9a60c69a4aca7eeaa45d124526e389f7aec8e62f38" +dependencies = [ + "memchr", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -460,6 +491,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "elasticlunr-rs" version = "3.0.2" @@ -737,6 +774,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "html-escape" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476" +dependencies = [ + "utf8-width", +] + [[package]] name = "html5ever" version = "0.26.0" @@ -1236,6 +1282,7 @@ dependencies = [ "serde_json", "sha2", "shlex", + "snapbox", "tempfile", "tokio", "toml", @@ -1920,6 +1967,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + [[package]] name = "siphasher" version = "0.3.11" @@ -1947,6 +2000,37 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +[[package]] +name = "snapbox" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96dcfc4581e3355d70ac2ee14cfdf81dce3d85c85f1ed9e2c1d3013f53b3436b" +dependencies = [ + "anstream", + "anstyle", + "anstyle-svg", + "content_inspector", + "dunce", + "filetime", + "normalize-line-endings", + "regex", + "serde", + "serde_json", + "similar", + "snapbox-macros", + "tempfile", + "walkdir", +] + +[[package]] +name = "snapbox-macros" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16569f53ca23a41bb6f62e0a5084aa1661f4814a67fa33696a79073e03a664af" +dependencies = [ + "anstream", +] + [[package]] name = "socket2" version = "0.5.8" @@ -2261,6 +2345,12 @@ version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "url" version = "2.5.4" @@ -2284,6 +2374,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" +[[package]] +name = "utf8-width" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" + [[package]] name = "utf8_iter" version = "1.0.4" diff --git a/Cargo.toml b/Cargo.toml index 4c43d0eb..00afec38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,7 @@ assert_cmd = "2.0.11" predicates = "3.0.3" select = "0.6.0" semver = "1.0.17" +snapbox = { version = "0.6.21", features = ["diff", "dir", "term-svg", "regex", "json"] } pretty_assertions = "1.3.0" walkdir = "2.3.3" diff --git a/tests/testsuite/README.md b/tests/testsuite/README.md new file mode 100644 index 00000000..73c594e3 --- /dev/null +++ b/tests/testsuite/README.md @@ -0,0 +1,43 @@ +# Testsuite + +## Introduction + +This is the main testsuite for exercising all functionality of mdBook. + +Tests should be organized into modules based around major features. Tests should use `BookTest` to drive the test. `BookTest` will set up a temp directory, and provides a variety of methods to help create a build books. + +## Basic structure of a test + +Using `BookTest`, you typically use it to copy a directory into a temp directory, and then run mdbook commands in that temp directory. You can run the `mdbook` executable, or use the mdbook API to perform whatever tasks you need. Running the executable has the benefit of being able to validate the console output. + +See `build::basic_build` for a simple test example. I recommend reviewing the methods on `BookTest` to learn more, and reviewing some of the existing tests to get a feel for how they are structured. + +For example, let's say you are creating a new theme test. In the `testsuite/theme` directory, create a new directory with the book source that you want to exercise. At a minimum, this needs a `src/SUMMARY.md`, but often you'll also want `book.toml`. Then, in `testsuite/theme.rs`, add a test with `BookTest::from_dir("theme/mytest")`, and then use the methods to perform whatever actions you want. + +`BookTest` is designed to be able to chain a series of actions. For example, you can do something like: + +```rust +BookTest::from_dir("theme/mytest") + .build() + .check_main_file("book/index.html", str![["file contents"]]) + .change_file("src/index.md", "new contents") + .build() + .check_main_file("book/index.html", str![["new contents"]]); +``` + +## Snapbox + +The testsuite uses [`snapbox`] to drive most of the tests. This library provides the ability to compare strings using a variety of methods. These strings are written in the source code using either the [`str!`] or [`file!`] macros. + +The magic is that you can set the `SNAPSHOTS=overwrite` environment variable, and snapbox will automatically update the strings contents of `str!`, or the file contents of `file!`. This makes it easier to update tests. Snapbox provides nice diffing output, and quite a few other features. + +Expected contents can have wildcards like `...` (matches any lines) or `[..]` (matches any characters on a line). See [snapbox filters] for more info and other filters. + +Typically when writing a test, I'll just start with an empty `str!` or `file!`, and let snapbox fill it in. Then I review the contents to make sure they are what I expect. + +Note that there is some normalization applied to the strings. See `book_test::assert` for how some of these normalizations happen. + +[`snapbox`]: https://docs.rs/snapbox/latest/snapbox/ +[`str!`]: https://docs.rs/snapbox/latest/snapbox/macro.str.html +[`file!`]: https://docs.rs/snapbox/latest/snapbox/macro.file.html +[snapbox filters]: https://docs.rs/snapbox/latest/snapbox/assert/struct.Assert.html#method.eq diff --git a/tests/testsuite/book_test.rs b/tests/testsuite/book_test.rs new file mode 100644 index 00000000..427c38d1 --- /dev/null +++ b/tests/testsuite/book_test.rs @@ -0,0 +1,442 @@ +//! Utility for building and running tests against mdbook. + +use mdbook::book::BookBuilder; +use mdbook::MDBook; +use snapbox::IntoData; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::atomic::{AtomicU32, Ordering}; + +/// Test number used for generating unique temp directory names. +static NEXT_TEST_ID: AtomicU32 = AtomicU32::new(0); + +#[derive(Clone, Copy, Eq, PartialEq)] +enum StatusCode { + Success, + Failure, + Code(i32), +} + +/// Main helper for driving mdbook tests. +pub struct BookTest { + /// The temp directory where the test should perform its work. + pub dir: PathBuf, + assert: snapbox::Assert, + /// This indicates whether or not the book has been built. + built: bool, +} + +impl BookTest { + /// Creates a new test, copying the contents from the given directory into + /// a temp directory. + pub fn from_dir(dir: &str) -> BookTest { + // Copy this test book to a temp directory. + let dir = Path::new("tests/testsuite").join(dir); + assert!(dir.exists(), "{dir:?} should exist"); + let tmp = Self::new_tmp(); + mdbook::utils::fs::copy_files_except_ext( + &dir, + &tmp, + true, + Some(&PathBuf::from("book")), + &[], + ) + .unwrap_or_else(|e| panic!("failed to copy test book {dir:?} to {tmp:?}: {e:?}")); + Self::new(tmp) + } + + /// Creates a new test with an empty temp directory. + pub fn empty() -> BookTest { + Self::new(Self::new_tmp()) + } + + /// Creates a new test with the given function to initialize a new book. + /// + /// The book itself is not built. + pub fn init(f: impl Fn(&mut BookBuilder)) -> BookTest { + let tmp = Self::new_tmp(); + let mut bb = MDBook::init(&tmp); + f(&mut bb); + bb.build() + .unwrap_or_else(|e| panic!("failed to initialize book at {tmp:?}: {e:?}")); + Self::new(tmp) + } + + fn new_tmp() -> PathBuf { + let id = NEXT_TEST_ID.fetch_add(1, Ordering::SeqCst); + let tmp = Path::new(env!("CARGO_TARGET_TMPDIR")) + .join("ts") + .join(format!("t{id}")); + if tmp.exists() { + std::fs::remove_dir_all(&tmp) + .unwrap_or_else(|e| panic!("failed to remove {tmp:?}: {e:?}")); + } + std::fs::create_dir_all(&tmp).unwrap_or_else(|e| panic!("failed to create {tmp:?}: {e:?}")); + tmp + } + + fn new(dir: PathBuf) -> BookTest { + let assert = assert(&dir); + BookTest { + dir, + assert, + built: false, + } + } + + /// Checks the contents of an HTML file that it has the given contents + /// between the `
` tag. + /// + /// Normally the contents outside of the `
` tag aren't interesting, + /// and they add a significant amount of noise. + pub fn check_main_file(&mut self, path: &str, expected: impl IntoData) -> &mut Self { + if !self.built { + self.build(); + } + let full_path = self.dir.join(path); + let actual = read_to_string(&full_path); + let start = actual + .find("
") + .unwrap_or_else(|| panic!("didn't find
in:\n{actual}")); + let end = actual.find("
").unwrap(); + let contents = actual[start + 6..end - 7].trim(); + self.assert.eq(contents, expected); + self + } + + /// Checks the summary contents of `toc.js` against the expected value. + pub fn check_toc_js(&mut self, expected: impl IntoData) -> &mut Self { + if !self.built { + self.build(); + } + let inner = self.toc_js_html(); + // Would be nice if this were prettified, but a primitive wrapping will do for now. + let inner = inner.replace("><", ">\n<"); + self.assert.eq(inner, expected); + self + } + + /// Returns the summary contents from `toc.js`. + pub fn toc_js_html(&self) -> String { + let full_path = self.dir.join("book/toc.js"); + let actual = read_to_string(&full_path); + let inner = actual + .lines() + .filter_map(|line| { + let line = line.trim().strip_prefix("this.innerHTML = '")?; + let line = line.strip_suffix("';")?; + Some(line) + }) + .next() + .expect("should have innerHTML"); + inner.to_string() + } + + /// Checks that the contents of the given file matches the expected value. + pub fn check_file(&mut self, path: &str, expected: impl IntoData) -> &mut Self { + if !self.built { + self.build(); + } + let path = self.dir.join(path); + let actual = read_to_string(&path); + self.assert.eq(actual, expected); + self + } + + /// Checks that the given file contains the given string somewhere. + pub fn check_file_contains(&mut self, path: &str, expected: &str) -> &mut Self { + if !self.built { + self.build(); + } + let path = self.dir.join(path); + let actual = read_to_string(&path); + assert!( + actual.contains(expected), + "Did not find {expected:?} in {path:?}\n\n{actual}", + ); + self + } + + /// Checks that the given file does not contain the given string anywhere. + /// + /// Beware that using this is fragile, as it may be unable to catch + /// regressions (it can't tell the difference between success, or the + /// string being looked for changed). + pub fn check_file_doesnt_contain(&mut self, path: &str, string: &str) -> &mut Self { + if !self.built { + self.build(); + } + let path = self.dir.join(path); + let actual = read_to_string(&path); + assert!( + !actual.contains(string), + "Unexpectedly found {string:?} in {path:?}\n\n{actual}", + ); + self + } + + /// Checks that the list of files at the given path matches the given value. + pub fn check_file_list(&mut self, path: &str, expected: impl IntoData) -> &mut Self { + let mut all_paths: Vec<_> = walkdir::WalkDir::new(&self.dir.join(path)) + .into_iter() + // Skip the outer directory. + .skip(1) + .map(|e| { + e.unwrap() + .into_path() + .strip_prefix(&self.dir) + .unwrap() + .to_str() + .unwrap() + .replace('\\', "/") + }) + .collect(); + all_paths.sort(); + let actual = all_paths.join("\n"); + self.assert.eq(actual, expected); + self + } + + /// Loads an [`MDBook`] from the temp directory. + pub fn load_book(&self) -> MDBook { + MDBook::load(&self.dir).unwrap_or_else(|e| panic!("book failed to load: {e:?}")) + } + + /// Builds the book in the temp directory. + pub fn build(&mut self) -> &mut Self { + let book = self.load_book(); + book.build() + .unwrap_or_else(|e| panic!("book failed to build: {e:?}")); + self.built = true; + self + } + + /// Runs the `mdbook` binary in the temp directory. + /// + /// This runs `mdbook` with the given args. The args are split on spaces + /// (if you need args with spaces, use the `args` method). The given + /// callback receives a [`BookCommand`] for you to customize how the + /// executable is run. + pub fn run(&mut self, args: &str, f: impl Fn(&mut BookCommand)) -> &mut Self { + let mut cmd = BookCommand { + assert: self.assert.clone(), + dir: self.dir.clone(), + args: split_args(args), + env: BTreeMap::new(), + expect_status: StatusCode::Success, + expect_stderr_data: None, + expect_stdout_data: None, + }; + f(&mut cmd); + cmd.run(); + self + } + + /// Change a file's contents in the given path. + pub fn change_file(&mut self, path: impl AsRef, body: &str) -> &mut Self { + let path = self.dir.join(path); + std::fs::write(&path, body).unwrap_or_else(|e| panic!("failed to write {path:?}: {e:?}")); + self + } + + /// Builds a Rust program with the given src. + /// + /// The given path should be the path where to output the executable in + /// the temp directory. + pub fn rust_program(&mut self, path: &str, src: &str) -> &mut Self { + let rs = self.dir.join(path).with_extension("rs"); + let parent = rs.parent().unwrap(); + if !parent.exists() { + std::fs::create_dir_all(&parent).unwrap(); + } + std::fs::write(&rs, src).unwrap_or_else(|e| panic!("failed to write {rs:?}: {e:?}")); + let status = std::process::Command::new("rustc") + .arg(&rs) + .current_dir(&parent) + .status() + .expect("rustc should run"); + assert!(status.success()); + self + } +} + +/// A builder for preparing to run the `mdbook` executable. +/// +/// By default, it expects the process to succeed. +pub struct BookCommand { + pub dir: PathBuf, + assert: snapbox::Assert, + args: Vec, + env: BTreeMap>, + expect_status: StatusCode, + expect_stderr_data: Option, + expect_stdout_data: Option, +} + +impl BookCommand { + /// Indicates that the process should fail. + pub fn expect_failure(&mut self) -> &mut Self { + self.expect_status = StatusCode::Failure; + self + } + + /// Indicates the process should fail with the given exit code. + pub fn expect_code(&mut self, code: i32) -> &mut Self { + self.expect_status = StatusCode::Code(code); + self + } + + /// Verifies that stderr matches the given value. + pub fn expect_stderr(&mut self, expected: impl snapbox::IntoData) -> &mut Self { + self.expect_stderr_data = Some(expected.into_data()); + self + } + + /// Verifies that stdout matches the given value. + pub fn expect_stdout(&mut self, expected: impl snapbox::IntoData) -> &mut Self { + self.expect_stdout_data = Some(expected.into_data()); + self + } + + /// Adds arguments to the command to run. + pub fn args(&mut self, args: &[&str]) -> &mut Self { + self.args.extend(args.into_iter().map(|t| t.to_string())); + self + } + + /// Specifies an environment variable to set on the executable. + pub fn env>(&mut self, key: &str, value: T) -> &mut Self { + self.env.insert(key.to_string(), Some(value.into())); + self + } + + /// Runs the command, and verifies the output. + fn run(&mut self) { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_mdbook")); + cmd.current_dir(&self.dir) + .args(&self.args) + .env_remove("RUST_LOG") + // Don't read the system git config which is out of our control. + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", &self.dir) + .env("GIT_CONFIG_SYSTEM", &self.dir) + .env_remove("GIT_AUTHOR_EMAIL") + .env_remove("GIT_AUTHOR_NAME") + .env_remove("GIT_COMMITTER_EMAIL") + .env_remove("GIT_COMMITTER_NAME"); + + for (k, v) in &self.env { + match v { + Some(v) => cmd.env(k, v), + None => cmd.env_remove(k), + }; + } + + let output = cmd.output().expect("mdbook should be runnable"); + let stdout = std::str::from_utf8(&output.stdout).expect("stdout is not utf8"); + let stderr = std::str::from_utf8(&output.stderr).expect("stderr is not utf8"); + let render_output = || format!("\n--- stdout\n{stdout}\n--- stderr\n{stderr}"); + match (self.expect_status, output.status.success()) { + (StatusCode::Success, false) => { + panic!("mdbook failed, but expected success{}", render_output()) + } + (StatusCode::Failure, true) => { + panic!("mdbook succeeded, but expected failure{}", render_output()) + } + (StatusCode::Code(expected), _) => match output.status.code() { + Some(actual) => assert_eq!( + actual, expected, + "process exit code did not match as expected" + ), + None => panic!("process exited via signal {:?}", output.status), + }, + _ => {} + } + self.expect_status = StatusCode::Success; // Reset to default. + if let Some(expect_stderr_data) = &self.expect_stderr_data { + if let Err(e) = self.assert.try_eq( + Some(&"stderr"), + stderr.into_data(), + expect_stderr_data.clone(), + ) { + panic!("{e}"); + } + } + if let Some(expect_stdout_data) = &self.expect_stdout_data { + if let Err(e) = self.assert.try_eq( + Some(&"stdout"), + stdout.into_data(), + expect_stdout_data.clone(), + ) { + panic!("{e}"); + } + } + } +} + +fn split_args(s: &str) -> Vec { + s.split_whitespace() + .map(|arg| { + if arg.contains(&['"', '\''][..]) { + panic!("shell-style argument parsing is not supported"); + } + String::from(arg) + }) + .collect() +} + +static LITERAL_REDACTIONS: &[(&str, &str)] = &[ + // Unix message for an entity was not found + ("[NOT_FOUND]", "No such file or directory (os error 2)"), + // Windows message for an entity was not found + ( + "[NOT_FOUND]", + "The system cannot find the file specified. (os error 2)", + ), + ( + "[NOT_FOUND]", + "The system cannot find the path specified. (os error 3)", + ), + ("[NOT_FOUND]", "program not found"), + // Unix message for exit status + ("[EXIT_STATUS]", "exit status"), + // Windows message for exit status + ("[EXIT_STATUS]", "exit code"), + ("[TAB]", "\t"), + ("[EXE]", std::env::consts::EXE_SUFFIX), +]; + +/// This makes it easier to write regex replacements that are guaranteed to only +/// get compiled once +macro_rules! regex { + ($re:literal $(,)?) => {{ + static RE: std::sync::OnceLock = std::sync::OnceLock::new(); + RE.get_or_init(|| regex::Regex::new($re).unwrap()) + }}; +} + +fn assert(root: &Path) -> snapbox::Assert { + let mut subs = snapbox::Redactions::new(); + subs.insert("[ROOT]", root.to_path_buf()).unwrap(); + subs.insert( + "[TIMESTAMP]", + regex!(r"(?m)(?20\d\d-\d{2}-\d{2} \d{2}:\d{2}:\d{2})"), + ) + .unwrap(); + subs.insert("[VERSION]", mdbook::MDBOOK_VERSION).unwrap(); + + subs.extend(LITERAL_REDACTIONS.into_iter().cloned()) + .unwrap(); + + snapbox::Assert::new() + .action_env(snapbox::assert::DEFAULT_ACTION_ENV) + .redact_with(subs) +} + +/// Helper to read a string from the filesystem. +#[track_caller] +pub fn read_to_string>(path: P) -> String { + let path = path.as_ref(); + std::fs::read_to_string(path).unwrap_or_else(|e| panic!("could not read file {path:?}: {e:?}")) +} diff --git a/tests/testsuite/main.rs b/tests/testsuite/main.rs new file mode 100644 index 00000000..8ed64dd3 --- /dev/null +++ b/tests/testsuite/main.rs @@ -0,0 +1,10 @@ +//! Main testsuite for exercising all functionality of mdBook. +//! +//! See README.md for documentation. + +mod book_test; + +mod prelude { + pub use crate::book_test::BookTest; + pub use snapbox::str; +}