Improve wikilink parsing and checking

README.md

@@ -18,6 +18,7 @@ Available as a command-line utility, a library and a [GitHub Action](https://git
 
 <!-- START doctoc generated TOC please keep comment here to allow auto update -->
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
 ## Table of Contents
 
 - [Development](#development)
@@ -660,7 +661,7 @@ Options:
           and existing cookies will be updated.
 
       --include-wikilinks
-          Check WikiLinks in Markdown files
+          Check WikiLinks in Markdown files, this requires specifying --base-url
 
   -h, --help
           Print help (see a summary with '-h')

fixtures/wiki/Dash-Usage.md

@@ -0,0 +1 @@
+# Header

fixtures/wiki/Space Usage.md

@@ -0,0 +1 @@
+# Header

fixtures/wiki/Underscore_Usage.md

@@ -0,0 +1 @@
+# Header

fixtures/wiki/Usage.md

@@ -0,0 +1 @@
+# Header

fixtures/wiki/obsidian-style-plus-headers.md

@@ -0,0 +1,8 @@
+[[#LocalHeader]]
+
+# LocalHeader
+
+[[Usage#Header|HeaderRenaming]]
+[[Space Usage#Header|HeaderRenaming]]
+[[Space Usage DifferentDirectory#Header|HeaderRenaming]]
+[[DifferentDirectory#Header|HeaderRenaming]]

fixtures/wiki/obsidian-style.md

@@ -0,0 +1,4 @@
+[[Usage]]
+[[Space Usage]]
+[[Space Usage DifferentDirectory]]
+[[DifferentDirectory]]

fixtures/wiki/subdirectory/Different-Directory-Dash.md

@@ -0,0 +1 @@
+# Header

fixtures/wiki/subdirectory/DifferentDirectory.md

@@ -0,0 +1 @@
+# Header

fixtures/wiki/subdirectory/Different_Directory_Underscore.md

@@ -0,0 +1 @@
+# Header

fixtures/wiki/subdirectory/Space Usage DifferentDirectory.md

@@ -0,0 +1 @@
+# Header

fixtures/wiki/wikilink-style.md

@@ -0,0 +1,19 @@
+[[#LocalHeader]]
+
+[[Usage]]
+[[Space Usage]]
+[[Dash Usage]]
+[[Underscore Usage]]
+[[DifferentDirectory]]
+[[Different Directory Dash]]
+[[Different Directory Underscore]]
+
+[[Usage#Header|HeaderRenaming]]
+[[Space Usage#Header|HeaderRenaming]]
+[[Dash Usage#Header|HeaderRenaming]]
+[[Underscore Usage#Header|HeaderRenaming]]
+[[DifferentDirectory#Header|HeaderRenaming]]
+[[Different Directory Dash#Header|HeaderRenaming]]
+[[Different Directory Underscore#Header|HeaderRenaming]]
+
+# LocalHeader

lychee-bin/src/client.rs

@@ -55,6 +55,7 @@ pub(crate) fn create(cfg: &Config, cookie_jar: Option<&Arc<CookieStoreMutex>>) -
         .include_fragments(cfg.include_fragments)
         .fallback_extensions(cfg.fallback_extensions.clone())
         .index_files(cfg.index_files.clone())
+        .include_wikilinks(cfg.include_wikilinks)
         .build()
         .client()
         .context("Failed to create request client")

lychee-bin/src/options.rs

@@ -845,7 +845,8 @@ and existing cookies will be updated."
     pub(crate) cookie_jar: Option<PathBuf>,
 
     #[allow(clippy::doc_markdown)]
-    /// Check WikiLinks in Markdown files
+    /// Check WikiLinks in Markdown files, this requires specifying --base-url
+    #[clap(requires = "base_url")]
     #[arg(long)]
     #[serde(default)]
     pub(crate) include_wikilinks: bool,

lychee-bin/tests/cli.rs

@@ -2501,6 +2501,8 @@ mod cli {
         let mut cmd = main_command();
         cmd.arg("--dump")
             .arg("--include-wikilinks")
+            .arg("--base-url")
+            .arg(fixtures_path())
             .arg(test_path)
             .assert()
             .success()
@@ -2964,6 +2966,58 @@ mod cli {
             .stdout(contains("https://example.org")); // Should extract the link as plaintext
     }
 
+    #[test]
+    fn test_wikilink_fixture_obsidian_style() {
+        let input = fixtures_path().join("wiki/obsidian-style.md");
+
+        // testing without fragments should not yield failures
+        main_command()
+            .arg(&input)
+            .arg("--include-wikilinks")
+            .arg("--fallback-extensions")
+            .arg("md")
+            .arg("--base-url")
+            .arg(fixtures_path())
+            .assert()
+            .success()
+            .stdout(contains("4 OK"));
+    }
+
+    #[test]
+    fn test_wikilink_fixture_with_fragments_obsidian_style_fixtures_excluded() {
+        let input = fixtures_path().join("wiki/obsidian-style-plus-headers.md");
+
+        // fragments should resolve all headers
+        main_command()
+            .arg(&input)
+            .arg("--include-wikilinks")
+            .arg("--fallback-extensions")
+            .arg("md")
+            .arg("--base-url")
+            .arg(fixtures_path())
+            .assert()
+            .success()
+            .stdout(contains("4 OK"));
+    }
+
+    #[test]
+    fn test_wikilink_fixture_with_fragments_obsidian_style() {
+        let input = fixtures_path().join("wiki/obsidian-style-plus-headers.md");
+
+        // fragments should resolve all headers
+        main_command()
+            .arg(&input)
+            .arg("--include-wikilinks")
+            .arg("--include-fragments")
+            .arg("--fallback-extensions")
+            .arg("md")
+            .arg("--base-url")
+            .arg(fixtures_path())
+            .assert()
+            .success()
+            .stdout(contains("4 OK"));
+    }
+
     /// An input which matches nothing should print a warning and continue.
     #[test]
     fn test_input_matching_nothing_warns() -> Result<()> {

lychee-lib/Cargo.toml

@@ -61,6 +61,7 @@ tokio = { version = "1.47.1", features = ["full"] }
 toml = "0.9.5"
 typed-builder = "0.22.0"
 url = { version = "2.5.7", features = ["serde"] }
+walkdir = "2.5.0"
 
 [dependencies.par-stream]
 version = "0.10.2"

lychee-lib/src/checker/file.rs

@@ -1,8 +1,9 @@
 use http::StatusCode;
-use log::warn;
+use log::{trace, warn};
 use std::borrow::Cow;
 use std::path::{Path, PathBuf};
 
+use crate::utils::wikilink_checker::WikilinkChecker;
 use crate::{
     Base, ErrorKind, Status, Uri,
     utils::fragment_checker::{FragmentChecker, FragmentInput},
@@ -32,8 +33,12 @@ pub(crate) struct FileChecker {
     index_files: Option<Vec<String>>,
     /// Whether to check for the existence of fragments (e.g., `#section-id`) in HTML files.
     include_fragments: bool,
+    /// Whether to check for the existence of files linked to by Wikilinks
+    include_wikilinks: bool,
     /// Utility for performing fragment checks in HTML files.
     fragment_checker: FragmentChecker,
+    /// Utility for checking wikilinks, indexes files in a given directory
+    wikilink_checker: Option<WikilinkChecker>,
 }
 
 impl FileChecker {
@@ -50,13 +55,16 @@ impl FileChecker {
         fallback_extensions: Vec<String>,
         index_files: Option<Vec<String>>,
         include_fragments: bool,
+        include_wikilinks: bool,
     ) -> Self {
         Self {
-            base,
+            base: base.clone(),
             fallback_extensions,
             index_files,
             include_fragments,
+            include_wikilinks,
             fragment_checker: FragmentChecker::new(),
+            wikilink_checker: WikilinkChecker::new(base),
         }
     }
 
@@ -73,6 +81,13 @@ impl FileChecker {
     ///
     /// Returns a `Status` indicating the result of the check.
     pub(crate) async fn check(&self, uri: &Uri) -> Status {
+        // only populate the wikilink filenames if the feature is enabled
+        if self.include_wikilinks {
+            match self.setup_wikilinks() {
+                Ok(()) => (),
+                Err(e) => return Status::Error(e),
+            }
+        }
         let Ok(path) = uri.url.to_file_path() else {
             return ErrorKind::InvalidFilePath(uri.clone()).into();
         };
@@ -134,8 +149,12 @@ impl FileChecker {
     ) -> Result<Cow<'a, Path>, ErrorKind> {
         let path = match path.metadata() {
             // for non-existing paths, attempt fallback extensions
+            // if fallback extensions don't help, try wikilinks
             Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
-                self.apply_fallback_extensions(path, uri).map(Cow::Owned)
+                match self.apply_fallback_extensions(path, uri).map(Cow::Owned) {
+                    Ok(val) => Ok(val),
+                    Err(_) => self.apply_wikilink_check(path, uri).map(Cow::Owned),
+                }
             }
 
             // other IO errors are unexpected and should fail the check
@@ -313,6 +332,34 @@ impl FileChecker {
             }
         }
     }
+
+    // Initializes the index of the wikilink checker
+    fn setup_wikilinks(&self) -> Result<(), ErrorKind> {
+        match &self.wikilink_checker {
+            Some(checker) => checker.setup_wikilinks_index(),
+            None => Err(ErrorKind::WikilinkCheckerInit(
+                "Initialization failed, no checker instantiated".to_string(),
+            )),
+        }
+    }
+
+    // Tries to resolve a link by looking up the filename in the wikilink index
+    fn apply_wikilink_check(&self, path: &Path, uri: &Uri) -> Result<PathBuf, ErrorKind> {
+        let mut path_buf = path.to_path_buf();
+        for ext in &self.fallback_extensions {
+            path_buf.set_extension(ext);
+            if let Some(checker) = &self.wikilink_checker {
+                match checker.contains_path(&path_buf) {
+                    None => {
+                        trace!("Tried to find wikilink {} at {}", uri, path_buf.display());
+                    }
+                    Some(resolved_path) => return Ok(resolved_path),
+                }
+            }
+        }
+
+        Err(ErrorKind::InvalidFilePath(uri.clone()))
+    }
 }
 
 #[cfg(test)]
@@ -372,7 +419,7 @@ mod tests {
     #[tokio::test]
     async fn test_default() {
         // default behaviour accepts dir links as long as the directory exists.
-        let checker = FileChecker::new(None, vec![], None, true);
+        let checker = FileChecker::new(None, vec![], None, true, false);
 
         assert_filecheck!(&checker, "filechecker/index_dir", Status::Ok(_));
 
@@ -430,6 +477,7 @@ mod tests {
             vec![],
             Some(vec!["index.html".to_owned(), "index.md".to_owned()]),
             true,
+            false,
         );
 
         assert_resolves!(
@@ -468,6 +516,7 @@ mod tests {
             vec!["html".to_owned()],
             Some(vec!["index".to_owned()]),
             false,
+            false,
         );
 
         // this test case has a subdir 'same_name' and a file 'same_name.html'.
@@ -492,7 +541,7 @@ mod tests {
     #[tokio::test]
     async fn test_empty_index_list_corner() {
         // empty index_files list will reject all directory links
-        let checker_no_indexes = FileChecker::new(None, vec![], Some(vec![]), false);
+        let checker_no_indexes = FileChecker::new(None, vec![], Some(vec![]), false, false);
         assert_resolves!(
             &checker_no_indexes,
             "filechecker/index_dir",
@@ -516,7 +565,7 @@ mod tests {
             "..".to_owned(),
             "/".to_owned(),
         ];
-        let checker_dir_indexes = FileChecker::new(None, vec![], Some(dir_names), false);
+        let checker_dir_indexes = FileChecker::new(None, vec![], Some(dir_names), false, false);
         assert_resolves!(
             &checker_dir_indexes,
             "filechecker/index_dir",
@@ -537,6 +586,7 @@ mod tests {
             vec![],
             Some(vec!["../index_dir/index.html".to_owned()]),
             true,
+            false,
         );
         assert_resolves!(
             &checker_dotdot,
@@ -550,7 +600,8 @@ mod tests {
             .to_str()
             .expect("expected utf-8 fixtures path")
             .to_owned();
-        let checker_absolute = FileChecker::new(None, vec![], Some(vec![absolute_html]), true);
+        let checker_absolute =
+            FileChecker::new(None, vec![], Some(vec![absolute_html]), true, false);
         assert_resolves!(
             &checker_absolute,
             "filechecker/empty_dir#fragment",
@@ -560,7 +611,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_fallback_extensions_on_directories() {
-        let checker = FileChecker::new(None, vec!["html".to_owned()], None, true);
+        let checker = FileChecker::new(None, vec!["html".to_owned()], None, true, false);
 
         // fallback extensions should be applied when directory links are resolved
         // to directories (i.e., the default index_files behavior or if `.`

lychee-lib/src/client.rs

@@ -299,6 +299,9 @@ pub struct ClientBuilder {
     /// Enable the checking of fragments in links.
     include_fragments: bool,
 
+    /// Enable the checking of wikilinks in markdown files
+    include_wikilinks: bool,
+
     /// Requests run through this chain where each item in the chain
     /// can modify the request. A chained item can also decide to exit
     /// early and return a status, so that subsequent chain items are
@@ -424,6 +427,7 @@ impl ClientBuilder {
                 self.fallback_extensions,
                 self.index_files,
                 self.include_fragments,
+                self.include_wikilinks,
             ),
         })
     }