diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..83ae8aa --- /dev/null +++ b/.envrc @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +if type -P lorri &>/dev/null; then + eval "$(lorri direnv)" +else + use flake +fi diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..af3c0de --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,57 @@ +# Copilot Instructions for ixx + +## Project Overview +- **ixx** is a Rust-based search engine and index creation tool for NüschtOS Search. +- The workspace contains multiple Rust crates: `ixx`, `libixx`, and `fixx` (WASM target). +- Main CLI entrypoint: `ixx/src/main.rs`. +- Index logic and binary format: `libixx/src/index.rs`. +- WASM build: `fixx` crate, output in `fixx/pkg`. + +## Build & Test Workflows +- **Build all crates:** + - Standard: `cargo build --workspace` + - WASM: `wasm-pack build --release fixx --target web` (output: `fixx/pkg`) +- **Run CLI:** + - `cargo run --package ixx -- [args]` +- **Testing:** + - `cargo test --workspace` +- **Benchmarks:** + - `cargo bench --package libixx` + +## CLI Usage Patterns +- Subcommands: `index`, `search`, `meta` (see `ixx/src/args.rs`) +- Example: `ixx index [--options-index-output ...]` +- Output files: index and chunk files for both options and packages +- Search supports output formats: `--format json|text` + +## Index Format & Data Flow +- Indexes are binary files with magic header `ixx02` (see `libixx/src/index.rs`). +- Entries are grouped by scopes; labels use in-place or reference encoding for compression. +- Search uses Levenshtein distance for fuzzy matching. +- Data flows: CLI parses args → calls action module → reads/writes index via `libixx`. + +## Conventions & Patterns +- All index and chunk files are written/read via `libixx` APIs. +- Option and package indexes are separated (see default paths in `ixx/src/args.rs`). +- Use `anyhow` for error handling, `serde` for serialization. +- Prefer explicit subcommand modules for CLI logic (`ixx/src/action/`). +- WASM build is only for the `fixx` crate; main logic is in Rust. + +## Integration Points +- External: `wasm-pack`, `tokio`, `clap`, `serde`, `binrw`, `levenshtein`. +- WASM output integrates with web frontends via `fixx/pkg/fixx.js`. + +## Key Files & Directories +- `ixx/src/main.rs`, `ixx/src/args.rs`, `ixx/src/action/` +- `libixx/src/index.rs` (index format, search logic) +- `fixx/pkg/` (WASM output) +- `Cargo.toml` (workspace, dependencies) + +## Example Workflow +1. Build index: `cargo run --package ixx -- index ` +2. Search: `cargo run --package ixx -- search --query --format json` +3. Build WASM: `wasm-pack build --release fixx --target web` + +--- + +For questions, see the main [README.md](../README.md) or join Matrix chat. diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2b8d764..f327790 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,7 +19,7 @@ jobs: fail-fast: false matrix: module: [ libixx, ixx, fixx ] - target: [ "25.05", "unstable" ] + target: [ "25.11", "unstable" ] steps: - uses: actions/checkout@v5 diff --git a/.gitignore b/.gitignore index 7594ebe..903e559 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ -/target -/result -/fixx/pkg /data +/fixx/pkg +/index.ixx +/result +/target diff --git a/.rustfmt.toml b/.rustfmt.toml index b196eaa..44b6522 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -1 +1,2 @@ +max_width = 110 tab_spaces = 2 diff --git a/Cargo.lock b/Cargo.lock index 4f7e075..57d4ee7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,18 +4,33 @@ version = 4 [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" -version = "0.6.19" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" dependencies = [ "anstyle", "anstyle-parse", @@ -28,9 +43,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.11" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" @@ -43,18 +58,18 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" +checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" dependencies = [ "windows-sys", ] [[package]] name = "anstyle-wincon" -version = "3.0.9" +version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" +checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" dependencies = [ "anstyle", "once_cell_polyfill", @@ -63,9 +78,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.99" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "array-init" @@ -73,6 +88,12 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + [[package]] name = "binrw" version = "0.15.0" @@ -105,30 +126,70 @@ checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "bytemuck" -version = "1.23.1" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c76a5792e44e4abe34d3abf15636779261d45a7450612059293d1d2cfc63422" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.30" +version = "1.2.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7" +checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe" dependencies = [ + "find-msvc-tools", "shlex", ] [[package]] name = "cfg-if" -version = "1.0.1" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "ciborium" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] [[package]] name = "clap" -version = "4.5.47" +version = "4.5.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eac00902d9d136acd712710d71823fb8ac8004ca445a89e73a41d45aa712931" +checksum = "4c26d721170e0295f191a69bd9a1f93efcdb0aff38684b61ab5750468972e5f5" dependencies = [ "clap_builder", "clap_derive", @@ -136,9 +197,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.47" +version = "4.5.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad9bbf750e73b5884fb8a211a9424a1906c1e156724260fdae972f31d70e1d6" +checksum = "75835f0c7bf681bfd05abe44e965760fea999a5286c6eb2d59883634fd02011a" dependencies = [ "anstream", "anstyle", @@ -148,9 +209,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.47" +version = "4.5.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" dependencies = [ "heck", "proc-macro2", @@ -160,9 +221,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" [[package]] name = "colorchoice" @@ -170,6 +231,72 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "criterion" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0dfe5e9e71bdcf4e4954f7d14da74d1cdb92a3a07686452d1509652684b1aab" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de36c2bee19fba779808f92bf5d9b0fa5a40095c277aba10c458a12b35d21d6" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "displaydoc" version = "0.2.5" @@ -193,6 +320,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "find-msvc-tools" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" + [[package]] name = "fixx" version = "0.0.0-git" @@ -210,11 +343,22 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" -version = "0.15.4" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" [[package]] name = "heck" @@ -224,9 +368,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "icu_collections" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" dependencies = [ "displaydoc", "potential_utf", @@ -237,9 +381,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" dependencies = [ "displaydoc", "litemap", @@ -250,11 +394,10 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" dependencies = [ - "displaydoc", "icu_collections", "icu_normalizer_data", "icu_properties", @@ -265,42 +408,38 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" [[package]] name = "icu_properties" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99" dependencies = [ - "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", "icu_provider", - "potential_utf", "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" +checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899" [[package]] name = "icu_provider" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" dependencies = [ "displaydoc", "icu_locale_core", - "stable_deref_trait", - "tinystr", "writeable", "yoke", "zerofrom", @@ -331,9 +470,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.10.0" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" +checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" dependencies = [ "equivalent", "hashbrown", @@ -341,9 +480,18 @@ dependencies = [ [[package]] name = "is_terminal_polyfill" -version = "1.70.1" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] [[package]] name = "itoa" @@ -359,18 +507,44 @@ dependencies = [ "clap", "libixx", "markdown", + "regex", "serde", "serde_json", + "tokio", "tree-sitter-highlight", "tree-sitter-nix", "url", ] +[[package]] +name = "js-sys" +version = "0.3.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "levenshtein" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db13adb97ab515a3691f56e4dbab09283d0b86cb45abd991d8634a9d6f501760" + +[[package]] +name = "libc" +version = "0.2.177" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" + [[package]] name = "libixx" version = "0.0.0-git" dependencies = [ "binrw", + "criterion", + "levenshtein", "serde", "serde_json", "thiserror", @@ -379,15 +553,15 @@ dependencies = [ [[package]] name = "litemap" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" [[package]] name = "log" -version = "0.4.27" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" [[package]] name = "markdown" @@ -400,9 +574,18 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.5" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "num-traits" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] [[package]] name = "once_cell" @@ -412,15 +595,31 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "once_cell_polyfill" -version = "1.70.1" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oorandom" +version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "owo-colors" -version = "4.2.2" +version = "4.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48dd4f4a2c8405440fd0462561f0e5806bd0f77e86f51c761481bdd4018b545e" +checksum = "9c6901729fa79e91a0913333229e9ca5dc725089d1c363b2f4b4760709dc4a52" + +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] [[package]] name = "percent-encoding" @@ -428,38 +627,92 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "potential_utf" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" dependencies = [ "zerovec", ] [[package]] name = "proc-macro2" -version = "1.0.95" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.40" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "regex" -version = "1.11.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", @@ -469,9 +722,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.9" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", @@ -480,15 +733,15 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "rustversion" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" @@ -496,11 +749,20 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "serde" -version = "1.0.222" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aab69e3f5be1836a1fe0aca0b286e5a5b38f262d6c9e8acd2247818751fcc8fb" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", "serde_derive", @@ -508,18 +770,18 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.222" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f8ebec5eea07db7df9342aa712db2138f019d9ab3454a60a680579a6f841b80" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.222" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5f61630fe26d0ff555e6c37dc445ab2f15871c8a11ace3cf471b3195d3e4f49" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -554,9 +816,9 @@ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "stable_deref_trait" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "streaming-iterator" @@ -572,9 +834,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.104" +version = "2.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +checksum = "2f17c7e013e88258aa9543dcbe81aca68a667a9ac37cd69c9fbc07858bfe0e2f" dependencies = [ "proc-macro2", "quote", @@ -594,18 +856,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", @@ -614,19 +876,51 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" dependencies = [ "displaydoc", "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tokio" +version = "1.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +dependencies = [ + "bytes", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tree-sitter" -version = "0.25.9" +version = "0.25.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccd2a058a86cfece0bf96f7cce1021efef9c8ed0e892ab74639173e5ed7a34fa" +checksum = "78f873475d258561b06f1c595d93308a7ed124d9977cb26b148c2084a4a3cc87" dependencies = [ "cc", "regex", @@ -638,9 +932,9 @@ dependencies = [ [[package]] name = "tree-sitter-highlight" -version = "0.25.9" +version = "0.25.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89fa29ef0aa9595898934922482181db525a45657ab5bb7d1127ffde1682826" +checksum = "adc5f880ad8d8f94e88cb81c3557024cf1a8b75e3b504c50481ed4f5a6006ff3" dependencies = [ "regex", "streaming-iterator", @@ -666,15 +960,15 @@ dependencies = [ [[package]] name = "unicode-id" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10103c57044730945224467c09f71a4db0071c123a0648cc3e818913bde6b561" +checksum = "70ba288e709927c043cbe476718d37be306be53fb1fafecd0dbe36d072be2580" [[package]] name = "unicode-ident" -version = "1.0.18" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "url" @@ -700,11 +994,21 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasm-bindgen" -version = "0.2.101" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e14915cadd45b529bb8d1f343c4ed0ac1de926144b746e2710f9cd05df6603b" +checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" dependencies = [ "cfg-if", "once_cell", @@ -715,9 +1019,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.101" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28d1ba982ca7923fd01448d5c30c6864d0a14109560296a162f80f305fb93bb" +checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" dependencies = [ "bumpalo", "log", @@ -729,9 +1033,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.101" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c3d463ae3eff775b0c45df9da45d68837702ac35af998361e2c84e7c5ec1b0d" +checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -739,9 +1043,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.101" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bb4ce89b08211f923caf51d527662b75bdc9c9c7aab40f86dcb9fb85ac552aa" +checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" dependencies = [ "proc-macro2", "quote", @@ -752,28 +1056,76 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.101" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f143854a3b13752c6950862c906306adb27c7e839f7414cec8fea35beab624c1" +checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" dependencies = [ "unicode-ident", ] +[[package]] +name = "web-sys" +version = "0.3.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9367c417a924a74cae129e6a2ae3b47fabb1f8995595ab474029da749a8be120" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-sys" -version = "0.59.0" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ "windows-targets", ] [[package]] name = "windows-targets" -version = "0.52.6" +version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ + "windows-link", "windows_aarch64_gnullvm", "windows_aarch64_msvc", "windows_i686_gnu", @@ -786,65 +1138,64 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.6" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" [[package]] name = "windows_aarch64_msvc" -version = "0.52.6" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" [[package]] name = "windows_i686_gnu" -version = "0.52.6" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" [[package]] name = "windows_i686_gnullvm" -version = "0.52.6" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" [[package]] name = "windows_i686_msvc" -version = "0.52.6" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" [[package]] name = "windows_x86_64_gnu" -version = "0.52.6" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.6" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" [[package]] name = "windows_x86_64_msvc" -version = "0.52.6" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "writeable" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] name = "yoke" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" dependencies = [ - "serde", "stable_deref_trait", "yoke-derive", "zerofrom", @@ -852,9 +1203,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", @@ -862,6 +1213,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zerofrom" version = "0.1.6" @@ -885,9 +1256,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" dependencies = [ "displaydoc", "yoke", @@ -896,9 +1267,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.2" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" dependencies = [ "yoke", "zerofrom", @@ -907,9 +1278,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 3d87f59..81bd8d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,3 +13,6 @@ strip = true [profile.release.package."*"] codegen-units = 1 +[profile.bench] +debug = true +strip = false diff --git a/README.md b/README.md index bed959f..bfa2930 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ixx is the search engine and index creation tool used by [NüschtOS Search](http ## Building fixx ``` -wasm-pack build --release fixx --target web +wasm-pack build --release fixx --target web --reference-types ``` The result will be in `fixx/pkg`. diff --git a/fixx/Cargo.toml b/fixx/Cargo.toml index bb0db2d..16b4fcd 100644 --- a/fixx/Cargo.toml +++ b/fixx/Cargo.toml @@ -10,8 +10,11 @@ crate-type = ["cdylib"] [dependencies] libixx = { path = "../libixx" } -wasm-bindgen = "0.2" +wasm-bindgen = "=0.2.104" # untill wasm-opt fixes it's stuff (caused by llvm update and new wasm features which was caused by rustc update) [package.metadata.wasm-pack.profile.release] -wasm-opt = ['-O', '--enable-bulk-memory'] +wasm-opt = [ + "-O4", + "--enable-bulk-memory", +] diff --git a/fixx/derivation.nix b/fixx/derivation.nix index 0ecfd15..713e623 100644 --- a/fixx/derivation.nix +++ b/fixx/derivation.nix @@ -1,9 +1,11 @@ { lib , rustPlatform , binaryen +, nodejs , rustc , wasm-pack -, wasm-bindgen-cli_0_2_100 +, wasm-bindgen-cli_0_2_104 +, release ? true }: let @@ -13,23 +15,33 @@ rustPlatform.buildRustPackage rec { pname = "fixx"; inherit (manifest) version; + outputs = [ "out" "dist" ]; + src = lib.cleanSource ../.; cargoLock.lockFile = ../Cargo.lock; nativeBuildInputs = [ binaryen + nodejs # for npm rustc.llvmPackages.lld wasm-pack - wasm-bindgen-cli_0_2_100 + wasm-bindgen-cli_0_2_104 ]; buildPhase = '' export HOME=$(mktemp -d) - (cd fixx && wasm-pack build --release --target web --scope nuschtos) + + cd fixx + wasm-pack build --${if release then "release" else "dev"} --target web --scope nuschtos --reference-types + cd pkg + npm pack + cd ../.. ''; installPhase = '' cp -r fixx/pkg $out + mkdir $dist + mv $out/nuschtos-fixx-*-git.tgz $dist/ ''; cargoBuildFlags = "-p ${pname}"; diff --git a/fixx/pkg/fixx.js b/fixx/pkg/fixx.js new file mode 100644 index 0000000..792b19a --- /dev/null +++ b/fixx/pkg/fixx.js @@ -0,0 +1,455 @@ +let wasm; + +let WASM_VECTOR_LEN = 0; + +let cachedUint8ArrayMemory0 = null; + +function getUint8ArrayMemory0() { + if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) { + cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer); + } + return cachedUint8ArrayMemory0; +} + +const cachedTextEncoder = new TextEncoder(); + +if (!('encodeInto' in cachedTextEncoder)) { + cachedTextEncoder.encodeInto = function (arg, view) { + const buf = cachedTextEncoder.encode(arg); + view.set(buf); + return { + read: arg.length, + written: buf.length + }; + } +} + +function passStringToWasm0(arg, malloc, realloc) { + + if (realloc === undefined) { + const buf = cachedTextEncoder.encode(arg); + const ptr = malloc(buf.length, 1) >>> 0; + getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf); + WASM_VECTOR_LEN = buf.length; + return ptr; + } + + let len = arg.length; + let ptr = malloc(len, 1) >>> 0; + + const mem = getUint8ArrayMemory0(); + + let offset = 0; + + for (; offset < len; offset++) { + const code = arg.charCodeAt(offset); + if (code > 0x7F) break; + mem[ptr + offset] = code; + } + + if (offset !== len) { + if (offset !== 0) { + arg = arg.slice(offset); + } + ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0; + const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len); + const ret = cachedTextEncoder.encodeInto(arg, view); + + offset += ret.written; + ptr = realloc(ptr, len, offset, 1) >>> 0; + } + + WASM_VECTOR_LEN = offset; + return ptr; +} + +function isLikeNone(x) { + return x === undefined || x === null; +} + +let cachedDataViewMemory0 = null; + +function getDataViewMemory0() { + if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) { + cachedDataViewMemory0 = new DataView(wasm.memory.buffer); + } + return cachedDataViewMemory0; +} + +let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); + +cachedTextDecoder.decode(); + +const MAX_SAFARI_DECODE_BYTES = 2146435072; +let numBytesDecoded = 0; +function decodeText(ptr, len) { + numBytesDecoded += len; + if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) { + cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); + cachedTextDecoder.decode(); + numBytesDecoded = len; + } + return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); +} + +function getStringFromWasm0(ptr, len) { + ptr = ptr >>> 0; + return decodeText(ptr, len); +} + +function takeFromExternrefTable0(idx) { + const value = wasm.__wbindgen_export_2.get(idx); + wasm.__wbindgen_export_4(idx); + return value; +} + +function passArray8ToWasm0(arg, malloc) { + const ptr = malloc(arg.length * 1, 1) >>> 0; + getUint8ArrayMemory0().set(arg, ptr / 1); + WASM_VECTOR_LEN = arg.length; + return ptr; +} + +function getArrayJsValueFromWasm0(ptr, len) { + ptr = ptr >>> 0; + const mem = getDataViewMemory0(); + const result = []; + for (let i = ptr; i < ptr + 4 * len; i += 4) { + result.push(wasm.__wbindgen_export_2.get(mem.getUint32(i, true))); + } + wasm.__wbindgen_export_5(ptr, len); + return result; +} + +const IndexFinalization = (typeof FinalizationRegistry === 'undefined') + ? { register: () => {}, unregister: () => {} } + : new FinalizationRegistry(ptr => wasm.__wbg_index_free(ptr >>> 0, 1)); + +export class Index { + + static __wrap(ptr) { + ptr = ptr >>> 0; + const obj = Object.create(Index.prototype); + obj.__wbg_ptr = ptr; + IndexFinalization.register(obj, obj.__wbg_ptr, obj); + return obj; + } + + __destroy_into_raw() { + const ptr = this.__wbg_ptr; + this.__wbg_ptr = 0; + IndexFinalization.unregister(this); + return ptr; + } + + free() { + const ptr = this.__destroy_into_raw(); + wasm.__wbg_index_free(ptr, 0); + } + /** + * @returns {number} + */ + chunk_size() { + const ret = wasm.index_chunk_size(this.__wbg_ptr); + return ret >>> 0; + } + /** + * @param {number} scope_id + * @param {string} name + * @returns {number | undefined} + */ + get_idx_by_name(scope_id, name) { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + wasm.index_get_idx_by_name(retptr, this.__wbg_ptr, scope_id, name); + var r0 = getDataViewMemory0().getFloat64(retptr + 8 * 0, true); + var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); + var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true); + if (r3) { + throw takeFromExternrefTable0(r2); + } + return r0 === 0x100000001 ? undefined : r0; + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } + } + /** + * @param {Uint8Array} buf + * @returns {Index} + */ + static read(buf) { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passArray8ToWasm0(buf, wasm.__wbindgen_export_0); + const len0 = WASM_VECTOR_LEN; + wasm.index_read(retptr, ptr0, len0); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); + if (r2) { + throw takeFromExternrefTable0(r1); + } + return Index.__wrap(r0); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } + } + /** + * @returns {number} + */ + size() { + const ret = wasm.index_size(this.__wbg_ptr); + return ret >>> 0; + } + /** + * @returns {string[]} + */ + scopes() { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + wasm.index_scopes(retptr, this.__wbg_ptr); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); + var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true); + if (r3) { + throw takeFromExternrefTable0(r2); + } + var v1 = getArrayJsValueFromWasm0(r0, r1).slice(); + wasm.__wbindgen_export_3(r0, r1 * 4, 4); + return v1; + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } + } + /** + * @param {number | null | undefined} scope_id + * @param {string} query + * @param {number} max_results + * @returns {SearchedOption[]} + */ + search(scope_id, query, max_results) { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + wasm.index_search(retptr, this.__wbg_ptr, isLikeNone(scope_id) ? 0xFFFFFF : scope_id, query, max_results); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); + var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true); + if (r3) { + throw takeFromExternrefTable0(r2); + } + var v1 = getArrayJsValueFromWasm0(r0, r1).slice(); + wasm.__wbindgen_export_3(r0, r1 * 4, 4); + return v1; + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } + } +} +if (Symbol.dispose) Index.prototype[Symbol.dispose] = Index.prototype.free; + +const SearchedOptionFinalization = (typeof FinalizationRegistry === 'undefined') + ? { register: () => {}, unregister: () => {} } + : new FinalizationRegistry(ptr => wasm.__wbg_searchedoption_free(ptr >>> 0, 1)); + +export class SearchedOption { + + static __wrap(ptr) { + ptr = ptr >>> 0; + const obj = Object.create(SearchedOption.prototype); + obj.__wbg_ptr = ptr; + SearchedOptionFinalization.register(obj, obj.__wbg_ptr, obj); + return obj; + } + + __destroy_into_raw() { + const ptr = this.__wbg_ptr; + this.__wbg_ptr = 0; + SearchedOptionFinalization.unregister(this); + return ptr; + } + + free() { + const ptr = this.__destroy_into_raw(); + wasm.__wbg_searchedoption_free(ptr, 0); + } + /** + * @returns {number} + */ + idx() { + const ret = wasm.index_chunk_size(this.__wbg_ptr); + return ret >>> 0; + } + /** + * @returns {string} + */ + name() { + let deferred1_0; + let deferred1_1; + try { + const ptr = this.__destroy_into_raw(); + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + wasm.searchedoption_name(retptr, ptr); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + deferred1_0 = r0; + deferred1_1 = r1; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_export_3(deferred1_0, deferred1_1, 1); + } + } + /** + * @returns {number} + */ + scope_id() { + const ret = wasm.searchedoption_scope_id(this.__wbg_ptr); + return ret; + } +} +if (Symbol.dispose) SearchedOption.prototype[Symbol.dispose] = SearchedOption.prototype.free; + +const EXPECTED_RESPONSE_TYPES = new Set(['basic', 'cors', 'default']); + +async function __wbg_load(module, imports) { + if (typeof Response === 'function' && module instanceof Response) { + if (typeof WebAssembly.instantiateStreaming === 'function') { + try { + return await WebAssembly.instantiateStreaming(module, imports); + + } catch (e) { + const validResponse = module.ok && EXPECTED_RESPONSE_TYPES.has(module.type); + + if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') { + console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e); + + } else { + throw e; + } + } + } + + const bytes = await module.arrayBuffer(); + return await WebAssembly.instantiate(bytes, imports); + + } else { + const instance = await WebAssembly.instantiate(module, imports); + + if (instance instanceof WebAssembly.Instance) { + return { instance, module }; + + } else { + return instance; + } + } +} + +function __wbg_get_imports() { + const imports = {}; + imports.wbg = {}; + imports.wbg.__wbg_searchedoption_new = function(arg0) { + const ret = SearchedOption.__wrap(arg0); + return ret; + }; + imports.wbg.__wbg_wbindgenstringget_0f16a6ddddef376f = function(arg0, arg1) { + const obj = arg1; + const ret = typeof(obj) === 'string' ? obj : undefined; + var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_export_0, wasm.__wbindgen_export_1); + var len1 = WASM_VECTOR_LEN; + getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); + getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); + }; + imports.wbg.__wbg_wbindgenthrow_451ec1a8469d7eb6 = function(arg0, arg1) { + throw new Error(getStringFromWasm0(arg0, arg1)); + }; + imports.wbg.__wbindgen_cast_2241b6af4c4b2941 = function(arg0, arg1) { + // Cast intrinsic for `Ref(String) -> Externref`. + const ret = getStringFromWasm0(arg0, arg1); + return ret; + }; + imports.wbg.__wbindgen_init_externref_table = function() { + const table = wasm.__wbindgen_export_2; + const offset = table.grow(4); + table.set(0, undefined); + table.set(offset + 0, undefined); + table.set(offset + 1, null); + table.set(offset + 2, true); + table.set(offset + 3, false); + ; + }; + + return imports; +} + +function __wbg_init_memory(imports, memory) { + +} + +function __wbg_finalize_init(instance, module) { + wasm = instance.exports; + __wbg_init.__wbindgen_wasm_module = module; + cachedDataViewMemory0 = null; + cachedUint8ArrayMemory0 = null; + + + wasm.__wbindgen_start(); + return wasm; +} + +function initSync(module) { + if (wasm !== undefined) return wasm; + + + if (typeof module !== 'undefined') { + if (Object.getPrototypeOf(module) === Object.prototype) { + ({module} = module) + } else { + console.warn('using deprecated parameters for `initSync()`; pass a single object instead') + } + } + + const imports = __wbg_get_imports(); + + __wbg_init_memory(imports); + + if (!(module instanceof WebAssembly.Module)) { + module = new WebAssembly.Module(module); + } + + const instance = new WebAssembly.Instance(module, imports); + + return __wbg_finalize_init(instance, module); +} + +async function __wbg_init(module_or_path) { + if (wasm !== undefined) return wasm; + + + if (typeof module_or_path !== 'undefined') { + if (Object.getPrototypeOf(module_or_path) === Object.prototype) { + ({module_or_path} = module_or_path) + } else { + console.warn('using deprecated parameters for the initialization function; pass a single object instead') + } + } + + if (typeof module_or_path === 'undefined') { + module_or_path = new URL('fixx_bg.wasm', import.meta.url); + } + const imports = __wbg_get_imports(); + + if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) { + module_or_path = fetch(module_or_path); + } + + __wbg_init_memory(imports); + + const { instance, module } = await __wbg_load(await module_or_path, imports); + + return __wbg_finalize_init(instance, module); +} + +export { initSync }; +export default __wbg_init; diff --git a/fixx/src/lib.rs b/fixx/src/lib.rs index cf32b29..86162b3 100644 --- a/fixx/src/lib.rs +++ b/fixx/src/lib.rs @@ -17,9 +17,10 @@ impl Index { pub fn read(buf: Vec) -> Result { libixx::Index::read(&buf) .map(Self) - .map_err(|err| format!("{:?}", err)) + .map_err(|err| format!("{err:?}")) } + #[must_use] pub fn chunk_size(&self) -> u32 { self.0.meta().chunk_size } @@ -32,48 +33,62 @@ impl Index { .iter() .map(|scope| String::try_from(scope.clone())) .collect::, FromUtf8Error>>() - .map_err(|err| format!("{:?}", err)) + .map_err(|err| format!("{err:?}")) } pub fn search( &self, scope_id: Option, - query: String, + #[wasm_bindgen(unchecked_param_type = "string")] query: &JsValue, max_results: usize, ) -> Result, String> { - match self.0.search(scope_id, &query, max_results) { + let query_str = query + .as_string() + .ok_or_else(|| "Invalid query: expected a string".to_string())?; + match self.0.search(scope_id, &query_str, max_results) { Ok(options) => Ok( options .into_iter() - .map(|(idx, scope_id, name)| SearchedOption { - idx, - scope_id, - name, - }) + .map(|(idx, scope_id, name)| SearchedOption { idx, scope_id, name }) .collect(), ), - Err(err) => Err(format!("{:?}", err)), + Err(err) => Err(format!("{err:?}")), } } - pub fn get_idx_by_name(&self, scope_id: u8, name: String) -> Result, String> { + pub fn get_idx_by_name( + &self, + scope_id: u8, + #[wasm_bindgen(unchecked_param_type = "string")] name: &JsValue, + ) -> Result, String> { + let name_str = name + .as_string() + .ok_or_else(|| "Invalid name: expected a string".to_string())?; self .0 - .get_idx_by_name(scope_id, &name) - .map_err(|err| format!("{:?}", err)) + .get_idx_by_name(scope_id, &name_str) + .map_err(|err| format!("{err:?}")) + } + + #[must_use] + pub fn size(&self) -> usize { + self.0.size() } } #[wasm_bindgen] impl SearchedOption { + #[must_use] pub fn idx(&self) -> usize { self.idx } + #[must_use] pub fn scope_id(&self) -> u8 { self.scope_id } + #[must_use] pub fn name(self) -> String { self.name } diff --git a/flake.lock b/flake.lock index d680c26..4ef7d21 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1757745802, - "narHash": "sha256-hLEO2TPj55KcUFUU1vgtHE9UEIOjRcH/4QbmfHNF820=", + "lastModified": 1764517877, + "narHash": "sha256-pp3uT4hHijIC8JUK5MEqeAWmParJrgBVzHLNfJDZxg4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c23193b943c6c689d70ee98ce3128239ed9e32d1", + "rev": "2d293cbfa5a793b4c50d17c05ef9e385b90edf6c", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 622c899..8c39a4f 100644 --- a/flake.nix +++ b/flake.nix @@ -17,8 +17,13 @@ devShells.default = pkgs.mkShell { nativeBuildInputs = with pkgs; [ cargo + cargo-flamegraph + clippy + nodejs # for npm + pnpm rustc rustc.llvmPackages.lld + rustfmt wasm-pack ]; diff --git a/ixx/Cargo.toml b/ixx/Cargo.toml index 955290b..1a7ad3a 100644 --- a/ixx/Cargo.toml +++ b/ixx/Cargo.toml @@ -5,14 +5,20 @@ edition = "2024" repository = "https://github.com/NuschtOS/ixx/" license = "MIT OR Apache-2.0" +[dependencies.tokio] +version = "1.48" +default-features = false +features = ["macros", "rt-multi-thread", "fs", "io-util"] + [dependencies] -serde = { version = "1.0", features = ["derive"] } +anyhow = "1.0" clap = { version = "4.5", features = ["derive"] } -url = { version = "2.5", features = ["serde"] } libixx = { path = "../libixx" } markdown = "1.0" +regex = "1.12" +serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -anyhow = "1.0" +url = { version = "2.5", features = ["serde"] } tree-sitter-highlight = "0.25" # when updating commit, also update HIGHLIGHT_NAMES in highlight.rs diff --git a/ixx/src/action/index.rs b/ixx/src/action/index.rs deleted file mode 100644 index e52bdb9..0000000 --- a/ixx/src/action/index.rs +++ /dev/null @@ -1,246 +0,0 @@ -use std::{collections::HashMap, fs::File, path::PathBuf}; - -use anyhow::anyhow; -use libixx::Index; -use serde::Deserialize; -use url::Url; - -use crate::{ - args::IndexModule, - option::{self, Declaration}, -}; - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct Config { - scopes: Vec, -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct Scope { - name: Option, - options_json: PathBuf, - url_prefix: Url, - options_prefix: Option, -} - -struct Entry { - name: String, - scope: u8, - option: libixx::Option, -} - -pub(crate) fn index(module: IndexModule) -> anyhow::Result<()> { - let mut raw_options: Vec = vec![]; - - let config_file = File::open(module.config)?; - let config: Config = serde_json::from_reader(config_file)?; - - let mut index = Index::new(module.chunk_size); - - for scope in config.scopes { - println!("Parsing {}", scope.options_json.to_string_lossy()); - let file = File::open(scope.options_json)?; - let options: HashMap = serde_json::from_reader(file)?; - - let scope_idx = index.push_scope( - scope - .name - .map(|x| x.to_string()) - .unwrap_or_else(|| scope.url_prefix.to_string()), - ); - - for (name, option) in options { - // internal options which cannot be hidden when importing existing options.json - if name == "_module.args" { - continue; - } - - // skip modular services until upstream doc rendering is fixed - // https://github.com/NixOS/nixpkgs/issues/432550 - if name.starts_with(" format!("{}.{}", prefix, name), - None => name, - }; - let option = into_option(&scope.url_prefix, &name, option)?; - raw_options.push(Entry { - name, - scope: scope_idx, - option, - }); - } - } - - println!("Read {} options", raw_options.len()); - - raw_options.sort_by(|a, b| a.name.cmp(&b.name)); - - println!("Sorted options"); - - for entry in &raw_options { - index.push(entry.scope, &entry.name); - } - - println!("Writing index to {}", module.index_output.to_string_lossy()); - - let mut output = File::create(module.index_output)?; - index.write_into(&mut output)?; - - println!("Writing meta to {}", module.meta_output.to_string_lossy()); - - if !module.meta_output.exists() { - std::fs::create_dir(&module.meta_output)?; - } - - let options = raw_options - .into_iter() - .map(|entry| entry.option) - .collect::>(); - - for (idx, chunk) in options.chunks(module.chunk_size as usize).enumerate() { - let mut file = File::create(module.meta_output.join(format!("{}.json", idx)))?; - serde_json::to_writer(&mut file, chunk)?; - } - - Ok(()) -} - -fn into_option( - url_prefix: &Url, - name: &str, - option: option::Option, -) -> anyhow::Result { - Ok(libixx::Option { - declarations: option - .declarations - .into_iter() - .map(|declaration| update_declaration(url_prefix, declaration)) - .collect::>()?, - default: option.default.map(|option| option.render()), - description: markdown::to_html(&option.description), - example: option.example.map(|example| example.render()), - read_only: option.read_only, - r#type: option.r#type, - name: name.to_string(), - }) -} - -fn update_declaration(url_prefix: &Url, declaration: Declaration) -> anyhow::Result { - let mut url = match declaration { - Declaration::StorePath(path) => { - if path.starts_with("/") { - let idx = path - .match_indices('/') - .nth(3) - .ok_or_else(|| anyhow!("Invalid store path: {}", path))? - .0 - // +1 to also remove the / itself, when we join it with a url, the path in the url would - // get removed if we won't remove it. - + 1; - url_prefix.join(path.split_at(idx).1)? - } else { - url_prefix.join(&path)? - } - } - Declaration::Url { name: _, url } => url, - }; - - if !url.path().ends_with(".nix") { - if url.path().ends_with("/") { - url = url.join("default.nix")?; - } else { - url = url.join(&format!( - "{}/default.nix", - url - .path_segments() - .map(|segments| segments.last().unwrap_or("")) - .unwrap_or(""), - ))?; - } - } - - Ok(url) -} - -#[cfg(test)] -mod test { - use url::Url; - - use crate::{action::index::update_declaration, option::Declaration}; - - #[test] - fn test_update_declaration() { - assert_eq!( - update_declaration( - &Url::parse("https://example.com/some/path").unwrap(), - Declaration::StorePath( - "/nix/store/vgvk6q3zsjgb66f8s5cm8djz6nmcag1i-source/modules/initrd.nix".to_string() - ) - ) - .unwrap(), - Url::parse("https://example.com/some/modules/initrd.nix").unwrap() - ); - - assert_eq!( - update_declaration( - &Url::parse("https://example.com/some/path/").unwrap(), - Declaration::StorePath( - "/nix/store/vgvk6q3zsjgb66f8s5cm8djz6nmcag1i-source/modules/initrd.nix".to_string() - ) - ) - .unwrap(), - Url::parse("https://example.com/some/path/modules/initrd.nix").unwrap() - ); - - assert_eq!( - update_declaration( - &Url::parse("https://example.com/some/path/").unwrap(), - Declaration::StorePath( - "/nix/store/vgvk6q3zsjgb66f8s5cm8djz6nmcag1i-source-idk/modules/initrd.nix".to_string() - ) - ) - .unwrap(), - Url::parse("https://example.com/some/path/modules/initrd.nix").unwrap() - ); - - // Suffix default.nix if url is referencing folder - assert_eq!( - update_declaration( - &Url::parse("https://example.com/some/path").unwrap(), - Declaration::Url { - name: "idk".to_string(), - url: Url::parse("https://example.com/some/path").unwrap(), - } - ) - .unwrap(), - Url::parse("https://example.com/some/path/default.nix").unwrap() - ); - - assert_eq!( - update_declaration( - &Url::parse("https://example.com/some/path").unwrap(), - Declaration::Url { - name: "idk".to_string(), - url: Url::parse("https://example.com/some/path/").unwrap(), - } - ) - .unwrap(), - Url::parse("https://example.com/some/path/default.nix").unwrap() - ); - - // nixpkgs edge case - assert_eq!( - update_declaration( - &Url::parse("https://example.com/some/path/").unwrap(), - Declaration::StorePath("nixos/hello/world.nix".to_string()), - ) - .unwrap(), - Url::parse("https://example.com/some/path/nixos/hello/world.nix").unwrap() - ); - } -} diff --git a/ixx/src/action/index/mod.rs b/ixx/src/action/index/mod.rs new file mode 100644 index 0000000..fc4677f --- /dev/null +++ b/ixx/src/action/index/mod.rs @@ -0,0 +1,267 @@ +use std::{collections::HashMap, path::PathBuf}; + +use anyhow::Context; +use serde::{Deserialize, Serialize}; +use tokio::{fs::File, io::AsyncWriteExt, join}; +use url::Url; + +use crate::{ + Declaration, + action::index::{options::index_options, packages::index_packages}, + args::IndexModule, +}; + +mod options; +mod packages; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub(crate) struct Config { + scopes: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub(crate) struct Scope { + name: Option, + license_mapping: HashMap, + maintainer_mapping: HashMap, + team_mapping: HashMap, + options_json: Option, + packages_jsons: Option>, + url_prefix: Url, + options_prefix: Option, +} + +struct OptionEntry { + name: String, + scope: u8, + option: libixx::Option, +} + +struct PackageEntry { + name: String, + scope: u8, + option: libixx::Package, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct License { + free: bool, + full_name: String, + redistributable: bool, + spdx_id: Option, + url: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct Maintainer { + email: Option, + matrix: Option, + github: String, + name: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct Team { + members: Vec, + scope: String, +} + +#[derive(Serialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct Meta { + scopes: HashMap, +} + +#[derive(Serialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct ScopeMeta { + licenses: HashMap, + maintainers: HashMap, + teams: HashMap, +} + +pub(crate) async fn index(module: IndexModule) -> anyhow::Result<()> { + let config: Config = { + let raw_config = tokio::fs::read_to_string(&module.config) + .await + .with_context(|| format!("Failed to read config file: {}", module.config.to_string_lossy()))?; + serde_json::from_str(&raw_config) + .with_context(|| format!("Failed to parse config file: {}", module.config.to_string_lossy()))? + }; + + let (options_result, packages_result, meta_result) = join!( + index_options(&module, &config), + index_packages(&module, &config), + async { + let meta = Meta { + scopes: config + .scopes + .iter() + .enumerate() + .map(|(idx, scope)| { + ( + idx as u8, + ScopeMeta { + licenses: scope.license_mapping.clone(), + maintainers: scope.maintainer_mapping.clone(), + teams: scope.team_mapping.clone(), + }, + ) + }) + .collect(), + }; + + let raw_meta = serde_json::to_string(&meta)?; + let mut meta_file = File::create(&module.meta_output).await?; + + meta_file.write_all(raw_meta.as_bytes()).await?; + + Ok::<_, anyhow::Error>(()) + } + ); + + options_result?; + packages_result?; + meta_result?; + + Ok(()) +} + +fn update_declaration(url_prefix: &Url, declaration: Declaration) -> anyhow::Result { + let mut url = match declaration { + Declaration::StorePath(path) => { + let mut url_path; + if path.starts_with('/') { + let idx = path + .match_indices('/') + .nth(3) + .ok_or_else(|| anyhow::anyhow!("Invalid store path: {path}"))? + .0 + // +1 to also remove the / itself, when we join it with a url, the path in the url would + // get removed if we won't remove it. + + 1; + url_path = path.split_at(idx).1.to_owned(); + } else { + url_path = path; + } + + if let Some((path, line)) = url_path.split_once(':') { + url_path = format!("{path}#L{line}"); + } + + url_prefix.join(&url_path)? + } + Declaration::Url { name: _, url } => url, + }; + + if !url.path().ends_with(".nix") { + if url.path().ends_with('/') { + url = url.join("default.nix")?; + } else { + url = url.join(&format!( + "{}/default.nix", + url + .path_segments() + .map_or("", |mut segments| segments.next_back().unwrap_or("")), + ))?; + } + } + + Ok(url) +} + +#[cfg(test)] +mod test { + use url::Url; + + use crate::{Declaration, action::index::update_declaration}; + + #[test] + fn test_update_declaration() { + assert_eq!( + update_declaration( + &Url::parse("https://example.com/some/path").unwrap(), + Declaration::StorePath( + "/nix/store/vgvk6q3zsjgb66f8s5cm8djz6nmcag1i-source/modules/initrd.nix".to_string() + ) + ) + .unwrap(), + Url::parse("https://example.com/some/modules/initrd.nix").unwrap() + ); + + assert_eq!( + update_declaration( + &Url::parse("https://example.com/some/path/").unwrap(), + Declaration::StorePath( + "/nix/store/vgvk6q3zsjgb66f8s5cm8djz6nmcag1i-source/modules/initrd.nix".to_string() + ) + ) + .unwrap(), + Url::parse("https://example.com/some/path/modules/initrd.nix").unwrap() + ); + + assert_eq!( + update_declaration( + &Url::parse("https://example.com/some/path/").unwrap(), + Declaration::StorePath( + "/nix/store/vgvk6q3zsjgb66f8s5cm8djz6nmcag1i-source-idk/modules/initrd.nix".to_string() + ) + ) + .unwrap(), + Url::parse("https://example.com/some/path/modules/initrd.nix").unwrap() + ); + + // package position + assert_eq!( + update_declaration( + &Url::parse("https://example.com/some/path/").unwrap(), + Declaration::StorePath( + "/nix/store/pb93n2bk2zpyn1sqpkm3gyhra26zy4ps-source/pkgs/by-name/he/hello/package.nix:47" + .to_string() + ) + ) + .unwrap(), + Url::parse("https://example.com/some/path/pkgs/by-name/he/hello/package.nix#L47").unwrap() + ); + + // Suffix default.nix if url is referencing folder + assert_eq!( + update_declaration( + &Url::parse("https://example.com/some/path").unwrap(), + Declaration::Url { + name: "idk".to_string(), + url: Url::parse("https://example.com/some/path").unwrap(), + } + ) + .unwrap(), + Url::parse("https://example.com/some/path/default.nix").unwrap() + ); + + assert_eq!( + update_declaration( + &Url::parse("https://example.com/some/path").unwrap(), + Declaration::Url { + name: "idk".to_string(), + url: Url::parse("https://example.com/some/path/").unwrap(), + } + ) + .unwrap(), + Url::parse("https://example.com/some/path/default.nix").unwrap() + ); + + // nixpkgs edge case + assert_eq!( + update_declaration( + &Url::parse("https://example.com/some/path/").unwrap(), + Declaration::StorePath("nixos/hello/world.nix".to_string()), + ) + .unwrap(), + Url::parse("https://example.com/some/path/nixos/hello/world.nix").unwrap() + ); + } +} diff --git a/ixx/src/action/index/options.rs b/ixx/src/action/index/options.rs new file mode 100644 index 0000000..ad88c01 --- /dev/null +++ b/ixx/src/action/index/options.rs @@ -0,0 +1,167 @@ +use std::{collections::HashMap, io::Cursor}; + +use anyhow::Context; +use libixx::{Index, IndexBuilder}; +use tokio::{fs::File, io::AsyncWriteExt, task::JoinSet}; +use url::Url; + +use crate::{ + action::index::{Config, OptionEntry, update_declaration}, + args::IndexModule, + option::{self, Content}, +}; + +pub(crate) async fn index_options(module: &IndexModule, config: &Config) -> anyhow::Result<()> { + let mut raw_options: Vec = vec![]; + + let mut index_builder = IndexBuilder::new(module.chunk_size); + + for scope in &config.scopes { + let options_json = match &scope.options_json { + Some(options_jsons) => options_jsons, + None => { + continue; + } + }; + + println!("Parsing {}", options_json.to_string_lossy()); + let options: HashMap = { + let raw_options = tokio::fs::read_to_string(&options_json) + .await + .with_context(|| format!("Failed to read options json: {}", options_json.to_string_lossy()))?; + serde_json::from_str(&raw_options) + .with_context(|| format!("Failed to parse options json: {}", options_json.to_string_lossy()))? + }; + + let scope_idx = index_builder.push_scope( + scope + .name + .as_ref() + .map_or_else(|| scope.url_prefix.to_string(), ToString::to_string), + ); + + for (name, option) in options { + // internal options which cannot be hidden when importing existing options.json + if name == "_module.args" { + continue; + } + + // skip modular services until upstream doc rendering is fixed + // https://github.com/NixOS/nixpkgs/issues/432550 + if name.starts_with(" format!("{prefix}.{name}"), + None => name, + }; + + let option = into_option(&scope.url_prefix, &name, option)?; + + raw_options.push(OptionEntry { + name, + scope: scope_idx, + option, + }); + } + } + + println!("Read {} options", raw_options.len()); + if raw_options.is_empty() { + return Ok(()); + } + + println!("Sorting options"); + raw_options.sort_by(|a, b| a.name.cmp(&b.name)); + + println!("Building options index"); + for entry in &raw_options { + index_builder.push(entry.scope, &entry.name); + } + + println!( + "Writing options index to {}", + module.options_index_output.to_string_lossy() + ); + + { + let index_buf = { + let mut buf = Vec::new(); + let index: Index = index_builder.into(); + index.write_into(&mut Cursor::new(&mut buf))?; + buf + }; + + let mut index_output = File::create(&module.options_index_output) + .await + .with_context(|| { + format!( + "Failed to create {}", + module.options_index_output.to_string_lossy() + ) + })?; + + index_output.write_all(index_buf.as_slice()).await?; + } + + println!( + "Writing options chunks to {}", + module.options_chunks_output.to_string_lossy() + ); + + if !module.options_chunks_output.exists() { + std::fs::create_dir(&module.options_chunks_output).with_context(|| { + format!( + "Failed to create dir {}", + module.options_chunks_output.to_string_lossy() + ) + })?; + } + + let options = raw_options + .into_iter() + .map(|entry| entry.option) + .collect::>(); + + let mut join_set = JoinSet::new(); + + for (idx, chunk) in options.chunks(module.chunk_size as usize).enumerate() { + let path = module.options_chunks_output.join(format!("{idx}.json")); + + let chunk_string = serde_json::to_string(chunk) + .with_context(|| format!("Failed to write to {}", path.to_string_lossy()))?; + + join_set.spawn(async move { + let mut file = File::create(&path) + .await + .with_context(|| format!("Failed to create {}", path.to_string_lossy()))?; + + file.write_all(chunk_string.as_bytes()).await?; + + Ok::<_, anyhow::Error>(()) + }); + } + + while let Some(result) = join_set.join_next().await { + result??; + } + + Ok(()) +} + +fn into_option(url_prefix: &Url, name: &str, option: option::Option) -> anyhow::Result { + Ok(libixx::Option { + declarations: option + .declarations + .into_iter() + .map(|declaration| update_declaration(url_prefix, declaration)) + .collect::>()?, + default: option.default.map(Content::render), + description: markdown::to_html(&option.description), + example: option.example.map(Content::render), + read_only: option.read_only, + r#type: option.r#type, + name: name.to_string(), + }) +} diff --git a/ixx/src/action/index/packages.rs b/ixx/src/action/index/packages.rs new file mode 100644 index 0000000..6291192 --- /dev/null +++ b/ixx/src/action/index/packages.rs @@ -0,0 +1,234 @@ +use std::{ + io::Cursor, + sync::{Arc, LazyLock}, +}; + +use anyhow::Context; +use libixx::{Index, IndexBuilder}; +use regex::{Captures, Regex}; +use tokio::{fs::File, io::AsyncWriteExt, task::JoinSet}; +use url::Url; + +use crate::{ + action::index::{Config, PackageEntry, update_declaration}, + args::IndexModule, + package::{self, OneOrMany}, +}; + +pub(crate) async fn index_packages(module: &IndexModule, config: &Config) -> anyhow::Result<()> { + let mut raw_packages: Vec = vec![]; + + let mut index_builder = IndexBuilder::new(module.chunk_size); + + for scope in &config.scopes { + let packages_jsons = match &scope.packages_jsons { + Some(packages_jsons) => packages_jsons, + None => { + continue; + } + }; + + let scope_idx = index_builder.push_scope( + scope + .name + .as_ref() + .map_or_else(|| scope.url_prefix.to_string(), ToString::to_string), + ); + + let mut join_set = JoinSet::new(); + + let url_prefix = Arc::new(scope.url_prefix.clone()); + + for packages_json in packages_jsons { + let packages_json = packages_json.clone(); + let url_prefix = url_prefix.clone(); + join_set.spawn(async move { + println!("Parsing {}", packages_json.to_string_lossy()); + let packages: Vec = { + let raw_packages = tokio::fs::read_to_string(&packages_json).await.with_context(|| { + format!( + "Failed to read packages json: {}", + packages_json.to_string_lossy() + ) + })?; + serde_json::from_str(&raw_packages).with_context(|| { + format!( + "Failed to parse packages json: {}", + packages_json.to_string_lossy() + ) + })? + }; + + let packages = packages + .into_iter() + .map(|package| { + Ok::<_, anyhow::Error>(PackageEntry { + name: package.attr_name.clone(), + scope: scope_idx, + option: into_package(&url_prefix, package)?, + }) + }) + .collect::, _>>()?; + + Ok::<_, anyhow::Error>(packages) + }); + + while let Some(result) = join_set.join_next().await { + raw_packages.extend(result??); + } + } + } + + println!("Read {} packages", raw_packages.len()); + if raw_packages.is_empty() { + return Ok(()); + } + + println!("Sorting packages"); + raw_packages.sort_by(|a, b| a.name.cmp(&b.name)); + + for entry in &raw_packages { + index_builder.push(entry.scope, &entry.name); + } + + println!( + "Writing packages index to {}", + module.packages_index_output.to_string_lossy() + ); + + { + let index_buf = { + let mut buf = Vec::new(); + let index: Index = index_builder.into(); + index.write_into(&mut Cursor::new(&mut buf))?; + buf + }; + + let mut index_output = File::create(&module.packages_index_output) + .await + .with_context(|| { + format!( + "Failed to create {}", + module.packages_index_output.to_string_lossy() + ) + })?; + + index_output.write_all(index_buf.as_slice()).await?; + } + + println!( + "Writing packages chunks to {}", + module.packages_chunks_output.to_string_lossy() + ); + + if !module.packages_chunks_output.exists() { + std::fs::create_dir(&module.packages_chunks_output).with_context(|| { + format!( + "Failed to create dir {}", + module.packages_chunks_output.to_string_lossy() + ) + })?; + } + + let packages = raw_packages + .into_iter() + .map(|entry| entry.option) + .collect::>(); + + let mut join_set = JoinSet::new(); + + for (idx, chunk) in packages.chunks(module.chunk_size as usize).enumerate() { + let path = module.packages_chunks_output.join(format!("{idx}.json")); + + let meta_string = serde_json::to_string(chunk) + .with_context(|| format!("Failed to write to {}", path.to_string_lossy()))?; + + join_set.spawn(async move { + let mut file = File::create(&path) + .await + .with_context(|| format!("Failed to create {}", path.to_string_lossy()))?; + + file.write_all(meta_string.as_bytes()).await?; + + Ok::<_, anyhow::Error>(()) + }); + } + + while let Some(result) = join_set.join_next().await { + result??; + } + + Ok(()) +} + +fn into_package(url_prefix: &Url, package: package::Package) -> anyhow::Result { + static CVE_REGEX: LazyLock = LazyLock::new(|| Regex::new(r"CVE-(\d{4})-(\d+)").unwrap()); + static GHSA_REGEX: LazyLock = + LazyLock::new(|| Regex::new(r"GHSA((?:-[23456789cfghjmpqrvwx]{4}){3})").unwrap()); + + Ok(libixx::Package { + attr_name: package.attr_name, + broken: package.broken, + cpe: package.cpe, + disabled: package.disabled, + possible_cpes: package.possible_cpes.unwrap_or_default(), + purl: package.purl, + declaration: package + .declaration + .map(|declaration| update_declaration(url_prefix, declaration)) + .transpose()?, + description: package + .description + .map(|description| markdown::to_html(&description)), + long_description: package + .long_description + .map(|description| markdown::to_html(&description)), + eval_error: package.eval_error, + homepages: match package.homepage { + None => vec![], + Some(OneOrMany::One(homepage)) => Url::parse(&homepage) + .with_context(|| format!("Failed to parse URL '{homepage}'")) + .ok() + .into_iter() + .collect(), + Some(OneOrMany::Many(homepages)) => homepages + .into_iter() + .filter_map(|homepage| { + Url::parse(&homepage) + .with_context(|| format!("Failed to parse URL '{homepage}'")) + .ok() + }) + .collect(), + }, + known_vulnerabilities: package + .known_vulnerabilities + .unwrap_or_default() + .into_iter() + .map(|vulnerability| { + let vulnerability = markdown::to_html(&vulnerability); + let vulnerability = CVE_REGEX.replace_all(&vulnerability, |caps: &Captures| { + format!( + "CVE-{0}-{1}", + &caps[1], &caps[2] + ) + }); + + GHSA_REGEX + .replace_all(&vulnerability, |caps: &Captures| { + format!( + "GHSA{0}", + &caps[1] + ) + }) + .to_string() + }) + .collect(), + licenses: package.licenses.unwrap_or_default(), + maintainers: package.maintainers.unwrap_or_default(), + name: package.name, + outputs: package.outputs.unwrap_or_default(), + pname: package.pname, + teams: package.teams.unwrap_or_default(), + version: package.version, + }) +} diff --git a/ixx/src/action/meta.rs b/ixx/src/action/meta.rs index 5a5b719..75d4353 100644 --- a/ixx/src/action/meta.rs +++ b/ixx/src/action/meta.rs @@ -27,17 +27,19 @@ pub(crate) fn meta(module: MetaModule) -> anyhow::Result<()> { .scopes .iter() .enumerate() - .map(|(i, scope)| Scope { - id: i as u8, - name: scope.to_string(), + .map(|(i, scope)| { + Ok::<_, anyhow::Error>(Scope { + id: i as u8, + name: scope.clone().try_into()?, + }) }) - .collect(), + .collect::, _>>()?, }; match module.format { Format::Json => { let json_output = serde_json::to_string_pretty(&meta)?; - println!("{}", json_output); + println!("{json_output}"); } Format::Text => { println!("chunk_size: {}", meta.chunk_size); diff --git a/ixx/src/action/mod.rs b/ixx/src/action/mod.rs index 6f76f0d..bfebf34 100644 --- a/ixx/src/action/mod.rs +++ b/ixx/src/action/mod.rs @@ -1,3 +1,3 @@ pub(crate) mod index; -pub(crate) mod search; pub(crate) mod meta; +pub(crate) mod search; diff --git a/ixx/src/action/search.rs b/ixx/src/action/search.rs index a8e2e0e..5c9e79e 100644 --- a/ixx/src/action/search.rs +++ b/ixx/src/action/search.rs @@ -22,19 +22,15 @@ pub(crate) fn search(module: SearchModule) -> anyhow::Result<()> { Format::Json => { let entries: Vec = result .into_iter() - .map(|(idx, scope_id, name)| Entry { - idx, - scope_id, - name, - }) + .map(|(idx, scope_id, name)| Entry { idx, scope_id, name }) .collect(); let json_output = serde_json::to_string_pretty(&entries)?; - println!("{}", json_output); + println!("{json_output}"); } Format::Text => { for (idx, scope_id, name) in result { - println!("idx: {}, scope_id: {}, name: {}", idx, scope_id, name); + println!("idx: {idx}, scope_id: {scope_id}, name: {name}"); } } } diff --git a/ixx/src/args.rs b/ixx/src/args.rs index fcfea03..c742fc4 100644 --- a/ixx/src/args.rs +++ b/ixx/src/args.rs @@ -9,8 +9,11 @@ pub(super) struct Args { #[derive(Subcommand)] pub(super) enum Action { + #[clap(about = "Build the index")] Index(IndexModule), + #[clap(about = "Search the index for packages or options")] Search(SearchModule), + #[clap(about = "Show index metadata")] Meta(MetaModule), } @@ -24,13 +27,22 @@ pub(super) enum Format { pub(super) struct IndexModule { pub(super) config: PathBuf, - #[clap(short, long, default_value = "index.ixx")] - pub(super) index_output: PathBuf, + #[clap(long, default_value = "options/index.ixx")] + pub(super) options_index_output: PathBuf, + + #[clap(long, default_value = "options/chunks")] + pub(crate) options_chunks_output: PathBuf, + + #[clap(long, default_value = "packages/index.ixx")] + pub(super) packages_index_output: PathBuf, + + #[clap(long, default_value = "packages/chunks")] + pub(crate) packages_chunks_output: PathBuf, - #[clap(short, long, default_value = "meta")] + #[clap(long, default_value = "meta.json")] pub(crate) meta_output: PathBuf, - #[clap(short, long, default_value = "100")] + #[clap(long, default_value = "100")] pub(super) chunk_size: u32, } @@ -41,7 +53,7 @@ pub(super) struct SearchModule { #[clap(short, long, default_value = "index.ixx")] pub(super) index: PathBuf, - #[clap(short, long)] + #[clap(short, long, default_value = "0")] pub(super) scope_id: Option, #[clap(short, long, default_value = "10")] diff --git a/ixx/src/main.rs b/ixx/src/main.rs index e6d57c0..f1481fb 100644 --- a/ixx/src/main.rs +++ b/ixx/src/main.rs @@ -1,19 +1,35 @@ use args::{Action, Args}; use clap::Parser; +use serde::Deserialize; +use url::Url; mod action; mod args; mod option; +mod package; pub(crate) mod utils; -fn main() -> anyhow::Result<()> { +#[tokio::main] +async fn main() -> anyhow::Result<()> { let args = Args::parse(); match args.action { - Action::Index(module) => action::index::index(module), + Action::Index(module) => action::index::index(module).await, Action::Search(module) => action::search::search(module), Action::Meta(module) => action::meta::meta(module), }?; Ok(()) } + +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase", untagged)] +pub enum Declaration { + /// Example Value: `/nix/store/vgvk6q3zsjgb66f8s5cm8djz6nmcag1i-source/modules/initrd.nix` + StorePath(String), + Url { + name: String, + url: Url, + }, +} diff --git a/ixx/src/option.rs b/ixx/src/option.rs index c160d12..d88e24a 100644 --- a/ixx/src/option.rs +++ b/ixx/src/option.rs @@ -1,10 +1,9 @@ use serde::{Deserialize, Serialize}; -use url::Url; -use crate::utils::highlight; +use crate::{Declaration, utils::highlight}; -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[derive(Default, Debug, Clone, PartialEq, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] pub struct Option { pub declarations: Vec, pub description: String, @@ -17,33 +16,22 @@ pub struct Option { } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", untagged)] -pub enum Declaration { - /// Example Value: `/nix/store/vgvk6q3zsjgb66f8s5cm8djz6nmcag1i-source/modules/initrd.nix` - StorePath(String), - Url { - name: String, - url: Url, - }, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields, rename_all = "camelCase")] #[serde(tag = "_type")] pub enum Content { LiteralExpression { text: String, + // nixvim uses this in programs.nixvim.dependencies.coreutils.package + path: std::option::Option>, }, #[serde(rename = "literalMD")] - Markdown { - text: String, - }, + Markdown { text: String }, } impl Content { pub(crate) fn render(self) -> String { match self { - Self::LiteralExpression { text } => highlight(text.trim()), + Self::LiteralExpression { text, .. } => highlight(text.trim()), Self::Markdown { text } => markdown::to_html(text.trim()), } } diff --git a/ixx/src/package.rs b/ixx/src/package.rs new file mode 100644 index 0000000..6e9f069 --- /dev/null +++ b/ixx/src/package.rs @@ -0,0 +1,35 @@ +use serde::Deserialize; + +use crate::Declaration; + +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct Package { + pub attr_name: String, + pub broken: Option, + pub cpe: Option, + pub disabled: Option, + pub possible_cpes: Option>, + pub purl: Option, + pub declaration: Option, + pub description: Option, + pub long_description: Option, + pub eval_error: Option, + pub homepage: Option>, + pub known_vulnerabilities: Option>, + pub licenses: Option>, + pub maintainers: Option>, + pub name: Option, + pub outputs: Option>, + pub pname: Option, + pub teams: Option>, + pub version: Option, +} + +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[serde(untagged)] +pub enum OneOrMany { + One(T), + Many(Vec), +} diff --git a/libixx/Cargo.toml b/libixx/Cargo.toml index e917212..e254ef7 100644 --- a/libixx/Cargo.toml +++ b/libixx/Cargo.toml @@ -5,12 +5,20 @@ edition = "2024" repository = "https://github.com/NuschtOS/ixx/" license = "MIT OR Apache-2.0" +[lib] +bench = true + [dependencies] +binrw = "0.15" +levenshtein = "1.0" serde = { version = "1.0", features = ["derive"] } -url = { version = "2.5", features = ["serde"] } thiserror = "2.0" -binrw = "0.15" +url = { version = "2.5", features = ["serde"] } [dev-dependencies] serde_json = "1.0" +criterion = "0.8" +[[bench]] +name = "search" +harness = false diff --git a/libixx/benches/search.rs b/libixx/benches/search.rs new file mode 100644 index 0000000..85de267 --- /dev/null +++ b/libixx/benches/search.rs @@ -0,0 +1,56 @@ +use criterion::{Criterion, criterion_group, criterion_main}; +use libixx::Index; +use std::{fs::File, hint::black_box}; + +fn criterion_benchmark(c: &mut Criterion) { + let mut file = match File::open("../index.ixx") { + Ok(f) => f, + Err(e) => { + eprintln!( + "index.ixx is missing, you can download one from https://HEAD.nuschtos-search.pages.dev/data/packages/index.ixx and place it in the root of the project: {}", + e + ); + std::process::exit(1); + } + }; + let index = Index::read_from(&mut file).unwrap(); + + c.bench_function("search for hello", |b| { + b.iter(|| index.search(None, black_box("hello"), 500)) + }); + + c.bench_function("search for zoo", |b| { + b.iter(|| index.search(None, black_box("zoo"), 500)) + }); + + c.bench_function("search for python313Packages.cryptography", |b| { + b.iter(|| index.search(None, black_box("python313Packages.cryptography"), 500)) + }); + + c.bench_function("search for python3*.crypto*", |b| { + b.iter(|| index.search(None, black_box("python3*.crypto*"), 500)) + }); + + c.bench_function( + "search for haskell.packages.ghc9103.Facebook-Password-Hacker-Online-Latest-Version +", + |b| { + b.iter(|| { + index.search( + None, + black_box("haskell.packages.ghc9103.Facebook-Password-Hacker-Online-Latest-Version"), + 500, + ) + }) + }, + ); + + c.bench_function( + "search for haskell.packages.ghc*.Facebook-*-Version +", + |b| b.iter(|| index.search(None, black_box("haskell.packages.ghc*.Facebook-*-Version"), 500)), + ); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/libixx/index.nuscht b/libixx/index.nuscht deleted file mode 100644 index 4279e82..0000000 Binary files a/libixx/index.nuscht and /dev/null differ diff --git a/libixx/src/error.rs b/libixx/src/error.rs index b8e85a1..5b3f523 100644 --- a/libixx/src/error.rs +++ b/libixx/src/error.rs @@ -1,4 +1,4 @@ -use std::string::FromUtf8Error; +use std::str::Utf8Error; use thiserror::Error; @@ -12,5 +12,5 @@ pub enum IxxError { #[error("(de)serialization failed")] Binrw(#[from] binrw::Error), #[error("invalid utf8")] - FromUtf8Error(#[from] FromUtf8Error), + FromUtf8Error(#[from] Utf8Error), } diff --git a/libixx/src/index.rs b/libixx/src/index.rs index f5b4e2d..97cf99d 100644 --- a/libixx/src/index.rs +++ b/libixx/src/index.rs @@ -1,18 +1,29 @@ -use std::io::{Cursor, Read, Seek, Write}; +use std::{ + collections::HashMap, + io::{Cursor, Read, Seek, Write}, + string::FromUtf8Error, +}; -use binrw::{binrw, BinRead, BinWrite, Endian, NullString}; +use binrw::{BinRead, BinWrite, Endian, VecArgs, binrw}; -use crate::IxxError; +use levenshtein::levenshtein; + +use crate::{IxxError, string_view::StringView}; + +pub struct IndexBuilder { + index: Index, + label_cache: HashMap, (usize, u8)>, +} #[binrw] -#[brw(magic = b"ixx01")] +#[brw(magic = b"ixx02")] #[derive(Debug, Clone, PartialEq)] pub struct Index { - meta: Meta, - #[bw(calc = options.len() as u32)] + pub(crate) meta: Meta, + #[bw(calc = entries.len() as u32)] count: u32, #[br(count = count)] - options: Vec, + pub(crate) entries: Vec, } #[binrw] @@ -22,47 +33,212 @@ pub struct Meta { #[bw(calc = scopes.len() as u8)] scope_count: u8, #[br(count = scope_count)] - pub scopes: Vec, + pub scopes: Vec, +} + +#[binrw] +#[derive(Debug, Clone, PartialEq)] +pub struct PascalString { + #[bw(calc = data.len() as u8)] + len: u8, + #[br(count = len)] + pub(crate) data: Vec, } #[binrw] #[derive(Default, Debug, Clone, PartialEq)] -pub struct OptionEntry { +pub struct Entry { /// index in the scopes Vec - scope_id: u8, - #[bw(calc = labels.len() as u16)] - count: u16, + pub(crate) scope_id: u8, + #[bw(calc = labels.len() as u8)] + count: u8, #[br(count = count)] - labels: Vec