Skip to content

perf: key gen cache by emit configuration #30791

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

perf: key gen cache by emit configuration #30791

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
93035a1
perf: key gen cache by emit configuration
lucacasonato Sep 19, 2025
6940cb8
fix
lucacasonato Sep 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion libs/resolver/cache/deno_dir.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ impl<TSys: DiskCacheSys> DenoDir<TSys> {
#[cfg(not(target_arch = "wasm32"))]
assert!(root.is_absolute());

let gen_path = root.join("gen");
let gen_path = root.join("gen_v2");
Self {
root,
gen_cache: DiskCache::new(sys, gen_path),
Expand Down
89 changes: 69 additions & 20 deletions libs/resolver/cache/emit.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,13 @@ impl<TSys: EmitCacheSys> EmitCache<TSys> {
}
};

log::debug!("Emit cache version: {:?}", cache_version);
log::debug!(
"Emit cache xxhash64 check: {}",
twox_hash::XxHash64::oneshot(0, b"test")
);
log::debug!("Emit cache mode: {:?}", mode);

Self {
disk_cache,
emit_failed_flag: Default::default(),
Expand All @@ -69,22 +76,35 @@ impl<TSys: EmitCacheSys> EmitCache<TSys> {
pub fn get_emit_code(
&self,
specifier: &Url,
transpile_and_emit_options_hash: u64,
expected_source_hash: u64,
) -> Option<String> {
if matches!(self.mode, Mode::Disable) {
return None;
}

let emit_filename = self.get_emit_filename(specifier)?;
let emit_filename =
self.get_emit_filename(specifier, transpile_and_emit_options_hash)?;
let bytes = self.disk_cache.get(&emit_filename).ok()?;
self
.file_serializer
.deserialize(bytes, expected_source_hash)
}

/// Sets the emit code in the cache.
pub fn set_emit_code(&self, specifier: &Url, source_hash: u64, code: &[u8]) {
if let Err(err) = self.set_emit_code_result(specifier, source_hash, code) {
pub fn set_emit_code(
&self,
specifier: &Url,
transpile_and_emit_options_hash: u64,
source_hash: u64,
code: &[u8],
) {
if let Err(err) = self.set_emit_code_result(
specifier,
transpile_and_emit_options_hash,
source_hash,
code,
) {
// might error in cases such as a readonly file system
log::debug!("Error saving emit data ({}): {}", specifier, err);
// assume the cache can't be written to and disable caching to it
Expand All @@ -95,6 +115,7 @@ impl<TSys: EmitCacheSys> EmitCache<TSys> {
fn set_emit_code_result(
&self,
specifier: &Url,
transpile_and_emit_options_hash: u64,
source_hash: u64,
code: &[u8],
) -> Result<(), AnyError> {
Expand All @@ -104,18 +125,26 @@ impl<TSys: EmitCacheSys> EmitCache<TSys> {
}

let emit_filename = self
.get_emit_filename(specifier)
.get_emit_filename(specifier, transpile_and_emit_options_hash)
.ok_or_else(|| anyhow!("Could not get emit filename."))?;
let cache_data = self.file_serializer.serialize(code, source_hash);
self.disk_cache.set(&emit_filename, &cache_data)?;

Ok(())
}

fn get_emit_filename(&self, specifier: &Url) -> Option<PathBuf> {
self
fn get_emit_filename(
&self,
specifier: &Url,
transpile_and_emit_options_hash: u64,
) -> Option<PathBuf> {
let suffix = self
.disk_cache
.get_cache_filename_with_extension(specifier, "js")
.get_cache_filename_with_extension(specifier, "js")?;
let mut path =
PathBuf::from(format!("{:016x}", transpile_and_emit_options_hash));
path.push(suffix);
Some(path)
}
}

Expand All @@ -141,11 +170,22 @@ impl EmitFileSerializer {
// verify the meta data file is for this source and CLI version
let source_hash = source_hash.parse::<u64>().ok()?;
if source_hash != expected_source_hash {
log::debug!(
"Emit cache source hash mismatch (expected: {}, actual: {})",
expected_source_hash,
source_hash
);
return None;
}
let emit_hash = emit_hash.parse::<u64>().ok()?;
// prevent using an emit from a different cli version or emits that were tampered with
if emit_hash != self.compute_emit_hash(content) {
let expected_emit_hash = self.compute_emit_hash(content);
if emit_hash != expected_emit_hash {
log::debug!(
"Emit cache emit hash mismatch (expected: {}, actual: {})",
expected_emit_hash,
emit_hash
);
return None;
}

Expand Down Expand Up @@ -212,19 +252,19 @@ mod test {
temp_dir.path().join("file2.ts").as_path(),
)
.unwrap();
assert_eq!(cache.get_emit_code(&specifier1, 1), None);
assert_eq!(cache.get_emit_code(&specifier1, 1, 1), None);
let emit_code1 = "text1".to_string();
let emit_code2 = "text2".to_string();
cache.set_emit_code(&specifier1, 10, emit_code1.as_bytes());
cache.set_emit_code(&specifier2, 2, emit_code2.as_bytes());
cache.set_emit_code(&specifier1, 1, 10, emit_code1.as_bytes());
cache.set_emit_code(&specifier2, 1, 2, emit_code2.as_bytes());
// providing the incorrect source hash
assert_eq!(cache.get_emit_code(&specifier1, 5), None);
assert_eq!(cache.get_emit_code(&specifier1, 1, 5), None);
// providing the correct source hash
assert_eq!(
cache.get_emit_code(&specifier1, 10),
cache.get_emit_code(&specifier1, 1, 10),
Some(emit_code1.clone()),
);
assert_eq!(cache.get_emit_code(&specifier2, 2), Some(emit_code2));
assert_eq!(cache.get_emit_code(&specifier2, 1, 2), Some(emit_code2));

// try changing the cli version (should not load previous ones)
let cache = EmitCache {
Expand All @@ -235,8 +275,8 @@ mod test {
emit_failed_flag: Default::default(),
mode: Mode::Normal,
};
assert_eq!(cache.get_emit_code(&specifier1, 10), None);
cache.set_emit_code(&specifier1, 5, emit_code1.as_bytes());
assert_eq!(cache.get_emit_code(&specifier1, 1, 10), None);
cache.set_emit_code(&specifier1, 1, 5, emit_code1.as_bytes());

// recreating the cache should still load the data because the CLI version is the same
let cache = EmitCache {
Expand All @@ -247,12 +287,21 @@ mod test {
emit_failed_flag: Default::default(),
mode: Mode::Normal,
};
assert_eq!(cache.get_emit_code(&specifier1, 5), Some(emit_code1));
assert_eq!(cache.get_emit_code(&specifier1, 1, 5), Some(emit_code1));

// adding when already exists should not cause issue
let emit_code3 = "asdf".to_string();
cache.set_emit_code(&specifier1, 20, emit_code3.as_bytes());
assert_eq!(cache.get_emit_code(&specifier1, 5), None);
assert_eq!(cache.get_emit_code(&specifier1, 20), Some(emit_code3));
cache.set_emit_code(&specifier1, 1, 20, emit_code3.as_bytes());
assert_eq!(cache.get_emit_code(&specifier1, 1, 5), None);
assert_eq!(
cache.get_emit_code(&specifier1, 1, 20).as_deref(),
Some(&*emit_code3)
);

// different transpile options should not conflict
let emit_code4 = "qwerty".to_string();
cache.set_emit_code(&specifier1, 2, 20, emit_code4.as_bytes());
assert_eq!(cache.get_emit_code(&specifier1, 1, 20), Some(emit_code3));
assert_eq!(cache.get_emit_code(&specifier1, 2, 20), Some(emit_code4));
}
}
18 changes: 14 additions & 4 deletions libs/resolver/emit.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,11 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: EmitterSys>
.transpile_options()?;
let source_hash =
self.get_source_hash(module_kind, transpile_and_emit_options, source);
Ok(self.emit_cache.get_emit_code(specifier, source_hash))
Ok(self.emit_cache.get_emit_code(
specifier,
transpile_and_emit_options.pre_computed_hash,
source_hash,
))
}

pub async fn maybe_emit_source(
Expand Down Expand Up @@ -208,6 +212,7 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: EmitterSys>
let transpiled_source = emit()?;
helper.post_emit_parsed_source(
&specifier,
transpile_and_emit_options,
&transpiled_source,
source_hash,
);
Expand Down Expand Up @@ -263,6 +268,7 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: EmitterSys>
.text;
helper.post_emit_parsed_source(
specifier,
transpile_and_emit_options,
&transpiled_source,
source_hash,
);
Expand Down Expand Up @@ -487,9 +493,11 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: EmitterSys>
.0
.get_source_hash(module_kind, transpile_and_emit_options, source);

if let Some(emit_code) =
self.0.emit_cache.get_emit_code(specifier, source_hash)
{
if let Some(emit_code) = self.0.emit_cache.get_emit_code(
specifier,
transpile_and_emit_options.pre_computed_hash,
source_hash,
) {
PreEmitResult::Cached(emit_code)
} else {
PreEmitResult::NotCached { source_hash }
Expand All @@ -499,11 +507,13 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: EmitterSys>
pub fn post_emit_parsed_source(
&self,
specifier: &Url,
transpile_and_emit_options: &TranspileAndEmitOptions,
transpiled_source: &str,
source_hash: u64,
) {
self.0.emit_cache.set_emit_code(
specifier,
transpile_and_emit_options.pre_computed_hash,
source_hash,
transpiled_source.as_bytes(),
);
Expand Down
Loading