diff --git a/apps/demo/src/main.ts b/apps/demo/src/main.ts index b48f25917..07085041e 100644 --- a/apps/demo/src/main.ts +++ b/apps/demo/src/main.ts @@ -79,6 +79,7 @@ const poolManager = (() => { const manager = createWorkerAPI({ theme: DEFAULT_THEMES, langs: ['typescript', 'tsx'], + preferredHighlighter: 'shiki-wasm', }); void manager.initialize().then(() => { console.log('WorkerPoolManager initialized, with:', manager.getStats()); diff --git a/apps/docs/app/docs/ReactAPI/constants.ts b/apps/docs/app/docs/ReactAPI/constants.ts index 1348a25bf..9b11153c5 100644 --- a/apps/docs/app/docs/ReactAPI/constants.ts +++ b/apps/docs/app/docs/ReactAPI/constants.ts @@ -44,6 +44,11 @@ interface DiffOptions { // 'dark' or 'light' - forces specific theme themeType: 'system', + // Choose the Shiki engine: + // 'shiki-js' (default) - JavaScript regex engine + // 'shiki-wasm' - WASM Oniguruma engine + preferredHighlighter: 'shiki-js', + // ───────────────────────────────────────────────────────────── // DIFF DISPLAY // ───────────────────────────────────────────────────────────── @@ -482,6 +487,11 @@ interface FileOptions { // 'dark' or 'light' - forces specific theme themeType: 'system', + // Choose the Shiki engine: + // 'shiki-js' (default) - JavaScript regex engine + // 'shiki-wasm' - WASM Oniguruma engine + preferredHighlighter: 'shiki-js', + // ───────────────────────────────────────────────────────────── // LAYOUT & DISPLAY // ───────────────────────────────────────────────────────────── diff --git a/apps/docs/app/docs/VanillaAPI/constants.ts b/apps/docs/app/docs/VanillaAPI/constants.ts index 3446f3267..297e78097 100644 --- a/apps/docs/app/docs/VanillaAPI/constants.ts +++ b/apps/docs/app/docs/VanillaAPI/constants.ts @@ -115,6 +115,11 @@ const instance = new FileDiff({ // 'dark' or 'light' - forces specific theme themeType: 'system', + // Choose the Shiki engine: + // 'shiki-js' (default) - JavaScript regex engine + // 'shiki-wasm' - WASM Oniguruma engine + preferredHighlighter: 'shiki-js', + // ───────────────────────────────────────────────────────────── // DIFF DISPLAY // ───────────────────────────────────────────────────────────── @@ -337,6 +342,11 @@ const instance = new File({ // 'dark' or 'light' - forces specific theme themeType: 'system', + // Choose the Shiki engine: + // 'shiki-js' (default) - JavaScript regex engine + // 'shiki-wasm' - WASM Oniguruma engine + preferredHighlighter: 'shiki-js', + // ───────────────────────────────────────────────────────────── // LAYOUT & DISPLAY // ───────────────────────────────────────────────────────────── diff --git a/apps/docs/app/docs/WorkerPool/constants.ts b/apps/docs/app/docs/WorkerPool/constants.ts index 107ce1b96..098a7f385 100644 --- a/apps/docs/app/docs/WorkerPool/constants.ts +++ b/apps/docs/app/docs/WorkerPool/constants.ts @@ -292,6 +292,8 @@ export function HighlightProvider({ children }: { children: ReactNode }) { }} highlighterOptions={{ theme: { dark: 'pierre-dark', light: 'pierre-light' }, + // Optional: pick the Shiki engine ('shiki-js' is default) + // preferredHighlighter: 'shiki-wasm', // Optionally preload languages to avoid lazy-loading delays langs: ['typescript', 'javascript', 'css', 'html'], }} @@ -363,6 +365,8 @@ const workerPool = getOrCreateWorkerPoolSingleton({ }, highlighterOptions: { theme: { dark: 'pierre-dark', light: 'pierre-light' }, + // Optional: pick the Shiki engine ('shiki-js' is default) + // preferredHighlighter: 'shiki-wasm', // Optionally preload languages to avoid lazy-loading delays langs: ['typescript', 'javascript', 'css', 'html'], }, @@ -418,6 +422,7 @@ new WorkerPoolManager(poolOptions, highlighterOptions) // - theme?: DiffsThemeNames | ThemesType - Theme name or { dark, light } object // - lineDiffType?: 'word' | 'word-alt' | 'char' - How to diff lines (default: 'word-alt') // - tokenizeMaxLineLength?: number - Max line length to tokenize (default: 1000) +// - preferredHighlighter?: 'shiki-js' | 'shiki-wasm' - Highlighter engine (default: 'shiki-js') // - langs?: SupportedLanguages[] - Array of languages to preload // Methods: diff --git a/apps/docs/app/docs/WorkerPool/content.mdx b/apps/docs/app/docs/WorkerPool/content.mdx index 76ebc06be..efcf98a35 100644 --- a/apps/docs/app/docs/WorkerPool/content.mdx +++ b/apps/docs/app/docs/WorkerPool/content.mdx @@ -125,7 +125,9 @@ you can then manually pass to all your File/FileDiff instances. be ignored. To change render options after WorkerPoolManager instantiates, use the `setRenderOptions()` method on the `WorkerPoolManager`. **Note:** Changing render options will force all mounted components to re-render and will clear - the render cache. + the render cache. If you need to control which Shiki engine is used, set + `preferredHighlighter` when initializing the pool (`'shiki-js'` by default, + `'shiki-wasm'` optional). #### React diff --git a/bun.lock b/bun.lock index e0b6ef335..310b372b4 100644 --- a/bun.lock +++ b/bun.lock @@ -24,7 +24,6 @@ "version": "0.0.0", "dependencies": { "@pierre/diffs": "workspace:*", - "@pierre/icons": "catalog:", "react": "catalog:", "react-dom": "catalog:", "shiki": "catalog:", @@ -98,10 +97,8 @@ }, "packages/diffs": { "name": "@pierre/diffs", - "version": "1.1.0-beta.9", + "version": "1.1.0-beta.10", "dependencies": { - "@shikijs/core": "^3.0.0", - "@shikijs/engine-javascript": "^3.0.0", "@shikijs/transformers": "^3.0.0", "diff": "catalog:", "hast-util-to-html": "catalog:", diff --git a/packages/diffs/package.json b/packages/diffs/package.json index cd15d8955..282d0e23c 100644 --- a/packages/diffs/package.json +++ b/packages/diffs/package.json @@ -64,8 +64,6 @@ "prepublishOnly": "bun run build" }, "dependencies": { - "@shikijs/core": "^3.0.0", - "@shikijs/engine-javascript": "^3.0.0", "@shikijs/transformers": "^3.0.0", "diff": "catalog:", "hast-util-to-html": "catalog:", diff --git a/packages/diffs/src/highlighter/shared_highlighter.ts b/packages/diffs/src/highlighter/shared_highlighter.ts index e8f4bdaa1..0299e9b43 100644 --- a/packages/diffs/src/highlighter/shared_highlighter.ts +++ b/packages/diffs/src/highlighter/shared_highlighter.ts @@ -1,8 +1,13 @@ -import { createHighlighter, createJavaScriptRegexEngine } from 'shiki'; +import { + createHighlighter, + createJavaScriptRegexEngine, + createOnigurumaEngine, +} from 'shiki'; import type { DiffsHighlighter, DiffsThemeNames, + HighlighterTypes, SupportedLanguages, ThemeRegistrationResolved, } from '../types'; @@ -25,16 +30,21 @@ let highlighter: CachedOrLoadingHighlighterType; interface HighlighterOptions { themes: DiffsThemeNames[]; langs: SupportedLanguages[]; + preferredHighlighter?: HighlighterTypes; } export async function getSharedHighlighter({ themes, langs, + preferredHighlighter = 'shiki-js', }: HighlighterOptions): Promise { highlighter ??= createHighlighter({ themes: [], langs: ['text'], - engine: createJavaScriptRegexEngine(), + engine: + preferredHighlighter === 'shiki-wasm' + ? createOnigurumaEngine(import('shiki/wasm')) + : createJavaScriptRegexEngine(), }) as Promise; const instance = isHighlighterLoading(highlighter) diff --git a/packages/diffs/src/renderers/DiffHunksRenderer.ts b/packages/diffs/src/renderers/DiffHunksRenderer.ts index e3e61343a..1c02cebc8 100644 --- a/packages/diffs/src/renderers/DiffHunksRenderer.ts +++ b/packages/diffs/src/renderers/DiffHunksRenderer.ts @@ -104,7 +104,7 @@ interface ProcessContext { } type OptionsWithDefaults = Required< - Omit + Omit >; export interface HunksRenderResult { @@ -513,6 +513,8 @@ export class DiffHunksRenderer { result: ThemedDiffResult, options: RenderDiffOptions ): void { + // NOTE(amadeus): This is a bad assumption, and I should figure out + // something better... // If renderCache was blown away, we can assume we've run cleanUp() if (this.renderCache == null) { return; diff --git a/packages/diffs/src/shiki-stream/stream.ts b/packages/diffs/src/shiki-stream/stream.ts index 9cfd3244f..629beab86 100644 --- a/packages/diffs/src/shiki-stream/stream.ts +++ b/packages/diffs/src/shiki-stream/stream.ts @@ -1,4 +1,4 @@ -import type { ThemedToken } from '@shikijs/core'; +import type { ThemedToken } from 'shiki/core'; import { ShikiStreamTokenizer } from './tokenizer'; import type { CodeToTokenTransformStreamOptions, RecallToken } from './types'; diff --git a/packages/diffs/src/shiki-stream/tokenizer.ts b/packages/diffs/src/shiki-stream/tokenizer.ts index 1399f790d..7e41d4ca7 100644 --- a/packages/diffs/src/shiki-stream/tokenizer.ts +++ b/packages/diffs/src/shiki-stream/tokenizer.ts @@ -1,4 +1,4 @@ -import type { GrammarState, ThemedToken } from '@shikijs/core'; +import type { GrammarState, ThemedToken } from 'shiki/core'; import type { ShikiStreamTokenizerEnqueueResult, diff --git a/packages/diffs/src/shiki-stream/types.ts b/packages/diffs/src/shiki-stream/types.ts index 68d526c7d..37c174c3e 100644 --- a/packages/diffs/src/shiki-stream/types.ts +++ b/packages/diffs/src/shiki-stream/types.ts @@ -3,7 +3,7 @@ import type { HighlighterCore, HighlighterGeneric, ThemedToken, -} from '@shikijs/core'; +} from 'shiki/core'; /** * A special token that indicates the number of tokens to be removed from the previous streamed tokens. diff --git a/packages/diffs/src/types.ts b/packages/diffs/src/types.ts index 49c22d3a0..6318eb2ae 100644 --- a/packages/diffs/src/types.ts +++ b/packages/diffs/src/types.ts @@ -32,6 +32,8 @@ export interface FileContents { cacheKey?: string; } +export type HighlighterTypes = 'shiki-js' | 'shiki-wasm'; + export type { BundledLanguage, CodeToHastOptions, @@ -350,6 +352,7 @@ export interface BaseCodeOptions { disableVirtualizationBuffers?: boolean; // Shiki config options, ignored if you're using a WorkerPoolManager + preferredHighlighter?: HighlighterTypes; useCSSClasses?: boolean; tokenizeMaxLineLength?: number; diff --git a/packages/diffs/src/utils/getHighlighterOptions.ts b/packages/diffs/src/utils/getHighlighterOptions.ts index 6a29e412f..8f0982829 100644 --- a/packages/diffs/src/utils/getHighlighterOptions.ts +++ b/packages/diffs/src/utils/getHighlighterOptions.ts @@ -1,21 +1,29 @@ -import type { DiffsThemeNames, SupportedLanguages, ThemesType } from '../types'; +import type { + DiffsThemeNames, + HighlighterTypes, + SupportedLanguages, + ThemesType, +} from '../types'; import { getThemes } from './getThemes'; interface HighlighterOptionsShape { theme?: DiffsThemeNames | ThemesType; + preferredHighlighter?: HighlighterTypes; } interface GetHighlighterOptionsReturn { langs: SupportedLanguages[]; themes: DiffsThemeNames[]; + preferredHighlighter: HighlighterTypes; } export function getHighlighterOptions( lang: SupportedLanguages | undefined, - options: HighlighterOptionsShape + { theme, preferredHighlighter = 'shiki-js' }: HighlighterOptionsShape ): GetHighlighterOptionsReturn { return { langs: [lang ?? 'text'], - themes: getThemes(options.theme), + themes: getThemes(theme), + preferredHighlighter, }; } diff --git a/packages/diffs/src/worker/WorkerPoolManager.ts b/packages/diffs/src/worker/WorkerPoolManager.ts index 27208d970..62e6b49c3 100644 --- a/packages/diffs/src/worker/WorkerPoolManager.ts +++ b/packages/diffs/src/worker/WorkerPoolManager.ts @@ -13,6 +13,7 @@ import type { DiffsHighlighter, FileContents, FileDiffMetadata, + HighlighterTypes, HunkExpansionRegion, RenderDiffOptions, RenderDiffResult, @@ -69,6 +70,7 @@ interface ThemeSubscriber { export class WorkerPoolManager { private highlighter: DiffsHighlighter | undefined; + private readonly preferredHighlighter: HighlighterTypes; private renderOptions: WorkerRenderingOptions; private initialized: Promise | boolean = false; private workers: ManagedWorker[] = []; @@ -96,8 +98,10 @@ export class WorkerPoolManager { theme = DEFAULT_THEMES, lineDiffType = 'word-alt', tokenizeMaxLineLength = 1000, + preferredHighlighter = 'shiki-js', }: WorkerInitializationRenderOptions ) { + this.preferredHighlighter = preferredHighlighter; this.renderOptions = { theme, lineDiffType, tokenizeMaxLineLength }; this.fileCache = new LRUMapPkg.LRUMap(options.totalASTLRUCacheSize ?? 100); this.diffCache = new LRUMapPkg.LRUMap(options.totalASTLRUCacheSize ?? 100); @@ -182,7 +186,11 @@ export class WorkerPoolManager { await this.setRenderOptionsOnWorkers(newRenderOptions, resolvedThemes); } else { const [highlighter] = await Promise.all([ - getSharedHighlighter({ themes: themeNames, langs: ['text'] }), + getSharedHighlighter({ + themes: themeNames, + langs: ['text'], + preferredHighlighter: this.preferredHighlighter, + }), this.setRenderOptionsOnWorkers(newRenderOptions, resolvedThemes), ]); this.highlighter = highlighter; @@ -330,7 +338,11 @@ export class WorkerPoolManager { } const [highlighter] = await Promise.all([ - getSharedHighlighter({ themes, langs: ['text', ...languages] }), + getSharedHighlighter({ + themes, + langs: ['text', ...languages], + preferredHighlighter: this.preferredHighlighter, + }), this.initializeWorkers(resolvedThemes, resolvedLanguages), ]); @@ -400,6 +412,7 @@ export class WorkerPoolManager { type: 'initialize', id, renderOptions: this.renderOptions, + preferredHighlighter: this.preferredHighlighter, resolvedThemes, resolvedLanguages, }, diff --git a/packages/diffs/src/worker/types.ts b/packages/diffs/src/worker/types.ts index 9b79d49d3..d3be5dfa5 100644 --- a/packages/diffs/src/worker/types.ts +++ b/packages/diffs/src/worker/types.ts @@ -2,6 +2,7 @@ import type { DiffsThemeNames, FileContents, FileDiffMetadata, + HighlighterTypes, LanguageRegistration, LineDiffTypes, RenderDiffOptions, @@ -59,6 +60,7 @@ export interface InitializeWorkerRequest { type: 'initialize'; id: WorkerRequestId; renderOptions: WorkerRenderingOptions; + preferredHighlighter: HighlighterTypes; resolvedThemes: ThemeRegistrationResolved[]; resolvedLanguages?: ResolvedLanguage[]; } @@ -152,6 +154,7 @@ export interface WorkerPoolOptions { export interface WorkerInitializationRenderOptions extends Partial { langs?: SupportedLanguages[]; + preferredHighlighter?: HighlighterTypes; } export interface InitializeWorkerTask { diff --git a/packages/diffs/src/worker/worker.ts b/packages/diffs/src/worker/worker.ts index 7cfcbfda3..6e317526e 100644 --- a/packages/diffs/src/worker/worker.ts +++ b/packages/diffs/src/worker/worker.ts @@ -1,11 +1,13 @@ -import { createJavaScriptRegexEngine } from '@shikijs/engine-javascript'; -import { createHighlighterCoreSync } from 'shiki/core'; +import { createHighlighterCore } from 'shiki/core'; +import { createJavaScriptRegexEngine } from 'shiki/engine/javascript'; +import { createOnigurumaEngine } from 'shiki/engine/oniguruma'; import { DEFAULT_THEMES } from '../constants'; import { attachResolvedLanguages } from '../highlighter/languages/attachResolvedLanguages'; import { attachResolvedThemes } from '../highlighter/themes/attachResolvedThemes'; import type { DiffsHighlighter, + HighlighterTypes, RenderDiffOptions, RenderFileOptions, ThemedDiffResult, @@ -27,7 +29,7 @@ import type { WorkerRequestId, } from './types'; -let highlighter: DiffsHighlighter | undefined; +let highlighter: Promise | DiffsHighlighter | undefined; let renderOptions: WorkerRenderingOptions = { theme: DEFAULT_THEMES, tokenizeMaxLineLength: 1000, @@ -40,21 +42,23 @@ self.addEventListener('error', (event) => { // Handle incoming messages from the main thread self.addEventListener('message', (event: MessageEvent) => { - const request = event.data; + void handleMessage(event.data); +}); +async function handleMessage(request: WorkerRequest) { try { switch (request.type) { case 'initialize': - handleInitialize(request); + await handleInitialize(request); break; case 'set-render-options': - handleSetRenderOptions(request); + await handleSetRenderOptions(request); break; case 'file': - handleRenderFile(request); + await handleRenderFile(request); break; case 'diff': - handleRenderDiff(request); + await handleRenderDiff(request); break; default: throw new Error( @@ -65,15 +69,19 @@ self.addEventListener('message', (event: MessageEvent) => { console.error('Worker error:', error); sendError(request.id, error); } -}); +} -function handleInitialize({ +async function handleInitialize({ id, renderOptions: options, + preferredHighlighter, resolvedThemes, resolvedLanguages, -}: InitializeWorkerRequest) { - const highlighter = getHighlighter(); +}: InitializeWorkerRequest): Promise { + let highlighter = getHighlighter(preferredHighlighter); + if ('then' in highlighter) { + highlighter = await highlighter; + } attachResolvedThemes(resolvedThemes, highlighter); if (resolvedLanguages != null) { attachResolvedLanguages(resolvedLanguages, highlighter); @@ -87,12 +95,15 @@ function handleInitialize({ } satisfies InitializeSuccessResponse); } -function handleSetRenderOptions({ +async function handleSetRenderOptions({ id, renderOptions: options, resolvedThemes, -}: SetRenderOptionsWorkerRequest) { - const highlighter = getHighlighter(); +}: SetRenderOptionsWorkerRequest): Promise { + let highlighter = getHighlighter(); + if ('then' in highlighter) { + highlighter = await highlighter; + } attachResolvedThemes(resolvedThemes, highlighter); renderOptions = options; postMessage({ @@ -103,8 +114,15 @@ function handleSetRenderOptions({ }); } -function handleRenderFile({ id, file, resolvedLanguages }: RenderFileRequest) { - const highlighter = getHighlighter(); +async function handleRenderFile({ + id, + file, + resolvedLanguages, +}: RenderFileRequest): Promise { + let highlighter = getHighlighter(); + if ('then' in highlighter) { + highlighter = await highlighter; + } // Load resolved languages if provided if (resolvedLanguages != null) { attachResolvedLanguages(resolvedLanguages, highlighter); @@ -120,8 +138,15 @@ function handleRenderFile({ id, file, resolvedLanguages }: RenderFileRequest) { ); } -function handleRenderDiff({ id, diff, resolvedLanguages }: RenderDiffRequest) { - const highlighter = getHighlighter(); +async function handleRenderDiff({ + id, + diff, + resolvedLanguages, +}: RenderDiffRequest): Promise { + let highlighter = getHighlighter(); + if ('then' in highlighter) { + highlighter = await highlighter; + } // Load resolved languages if provided if (resolvedLanguages != null) { attachResolvedLanguages(resolvedLanguages, highlighter); @@ -130,12 +155,17 @@ function handleRenderDiff({ id, diff, resolvedLanguages }: RenderDiffRequest) { sendDiffSuccess(id, result, renderOptions); } -function getHighlighter(): DiffsHighlighter { - highlighter ??= createHighlighterCoreSync({ +function getHighlighter( + preferredHighlighter: HighlighterTypes = 'shiki-js' +): Promise | DiffsHighlighter { + highlighter ??= createHighlighterCore({ themes: [], langs: [], - engine: createJavaScriptRegexEngine(), - }) as DiffsHighlighter; + engine: + preferredHighlighter === 'shiki-wasm' + ? createOnigurumaEngine(import('shiki/wasm')) + : createJavaScriptRegexEngine(), + }) as Promise; return highlighter; } diff --git a/packages/diffs/test/sharedHighlighter.test.ts b/packages/diffs/test/sharedHighlighter.test.ts new file mode 100644 index 000000000..bcdedd917 --- /dev/null +++ b/packages/diffs/test/sharedHighlighter.test.ts @@ -0,0 +1,50 @@ +import { afterEach, describe, expect, test } from 'bun:test'; + +import { + disposeHighlighter, + getHighlighterIfLoaded, + getSharedHighlighter, +} from '../src/highlighter/shared_highlighter'; + +afterEach(async () => { + await disposeHighlighter(); +}); + +describe('shared highlighter engine selection', () => { + test('returns a cached highlighter instance until disposed', async () => { + const first = await getSharedHighlighter({ + themes: ['pierre-dark'], + langs: ['text'], + preferredHighlighter: 'shiki-js', + }); + + const second = await getSharedHighlighter({ + themes: ['pierre-dark'], + langs: ['text'], + preferredHighlighter: 'shiki-wasm', + }); + + expect(second).toBe(first); + expect(getHighlighterIfLoaded()).toBe(first); + }); + + test('can dispose and reinitialize with a different preferredHighlighter', async () => { + const jsHighlighter = await getSharedHighlighter({ + themes: ['pierre-dark'], + langs: ['text'], + preferredHighlighter: 'shiki-js', + }); + + await disposeHighlighter(); + expect(getHighlighterIfLoaded()).toBeUndefined(); + + const wasmHighlighter = await getSharedHighlighter({ + themes: ['pierre-dark'], + langs: ['text'], + preferredHighlighter: 'shiki-wasm', + }); + + expect(wasmHighlighter).not.toBe(jsHighlighter); + expect(getHighlighterIfLoaded()).toBe(wasmHighlighter); + }); +});