Introduce grammar patch from the simplify branch

src/core/registry.js

@@ -1,8 +1,8 @@
 import { kebabToCamelCase } from '../shared/util.js';
-import { cloneGrammar } from '../util/extend.js';
+import { extend } from '../util/extend.js';
+import { grammarPatch } from '../util/grammar-patch.js';
 import { forEach, toArray } from '../util/iterables.js';
-import { extend } from '../util/language-util.js';
-import { defineLazyProperty } from '../util/objects.js';
+import { deepClone, defineLazyProperty } from '../util/objects.js';
 
 /**
  * TODO: docs
@@ -221,7 +221,7 @@ export class Registry {
 
 		const base = entry?.proto.base;
 		// We need this so that any code modifying the base grammar doesn't affect other instances
-		const baseGrammar = base && cloneGrammar(required(base.id), base.id);
+		const baseGrammar = base && deepClone(required(base.id));
 
 		const requiredLanguages = toArray(
 			/** @type {LanguageProto | LanguageProto[] | undefined} */ (entry?.proto.require)
@@ -240,7 +240,7 @@ export class Registry {
 		else {
 			const options = {
 				getOptionalLanguage: id => this.getLanguage(id),
-				extend: (id, ref) => extend(required(id), id, ref),
+				extend: (id, ref) => extend(required(id), ref),
 				...(baseGrammar && { base: baseGrammar }),
 				...(requiredLanguages.length && { languages }),
 			};
@@ -249,10 +249,10 @@ export class Registry {
 		}
 
 		if (baseGrammar) {
-			evaluatedGrammar = extend(baseGrammar, base.id, evaluatedGrammar);
+			evaluatedGrammar = extend(baseGrammar, evaluatedGrammar);
 		}
 
-		return (entry.evaluatedGrammar = evaluatedGrammar);
+		return (entry.evaluatedGrammar = grammarPatch(evaluatedGrammar));
 	}
 }
 

src/core/tokenize/tokenize.js

@@ -42,13 +42,21 @@ export function tokenize (text, grammar) {
 	const tokenList = new LinkedList();
 	tokenList.addAfter(tokenList.head, text);
 
-	_matchGrammar.call(prism, text, tokenList, grammar, tokenList.head, 0);
+	_matchGrammar.call(
+		prism,
+		text,
+		tokenList,
+		/** @type {GrammarTokens} */ (grammar),
+		tokenList.head,
+		0
+	);
 
 	return tokenList.toArray();
 }
 
 /**
  * @typedef {import('../../types.d.ts').TokenStream} TokenStream
  * @typedef {import('../../types.d.ts').Grammar} Grammar
+ * @typedef {import('../../types.d.ts').GrammarTokens} GrammarTokens
  * @typedef {import('../prism.js').Prism} Prism
  */

src/languages/c.js

@@ -73,7 +73,7 @@ export default {
 		/* OpenCL host API */
 		const extensions = getOptionalLanguage('opencl-extensions');
 		if (extensions) {
-			insertBefore(base, 'keyword', extensions);
+			insertBefore(base, 'keyword', /** @type {GrammarTokens} */ (extensions));
 			delete base['type-opencl-host-cpp'];
 		}
 
@@ -104,4 +104,5 @@ export default {
 
 /**
  * @typedef {import('../types.d.ts').GrammarToken} GrammarToken
+ * @typedef {import('../types.d.ts').GrammarTokens} GrammarTokens
  */

src/languages/cpp.js

@@ -103,7 +103,11 @@ export default {
 		/* OpenCL host API */
 		const extensions = getOptionalLanguage('opencl-extensions');
 		if (extensions) {
-			insertBefore(cpp, 'keyword', extensions);
+			insertBefore(
+				cpp,
+				'keyword',
+				/** @type {import('../types.d.ts').GrammarTokens} */ (extensions)
+			);
 		}
 
 		const baseInside = { ...cpp };

src/languages/css.js

@@ -89,7 +89,11 @@ export default {
 
 		const extras = getOptionalLanguage('css-extras');
 		if (extras) {
-			insertBefore(css, 'function', extras);
+			insertBefore(
+				css,
+				'function',
+				/** @type {import('../types.d.ts').GrammarTokens} */ (extras)
+			);
 		}
 
 		return css;

src/languages/hlsl.js

@@ -12,7 +12,9 @@ export default {
 			// https://docs.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl-appendix-keywords
 			// https://docs.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl-appendix-reserved-words
 			'class-name': [
-				...toArray(base['class-name']),
+				...toArray(
+					/** @type {import('../types.d.ts').GrammarTokens} */ (base)['class-name']
+				),
 				/\b(?:AppendStructuredBuffer|BlendState|Buffer|ByteAddressBuffer|CompileShader|ComputeShader|ConsumeStructuredBuffer|DepthStencilState|DepthStencilView|DomainShader|GeometryShader|Hullshader|InputPatch|LineStream|OutputPatch|PixelShader|PointStream|RWBuffer|RWByteAddressBuffer|RWStructuredBuffer|RWTexture(?:1D|1DArray|2D|2DArray|3D)|RasterizerState|RenderTargetView|SamplerComparisonState|SamplerState|StructuredBuffer|Texture(?:1D|1DArray|2D|2DArray|2DMS|2DMSArray|3D|Cube|CubeArray)|TriangleStream|VertexShader)\b/,
 			],
 			'keyword': [

src/languages/php.js

@@ -352,7 +352,7 @@ export default {
 
 		const extras = getOptionalLanguage('php-extras');
 		if (extras) {
-			insertBefore(php, 'variable', extras);
+			insertBefore(php, 'variable', /** @type {GrammarTokens} */ (extras));
 		}
 
 		const embedded = embeddedIn('markup');
@@ -376,4 +376,5 @@ export default {
 
 /**
  * @typedef {import('../types.d.ts').Grammar} Grammar
+ * @typedef {import('../types.d.ts').GrammarTokens} GrammarTokens
  */

src/languages/typescript.js

@@ -8,8 +8,7 @@ export default {
 	require: javascript,
 	alias: 'ts',
 	grammar ({ extend }) {
-		/** @type {import('../types.d.ts').Grammar} */
-		const typeInside = {};
+		const typeInside = /** @type {import('../types.d.ts').Grammar} */ ({});
 
 		const typescript = extend('javascript', {
 			'class-name': {
@@ -24,7 +23,7 @@ export default {
 		});
 
 		typescript.keyword = [
-			...toArray(typescript.keyword),
+			...toArray(/** @type {import('../types.d.ts').GrammarTokens} */ (typescript).keyword),
 
 			// The keywords TypeScript adds to JavaScript
 			/\b(?:abstract|declare|is|keyof|out|readonly|require|satisfies)\b/,

src/types.d.ts

@@ -198,7 +198,35 @@ export type GrammarSpecial = {
 	$tokenize?: (code: string, grammar: Grammar, Prism: Prism) => TokenStream;
 };
 
-export type Grammar = GrammarTokens & GrammarSpecial;
+/**
+ * Tokens within $insert
+ */
+export type InsertableToken = (RegExpLike | GrammarToken | (RegExpLike | GrammarToken)[]) & {
+	$before?: TokenName | TokenName[];
+	$after?: TokenName | TokenName[];
+};
+
+/**
+ * A grammar that is defined as its delta from another grammar.
+ */
+export type GrammarPatch = {
+	$insert?: Partial<Record<TokenName, InsertableToken>>;
+	$insertBefore?: Partial<Record<TokenName, GrammarTokens>>;
+	$insertAfter?: Partial<Record<TokenName, GrammarTokens>>;
+	$delete?: TokenName[];
+	$merge?: Partial<
+		Record<TokenName, Partial<Omit<GrammarToken, 'pattern'>> & { pattern?: RegExpLike }>
+	>;
+};
+
+export interface Grammar extends GrammarSpecial, GrammarPatch {
+	[token: string]:
+		| RegExpLike
+		| GrammarToken
+		| (RegExpLike | GrammarToken)[]
+		| GrammarSpecial[keyof GrammarSpecial]
+		| GrammarPatch[keyof GrammarPatch];
+}
 
 export interface PlainObject {
 	[key: string]: unknown;

src/util/extend.js

@@ -1,3 +1,5 @@
+import { betterAssign, deepClone } from './objects.js';
+
 /**
  * Creates a deep copy of the language with the given id and appends the given tokens.
  *
@@ -13,9 +15,8 @@
  * Therefore, it is encouraged to order overwriting tokens according to the positions of the overwritten tokens.
  * Furthermore, all non-overwriting tokens should be placed after the overwriting ones.
  *
- * @param {Grammar} grammar The grammar of the language to extend.
- * @param {string} id The id of the language to extend.
- * @param {Grammar} reDef The new tokens to append.
+ * @param {Grammar} base The grammar of the language to extend.
+ * @param {Grammar} grammar The new tokens to append.
  * @returns {Grammar} The new language created.
  * @example
  * Prism.languages['css-with-colors'] = Prism.languages.extend('css', {
@@ -26,120 +27,67 @@
  *     'color': /\b(?:red|green|blue)\b/
  * });
  */
-export function extend (grammar, id, reDef) {
-	const lang = cloneGrammar(grammar, id);
+export function extend (base, grammar) {
+	const lang = deepClone(base);
+
+	for (const key in grammar) {
+		if (typeof key !== 'string' || key.startsWith('$')) {
+			// ignore special keys
+			continue;
+		}
 
-	for (const key in reDef) {
-		lang[key] = reDef[key];
+		lang[key] = grammar[key];
 	}
 
-	return lang;
-}
+	if (grammar.$insertBefore) {
+		lang.$insertBefore = betterAssign(lang.$insertBefore ?? {}, grammar.$insertBefore);
+	}
 
-/**
- * @param {Grammar} grammar
- * @param {string} id
- * @returns {Grammar}
- */
-export function cloneGrammar (grammar, id) {
-	/** @type {Grammar} */
-	const result = {};
+	if (grammar.$insertAfter) {
+		lang.$insertAfter = betterAssign(lang.$insertAfter ?? {}, grammar.$insertAfter);
+	}
 
-	/** @type {Map<Grammar, Grammar>} */
-	const visited = new Map();
+	if (grammar.$insert) {
+		// Syntactic sugar for $insertBefore/$insertAfter
+		for (const tokenName in grammar.$insert) {
+			const def = grammar.$insert[tokenName];
+			const { $before, $after, ...token } = def;
+			const relToken = $before || $after;
+			const all = $before ? '$insertBefore' : '$insertAfter';
+			lang[all] ??= {};
 
-	/**
-	 * @param {GrammarToken | RegExpLike} value
-	 */
-	function cloneToken (value) {
-		if (!value.pattern) {
-			return value;
-		}
-		else {
-			/** @type {GrammarToken} */
-			const copy = { pattern: value.pattern };
-			if (value.lookbehind) {
-				copy.lookbehind = value.lookbehind;
+			if (Array.isArray(relToken)) {
+				// Insert in multiple places
+				for (const t of relToken) {
+					lang[all][t][tokenName] = token;
+				}
 			}
-			if (value.greedy) {
-				copy.greedy = value.greedy;
+			else if (relToken) {
+				(lang[all][relToken] ??= {})[tokenName] = token;
 			}
-			if (value.alias) {
-				copy.alias = Array.isArray(value.alias) ? [...value.alias] : value.alias;
+			else {
+				lang[tokenName] = token;
 			}
-			if (value.inside) {
-				copy.inside = cloneRef(value.inside);
-			}
-			return copy;
 		}
 	}
 
-	/**
-	 * @param {GrammarTokens['string']} value
-	 */
-	function cloneTokens (value) {
-		if (!value) {
-			return undefined;
-		}
-		else if (Array.isArray(value)) {
-			return value.map(cloneToken);
+	if (grammar.$delete) {
+		if (lang.$delete) {
+			// base also had $delete
+			lang.$delete.push(...grammar.$delete);
 		}
 		else {
-			return cloneToken(value);
+			lang.$delete = [...grammar.$delete];
 		}
 	}
 
-	/**
-	 * @param {string | Grammar} ref
-	 */
-	function cloneRef (ref) {
-		if (ref === id) {
-			// self ref
-			return result;
-		}
-		else if (typeof ref === 'string') {
-			return ref;
-		}
-		else {
-			return clone(ref);
-		}
+	if (grammar.$merge) {
+		lang.$merge = betterAssign(lang.$merge ?? {}, grammar.$merge);
 	}
 
-	/**
-	 * @param {Grammar} value
-	 */
-	function clone (value) {
-		let mapped = visited.get(value);
-		if (mapped === undefined) {
-			mapped = value === grammar ? result : {};
-			visited.set(value, mapped);
-
-			// tokens
-			for (const [key, tokens] of Object.entries(value)) {
-				mapped[key] = cloneTokens(/** @type {GrammarToken[]} */ (tokens));
-			}
-
-			// rest
-			const r = value.$rest;
-			if (r != null) {
-				mapped.$rest = cloneRef(r);
-			}
-
-			// tokenize
-			const t = value.$tokenize;
-			if (t) {
-				mapped.$tokenize = t;
-			}
-		}
-		return mapped;
-	}
-
-	return clone(grammar);
+	return lang;
 }
 
 /**
  * @typedef {import('../types.d.ts').Grammar} Grammar
- * @typedef {import('../types.d.ts').GrammarToken} GrammarToken
- * @typedef {import('../types.d.ts').GrammarTokens} GrammarTokens
- * @typedef {import('../types.d.ts').RegExpLike} RegExpLike
  */