Introduce the concept of base language from the simplify branch (#3990)

* Introduce the concept of base language

Co-authored-by: Lea Verou <[email protected]>

* Improve the type system

So that authors defining a language shouldn’t add `!` everywhere they refer to  `base`.

---------

Co-authored-by: Lea Verou <[email protected]>

Author: DmitrySharabin

src/core/registry.ts

@@ -1,7 +1,14 @@
 import { kebabToCamelCase } from '../shared/util';
-import { forEach } from '../util/iterables';
+import { cloneGrammar } from '../util/extend';
+import { forEach, toArray } from '../util/iterables';
 import { extend } from '../util/language-util';
-import type { ComponentProto, Grammar } from '../types';
+import type {
+	ComponentProto,
+	Grammar,
+	GrammarOptions,
+	GrammarOptionsWithBase,
+	LanguageProto,
+} from '../types';
 import type { Prism } from './prism';
 
 interface Entry {
@@ -60,6 +67,13 @@ export class Registry {
 			// add aliases
 			forEach(proto.alias, alias => this.aliasMap.set(alias, id));
 
+			if ((proto as LanguageProto).base) {
+				proto.require = [
+					(proto as LanguageProto).base as ComponentProto,
+					...toArray(proto.require),
+				];
+			}
+
 			// dependencies
 			forEach(proto.require, register);
 
@@ -152,11 +166,6 @@ export class Registry {
 			return entry.evaluatedGrammar;
 		}
 
-		if (typeof grammar === 'object') {
-			// the grammar is a simple object, so we don't need to evaluate it
-			return (entry.evaluatedGrammar = grammar);
-		}
-
 		const required = (id: string): Grammar => {
 			const grammar = this.getLanguage(id);
 			if (!grammar) {
@@ -165,10 +174,33 @@ export class Registry {
 			return grammar;
 		};
 
-		return (entry.evaluatedGrammar = grammar({
-			getLanguage: required,
-			getOptionalLanguage: id => this.getLanguage(id),
-			extend: (id, ref) => extend(required(id), id, ref),
-		}));
+		const base = (entry?.proto as LanguageProto).base;
+		// We need this so that any code modifying the base grammar doesn't affect other instances
+		const baseGrammar = base && cloneGrammar(required(base.id), base.id);
+
+		let evaluatedGrammar: Grammar;
+		if (typeof grammar === 'object') {
+			// if the grammar is an object, we can use it directly
+			evaluatedGrammar = grammar;
+		}
+		else {
+			const options: GrammarOptions = {
+				getLanguage: required,
+				getOptionalLanguage: id => this.getLanguage(id),
+				extend: (id, ref) => extend(required(id), id, ref),
+				...(baseGrammar && { base: baseGrammar }),
+			};
+
+			const grammarFn = grammar as (
+				options: GrammarOptions | GrammarOptionsWithBase
+			) => Grammar;
+			evaluatedGrammar = grammarFn(options);
+		}
+
+		if (baseGrammar) {
+			evaluatedGrammar = extend(baseGrammar, base.id, evaluatedGrammar);
+		}
+
+		return (entry.evaluatedGrammar = evaluatedGrammar);
 	}
 }

src/types.d.ts

@@ -1,22 +1,45 @@
-import type { Prism } from './core/prism';
 import type { TokenStream } from './core/classes/token';
+import type { Prism } from './core/prism';
 
 export interface GrammarOptions {
 	readonly getLanguage: (id: string) => Grammar;
 	readonly getOptionalLanguage: (id: string) => Grammar | undefined;
 	readonly extend: (id: string, ref: GrammarTokens) => Grammar;
 }
+
+// Overload for when base is required
+export interface GrammarOptionsWithBase extends GrammarOptions {
+	readonly base: Grammar;
+}
+
 export interface ComponentProtoBase<Id extends string = string> {
 	id: Id;
 	require?: ComponentProto | readonly ComponentProto[];
 	optional?: string | readonly string[];
 	alias?: string | readonly string[];
 	effect?: (Prism: Prism & { plugins: Record<KebabToCamelCase<Id>, {}> }) => () => void;
 }
-export interface LanguageProto<Id extends string = string> extends ComponentProtoBase<Id> {
+
+// For languages that extend a base language
+export interface LanguageProtoWithBase<Id extends string = string> extends ComponentProtoBase<Id> {
+	grammar: Grammar | ((options: GrammarOptionsWithBase) => Grammar);
+	plugin?: undefined;
+	base: LanguageProto; // Required base
+}
+
+// For languages that don't extend a base language
+export interface LanguageProtoWithoutBase<Id extends string = string>
+	extends ComponentProtoBase<Id> {
 	grammar: Grammar | ((options: GrammarOptions) => Grammar);
 	plugin?: undefined;
+	base?: never; // Explicitly no base allowed
 }
+
+// Union type that allows TypeScript to discriminate
+export type LanguageProto<Id extends string = string> =
+	| LanguageProtoWithBase<Id>
+	| LanguageProtoWithoutBase<Id>;
+
 type PluginType<Name extends string> = unknown;
 export interface PluginProto<Id extends string = string> extends ComponentProtoBase<Id> {
 	grammar?: undefined;

src/util/extend.ts

@@ -38,7 +38,7 @@ export function extend (grammar: Grammar, id: string, reDef: Grammar): Grammar {
 	return lang;
 }
 
-function cloneGrammar (grammar: Grammar, id: string): Grammar {
+export function cloneGrammar (grammar: Grammar, id: string): Grammar {
 	const result: Grammar = {};
 
 	const visited = new Map<Grammar, Grammar>();

tests/core/registry.ts

@@ -1,5 +1,6 @@
 import { assert } from 'chai';
 import { Prism } from '../../src/core/prism';
+import type { GrammarOptions } from '../../src/types';
 
 describe('Registry', () => {
 	it('should resolve aliases', () => {
@@ -26,7 +27,7 @@ describe('Registry', () => {
 		components.add({
 			id: 'c',
 			optional: 'b',
-			grammar ({ getOptionalLanguage }) {
+			grammar ({ getOptionalLanguage }: GrammarOptions) {
 				return getOptionalLanguage('b') ?? {};
 			},
 		});