Introduce the concept of languages from the simplify branch (#4016)

It allows referencing required languages inside grammar definitions via the `language` property of an options object passed to the `grammar()` function. So, we don't need to expose the `getLanguage()` function to grammar authors.

- Don't pass `getLanguage()` to the `grammar()` function
- Don't mutate the language proto when the base grammar is specified (firstly, it's a bug, secondly, we don't want to define the base language as a property of the `languages` object—it's for the required languages only)
- Add new utility functions for working with objects (authored by @LeaVerou)
- Improve the type system to support the updated signature of `grammar()`
- Update definitions of languages previously relied on `getLanguage()`

Co-authored-by: Lea Verou <[email protected]>

Author: DmitrySharabin

src/core/registry.js

@@ -2,6 +2,7 @@ import { kebabToCamelCase } from '../shared/util.js';
 import { cloneGrammar } from '../util/extend.js';
 import { forEach, toArray } from '../util/iterables.js';
 import { extend } from '../util/language-util.js';
+import { defineLazyProperty } from '../util/objects.js';
 
 /**
  * TODO: docs
@@ -79,15 +80,18 @@ export class Registry {
 			// @ts-ignore - alias is always there
 			forEach(proto.alias, alias => this.aliasMap.set(alias, id));
 
+			// @ts-ignore
+			const required = [...toArray(proto.require)]; // don't mutate the original array
+
 			// @ts-ignore
 			if (proto.base) {
 				// @ts-ignore
-				proto.require = [proto.base, ...toArray(proto.require)];
+				required.unshift(proto.base);
 			}
 
 			// dependencies
 			// @ts-ignore
-			forEach(proto.require, register);
+			forEach(required, register);
 
 			// add plugin namespace
 			if (proto.plugin) {
@@ -219,6 +223,14 @@ export class Registry {
 		// We need this so that any code modifying the base grammar doesn't affect other instances
 		const baseGrammar = base && cloneGrammar(required(base.id), base.id);
 
+		const requiredLanguages = toArray(
+			/** @type {LanguageProto | LanguageProto[] | undefined} */ (entry?.proto.require)
+		);
+		const languages = /** @type {Record<string, Grammar>} */ ({});
+		for (const lang of requiredLanguages) {
+			defineLazyProperty(languages, lang.id, () => required(lang.id));
+		}
+
 		/** @type {Grammar} */
 		let evaluatedGrammar;
 		if (typeof grammar === 'object') {
@@ -227,10 +239,10 @@ export class Registry {
 		}
 		else {
 			const options = {
-				getLanguage: required,
 				getOptionalLanguage: id => this.getLanguage(id),
 				extend: (id, ref) => extend(required(id), id, ref),
 				...(baseGrammar && { base: baseGrammar }),
+				...(requiredLanguages.length && { languages }),
 			};
 
 			evaluatedGrammar = grammar(/** @type {any} */ (options));
@@ -253,6 +265,7 @@ export class Registry {
 
 /**
  * @typedef {import('../types.d.ts').ComponentProto} ComponentProto
+ * @typedef {import('../types.d.ts').LanguageProto} LanguageProto
  * @typedef {import('../types.d.ts').Grammar} Grammar
  * @typedef {import('./prism.js').Prism} Prism
  */

src/languages/apex.js

@@ -5,7 +5,7 @@ import sql from './sql.js';
 export default {
 	id: 'apex',
 	require: [clike, sql],
-	grammar ({ getLanguage }) {
+	grammar ({ languages }) {
 		const keywords =
 			/\b(?:abstract|activate|(?:after|before)(?=\s+[a-z])|and|any|array|as|asc|autonomous|begin|bigdecimal|blob|boolean|break|bulk|by|byte|case|cast|catch|char|class|collect|commit|const|continue|currency|date|datetime|decimal|default|delete|desc|do|double|else|end|enum|exception|exit|export|extends|final|finally|float|for|from|get(?=\s*[{};])|global|goto|group|having|hint|if|implements|import|in|inner|insert|instanceof|int|integer|interface|into|join|like|limit|list|long|loop|map|merge|new|not|null|nulls|number|object|of|on|or|outer|override|package|parallel|pragma|private|protected|public|retrieve|return|rollback|select|set|short|sObject|sort|static|string|super|switch|synchronized|system|testmethod|then|this|throw|time|transaction|transient|trigger|try|undelete|update|upsert|using|virtual|void|webservice|when|where|while|(?:inherited|with|without)\s+sharing)\b/i;
 
@@ -31,7 +31,7 @@ export default {
 			'punctuation': /[()\[\]{};,:.<>]/,
 		};
 
-		const clike = getLanguage('clike');
+		const clike = languages.clike;
 
 		return {
 			'comment': clike.comment,

src/languages/chaiscript.js

@@ -8,9 +8,7 @@ export default {
 	id: 'chaiscript',
 	base: clike,
 	require: cpp,
-	grammar ({ base, getLanguage }) {
-		const cpp = getLanguage('cpp');
-
+	grammar ({ base, languages }) {
 		insertBefore(base, 'operator', {
 			'parameter-type': {
 				// e.g. def foo(int x, Vector y) {...}
@@ -68,7 +66,7 @@ export default {
 			],
 			'keyword':
 				/\b(?:attr|auto|break|case|catch|class|continue|def|default|else|finally|for|fun|global|if|return|switch|this|try|var|while)\b/,
-			'number': [...toArray(cpp.number), /\b(?:Infinity|NaN)\b/],
+			'number': [...toArray(languages.cpp.number), /\b(?:Infinity|NaN)\b/],
 			'operator': />>=?|<<=?|\|\||&&|:[:=]?|--|\+\+|[=!<>+\-*/%|&^]=?|[?~]|`[^`\r\n]{1,4}`/,
 		};
 	},

src/languages/javadoc.js

@@ -8,9 +8,8 @@ export default {
 	id: 'javadoc',
 	base: javadoclike,
 	require: [markup, java],
-	grammar ({ base, getLanguage }) {
-		const java = getLanguage('java');
-		const { tag, entity } = getLanguage('markup');
+	grammar ({ base, languages }) {
+		const { tag, entity } = languages.markup;
 
 		const codeLinePattern = /(^(?:[\t ]*(?:\*\s*)*))[^*\s].*$/m;
 
@@ -45,7 +44,7 @@ export default {
 						},
 					},
 					'class-name': /\b[A-Z]\w*/,
-					'keyword': java.keyword,
+					'keyword': languages.java.keyword,
 					'punctuation': /[#()[\],.]/,
 				},
 			},

src/languages/jsdoc.js

@@ -8,9 +8,8 @@ export default {
 	id: 'jsdoc',
 	base: javadoclike,
 	require: [javascript, typescript],
-	grammar ({ base, getLanguage }) {
-		const javascript = getLanguage('javascript');
-		const typescript = getLanguage('typescript');
+	grammar ({ base, languages }) {
+		const { javascript, typescript } = languages;
 
 		const type = /\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})+\}/.source;
 		const parameterPrefix = '(@(?:arg|argument|param|property)\\s+(?:' + type + '\\s+)?)';

src/languages/xml-doc.js

@@ -4,8 +4,8 @@ import markup from './markup.js';
 export default {
 	id: 'xml-doc',
 	require: markup,
-	grammar ({ getLanguage }) {
-		const tag = getLanguage('markup').tag;
+	grammar ({ languages }) {
+		const tag = languages.markup.tag;
 
 		return {
 			'slash': {

src/types.d.ts

@@ -32,16 +32,10 @@ export type HooksRemove = <Name extends keyof HookEnv>(
 export type HooksRun = <Name extends keyof HookEnv>(name: Name, env: HookEnv[Name]) => void;
 
 export interface GrammarOptions {
-	readonly getLanguage: (id: string) => Grammar;
 	readonly getOptionalLanguage: (id: string) => Grammar | undefined;
 	readonly extend: (id: string, ref: GrammarTokens) => Grammar;
 }
 
-// Overload for when base is required
-export interface GrammarOptionsWithBase extends GrammarOptions {
-	readonly base: Grammar;
-}
-
 export interface ComponentProtoBase<Id extends string = string> {
 	id: Id;
 	require?: ComponentProto | readonly ComponentProto[];
@@ -50,25 +44,49 @@ export interface ComponentProtoBase<Id extends string = string> {
 	effect?: (Prism: Prism & { plugins: Record<KebabToCamelCase<Id>, {}> }) => () => void;
 }
 
-// For languages that extend a base language
-export interface LanguageProtoWithBase<Id extends string = string> extends ComponentProtoBase<Id> {
-	grammar: Grammar | ((options: GrammarOptionsWithBase) => Grammar);
+export type LanguageProto<Id extends string = string> =
+	| LanguageProtoPlain<Id>
+	| LanguageProtoWithBase<Id>
+	| LanguageProtoWithRequire<Id>
+	| LanguageProtoWithBaseAndRequire<Id>;
+
+interface LanguageProtoPlain<Id extends string = string> extends ComponentProtoBase<Id> {
+	grammar: Grammar | ((options: GrammarOptions) => Grammar);
+	plugin?: undefined;
+	base?: never; // Explicitly no base allowed
+	require?: never; // Explicitly no require allowed
+}
+
+interface LanguageProtoWithBase<Id extends string = string> extends ComponentProtoBase<Id> {
+	grammar: Grammar | ((options: GrammarOptions & { readonly base: Grammar }) => Grammar);
 	plugin?: undefined;
 	base: LanguageProto; // Required base
+	require?: never; // Explicitly no require allowed
 }
 
-// For languages that don't extend a base language
-export interface LanguageProtoWithoutBase<Id extends string = string>
-	extends ComponentProtoBase<Id> {
-	grammar: Grammar | ((options: GrammarOptions) => Grammar);
+interface LanguageProtoWithRequire<Id extends string = string> extends ComponentProtoBase<Id> {
+	grammar:
+		| Grammar
+		| ((options: GrammarOptions & { readonly languages: Record<string, Grammar> }) => Grammar);
 	plugin?: undefined;
 	base?: never; // Explicitly no base allowed
+	require: ComponentProto | readonly ComponentProto[]; // Required require
 }
 
-// Union type that allows TypeScript to discriminate
-export type LanguageProto<Id extends string = string> =
-	| LanguageProtoWithBase<Id>
-	| LanguageProtoWithoutBase<Id>;
+interface LanguageProtoWithBaseAndRequire<Id extends string = string>
+	extends ComponentProtoBase<Id> {
+	grammar:
+		| Grammar
+		| ((
+				options: GrammarOptions & {
+					readonly base: Grammar;
+					readonly languages: Record<string, Grammar>;
+				}
+		  ) => Grammar);
+	plugin?: undefined;
+	base: LanguageProto; // Required base
+	require: ComponentProto | readonly ComponentProto[]; // Required require
+}
 
 type PluginType<Name extends string> = unknown;
 export interface PluginProto<Id extends string = string> extends ComponentProtoBase<Id> {

src/util/objects.js

@@ -0,0 +1,45 @@
+/**
+ * @template {Record<string, any>} T
+ * @template {keyof T} K
+ * @param {T} obj
+ * @param {K} key
+ * @param {() => T[K]} getter
+ * @param {any} [waitFor]
+ * @returns {Promise<void>}
+ */
+export async function defineLazyProperty (obj, key, getter, waitFor) {
+	if (waitFor) {
+		await waitFor;
+	}
+
+	Object.defineProperty(obj, key, {
+		enumerable: true,
+		configurable: true,
+		get () {
+			const value = getter.call(this);
+			// Replace the getter with a writable property
+			defineSimpleProperty(this, key, value);
+			return value;
+		},
+		set (value) {
+			defineSimpleProperty(this, key, value);
+		},
+	});
+}
+
+/**
+ * @template {Record<string, any>} T
+ * @template {keyof T} K
+ * @param {T} obj
+ * @param {K} key
+ * @param {T[K]} value
+ * @returns {void}
+ */
+export function defineSimpleProperty (obj, key, value) {
+	Object.defineProperty(obj, key, {
+		value,
+		writable: true,
+		enumerable: true,
+		configurable: false,
+	});
+}