feat(core): markdown-parser can parse an existing stream of md tokens (#806)

Author: d3m1d0v

src/bundle/config/dynamicModifiers.ts

@@ -19,7 +19,7 @@ export function createDynamicModifiers(markupManager: MarkupManager): DynamicMod
             process: (token, _, rawMarkup) => {
                 const {map} = token;
 
-                if (map) {
+                if (map && rawMarkup !== null) {
                     const content = rawMarkup.split('\n').slice(map[0], map[1]).join('\n').trim();
                     const tokenId = v5(content, markupManager.getNamespace());
 

src/core/markdown/MarkdownParser.ts

@@ -77,17 +77,22 @@ export class MarkdownParser implements Parser {
         return this.tokenizer.linkify.match(text);
     }
 
-    parse(text: string) {
+    parse(src: string): Node;
+    parse(tokens: Token[]): Node;
+    parse(src: string | Token[]) {
         const time = Date.now();
 
         try {
             this.stack = [{type: this.schema.topNodeType, content: []}];
 
             let mdItTokens;
             try {
-                mdItTokens = this.tokenizer.parse(text, {});
+                mdItTokens = typeof src === 'string' ? this.tokenizer.parse(src, {}) : src;
                 if (this.dynamicModifier) {
-                    mdItTokens = this.dynamicModifier.processTokens(mdItTokens, text);
+                    mdItTokens = this.dynamicModifier.processTokens(
+                        mdItTokens,
+                        typeof src === 'string' ? src : null,
+                    );
                 }
             } catch (err) {
                 const e = err as Error;
@@ -377,7 +382,7 @@ function withoutTrailingNewline(str: string) {
 export type ProcessToken = (
     token: Token,
     index: number,
-    rawMarkup: string,
+    rawMarkup: string | null,
     allowedAttrs?: string[],
 ) => Token;
 export type ProcessNodeAttrs = (
@@ -455,7 +460,7 @@ export class MarkdownParserDynamicModifier {
         this.elementProcessors = new Map(Object.entries(config));
     }
 
-    processTokens(tokens: Token[], rawMarkup: string): Token[] {
+    processTokens(tokens: Token[], rawMarkup: string | null): Token[] {
         return tokens.map((token, index) => {
             const processor = this.elementProcessors.get(cropNodeName(token.type, openSuffix, ''));
             if (!processor || !processor.processToken || processor.processToken.length === 0) {

src/core/types/parser.ts

@@ -5,6 +5,8 @@ import type {Node} from 'prosemirror-model';
 export interface Parser {
     /** Parse raw markup to prosemirror's root node */
     parse(markup: string): Node;
+    /** Parse markdown-it tokens stream to prosemirror's root node */
+    parse(tokens: Token[]): Node;
     validateLink(url: string): boolean;
     normalizeLink(url: string): string;
     normalizeLinkText(url: string): string;