fix: Merge Includes

package.json

@@ -88,7 +88,8 @@
     "p-map": "4.0.0",
     "tapable": "2.2.1",
     "threads": "1.7.0",
-    "ts-dedent": "2.2.0"
+    "ts-dedent": "2.2.0",
+    "vitest": "^3.2.1"
   },
   "devDependencies": {
     "@aws-sdk/client-s3": "^3.758.0",

src/commands/build/features/output-md/index.ts

@@ -32,7 +32,7 @@
 };
 
 export type PreprocessConfig = {
-    preprocess: Partial<OutputMdConfig>;
+    preprocess: OutputMdConfig;
 };
 
 export class OutputMd {
@@ -48,7 +48,7 @@
                 hashIncludes: true,
             });
             const mergeIncludes = defined('mergeIncludes', args, config.preprocess || {}, {
-                mergeIncludes: false,
+                mergeIncludes: true,
             });
             const mergeAutotitles = defined('mergeAutotitles', args, config.preprocess || {}, {
                 mergeAutotitles: true,
@@ -89,12 +89,14 @@
                                 return processed.get(entry.path);
                             }
 
-                            const deps = await all(entry.deps.map((dep) => dump(dep, true)));
+                            const deps = await all(
+                                entry.deps.map((dep) => dump(dep, !config.mergeIncludes)),
+                            );
                             let content = entry.content;
                             const sheduler = new Sheduler();
 
-                            if (!config.mergeIncludes && config.hashIncludes) {
-                                sheduler.addStep(rehashIncludes(run, deps));
+                            if (config.hashIncludes) {
+                                sheduler.addStep(rehashIncludes(run, deps, config.mergeIncludes));
                             }
 
                             if (config.mergeAutotitles) {
@@ -104,7 +106,10 @@
                             await sheduler.shedule(entry);
                             content = await sheduler.process(content);
 
-                            const hash = config.hashIncludes ? rehashContent(content) : '';
+                            const hash =
+                                config.hashIncludes && !config.mergeIncludes
+                                    ? rehashContent(content)
+                                    : '';
                             const link = signlink(entry.path, hash);
                             const hashed = {...entry, content, hash};
 
@@ -168,7 +173,7 @@
     }
 
     private copyAssets(run: Run, service: Run['leading'] | Run['markdown'], cache: Set<string>) {
         return async (vfile: VFile<any>) => {
             const assets = await service.assets(vfile.path);
 
             await all(

src/commands/build/features/output-md/plugins/resolve-deps.ts

@@ -7,14 +7,57 @@ function rehashInclude(include: HashedGraphNode) {
     return replaceAll(include.match, include.link, signlink(include.link, include.hash));
 }
 
-export function rehashIncludes(_run: Run, deps: HashedGraphNode[]): StepFunction {
+function addIndent(content: string, numSpaces: number, skipFirstLine = false): string {
+    if (!content || numSpaces <= 0) {
+        return content;
+    }
+
+    const indent = ' '.repeat(numSpaces);
+    // Разбиваем строку, сохраняя разделители
+    const parts = content.split(/(\r\n|\r|\n)/);
+
+    let isFirstTextLine = true;
+    const result: string[] = [];
+    for (let i = 0; i < parts.length; i++) {
+        const part = parts[i];
+
+        // Если это окончание строки, просто добавляем его
+        if (part === '\r\n' || part === '\n' || part === '\r') {
+            result.push(part);
+            continue;
+        }
+        // Если это первая текстовая строка и нужно пропустить
+        if (isFirstTextLine && skipFirstLine) {
+            isFirstTextLine = false;
+            result.push(part);
+            continue;
+        }
+        isFirstTextLine = false;
+        // Добавляем отступ только к непустым строкам
+        result.push(part ? indent + part : part);
+    }
+    return result.join('');
+}
+
+export function rehashIncludes(
+    _run: Run,
+    deps: HashedGraphNode[],
+    mergeIncludes: boolean,
+): StepFunction {
     return async function (sheduler: Sheduler): Promise<void> {
         const actor = async (content: string, {dep}: StepContext): Promise<string> => {
             let result = content;
-            const {location} = dep as HashedGraphNode;
+            const {location, content: depContent, indent} = dep as HashedGraphNode;
 
-            const rehashed = rehashInclude(dep as HashedGraphNode);
-            result = result.slice(0, location[0]) + rehashed + result.slice(location[1]);
+            if (mergeIncludes) {
+                result =
+                    result.slice(0, location[0]) +
+                    addIndent(depContent, indent, true) +
+                    result.slice(location[1]);
+            } else {
+                const rehashed = rehashInclude(dep as HashedGraphNode);
+                result = result.slice(0, location[0]) + rehashed + result.slice(location[1]);
+            }
 
             return result;
         };

src/core/markdown/loader.spec.ts

@@ -242,6 +242,7 @@ describe('Markdown loader', () => {
                     link: './include.md',
                     location: [12, 42],
                     match: '{% include [](./include.md) %}',
+                    indent: 0,
                     hash: null,
                     search: null,
                 },
@@ -269,6 +270,7 @@ describe('Markdown loader', () => {
                     link: './include1.md',
                     location: [12, 43],
                     match: '{% include [](./include1.md) %}',
+                    indent: 0,
                     hash: null,
                     search: null,
                 },
@@ -278,6 +280,7 @@ describe('Markdown loader', () => {
                     location: [45, 99],
                     match: '{% include [some text (with) braces](./include2.md) %}',
                     hash: null,
+                    indent: 0,
                     search: null,
                 },
                 {
@@ -286,6 +289,7 @@ describe('Markdown loader', () => {
                     location: [105, 140],
                     match: '{% include [](./deep/include.md) %}',
                     hash: null,
+                    indent: 4,
                     search: null,
                 },
             ]);
@@ -312,6 +316,7 @@ describe('Markdown loader', () => {
                     link: './include1.md',
                     location: [12, 43],
                     match: '{% include [](./include1.md) %}',
+                    indent: 0,
                     hash: null,
                     search: null,
                 },
@@ -320,6 +325,7 @@ describe('Markdown loader', () => {
                     link: './include2.md',
                     location: [45, 99],
                     match: '{% include [some text (with) braces](./include2.md) %}',
+                    indent: 0,
                     hash: null,
                     search: null,
                 },

src/core/markdown/loader/resolve-deps.ts

@@ -10,18 +10,19 @@ export function resolveDependencies(this: LoaderContext, content: string) {
 
     // Include example: {% include [createfolder](create-folder.md) %}
     // Regexp result: [createfolder](create-folder.md)
-    const INCLUDE_CONTENTS = /{%\s*include\s*.+?%}/g;
+    const INCLUDE_CONTENTS = /(\r?\n\s+)?({%\s*include\s*.+?%})/g;
 
     let match;
     // eslint-disable-next-line no-cond-assign
     while ((match = INCLUDE_CONTENTS.exec(content))) {
         // Ugly workaround for include examples
         // TODO: rewrite all inspect code on markdown-it parsing with minimal set of plugins
+        match.index = match.index + (match[1]?.length || 0);
         if (content[match.index - 1] === '`') {
             continue;
         }
 
-        const link = findLink(match[0]) as string;
+        const link = findLink(match[2]) as string;
         // TODO: warn about non local urls
         const include = parseLocalUrl<IncludeInfo>(link);
 
@@ -30,6 +31,7 @@ export function resolveDependencies(this: LoaderContext, content: string) {
             include.link = link;
             include.match = content.slice(match.index, INCLUDE_CONTENTS.lastIndex);
             include.location = [match.index, INCLUDE_CONTENTS.lastIndex];
+            include.indent = match[1]?.replace(/\r?\n/g, '').length || 0;
 
             includes.push(include);
         }

src/core/markdown/types.ts

@@ -24,6 +24,7 @@ export type IncludeInfo = Pick<UrlWithStringQuery, 'hash' | 'search'> & {
     link: string;
     match: string;
     location: Location;
+    indent: number;
 };
 
 export type AssetInfo = Pick<UrlWithStringQuery, 'hash' | 'search'> & {

tests/e2e/__snapshots__/include-toc.test.ts.snap

@@ -2,31 +2,34 @@
 
 exports[`Include toc > Nested toc inclusions with mixed including modes > filelist 1`] = `
 "[
-  "product1/_includes/inc-hash.md",
   "product1/article1.md",
   "product1/toc.yaml",
-  "product2/overlay1/_includes/inc-hash.md",
   "product2/overlay1/article1.md",
-  "product2/overlay2/_includes/inc-hash.md",
   "product2/overlay2/article1.md",
-  "product2/overlay3/_includes/inc-hash.md",
   "product2/overlay3/article1.md",
   "product2/p2.md",
   "product2/toc.yaml"
 ]"
 `;
 
-exports[`Include toc > Nested toc inclusions with mixed including modes 1`] = `"This is the core include."`;
-
-exports[`Include toc > Nested toc inclusions with mixed including modes 2`] = `
+exports[`Include toc > Nested toc inclusions with mixed including modes 1`] = `
 "---
 metadata:
   - name: generator
     content: Diplodoc Platform vDIPLODOC-VERSION
 ---
 This is the core content of Article 1.
 
-{% include [x](_includes/inc-hash.md) %}
+This is the core include.
+"
+`;
+
+exports[`Include toc > Nested toc inclusions with mixed including modes 2`] = `
+"title: Product 1 title
+items:
+  - name: Article1
+    href: article1.md
+path: product1/toc.yaml
 "
 `;
 
@@ -52,66 +55,45 @@ path: toc.yaml
 `;
 
 exports[`Include toc > Nested toc inclusions with mixed including modes 3`] = `
-"title: Product 1 title
-items:
-  - name: Article1
-    href: article1.md
-path: product1/toc.yaml
-"
-`;
-
-exports[`Include toc > Nested toc inclusions with mixed including modes 4`] = `
-"This is the core include.
-"
-`;
-
-exports[`Include toc > Nested toc inclusions with mixed including modes 5`] = `
 "---
 metadata:
   - name: generator
     content: Diplodoc Platform vDIPLODOC-VERSION
 ---
 This is the overlay content of Article 1 for product 2.
 
-{% include [x](_includes/inc-hash.md) %}
-"
-`;
+This is the core include.
 
-exports[`Include toc > Nested toc inclusions with mixed including modes 6`] = `
-"This is the core include.
 "
 `;
 
-exports[`Include toc > Nested toc inclusions with mixed including modes 7`] = `
+exports[`Include toc > Nested toc inclusions with mixed including modes 4`] = `
 "---
 metadata:
   - name: generator
     content: Diplodoc Platform vDIPLODOC-VERSION
 ---
 This is the overlay number #2  of Article 1 content for product 2.
 
-{% include [x](_includes/inc-hash.md) %}
-"
-`;
+This is the core include.
 
-exports[`Include toc > Nested toc inclusions with mixed including modes 8`] = `
-"This is the core include.
 "
 `;
 
-exports[`Include toc > Nested toc inclusions with mixed including modes 9`] = `
+exports[`Include toc > Nested toc inclusions with mixed including modes 5`] = `
 "---
 metadata:
   - name: generator
     content: Diplodoc Platform vDIPLODOC-VERSION
 ---
 This is the core content of Article 1.
 
-{% include [x](_includes/inc-hash.md) %}
+This is the core include.
+
 "
 `;
 
-exports[`Include toc > Nested toc inclusions with mixed including modes 10`] = `
+exports[`Include toc > Nested toc inclusions with mixed including modes 6`] = `
 "---
 metadata:
   - name: generator
@@ -123,7 +105,7 @@ Check here link to [Article1 overlay 1](overlay1/article1.md)
 Check here link to [Article1 overlay 2](overlay2/article1.md)"
 `;
 
-exports[`Include toc > Nested toc inclusions with mixed including modes 11`] = `
+exports[`Include toc > Nested toc inclusions with mixed including modes 7`] = `
 "title: Product 2 title
 items:
   - name: P2 Article