feat: add sitemap and robots file

app/(marketing)/[slug]/page.tsx

@@ -80,7 +80,13 @@ export default async function MarketingPage({
               aria-label="Published date"
               className="text-gray-800 dark:text-gray-300"
             >
-              Last Updated: {data.lastUpdated}
+              Last Updated:{" "}
+              {data.lastUpdated.toLocaleDateString(undefined, {
+                weekday: "long",
+                year: "numeric",
+                month: "long",
+                day: "numeric",
+              })}
             </span>
           </div>
           <MarkdownContent content={content} />{" "}

app/robots.txt

@@ -0,0 +1,8 @@
+User-Agent: *
+Allow: /
+Disallow: /chat/
+Disallow: /writer/
+Disallow: /auth/
+Disallow: /api/
+
+Sitemap: https://pegna.ai/sitemap.xml

app/sitemap.ts

@@ -0,0 +1,45 @@
+import fs from "fs";
+import path from "path";
+import type { MetadataRoute } from "next";
+import matter from "gray-matter";
+
+function getMarketingPages(): MetadataRoute.Sitemap {
+  // Get all marketing pages
+  const contentDir = path.join(process.cwd(), "content/marketing");
+
+  try {
+    const files = fs.readdirSync(contentDir);
+    return files
+      .filter((file) => file.endsWith(".mdx"))
+      .map((file) => {
+        const filePath = path.join(
+          process.cwd(),
+          "content/marketing/",
+          `${file}`,
+        );
+        const fileContent = fs.readFileSync(filePath, "utf8");
+        const { data } = matter(fileContent);
+        return {
+          url: `${process.env.APP_BASE_URL}/${file.replace(/\.mdx$/, "")}`,
+          lastModified: data.lastUpdated || new Date(),
+          changeFrequency: "yearly",
+          priority: 1,
+        };
+      });
+  } catch (error) {
+    console.error("Error reading content directory:", error);
+    return [];
+  }
+}
+
+export default function sitemap(): MetadataRoute.Sitemap {
+  return [
+    {
+      url: `${process.env.APP_BASE_URL}/`,
+      lastModified: new Date(),
+      changeFrequency: "yearly",
+      priority: 1,
+    },
+    ...getMarketingPages(),
+  ];
+}

content/marketing/privacy.mdx

@@ -1,7 +1,7 @@
 ---
 title: Privacy Policy
 description: How we collect, use, and protect your personal information
-lastUpdated: April 5, 2025
+lastUpdated: 2025-04-05
 ---
 
 ## 1. Introduction

content/marketing/terms.mdx

@@ -1,7 +1,7 @@
 ---
 title: Terms and Conditions
 description: Legal terms and conditions for using our service
-lastUpdated: April 5, 2025
+lastUpdated: 2025-04-05
 ---
 
 ## 1. Introduction