Data fetch refactor

.github/workflows/external-data-storage-daily.yml

@@ -0,0 +1,42 @@
+name: External Data Storage (Daily)
+
+on:
+  schedule:
+    # Runs every day at 00:00 UTC
+    - cron: "0 0 * * *"
+  workflow_dispatch: # Can be dispatched manually
+
+jobs:
+  store-external-data:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: 20
+          cache: "pnpm"
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Install ts-node
+        run: pnpm add -g ts-node
+
+      - name: Run external data storage CRON (daily)
+        run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/externalDataStorageCRONDaily.ts
+        env:
+          UPSTASH_REDIS_REST_URL: ${{ secrets.UPSTASH_REDIS_REST_URL }}
+          UPSTASH_REDIS_REST_TOKEN: ${{ secrets.UPSTASH_REDIS_REST_TOKEN }}
+          NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL }}
+          NEXT_PUBLIC_SUPABASE_ANON_KEY: ${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY }}
+          EXTERNAL_DATA_TTL_SECONDS: ${{ secrets.EXTERNAL_DATA_TTL_SECONDS }}
+

.github/workflows/external-data-storage-hourly.yml

@@ -0,0 +1,42 @@
+name: External Data Storage (Hourly)
+
+on:
+  schedule:
+    # Runs every hour at minute 0 (e.g., 00:00, 01:00, 02:00, etc.)
+    - cron: "0 * * * *"
+  workflow_dispatch: # Can be dispatched manually
+
+jobs:
+  store-external-data:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: 20
+          cache: "pnpm"
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Install ts-node
+        run: pnpm add -g ts-node
+
+      - name: Run external data storage CRON (hourly)
+        run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/externalDataStorageCRONHourly.ts
+        env:
+          UPSTASH_REDIS_REST_URL: ${{ secrets.UPSTASH_REDIS_REST_URL }}
+          UPSTASH_REDIS_REST_TOKEN: ${{ secrets.UPSTASH_REDIS_REST_TOKEN }}
+          NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL }}
+          NEXT_PUBLIC_SUPABASE_ANON_KEY: ${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY }}
+          EXTERNAL_DATA_TTL_SECONDS: ${{ secrets.EXTERNAL_DATA_TTL_SECONDS }}
+

app/[locale]/[...slug]/page.tsx

@@ -11,23 +11,22 @@ import type { SlugPageParams } from "@/lib/types"
 import I18nProvider from "@/components/I18nProvider"
 import mdComponents from "@/components/MdComponents"
 
-import { dataLoader } from "@/lib/utils/data/dataLoader"
+import { extractGFIssues } from "@/lib/utils/data/extractExternalData"
+import { getExternalData } from "@/lib/utils/data/getExternalData"
 import { dateToString } from "@/lib/utils/date"
 import { getLayoutFromSlug } from "@/lib/utils/layout"
 import { checkPathValidity, getPostSlugs } from "@/lib/utils/md"
+import { every } from "@/lib/utils/time"
 import { getRequiredNamespacesForPage } from "@/lib/utils/translations"
 
 import { LOCALES_CODES } from "@/lib/constants"
 
 import SlugJsonLD from "./page-jsonld"
 
 import { componentsMapping, layoutMapping } from "@/layouts"
-import { fetchGFIs } from "@/lib/api/fetchGFIs"
 import { getPageData } from "@/lib/md/data"
 import { getMdMetadata } from "@/lib/md/metadata"
 
-const loadData = dataLoader([["gfissues", fetchGFIs]])
-
 export default async function Page({ params }: { params: SlugPageParams }) {
   const { locale, slug: slugArray } = params
 
@@ -40,7 +39,9 @@ export default async function Page({ params }: { params: SlugPageParams }) {
   // Enable static rendering
   setRequestLocale(locale)
 
-  const [gfissues] = await loadData()
+  // Fetch daily data (GitHub good first issues) with 24-hour revalidation
+  const dailyData = await getExternalData(["gfissues"], every("day"))
+  const gfissues = extractGFIssues(dailyData)
 
   const slug = slugArray.join("/")
 

app/[locale]/apps/[application]/page.tsx

@@ -30,31 +30,25 @@ import { ButtonLink } from "@/components/ui/buttons/Button"
 import { LinkBox, LinkOverlay } from "@/components/ui/link-box"
 import { Tag } from "@/components/ui/tag"
 
-import { APP_TAG_VARIANTS } from "@/lib/utils/apps"
+import { APP_TAG_VARIANTS, extractAppsData } from "@/lib/utils/apps"
 import { getAppPageContributorInfo } from "@/lib/utils/contributors"
-import { dataLoader } from "@/lib/utils/data/dataLoader"
+import { getExternalData } from "@/lib/utils/data/getExternalData"
 import { isValidDate } from "@/lib/utils/date"
 import { getMetadata } from "@/lib/utils/metadata"
+import { every } from "@/lib/utils/time"
 import {
   formatLanguageNames,
   getRequiredNamespacesForPage,
 } from "@/lib/utils/translations"
 import { slugify } from "@/lib/utils/url"
 import { formatStringList } from "@/lib/utils/wallets"
 
-import { BASE_TIME_UNIT } from "@/lib/constants"
-
 import AppCard from "../_components/AppCard"
 
 import ScreenshotSwiper from "./_components/ScreenshotSwiper"
 import AppsAppJsonLD from "./page-jsonld"
 
-import { fetchApps } from "@/lib/api/fetchApps"
-
-// 24 hours
-const REVALIDATE_TIME = BASE_TIME_UNIT * 24
-
-const loadData = dataLoader([["appsData", fetchApps]], REVALIDATE_TIME * 1000)
+// 24 hours revalidation for apps data
 
 const Page = async ({
   params,
@@ -72,8 +66,14 @@ const Page = async ({
   const requiredNamespaces = getRequiredNamespacesForPage("/apps")
   const messages = pick(allMessages, requiredNamespaces)
 
-  // const [application] = application
-  const [appsData] = await loadData()
+  // Fetch apps data with 24-hour revalidation
+  const appsDataRaw = await getExternalData(["appsData"], every("day"))
+  const appsData = extractAppsData(
+    appsDataRaw?.appsData as
+      | { value: Record<string, unknown> }
+      | { error: string }
+      | undefined
+  )
   const app = Object.values(appsData)
     .flat()
     .find((app) => slugify(app.name) === application)!
@@ -394,7 +394,14 @@ export async function generateMetadata({
 }) {
   const { locale, application } = params
 
-  const [appsData] = await loadData()
+  // Fetch apps data with 24-hour revalidation
+  const appsDataRaw = await getExternalData(["appsData"], every("day"))
+  const appsData = extractAppsData(
+    appsDataRaw?.appsData as
+      | { value: Record<string, unknown> }
+      | { error: string }
+      | undefined
+  )
 
   const app = Object.values(appsData)
     .flat()

app/[locale]/apps/categories/[catetgoryName]/page.tsx

@@ -27,34 +27,26 @@ import {
 } from "@/components/ui/breadcrumb"
 import TabNav from "@/components/ui/TabNav"
 
-import { getHighlightedApps } from "@/lib/utils/apps"
+import { extractAppsData, getHighlightedApps } from "@/lib/utils/apps"
 import { getAppPageContributorInfo } from "@/lib/utils/contributors"
-import { dataLoader } from "@/lib/utils/data/dataLoader"
+import { getExternalData } from "@/lib/utils/data/getExternalData"
 import { getMetadata } from "@/lib/utils/metadata"
+import { every } from "@/lib/utils/time"
 import { getRequiredNamespacesForPage } from "@/lib/utils/translations"
 
 import { appsCategories } from "@/data/apps/categories"
 
-import { BASE_TIME_UNIT } from "@/lib/constants"
-
 import AppsHighlight from "../../_components/AppsHighlight"
 import AppsTable from "../../_components/AppsTable"
 import SuggestAnApp from "../../_components/SuggestAnApp"
 
 import AppsCategoryJsonLD from "./page-jsonld"
 
-import { fetchApps } from "@/lib/api/fetchApps"
-
 const VALID_CATEGORIES = Object.values(AppCategoryEnum)
 
 const isValidCategory = (category: string): category is AppCategoryEnum =>
   VALID_CATEGORIES.includes(category as AppCategoryEnum)
 
-// 24 hours
-const REVALIDATE_TIME = BASE_TIME_UNIT * 24
-
-const loadData = dataLoader([["appsData", fetchApps]], REVALIDATE_TIME * 1000)
-
 const Page = async ({
   params,
 }: {
@@ -63,7 +55,14 @@ const Page = async ({
   const { locale, catetgoryName } = params
   setRequestLocale(locale)
 
-  const [appsData] = await loadData()
+  // Fetch apps data with 24-hour revalidation
+  const appsDataRaw = await getExternalData(["appsData"], every("day"))
+  const appsData = extractAppsData(
+    appsDataRaw?.appsData as
+      | { value: Record<string, unknown> }
+      | { error: string }
+      | undefined
+  )
 
   const t = await getTranslations({ locale, namespace: "page-apps" })
 

app/[locale]/apps/page.tsx

@@ -14,19 +14,20 @@ import MainArticle from "@/components/MainArticle"
 import SubpageCard from "@/components/SubpageCard"
 
 import {
+  extractAppsData,
   getDevconnectApps,
   getDiscoverApps,
   getHighlightedApps,
 } from "@/lib/utils/apps"
 import { getAppPageContributorInfo } from "@/lib/utils/contributors"
-import { dataLoader } from "@/lib/utils/data/dataLoader"
+import { extractCommunityPicks } from "@/lib/utils/data/extractExternalData"
+import { getExternalData } from "@/lib/utils/data/getExternalData"
 import { getMetadata } from "@/lib/utils/metadata"
+import { every } from "@/lib/utils/time"
 import { getRequiredNamespacesForPage } from "@/lib/utils/translations"
 
 import { appsCategories } from "@/data/apps/categories"
 
-import { BASE_TIME_UNIT } from "@/lib/constants"
-
 import AppCard from "./_components/AppCard"
 import AppsHighlight from "./_components/AppsHighlight"
 import CommunityPicks from "./_components/CommunityPicks"
@@ -35,26 +36,26 @@ import SuggestAnApp from "./_components/SuggestAnApp"
 import TopApps from "./_components/TopApps"
 import AppsJsonLD from "./page-jsonld"
 
-import { fetchApps } from "@/lib/api/fetchApps"
-import { fetchCommunityPicks } from "@/lib/api/fetchCommunityPicks"
-
-// 24 hours
-const REVALIDATE_TIME = BASE_TIME_UNIT * 24
-
-const loadData = dataLoader(
-  [
-    ["appsData", fetchApps],
-    ["communityPicks", fetchCommunityPicks],
-  ],
-  REVALIDATE_TIME * 1000
-)
-
 const Page = async ({ params }: { params: PageParams }) => {
   const { locale } = params
 
   setRequestLocale(locale)
 
-  const [appsData, communityPicks] = await loadData()
+  // Fetch daily data (apps and community picks) with 24-hour revalidation
+  const dailyData = await getExternalData(
+    ["appsData", "communityPicks"],
+    every("day")
+  )
+
+  // Extract apps data
+  const appsDataRaw = dailyData?.appsData as
+    | { value: Record<string, unknown> }
+    | { error: string }
+    | undefined
+  const appsData = extractAppsData(appsDataRaw)
+
+  // Extract community picks
+  const communityPicks = extractCommunityPicks(dailyData)
 
   // Get 3 random highlighted apps
   const highlightedApps = getHighlightedApps(appsData, 3)

app/[locale]/developers/local-environment/page.tsx

@@ -10,25 +10,37 @@ import type { CommitHistory, Lang, PageParams } from "@/lib/types"
 import I18nProvider from "@/components/I18nProvider"
 
 import { getAppPageContributorInfo } from "@/lib/utils/contributors"
-import { dataLoader } from "@/lib/utils/data/dataLoader"
+import { extractFrameworkGitHubData } from "@/lib/utils/data/extractExternalData"
+import { getExternalData } from "@/lib/utils/data/getExternalData"
 import { getMetadata } from "@/lib/utils/metadata"
+import { every } from "@/lib/utils/time"
 import { getRequiredNamespacesForPage } from "@/lib/utils/translations"
 
+import { frameworksList } from "@/data/frameworks/frameworks"
+
 import LocalEnvironmentPage from "./_components/local-environment"
 import LocalEnvironmentJsonLD from "./page-jsonld"
 
-import { getLocalEnvironmentFrameworkData } from "@/lib/api/ghRepoData"
-
-const loadData = dataLoader([
-  ["frameworksListData", getLocalEnvironmentFrameworkData],
-])
-
 const Page = async ({ params }: { params: PageParams }) => {
   const { locale } = params
 
   setRequestLocale(locale)
 
-  const [frameworksListData] = await loadData()
+  // Fetch daily data (framework GitHub data) with 24-hour revalidation
+  const dailyData = await getExternalData(["frameworkGitHubData"], every("day"))
+
+  // Extract framework GitHub data
+  const frameworkGitHubData = extractFrameworkGitHubData(dailyData)
+
+  // Combine static framework data with GitHub data
+  const frameworksListData = frameworksList.map((framework) => {
+    const githubData = frameworkGitHubData?.[framework.id]
+    return {
+      ...framework,
+      starCount: githubData?.starCount,
+      languages: githubData?.languages?.slice(0, 2),
+    }
+  })
 
   // Get i18n messages
   const allMessages = await getMessages({ locale })