Skip to content

Optimize queries with slim hydration and denormalized tag counts #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gavyncaldwell merged 5 commits into from
Mar 13, 2026
Merged

Optimize queries with slim hydration and denormalized tag counts #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8b6f5f1
add appCount column to tags table
gavyncaldwell Mar 13, 2026
825a5c1
compute tag app counts during seed import
gavyncaldwell Mar 13, 2026
f791254
use denormalized appCount for tag count queries
gavyncaldwell Mar 13, 2026
0822d6d
add slim list query helper for app cards
gavyncaldwell Mar 13, 2026
47170d0
convert list queries to use slim hydration
gavyncaldwell Mar 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
210 changes: 133 additions & 77 deletions db/queries.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import type { SourceType, TagType } from "./schema";
import {
alternatives,
appDownloads,
appSources,
apps,
appTags,
comparisonPairs,
Expand Down Expand Up @@ -41,6 +42,83 @@ function desktopOnlyAppIds(db: DrizzleDB) {
);
}

// ─── Slim List Helper ────────────────────────────────────────────────
// List pages only need card-relevant fields. Loading full sources
// (with metadata/packageName) and all tags wastes rows.

const appCardColumns = {
id: apps.id,
name: apps.name,
slug: apps.slug,
description: apps.description,
iconUrl: apps.iconUrl,
};

async function withSlimRelations(
db: DrizzleDB,
appRows: { id: string }[],
): Promise<
{
id: string;
name: string;
slug: string;
description: string | null;
iconUrl: string | null;
sources: { source: string; url: string }[];
tags: { name: string; slug: string; type: string }[];
}[]
> {
if (appRows.length === 0) return [];

const appIds = appRows.map((a) => a.id);

const [sources, platformTags] = await Promise.all([
db
.select({
appId: appSources.appId,
source: appSources.source,
url: appSources.url,
})
.from(appSources)
.where(inArray(appSources.appId, appIds)),
db
.select({
appId: appTags.appId,
name: tags.name,
slug: tags.slug,
type: tags.type,
})
.from(appTags)
.innerJoin(tags, eq(appTags.tagId, tags.id))
.where(and(inArray(appTags.appId, appIds), eq(tags.type, "platform"))),
]);

const sourcesByApp = new Map<string, { source: string; url: string }[]>();
for (const s of sources) {
const arr = sourcesByApp.get(s.appId) ?? [];
arr.push({ source: s.source, url: s.url });
sourcesByApp.set(s.appId, arr);
}

const tagsByApp = new Map<
string,
{ name: string; slug: string; type: string }[]
>();
for (const t of platformTags) {
const arr = tagsByApp.get(t.appId) ?? [];
arr.push({ name: t.name, slug: t.slug, type: t.type });
tagsByApp.set(t.appId, arr);
}

return (
appRows as ((typeof appRows)[number] & Record<string, unknown>)[]
).map((app) => ({
...(app as any),
sources: sourcesByApp.get(app.id) ?? [],
tags: tagsByApp.get(app.id) ?? [],
}));
}

// ─── Types ──────────────────────────────────────────────────────────

export type AppWithDetails = Awaited<ReturnType<typeof getAppBySlug>>;
Expand Down Expand Up @@ -79,18 +157,15 @@ export async function listApps(
conditions.push(inArray(apps.id, appIdsWithAllTags));
}

const results = await db.query.apps.findMany({
where: and(...conditions),
with: { sources: true, tags: { with: { tag: true } } },
limit,
offset,
orderBy: apps.name,
});
const rows = await db
.select(appCardColumns)
.from(apps)
.where(and(...conditions))
.limit(limit)
.offset(offset)
.orderBy(apps.name);

return results.map((app) => ({
...app,
tags: app.tags.map((at) => at.tag),
}));
return withSlimRelations(db, rows);
}

export async function getAppBySlug(db: DrizzleDB, slug: string) {
Expand Down Expand Up @@ -200,20 +275,16 @@ export async function listTagsByType(db: DrizzleDB, type: TagType) {
}

export async function listCategoriesWithApps(db: DrizzleDB) {
const rows = await db
return db
.select({
id: tags.id,
name: tags.name,
slug: tags.slug,
type: tags.type,
})
.from(tags)
.innerJoin(appTags, eq(tags.id, appTags.tagId))
.where(eq(tags.type, "category"))
.groupBy(tags.id)
.where(and(eq(tags.type, "category"), sql`${tags.appCount} > 0`))
.orderBy(tags.name);

return rows;
}

// ─── Tag / Category Page Queries ────────────────────────────────────
Expand Down Expand Up @@ -265,18 +336,15 @@ export async function listAppsByTag(
.from(appTags)
.where(eq(appTags.tagId, tagRow[0].id));

const results = await db.query.apps.findMany({
where: inArray(apps.id, appIdsWithTag),
with: { sources: true, tags: { with: { tag: true } } },
limit,
offset,
orderBy: apps.name,
});
const rows = await db
.select(appCardColumns)
.from(apps)
.where(inArray(apps.id, appIdsWithTag))
.limit(limit)
.offset(offset)
.orderBy(apps.name);

return results.map((app) => ({
...app,
tags: app.tags.map((at) => at.tag),
}));
return withSlimRelations(db, rows);
}

export async function listTagsWithCounts(db: DrizzleDB, type?: TagType) {
Expand All @@ -288,12 +356,10 @@ export async function listTagsWithCounts(db: DrizzleDB, type?: TagType) {
name: tags.name,
slug: tags.slug,
type: tags.type,
appCount: sql<number>`count(${appTags.appId})`,
appCount: tags.appCount,
})
.from(tags)
.leftJoin(appTags, eq(tags.id, appTags.tagId))
.where(conditions.length ? and(...conditions) : undefined)
.groupBy(tags.id)
.orderBy(tags.name);
}

Expand All @@ -302,16 +368,18 @@ export async function listTagsWithCounts(db: DrizzleDB, type?: TagType) {
export async function searchApps(db: DrizzleDB, query: string) {
const pattern = `%${query}%`;

const [appResults, propResults] = await Promise.all([
db.query.apps.findMany({
where: and(
like(apps.name, pattern),
notInArray(apps.id, desktopOnlyAppIds(db)),
),
with: { sources: true, tags: { with: { tag: true } } },
limit: 20,
orderBy: apps.name,
}),
const [appRows, propResults] = await Promise.all([
db
.select(appCardColumns)
.from(apps)
.where(
and(
like(apps.name, pattern),
notInArray(apps.id, desktopOnlyAppIds(db)),
),
)
.limit(20)
.orderBy(apps.name),
db
.select()
.from(proprietaryApps)
Expand All @@ -321,28 +389,22 @@ export async function searchApps(db: DrizzleDB, query: string) {
]);

return {
apps: appResults.map((app) => ({
...app,
tags: app.tags.map((at) => at.tag),
})),
apps: await withSlimRelations(db, appRows),
proprietaryApps: propResults,
};
}

// ─── Discovery Queries ──────────────────────────────────────────────

export async function getRecentApps(db: DrizzleDB) {
const results = await db.query.apps.findMany({
where: notInArray(apps.id, desktopOnlyAppIds(db)),
with: { sources: true, tags: { with: { tag: true } } },
orderBy: sql`${apps.createdAt} desc`,
limit: 20,
});
const rows = await db
.select(appCardColumns)
.from(apps)
.where(notInArray(apps.id, desktopOnlyAppIds(db)))
.orderBy(sql`${apps.createdAt} desc`)
.limit(20);

return results.map((app) => ({
...app,
tags: app.tags.map((at) => at.tag),
}));
return withSlimRelations(db, rows);
}

// ─── Desktop App Queries ────────────────────────────────────────────
Expand All @@ -366,18 +428,15 @@ export async function listDesktopApps(
.where(inArray(appTags.tagId, desktopTagIds))
.groupBy(appTags.appId);

const results = await db.query.apps.findMany({
where: inArray(apps.id, appsWithDesktopTag),
with: { sources: true, tags: { with: { tag: true } } },
limit,
offset,
orderBy: apps.name,
});
const rows = await db
.select(appCardColumns)
.from(apps)
.where(inArray(apps.id, appsWithDesktopTag))
.limit(limit)
.offset(offset)
.orderBy(apps.name);

return results.map((app) => ({
...app,
tags: app.tags.map((at) => at.tag),
}));
return withSlimRelations(db, rows);
}

// ─── Scan Query ─────────────────────────────────────────────────────
Expand Down Expand Up @@ -569,18 +628,15 @@ export async function listAppsByLicense(
const limit = Math.min(rawLimit, 100);
const offset = (page - 1) * limit;

const results = await db.query.apps.findMany({
where: eq(apps.license, license),
with: { sources: true, tags: { with: { tag: true } } },
limit,
offset,
orderBy: apps.name,
});
const rows = await db
.select(appCardColumns)
.from(apps)
.where(eq(apps.license, license))
.limit(limit)
.offset(offset)
.orderBy(apps.name);

return results.map((app) => ({
...app,
tags: app.tags.map((at) => at.tag),
}));
return withSlimRelations(db, rows);
}

// ─── Sitemap Queries ────────────────────────────────────────────────
Expand Down
1 change: 1 addition & 0 deletions db/schema.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ export const tags = sqliteTable(
name: text("name").notNull(),
slug: text("slug").notNull(),
type: text("type").$type<TagType>().notNull(),
appCount: integer("app_count").notNull().default(0),
},
(table) => ({
uniqueTag: uniqueIndex("tag_unique").on(table.slug, table.type),
Expand Down
12 changes: 12 additions & 0 deletions db/seed/import.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -464,6 +464,17 @@ async function upsertAlternatives() {
console.log(` ${stats.alternativesCreated} alternative mappings upserted`);
}

async function updateTagCounts() {
console.log("Updating tag counts...");
await client.execute(
"UPDATE tags SET app_count = (SELECT COUNT(*) FROM app_tags WHERE tag_id = tags.id)",
);
const result = await client.execute(
"SELECT COUNT(*) as total FROM tags WHERE app_count > 0",
);
console.log(` ${result.rows[0].total} tags with apps`);
}

async function main() {
console.log("\nSeed import");
console.log("═".repeat(50));
Expand All @@ -483,6 +494,7 @@ async function main() {
await upsertWebApps();
await upsertProprietaryApps();
await upsertAlternatives();
await updateTagCounts();

console.log(`\n${"═".repeat(50)}`);
console.log("Import complete:");
Expand Down
Loading