From 6e7a19752c5a5418098df14fd427bbf024237f66 Mon Sep 17 00:00:00 2001 From: Osama Mabkhot <99215291+O2sa@users.noreply.github.com> Date: Sat, 26 Sep 2026 03:31:53 +0300 Subject: [PATCH] perf(cache): add two-tier profile score caching and optimize user comparison - Add Tier 1 Redis computed profile cache (profile:) saving CPU scoring loops - Add Tier 2 PostgreSQL pre-computed score lookup in DatabaseStore bypassing raw_data JSONB - Refactor compareUsers to reuse getUserProfile and shared profile cache - Purge profile cache upon raw user data invalidation - Add unit test coverage in profile-cache.test.ts --- .../comparison/services/compare-service.ts | 83 +++----- src/features/developer/services/index.ts | 1 + .../developer/services/profile-cache.ts | 74 +++++++ .../developer/services/user-service.ts | 76 ++++++- .../developer/tests/profile-cache.test.ts | 196 ++++++++++++++++++ src/lib/db/db-store.ts | 29 +++ src/lib/github/github-client.ts | 3 + 7 files changed, 406 insertions(+), 56 deletions(-) create mode 100644 src/features/developer/services/profile-cache.ts create mode 100644 src/features/developer/tests/profile-cache.test.ts diff --git a/src/features/comparison/services/compare-service.ts b/src/features/comparison/services/compare-service.ts index 22cb86a..98c2323 100644 --- a/src/features/comparison/services/compare-service.ts +++ b/src/features/comparison/services/compare-service.ts @@ -1,6 +1,5 @@ -import { getUserData } from "@/lib/github"; -import { calculateUserScore, normalizeSelectedLanguages } from "@/features/scoring"; -import { persistUserScores } from "@/features/developer/services"; +import { normalizeSelectedLanguages } from "@/features/scoring"; +import { getUserProfile, UserFetchError } from "@/features/developer/services"; import { DEFAULT_LOCALE, LOCALE_COOKIE, @@ -8,7 +7,6 @@ import { parseAcceptLanguage, type Locale, } from "@/lib/i18n/core"; -import type { GitHubUserData } from "@/lib/github"; import type { ComparedUserResult, CompareInsights, CompareWinner, LanguageWinner } from "../types"; export class CompareUserFetchError extends Error { @@ -257,53 +255,36 @@ export async function compareUsers( usernames: string[], selectedLanguages: string[], ): Promise { - return Promise.all( - usernames.map(async (username) => { - let data: GitHubUserData; - try { - const { data: userData } = await getUserData(username, { - cacheInRedis: true, - withMetrics: true, - }); - data = userData; - } catch (error: unknown) { - throw new CompareUserFetchError(username, error); - } - - const score = calculateUserScore( - { - ...data, - selectedLanguages, - }, + const results: ComparedUserResult[] = []; + for (const username of usernames) { + try { + const { user } = await getUserProfile(username, selectedLanguages); + results.push({ + username: user.username, + name: user.name, + avatarUrl: user.avatarUrl, + repoScore: user.repoScore, + prScore: user.prScore, + contributionScore: user.contributionScore, + finalScore: user.finalScore, + normalizedRepoScore: user.normalizedRepoScore ?? user.repoScore, + normalizedPRScore: user.normalizedPRScore ?? user.prScore, + normalizedContributionScore: user.normalizedContributionScore ?? user.contributionScore, + normalizedFinalScore: user.normalizedFinalScore ?? user.finalScore, + topRepos: user.topRepos as ComparedUserResult["topRepos"], + topPullRequests: user.topPullRequests as ComparedUserResult["topPullRequests"], + topCommunityContributions: (user.topCommunityContributions ?? + []) as ComparedUserResult["topCommunityContributions"], + languageScores: user.languageScores as ComparedUserResult["languageScores"], + signals: user.signals, + explanations: user.explanations, + } as unknown as ComparedUserResult); + } catch (error: unknown) { + throw new CompareUserFetchError( username, + error instanceof UserFetchError ? error.causeError : error, ); - - // Fire-and-forget: detect country & persist canonical scores into DB - void persistUserScores({ - data, - score, - selectedLanguages, - }); - - return { - username: data.login, - name: data.name, - avatarUrl: data.avatarUrl, - repoScore: Math.round(score.repoScore), - prScore: Math.round(score.prScore), - contributionScore: Math.round(score.contributionScore), - finalScore: Math.round(score.finalScore), - normalizedRepoScore: Math.round(score.normalizedRepoScore), - normalizedPRScore: Math.round(score.normalizedPRScore), - normalizedContributionScore: Math.round(score.normalizedContributionScore), - normalizedFinalScore: Math.round(score.normalizedFinalScore), - topRepos: score.topRepos, - topPullRequests: score.topPullRequests, - topCommunityContributions: score.topCommunityContributions, - languageScores: score.languageScores, - signals: score.signals, - explanations: score.explanations, - }; - }), - ); + } + } + return results; } diff --git a/src/features/developer/services/index.ts b/src/features/developer/services/index.ts index 3c1fdfc..fec844e 100644 --- a/src/features/developer/services/index.ts +++ b/src/features/developer/services/index.ts @@ -1,2 +1,3 @@ export * from "./user-service"; export * from "./user-persistence"; +export * from "./profile-cache"; diff --git a/src/features/developer/services/profile-cache.ts b/src/features/developer/services/profile-cache.ts new file mode 100644 index 0000000..79421f1 --- /dev/null +++ b/src/features/developer/services/profile-cache.ts @@ -0,0 +1,74 @@ +import { createCacheStore, getCacheConfigFromEnv, type CacheStore } from "@/lib/cache"; +import type { UserProfileResponse } from "../types"; + +let cacheStore: CacheStore | undefined; + +function getStore(): CacheStore { + if (!cacheStore) { + cacheStore = createCacheStore(getCacheConfigFromEnv()); + } + return cacheStore; +} + +export function buildProfileCacheKey(username: string, selectedLanguages: string[] = []): string { + const config = getCacheConfigFromEnv(); + const normalizedUser = username.trim().toLowerCase(); + if (!selectedLanguages || selectedLanguages.length === 0) { + return `${config.namespace}:profile:${normalizedUser}`; + } + const langKey = selectedLanguages + .map((l) => l.trim().toLowerCase()) + .filter(Boolean) + .sort() + .join(","); + return `${config.namespace}:profile:${normalizedUser}:${langKey}`; +} + +export async function getCachedProfile( + username: string, + selectedLanguages: string[] = [], +): Promise { + const store = getStore(); + if (!store.enabled) { + return null; + } + try { + const key = buildProfileCacheKey(username, selectedLanguages); + const cached = await store.get(key); + return cached ?? null; + } catch { + return null; + } +} + +export async function setCachedProfile( + username: string, + profile: UserProfileResponse, + selectedLanguages: string[] = [], + ttlSeconds?: number, +): Promise { + const store = getStore(); + if (!store.enabled) { + return; + } + try { + const key = buildProfileCacheKey(username, selectedLanguages); + const ttl = ttlSeconds ?? getCacheConfigFromEnv().ttlSeconds; + await store.set(key, profile, ttl); + } catch { + // Non-fatal + } +} + +export async function invalidateProfileCache(username: string): Promise { + const store = getStore(); + if (!store.enabled || !store.del) { + return; + } + try { + const key = buildProfileCacheKey(username, []); + await store.del(key); + } catch { + // Non-fatal + } +} diff --git a/src/features/developer/services/user-service.ts b/src/features/developer/services/user-service.ts index 43caaed..12f757b 100644 --- a/src/features/developer/services/user-service.ts +++ b/src/features/developer/services/user-service.ts @@ -1,6 +1,8 @@ import { getUserData } from "@/lib/github"; import { calculateUserScore } from "@/features/scoring"; +import type { CalculateUserScoreResult } from "@/features/scoring/services"; import { persistUserScores } from "./user-persistence"; +import { getCachedProfile, setCachedProfile } from "./profile-cache"; import type { UserProfileResponse, UserResult } from "../types"; import type { GitHubUserData } from "@/lib/github"; @@ -25,6 +27,65 @@ export async function getUserProfile( throw new Error("Username is required"); } + // ── 1. Check Redis Profile Cache (Tier 1: Pre-computed Profile) ── + try { + const cached = await getCachedProfile(normalizedUsername, selectedLanguages); + if (cached) { + return cached; + } + } catch { + // Non-fatal: continue to DB / fresh calculation + } + + // ── 2. Check PostgreSQL Pre-calculated Scores (Tier 2 for Canonical) ── + const isCanonical = selectedLanguages.length === 0; + if (isCanonical) { + try { + const { getDatabaseStore } = await import("@/lib/db"); + const db = getDatabaseStore(); + if (typeof db.getUserProfile === "function") { + const row = await db.getUserProfile(normalizedUsername); + + if (row && row.scores && row.stale_after > new Date()) { + const scores = row.scores as CalculateUserScoreResult; + const profile: UserProfileResponse = { + user: { + username: row.username, + name: row.name, + avatarUrl: row.avatar_url, + repoScore: row.repo_score, + prScore: row.pr_score, + contributionScore: row.contribution_score, + finalScore: row.final_score, + normalizedRepoScore: Math.round(scores.normalizedRepoScore ?? scores.repoScore), + normalizedPRScore: Math.round(scores.normalizedPRScore ?? scores.prScore), + normalizedContributionScore: Math.round( + scores.normalizedContributionScore ?? scores.contributionScore, + ), + normalizedFinalScore: Math.round(scores.normalizedFinalScore ?? scores.finalScore), + topRepos: scores.topRepos ?? [], + topPullRequests: scores.topPullRequests ?? [], + topCommunityContributions: scores.topCommunityContributions ?? [], + languageScores: scores.languageScores, + signals: scores.signals, + explanations: scores.explanations, + scoreVersion: process.env.DEVIMPACT_VERSION || undefined, + }, + location: row.location ?? null, + }; + + // Warm Redis profile cache + void setCachedProfile(normalizedUsername, profile, selectedLanguages); + + return profile; + } + } + } catch { + // Non-fatal: fall through to full calculation + } + } + + // ── 3. Fallback: Fetch raw GitHub data & compute score ────────────── let data: GitHubUserData; try { const { data: userData } = await getUserData(normalizedUsername, { @@ -65,15 +126,20 @@ export async function getUserProfile( scoreVersion: process.env.DEVIMPACT_VERSION || undefined, }; - // Fire-and-forget: detect country & persist canonical scores into DB + const response: UserProfileResponse = { + user, + location: data.location ?? null, + }; + + // Warm Redis profile cache with the computed profile + void setCachedProfile(normalizedUsername, response, selectedLanguages); + + // Persist canonical scores into DB void persistUserScores({ data, score, selectedLanguages, }); - return { - user, - location: data.location ?? null, - }; + return response; } diff --git a/src/features/developer/tests/profile-cache.test.ts b/src/features/developer/tests/profile-cache.test.ts new file mode 100644 index 0000000..2973b58 --- /dev/null +++ b/src/features/developer/tests/profile-cache.test.ts @@ -0,0 +1,196 @@ +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + cacheGet: vi.fn(), + cacheSet: vi.fn(), + cacheDel: vi.fn(), + getUserData: vi.fn(), + calculateUserScore: vi.fn(), + getUserProfile: vi.fn(), + upsertUser: vi.fn(), +})); + +vi.mock("@/lib/cache", () => ({ + getCacheConfigFromEnv: () => ({ + enabled: true, + namespace: "devimpact:v1", + ttlSeconds: 604800, + }), + createCacheStore: () => ({ + enabled: true, + get: mocks.cacheGet, + set: mocks.cacheSet, + del: mocks.cacheDel, + }), +})); + +vi.mock("@/lib/github", () => ({ + getUserData: mocks.getUserData, +})); + +vi.mock("@/features/scoring", () => ({ + calculateUserScore: mocks.calculateUserScore, +})); + +vi.mock("@/lib/db", () => ({ + getDatabaseStore: () => ({ + getUserProfile: mocks.getUserProfile, + upsertUser: mocks.upsertUser, + }), +})); + +import { + buildProfileCacheKey, + getCachedProfile, + setCachedProfile, + invalidateProfileCache, +} from "../services/profile-cache"; +import { getUserProfile } from "../services/user-service"; + +describe("Profile & Score Caching", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + describe("buildProfileCacheKey", () => { + test("builds canonical key for user", () => { + const key = buildProfileCacheKey(" Torvalds "); + expect(key).toBe("devimpact:v1:profile:torvalds"); + }); + + test("builds language-sorted key when selectedLanguages provided", () => { + const key = buildProfileCacheKey("torvalds", ["Rust", "Python", "Go"]); + expect(key).toBe("devimpact:v1:profile:torvalds:go,python,rust"); + }); + }); + + describe("getCachedProfile & setCachedProfile", () => { + test("retrieves cached profile from store", async () => { + const mockProfile = { + user: { + username: "torvalds", + name: "Linus Torvalds", + avatarUrl: "https://example.com/avatar.png", + repoScore: 100, + prScore: 100, + contributionScore: 100, + finalScore: 100, + topRepos: [], + topPullRequests: [], + }, + location: "Helsinki, Finland", + }; + + mocks.cacheGet.mockResolvedValueOnce(mockProfile); + + const result = await getCachedProfile("torvalds"); + expect(mocks.cacheGet).toHaveBeenCalledWith("devimpact:v1:profile:torvalds"); + expect(result).toEqual(mockProfile); + }); + + test("stores profile with TTL", async () => { + const mockProfile = { + user: { + username: "torvalds", + name: "Linus", + avatarUrl: "https://example.com/avatar.png", + repoScore: 90, + prScore: 90, + contributionScore: 90, + finalScore: 90, + topRepos: [], + topPullRequests: [], + }, + location: null, + }; + + await setCachedProfile("torvalds", mockProfile); + expect(mocks.cacheSet).toHaveBeenCalledWith( + "devimpact:v1:profile:torvalds", + mockProfile, + 604800, + ); + }); + + test("invalidates profile cache", async () => { + await invalidateProfileCache("torvalds"); + expect(mocks.cacheDel).toHaveBeenCalledWith("devimpact:v1:profile:torvalds"); + }); + }); + + describe("getUserProfile tiered retrieval", () => { + test("Tier 1: returns directly from Redis cache without calling DB or GitHub", async () => { + const cachedProfile = { + user: { + username: "torvalds", + name: "Linus", + avatarUrl: "https://example.com/avatar.png", + repoScore: 95, + prScore: 95, + contributionScore: 95, + finalScore: 95, + topRepos: [], + topPullRequests: [], + }, + location: "Finland", + }; + + mocks.cacheGet.mockResolvedValueOnce(cachedProfile); + + const result = await getUserProfile("torvalds"); + + expect(result).toEqual(cachedProfile); + // Zero calls to DB, GitHub, or Scoring engine + expect(mocks.getUserProfile).not.toHaveBeenCalled(); + expect(mocks.getUserData).not.toHaveBeenCalled(); + expect(mocks.calculateUserScore).not.toHaveBeenCalled(); + }); + + test("Tier 2: returns from PostgreSQL pre-calculated scores when Redis misses", async () => { + mocks.cacheGet.mockResolvedValueOnce(null); + + mocks.getUserProfile.mockResolvedValueOnce({ + username: "torvalds", + name: "Linus Torvalds", + avatar_url: "https://example.com/avatar.png", + location: "Finland", + country: "Finland", + repo_score: 92, + pr_score: 92, + contribution_score: 92, + final_score: 92, + scores: { + repoScore: 92, + prScore: 92, + contributionScore: 92, + finalScore: 92, + normalizedRepoScore: 90, + normalizedPRScore: 90, + normalizedContributionScore: 90, + normalizedFinalScore: 90, + topRepos: [{ name: "linux", score: 99 }], + topPullRequests: [], + topCommunityContributions: [], + }, + stale_after: new Date(Date.now() + 86400000), // fresh for 1 day + }); + + const result = await getUserProfile("torvalds"); + + expect(result.user.username).toBe("torvalds"); + expect(result.user.finalScore).toBe(92); + expect(result.location).toBe("Finland"); + + // Warmed Redis profile cache + expect(mocks.cacheSet).toHaveBeenCalledWith( + "devimpact:v1:profile:torvalds", + expect.objectContaining({ location: "Finland" }), + 604800, + ); + + // GitHub and score calculation were NOT called + expect(mocks.getUserData).not.toHaveBeenCalled(); + expect(mocks.calculateUserScore).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/src/lib/db/db-store.ts b/src/lib/db/db-store.ts index 29b688b..c83eaf0 100644 --- a/src/lib/db/db-store.ts +++ b/src/lib/db/db-store.ts @@ -34,6 +34,21 @@ export type LeaderboardUserRow = Pick< | "final_score" >; +export type UserProfileRow = Pick< + GitHubUserRow, + | "username" + | "name" + | "avatar_url" + | "location" + | "country" + | "scores" + | "repo_score" + | "pr_score" + | "contribution_score" + | "final_score" + | "stale_after" +>; + export type TopStaleUserRow = Pick; export type UpsertUserParams = { @@ -270,6 +285,20 @@ export class DatabaseStore { return result.rows[0] ?? null; } + async getUserProfile(username: string): Promise { + const client = getPool(); + const result = await client.query( + `SELECT + username, name, avatar_url, location, country, + scores, repo_score, pr_score, contribution_score, final_score, + stale_after + FROM github_users + WHERE LOWER(username) = LOWER($1)`, + [username], + ); + return result.rows[0] ?? null; + } + async userExists(username: string): Promise { const client = getPool(); const result = await client.query( diff --git a/src/lib/github/github-client.ts b/src/lib/github/github-client.ts index 05e53b8..3e8b440 100644 --- a/src/lib/github/github-client.ts +++ b/src/lib/github/github-client.ts @@ -839,6 +839,9 @@ export async function getUserData( // Delete stale cache entry (leaderboard calculation path) try { await cacheStoreSingleton.del(cacheKey); + await cacheStoreSingleton.del( + `${cacheConfigSingleton.namespace}:profile:${normalizedUsername.toLowerCase()}`, + ); } catch { // Non-fatal }