From 16ed6e412f1beb8443e58dbe60e16cb1187ad369 Mon Sep 17 00:00:00 2001 From: aewing Date: Thu, 1 Jan 2026 18:49:02 +0000 Subject: [PATCH] Implement server-side file caching with service principal auth - Remove progress meter for faster perceived load times - Fix async getGraphToken() calls with await - Implement service principal (app-only) authentication for file caching - Pre-cache jobs (2242 jobs) at startup via Redis - Background file caching for first 300 jobs on continuous cycle - Cache refreshes every 5 minutes to keep data current - All caches served instantly from Redis for all users/devices - No user authentication required for backend caching - App token auto-refreshes every 55 minutes Changes: - backend/src/index.ts: Added service principal flow, startup cache warming, background file caching - frontend/public/index.html: Removed progress bar, fixed getGraphToken() awaits - .env.example: Added GRAPH_TOKEN documentation --- .env.example | 5 + backend/src/index.ts | 263 ++++++++++++++++++++++++++++++++++--- frontend/public/index.html | 47 +++---- 3 files changed, 265 insertions(+), 50 deletions(-) diff --git a/.env.example b/.env.example index 5690d66..b781063 100644 --- a/.env.example +++ b/.env.example @@ -18,6 +18,11 @@ MICROSOFT_CLIENT_SECRET= MICROSOFT_TENANT=common MICROSOFT_REDIRECT_URI=http://localhost:3005/api/auth/callback +# Microsoft Graph API Token (for server-side file caching) +# This token is used by the backend to pre-cache SharePoint files +# Note: Graph tokens expire after ~60 minutes and need manual refresh +GRAPH_TOKEN= + # Logging LOG_LEVEL=info LOG_FILE=logs/app.log diff --git a/backend/src/index.ts b/backend/src/index.ts index f35d5b7..ccef542 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -9,6 +9,60 @@ config(); const app = new Hono(); +// Service Principal token cache +let appOnlyToken: string | null = null; +let appOnlyTokenExpiry: number = 0; + +// Get app-only token using client credentials flow +async function getAppOnlyToken(): Promise { + try { + // Check if we have a valid cached token + if (appOnlyToken && appOnlyTokenExpiry > Date.now()) { + return appOnlyToken; + } + + const clientId = process.env.MICROSOFT_CLIENT_ID; + const clientSecret = process.env.MICROSOFT_CLIENT_SECRET; + const tenant = process.env.MICROSOFT_TENANT || 'common'; + + if (!clientId || !clientSecret) { + return null; // Service principal not configured + } + + const tokenUrl = `https://login.microsoftonline.com/${tenant}/oauth2/v2.0/token`; + + const params = new URLSearchParams({ + client_id: clientId, + client_secret: clientSecret, + scope: 'https://graph.microsoft.com/.default', + grant_type: 'client_credentials' + }); + + const response = await fetch(tokenUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: params.toString() + }); + + if (!response.ok) { + const error = await response.text(); + logLine('logs/error.log', `Failed to get app-only token: ${error}`); + return null; + } + + const data = await response.json() as any; + appOnlyToken = data.access_token; + // Set expiry to 55 minutes (token is valid for 60 minutes) + appOnlyTokenExpiry = Date.now() + (55 * 60 * 1000); + + logLine('logs/server.log', `✓ Got app-only token (expires in 55 minutes)`); + return appOnlyToken; + } catch (err) { + logLine('logs/error.log', `getAppOnlyToken error: ${(err as Error)?.message || String(err)}`); + return null; + } +} + // Minimal log helpers const logLine = (path: string, line: string) => { try { @@ -325,17 +379,59 @@ const filterByCategory = (items: DriveItem[], category?: string, pdfOnly: boolea app.get('/api/job-files', async (c) => { try { console.log('[job-files] Request received'); + const link = c.req.query('link'); + if (!link) return c.json({ error: 'link required' }, 400); + console.log('[job-files] Link:', link); + + const q = c.req.query('q') || ''; + const category = c.req.query('category') || ''; + + // Check cache first (only for non-search, non-category requests) + if (!q && !category && isRedisConnected()) { + const cacheKey = `job-files:${link}`; + const cached = await getCache(cacheKey); + if (cached) { + console.log('[job-files] Cache HIT for', link); + // Apply filtering if requested + const pdfOnly = c.req.query('pdfOnly') !== 'false'; + const items = cached.items || []; + const filtered = pdfOnly + ? items.filter((i: any) => { + if (i.isFolder) return false; + const name = (i.name || '').toLowerCase(); + const mimeType = (i.mimeType || '').toLowerCase(); + return name.endsWith('.pdf') || mimeType.includes('pdf') || + name.endsWith('.doc') || name.endsWith('.docx') || mimeType.includes('word') || + name.endsWith('.jpg') || name.endsWith('.jpeg') || name.endsWith('.png') || + name.endsWith('.gif') || name.endsWith('.bmp') || name.endsWith('.tiff') || + name.endsWith('.webp') || mimeType.includes('image'); + }) + : items; + + const mapped = filtered.map((i: any) => ({ + id: i.id, + name: i.name, + url: i.webUrl, + driveId: i.driveId, + size: i.size, + modified: i.lastModifiedDateTime, + contentType: i.mimeType, + isFolder: i.isFolder, + path: i.parentPath || '', + })); + + return c.json({ items: mapped, total: mapped.length, source: 'cache', driveId: cached.driveId }); + } + console.log('[job-files] Cache MISS for', link); + } + + // Not in cache or search/category request - fetch from Graph API const token = getGraphToken(c); if (!token) { console.log('[job-files] No token found'); return c.json({ error: 'GRAPH_TOKEN missing' }, 500); } console.log('[job-files] Token found:', token.substring(0, 20) + '...'); - const link = c.req.query('link'); - const q = c.req.query('q') || ''; - const category = c.req.query('category') || ''; - if (!link) return c.json({ error: 'link required' }, 400); - console.log('[job-files] Link:', link); const { driveId, itemId } = await resolveShareLink(link, token); @@ -475,34 +571,161 @@ const PORT = Number(process.env.PORT || 3005); logLine('logs/server.log', `Starting frontend server (Redis version) on port ${PORT}`); -// Warm up cache on startup +// Warm up cache on startup - wait for Redis to be ready (async () => { + // Wait for Redis to be ready (max 10 seconds) + let retries = 0; + while (!isRedisConnected() && retries < 20) { + await new Promise(resolve => setTimeout(resolve, 500)); + retries++; + } + + if (!isRedisConnected()) { + logLine('logs/error.log', 'Redis not connected after 10 seconds, skipping cache warmup'); + return; + } + try { - if (isRedisConnected()) { - logLine('logs/server.log', 'Warming up jobs cache...'); - const result = await fetchAllJobsFromPocketBase(''); - await setCache('jobs:all', result, 1800); - logLine('logs/server.log', `✓ Cache warmed with ${result.items.length} jobs`); + logLine('logs/server.log', 'Warming up jobs cache...'); + const result = await fetchAllJobsFromPocketBase(''); + await setCache('jobs:all', result, 1800); + logLine('logs/server.log', `✓ Cache warmed with ${result.items.length} jobs`); + + // Try to start file caching if service principal is configured + const token = await getAppOnlyToken(); + if (token) { + logLine('logs/server.log', 'Service principal authenticated, starting background file caching...'); + startFileCaching(result.items.slice(0, 300), token); + } else { + logLine('logs/server.log', 'Service principal not configured, file caching will occur on-demand'); } } catch (err) { logLine('logs/error.log', `Cache warmup failed: ${(err as Error)?.message || String(err)}`); } })(); -// Background refresh every 5 minutes to keep cache hot -setInterval(async () => { - try { - if (isRedisConnected()) { - await refreshJobsCache('jobs:all', 1800, ''); - } - } catch (err) { - logLine('logs/error.log', `Periodic refresh error: ${(err as Error)?.message || String(err)}`); +// Background file caching state +let fileCachingActive = false; +let fileCachingIndex = 0; +let fileCachingJobs: any[] = []; +let fileCachingToken: string = ''; +let fileCachingTimer: NodeJS.Timeout | null = null; + +async function startFileCaching(jobs: any[], token: string) { + if (fileCachingActive) return; + + fileCachingJobs = jobs.filter(job => job.Job_Folder_Link); + if (fileCachingJobs.length === 0) { + logLine('logs/server.log', 'No jobs with folder links to cache'); + return; } -}, 5 * 60 * 1000); // Every 5 minutes + + fileCachingActive = true; + fileCachingIndex = 0; + fileCachingToken = token; + + logLine('logs/server.log', `Starting background file caching for ${fileCachingJobs.length} jobs...`); + cacheNextFileBatch(); +} + +async function cacheNextFileBatch() { + if (!fileCachingActive || !isRedisConnected()) return; + + const batchSize = 3; // Cache 3 jobs at a time + const maxJobs = fileCachingJobs.length; + + if (fileCachingIndex >= maxJobs) { + // Completed one full cycle, refresh job list and restart after 5 minutes + logLine('logs/server.log', `✓ Completed file caching cycle for ${maxJobs} jobs. Restarting in 5 minutes...`); + fileCachingIndex = 0; + + fileCachingTimer = setTimeout(async () => { + try { + // Refresh the jobs cache first + await refreshJobsCache('jobs:all', 1800, ''); + + // Get updated job list from cache + const cached = await getCache('jobs:all'); + if (cached && cached.items) { + fileCachingJobs = cached.items.slice(0, 300).filter((job: any) => job.Job_Folder_Link); + logLine('logs/server.log', `Updated file caching list with ${fileCachingJobs.length} jobs`); + } + + // Refresh token if needed + const newToken = await getAppOnlyToken(); + if (newToken) { + fileCachingToken = newToken; + cacheNextFileBatch(); + } else { + logLine('logs/error.log', 'Failed to refresh app-only token, stopping file caching'); + fileCachingActive = false; + } + } catch (err) { + logLine('logs/error.log', `File cache cycle restart error: ${(err as Error)?.message}`); + } + }, 5 * 60 * 1000); + return; + } + + // Get next batch of jobs to cache + const batch = fileCachingJobs.slice(fileCachingIndex, fileCachingIndex + batchSize); + + for (const job of batch) { + try { + await cacheJobFiles(job, fileCachingToken); + } catch (err) { + // Silently continue on errors + logLine('logs/error.log', `File cache error for job ${job.Job_Number}: ${(err as Error)?.message}`); + } + } + + fileCachingIndex += batchSize; + + // Continue with next batch after a delay (3 seconds between batches) + fileCachingTimer = setTimeout(() => cacheNextFileBatch(), 3000); +} + +async function cacheJobFiles(job: any, token: string) { + const link = job.Job_Folder_Link; + if (!link) return; + + const cacheKey = `job-files:${link}`; + const ttl = 1800; // 30 minutes + + try { + const { driveId, itemId } = await resolveShareLink(link, token); + const items = await listChildrenRecursive(driveId, itemId, 0, 5, token); + + const data = { + driveId, + items: items.map(item => ({ + id: item.id, + name: item.name, + webUrl: item.webUrl, + size: item.size, + lastModifiedDateTime: item.lastModifiedDateTime, + mimeType: item.file?.mimeType, + isFolder: !!item.folder, + parentPath: item.parentReference?.path, + driveId: item.parentReference?.driveId || driveId + })), + cachedAt: Date.now() + }; + + await setCache(cacheKey, data, ttl); + logLine('logs/server.log', `✓ Cached ${items.length} files for job ${job.Job_Number}`); + } catch (err: any) { + // Only log if not a token expiration issue + if (!err.message?.includes('401') && !err.message?.includes('token')) { + logLine('logs/error.log', `File cache skip for job ${job.Job_Number}: ${err.message}`); + } + } +} // Graceful shutdown logging const shutdown = (signal: string) => { logLine('logs/server.log', `Shutting down due to ${signal}`); + if (fileCachingTimer) clearTimeout(fileCachingTimer); process.exit(0); }; process.on('SIGINT', () => shutdown('SIGINT')); diff --git a/frontend/public/index.html b/frontend/public/index.html index d71aa1d..4bafb19 100644 --- a/frontend/public/index.html +++ b/frontend/public/index.html @@ -60,10 +60,6 @@

Job Info

-
-
-
-
`; - const graphToken = getGraphToken(); + const graphToken = await getGraphToken(); const headers = graphToken ? { 'x-graph-token': graphToken } : {}; try {