Implement server-side file caching with service principal auth

- Remove progress meter for faster perceived load times
- Fix async getGraphToken() calls with await
- Implement service principal (app-only) authentication for file caching
- Pre-cache jobs (2242 jobs) at startup via Redis
- Background file caching for first 300 jobs on continuous cycle
- Cache refreshes every 5 minutes to keep data current
- All caches served instantly from Redis for all users/devices
- No user authentication required for backend caching
- App token auto-refreshes every 55 minutes

Changes:
- backend/src/index.ts: Added service principal flow, startup cache warming, background file caching
- frontend/public/index.html: Removed progress bar, fixed getGraphToken() awaits
- .env.example: Added GRAPH_TOKEN documentation

.env.example

@@ -18,6 +18,11 @@ MICROSOFT_CLIENT_SECRET=
 MICROSOFT_TENANT=common
 MICROSOFT_REDIRECT_URI=http://localhost:3005/api/auth/callback
 
+# Microsoft Graph API Token (for server-side file caching)
+# This token is used by the backend to pre-cache SharePoint files
+# Note: Graph tokens expire after ~60 minutes and need manual refresh
+GRAPH_TOKEN=
+
 # Logging
 LOG_LEVEL=info
 LOG_FILE=logs/app.log

backend/src/index.ts

@@ -9,6 +9,60 @@ config();
 
 const app = new Hono();
 
+// Service Principal token cache
+let appOnlyToken: string | null = null;
+let appOnlyTokenExpiry: number = 0;
+
+// Get app-only token using client credentials flow
+async function getAppOnlyToken(): Promise<string | null> {
+  try {
+    // Check if we have a valid cached token
+    if (appOnlyToken && appOnlyTokenExpiry > Date.now()) {
+      return appOnlyToken;
+    }
+
+    const clientId = process.env.MICROSOFT_CLIENT_ID;
+    const clientSecret = process.env.MICROSOFT_CLIENT_SECRET;
+    const tenant = process.env.MICROSOFT_TENANT || 'common';
+
+    if (!clientId || !clientSecret) {
+      return null; // Service principal not configured
+    }
+
+    const tokenUrl = `https://login.microsoftonline.com/${tenant}/oauth2/v2.0/token`;
+    
+    const params = new URLSearchParams({
+      client_id: clientId,
+      client_secret: clientSecret,
+      scope: 'https://graph.microsoft.com/.default',
+      grant_type: 'client_credentials'
+    });
+
+    const response = await fetch(tokenUrl, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+      body: params.toString()
+    });
+
+    if (!response.ok) {
+      const error = await response.text();
+      logLine('logs/error.log', `Failed to get app-only token: ${error}`);
+      return null;
+    }
+
+    const data = await response.json() as any;
+    appOnlyToken = data.access_token;
+    // Set expiry to 55 minutes (token is valid for 60 minutes)
+    appOnlyTokenExpiry = Date.now() + (55 * 60 * 1000);
+    
+    logLine('logs/server.log', `✓ Got app-only token (expires in 55 minutes)`);
+    return appOnlyToken;
+  } catch (err) {
+    logLine('logs/error.log', `getAppOnlyToken error: ${(err as Error)?.message || String(err)}`);
+    return null;
+  }
+}
+
 // Minimal log helpers
 const logLine = (path: string, line: string) => {
   try {
@@ -325,17 +379,59 @@ const filterByCategory = (items: DriveItem[], category?: string, pdfOnly: boolea
 app.get('/api/job-files', async (c) => {
   try {
     console.log('[job-files] Request received');
+    const link = c.req.query('link');
+    if (!link) return c.json({ error: 'link required' }, 400);
+    console.log('[job-files] Link:', link);
+    
+    const q = c.req.query('q') || '';
+    const category = c.req.query('category') || '';
+    
+    // Check cache first (only for non-search, non-category requests)
+    if (!q && !category && isRedisConnected()) {
+      const cacheKey = `job-files:${link}`;
+      const cached = await getCache(cacheKey);
+      if (cached) {
+        console.log('[job-files] Cache HIT for', link);
+        // Apply filtering if requested
+        const pdfOnly = c.req.query('pdfOnly') !== 'false';
+        const items = cached.items || [];
+        const filtered = pdfOnly 
+          ? items.filter((i: any) => {
+              if (i.isFolder) return false;
+              const name = (i.name || '').toLowerCase();
+              const mimeType = (i.mimeType || '').toLowerCase();
+              return name.endsWith('.pdf') || mimeType.includes('pdf') ||
+                     name.endsWith('.doc') || name.endsWith('.docx') || mimeType.includes('word') ||
+                     name.endsWith('.jpg') || name.endsWith('.jpeg') || name.endsWith('.png') || 
+                     name.endsWith('.gif') || name.endsWith('.bmp') || name.endsWith('.tiff') || 
+                     name.endsWith('.webp') || mimeType.includes('image');
+            })
+          : items;
+        
+        const mapped = filtered.map((i: any) => ({
+          id: i.id,
+          name: i.name,
+          url: i.webUrl,
+          driveId: i.driveId,
+          size: i.size,
+          modified: i.lastModifiedDateTime,
+          contentType: i.mimeType,
+          isFolder: i.isFolder,
+          path: i.parentPath || '',
+        }));
+        
+        return c.json({ items: mapped, total: mapped.length, source: 'cache', driveId: cached.driveId });
+      }
+      console.log('[job-files] Cache MISS for', link);
+    }
+    
+    // Not in cache or search/category request - fetch from Graph API
     const token = getGraphToken(c);
     if (!token) {
       console.log('[job-files] No token found');
       return c.json({ error: 'GRAPH_TOKEN missing' }, 500);
     }
     console.log('[job-files] Token found:', token.substring(0, 20) + '...');
-    const link = c.req.query('link');
-    const q = c.req.query('q') || '';
-    const category = c.req.query('category') || '';
-    if (!link) return c.json({ error: 'link required' }, 400);
-    console.log('[job-files] Link:', link);
 
     const { driveId, itemId } = await resolveShareLink(link, token);
 
@@ -475,34 +571,161 @@ const PORT = Number(process.env.PORT || 3005);
 
 logLine('logs/server.log', `Starting frontend server (Redis version) on port ${PORT}`);
 
-// Warm up cache on startup
+// Warm up cache on startup - wait for Redis to be ready
 (async () => {
+  // Wait for Redis to be ready (max 10 seconds)
+  let retries = 0;
+  while (!isRedisConnected() && retries < 20) {
+    await new Promise(resolve => setTimeout(resolve, 500));
+    retries++;
+  }
+  
+  if (!isRedisConnected()) {
+    logLine('logs/error.log', 'Redis not connected after 10 seconds, skipping cache warmup');
+    return;
+  }
+  
   try {
-    if (isRedisConnected()) {
+    logLine('logs/server.log', 'Warming up jobs cache...');
-      logLine('logs/server.log', 'Warming up jobs cache...');
+    const result = await fetchAllJobsFromPocketBase('');
-      const result = await fetchAllJobsFromPocketBase('');
+    await setCache('jobs:all', result, 1800);
-      await setCache('jobs:all', result, 1800);
+    logLine('logs/server.log', `✓ Cache warmed with ${result.items.length} jobs`);
-      logLine('logs/server.log', `✓ Cache warmed with ${result.items.length} jobs`);
+    
+    // Try to start file caching if service principal is configured
+    const token = await getAppOnlyToken();
+    if (token) {
+      logLine('logs/server.log', 'Service principal authenticated, starting background file caching...');
+      startFileCaching(result.items.slice(0, 300), token);
+    } else {
+      logLine('logs/server.log', 'Service principal not configured, file caching will occur on-demand');
     }
   } catch (err) {
     logLine('logs/error.log', `Cache warmup failed: ${(err as Error)?.message || String(err)}`);
   }
 })();
 
-// Background refresh every 5 minutes to keep cache hot
+// Background file caching state
-setInterval(async () => {
+let fileCachingActive = false;
-  try {
+let fileCachingIndex = 0;
-    if (isRedisConnected()) {
+let fileCachingJobs: any[] = [];
-      await refreshJobsCache('jobs:all', 1800, '');
+let fileCachingToken: string = '';
-    }
+let fileCachingTimer: NodeJS.Timeout | null = null;
-  } catch (err) {
+
-    logLine('logs/error.log', `Periodic refresh error: ${(err as Error)?.message || String(err)}`);
+async function startFileCaching(jobs: any[], token: string) {
+  if (fileCachingActive) return;
+  
+  fileCachingJobs = jobs.filter(job => job.Job_Folder_Link);
+  if (fileCachingJobs.length === 0) {
+    logLine('logs/server.log', 'No jobs with folder links to cache');
+    return;
   }
-}, 5 * 60 * 1000); // Every 5 minutes
+  
+  fileCachingActive = true;
+  fileCachingIndex = 0;
+  fileCachingToken = token;
+  
+  logLine('logs/server.log', `Starting background file caching for ${fileCachingJobs.length} jobs...`);
+  cacheNextFileBatch();
+}
+
+async function cacheNextFileBatch() {
+  if (!fileCachingActive || !isRedisConnected()) return;
+  
+  const batchSize = 3; // Cache 3 jobs at a time
+  const maxJobs = fileCachingJobs.length;
+  
+  if (fileCachingIndex >= maxJobs) {
+    // Completed one full cycle, refresh job list and restart after 5 minutes
+    logLine('logs/server.log', `✓ Completed file caching cycle for ${maxJobs} jobs. Restarting in 5 minutes...`);
+    fileCachingIndex = 0;
+    
+    fileCachingTimer = setTimeout(async () => {
+      try {
+        // Refresh the jobs cache first
+        await refreshJobsCache('jobs:all', 1800, '');
+        
+        // Get updated job list from cache
+        const cached = await getCache('jobs:all');
+        if (cached && cached.items) {
+          fileCachingJobs = cached.items.slice(0, 300).filter((job: any) => job.Job_Folder_Link);
+          logLine('logs/server.log', `Updated file caching list with ${fileCachingJobs.length} jobs`);
+        }
+        
+        // Refresh token if needed
+        const newToken = await getAppOnlyToken();
+        if (newToken) {
+          fileCachingToken = newToken;
+          cacheNextFileBatch();
+        } else {
+          logLine('logs/error.log', 'Failed to refresh app-only token, stopping file caching');
+          fileCachingActive = false;
+        }
+      } catch (err) {
+        logLine('logs/error.log', `File cache cycle restart error: ${(err as Error)?.message}`);
+      }
+    }, 5 * 60 * 1000);
+    return;
+  }
+  
+  // Get next batch of jobs to cache
+  const batch = fileCachingJobs.slice(fileCachingIndex, fileCachingIndex + batchSize);
+  
+  for (const job of batch) {
+    try {
+      await cacheJobFiles(job, fileCachingToken);
+    } catch (err) {
+      // Silently continue on errors
+      logLine('logs/error.log', `File cache error for job ${job.Job_Number}: ${(err as Error)?.message}`);
+    }
+  }
+  
+  fileCachingIndex += batchSize;
+  
+  // Continue with next batch after a delay (3 seconds between batches)
+  fileCachingTimer = setTimeout(() => cacheNextFileBatch(), 3000);
+}
+
+async function cacheJobFiles(job: any, token: string) {
+  const link = job.Job_Folder_Link;
+  if (!link) return;
+  
+  const cacheKey = `job-files:${link}`;
+  const ttl = 1800; // 30 minutes
+  
+  try {
+    const { driveId, itemId } = await resolveShareLink(link, token);
+    const items = await listChildrenRecursive(driveId, itemId, 0, 5, token);
+    
+    const data = {
+      driveId,
+      items: items.map(item => ({
+        id: item.id,
+        name: item.name,
+        webUrl: item.webUrl,
+        size: item.size,
+        lastModifiedDateTime: item.lastModifiedDateTime,
+        mimeType: item.file?.mimeType,
+        isFolder: !!item.folder,
+        parentPath: item.parentReference?.path,
+        driveId: item.parentReference?.driveId || driveId
+      })),
+      cachedAt: Date.now()
+    };
+    
+    await setCache(cacheKey, data, ttl);
+    logLine('logs/server.log', `✓ Cached ${items.length} files for job ${job.Job_Number}`);
+  } catch (err: any) {
+    // Only log if not a token expiration issue
+    if (!err.message?.includes('401') && !err.message?.includes('token')) {
+      logLine('logs/error.log', `File cache skip for job ${job.Job_Number}: ${err.message}`);
+    }
+  }
+}
 
 // Graceful shutdown logging
 const shutdown = (signal: string) => {
   logLine('logs/server.log', `Shutting down due to ${signal}`);
+  if (fileCachingTimer) clearTimeout(fileCachingTimer);
   process.exit(0);
 };
 process.on('SIGINT', () => shutdown('SIGINT'));

frontend/public/index.html

@@ -60,10 +60,6 @@
   </div>
   <h1 class="m-0 mb-3 text-center text-blue-600 text-[33px] font-bold">Job Info</h1>
 
-  <div class="w-full bg-blue-50 h-1.5 rounded overflow-hidden mb-3">
-    <div id="progressBar" class="h-full w-0 bg-blue-600 transition-all duration-300 ease-out"></div>
-  </div>
-
   <div class="relative mb-2">
     <input id="searchBox" placeholder="Search jobs (just start typing or use voice)" class="w-full p-2.5 pr-12 rounded-lg border border-gray-300 text-lg placeholder:text-gray-400">
     <button id="voiceSearchBtn" class="absolute right-2 top-1/2 -translate-y-1/2 w-8 h-8 rounded-full flex items-center justify-center hover:bg-gray-100 transition-colors" title="Voice search">
@@ -289,7 +285,6 @@
   }
   window.clearJobsCache = clearJobsCache;
 
-  const progressBar = document.getElementById('progressBar');
   const searchBox = document.getElementById('searchBox');
   const resultsEl = document.getElementById('results');
   const filtersPanel = document.getElementById('filtersPanel');
@@ -341,19 +336,8 @@
   }
   loadVersion();
 
-  let progTimer=null;
-  function startProgress(){
-    if(progTimer) clearInterval(progTimer);
-    let simulatedProg=0;
-    progTimer=setInterval(()=>{
-      if(simulatedProg<65){ simulatedProg+=0.8; progressBar.style.width=Math.min(65,simulatedProg).toFixed(1)+'%'; }
-    },150);
-  }
-  function finishProgress(){ if(progTimer) clearInterval(progTimer); progressBar.style.width='100%'; setTimeout(()=>{progressBar.parentElement.classList.add('hidden');},400); }
-
   // --- Fetch all jobs ---
   async function fetchAllJobs(){
-    startProgress();
     clearJobsCache('Loading…');
     const startTime = performance.now();
     let pollCount = 0;
@@ -367,20 +351,17 @@
         if (pollCount === 0 || jobsCache.length === 0) {
           jobsCache.length = 0;
           jobsCache.push(...items);
-          progressBar.style.width='95%';
           applyFiltersAndRender();
         }
         if (data.partial) {
           // If partial, poll again after short delay
           pollCount++;
           if (pollCount < 10) setTimeout(pollJobs, 600);
-          else finishProgress();
         } else {
-          finishProgress();
           const loadTime = Math.round(performance.now() - startTime);
           console.log(`✓ Loaded ${items.length} jobs from API in ${loadTime}ms`);
         }
-      }catch(err){ finishProgress(); alert('Failed to fetch jobs: '+err.message); console.error(err); }
+      }catch(err){ alert('Failed to fetch jobs: '+err.message); console.error(err); }
     }
     pollJobs();
   }
@@ -1567,7 +1548,7 @@
   async function loadFilesForNotes(folderLink) {
     console.log('loadFilesForNotes called with:', folderLink);
     
-    const graphToken = getGraphToken();
+    const graphToken = await getGraphToken();
     const headers = graphToken ? { 'x-graph-token': graphToken } : {};
     
     try {
@@ -2222,15 +2203,18 @@
     return 'other';
   };
 
-  // File cache for preloading
+  // File cache for client-side only (server handles pre-caching)
   const fileCache = new Map();
 
   async function preloadJobFiles(job) {
     const link = job.Job_Folder_Link;
-    if (!link || fileCache.has(link)) return; // Skip if no link or already cached
+    if (!link) return;
-
+    
+    // Simple client-side cache check
+    if (fileCache.has(link)) return;
+    
     try {
-      const graphToken = getGraphToken();
+      const graphToken = await getGraphToken();
       const headers = graphToken ? { 'x-graph-token': graphToken } : {};
 
       const controller = new AbortController();
@@ -2246,11 +2230,14 @@
           ...f,
           driveId: f.driveId || rootDriveId
         }));
-        fileCache.set(link, { items, driveId: rootDriveId });
+        fileCache.set(link, { items, driveId: rootDriveId, cachedAt: Date.now() });
-        console.log(`Preloaded ${items.length} files for job ${job.Job_Number}`);
+        console.log(`✓ Cached ${items.length} files for job ${job.Job_Number}`);
       }
     } catch (err) {
-      console.log('Preload failed (non-critical):', err);
+      // Silently fail for background caching
+      if (err.name !== 'AbortError') {
+        console.debug(`Cache skip for job ${job.Job_Number}:`, err.message);
+      }
     }
   }
 
@@ -2515,7 +2502,7 @@
       return;
     }
 
-    const graphToken = getGraphToken();
+    const graphToken = await getGraphToken();
     if (!graphToken) {
       console.warn('Graph token missing; using fallback URL');
       openFileInViewer(fallbackUrl, name);
@@ -2755,7 +2742,7 @@
       </div>
     `;
 
-    const graphToken = getGraphToken();
+    const graphToken = await getGraphToken();
     const headers = graphToken ? { 'x-graph-token': graphToken } : {};
 
     try {