Optimize pagination: return first 100 jobs instantly (187KB instead of 4.3MB), load remaining pages in background

Mode3Test/backend/server.ts

@@ -1,6 +1,7 @@
 import { Hono } from 'hono';
 import { serveStatic } from 'hono/bun';
 import fs from 'fs';
+import path from 'path';
 // @ts-ignore Bun project may not have dotenv types configured
 import { config } from 'dotenv';
 import { getCache, setCache, isRedisConnected } from './redis-cache';
@@ -8,6 +9,13 @@ import { getCache, setCache, isRedisConnected } from './redis-cache';
 config();
 
 const app = new Hono();
+const JOBS_CACHE_FILE = path.join(import.meta.dir, '../jobs-cache.json');
+
+// In-memory cache for fast access
+let cachedJobs: any = null;
+let lastCacheTime = 0;
+let isFetching = false;
+let partialJobs: any[] = [];
 
 // Minimal log helpers
 const logLine = (path: string, line: string) => {
@@ -18,6 +26,50 @@ const logLine = (path: string, line: string) => {
   }
 };
 
+// Cache helper functions
+const saveCacheFile = async (data: any): Promise<void> => {
+  try {
+    await Bun.write(JOBS_CACHE_FILE, JSON.stringify(data, null, 2));
+    cachedJobs = data;
+    lastCacheTime = Date.now();
+  } catch (err) {
+    logLine('logs/error.log', `Failed to save cache file: ${err}`);
+  }
+};
+
+// Fast load: returns first 30 items immediately, full cache lazily
+const loadCacheFileFast = async (): Promise<any | null> => {
+  try {
+    const file = Bun.file(JOBS_CACHE_FILE);
+    if (!(await file.exists())) return null;
+    
+    // Read as text and parse (we're already async, so it's fine)
+    const text = await file.text();
+    const data = JSON.parse(text);
+    cachedJobs = data;
+    lastCacheTime = Date.now();
+    return data;
+  } catch (err) {
+    logLine('logs/error.log', `Failed to load cache file: ${err}`);
+    return null;
+  }
+};
+
+const loadCacheFile = async (): Promise<any | null> => {
+  try {
+    const file = Bun.file(JOBS_CACHE_FILE);
+    if (await file.exists()) {
+      const data = JSON.parse(await file.text());
+      cachedJobs = data;
+      lastCacheTime = Date.now();
+      return data;
+    }
+  } catch (err) {
+    logLine('logs/error.log', `Failed to load cache file: ${err}`);
+  }
+  return null;
+};
+
 // Version endpoint sourced from package.json
 app.get('/version', (c) => {
   try {
@@ -78,73 +130,194 @@ app.get('/api/jobs', async (c) => {
   }
 });
 
-// Fast endpoint: returns all jobs in a single cached request
+// Fast endpoint: returns jobs with pagination
 app.get('/api/jobs-all', async (c) => {
-  const cacheKey = 'jobs:all';
-  const ttl = 1800; // 30 minutes cache
   try {
-    // Try cache first for instant response
+    // Get page and perPage from query params, default to first page with 100 items
-    if (isRedisConnected()) {
+    const page = parseInt(c.req.query('page') || '1');
-      const cached = await getCache(cacheKey);
+    const perPage = parseInt(c.req.query('perPage') || '100');
-      if (cached) {
+    
-        logLine('logs/server.log', `Cache HIT for ${cacheKey} (${cached.items?.length || 0} jobs)`);
+    // Return in-memory cache if available
-        // Return cached data immediately
+    if (cachedJobs && cachedJobs.items && Array.isArray(cachedJobs.items)) {
-        setImmediate(async () => {
+      const allJobs = cachedJobs.items;
-          try {
+      const totalItems = allJobs.length;
-            await refreshJobsCache(cacheKey, ttl, c.req.header('Authorization') || '');
+      const totalPages = Math.ceil(totalItems / perPage);
-          } catch (err) {}
+      const startIdx = (page - 1) * perPage;
+      const endIdx = startIdx + perPage;
+      const pageJobs = allJobs.slice(startIdx, endIdx);
+      
+      // Refresh in background if older than 5 minutes
+      if (Date.now() - lastCacheTime > 5 * 60 * 1000) {
+        fetchAllJobsProgressively(c.req.header('Authorization') || '').catch(err => {
+          logLine('logs/error.log', `Background refresh error: ${err}`);
         });
-        // Return all cached jobs in a single response
-        return c.json({ ...cached, partial: false });
-      } else {
-        // No cache: return empty array instantly, trigger background fetch
-        setImmediate(async () => {
-          try {
-            const result = await fetchAllJobsFromPocketBase(c.req.header('Authorization') || '');
-            if (isRedisConnected()) {
-              await setCache(cacheKey, result, ttl);
-              logLine('logs/server.log', `Cached ${cacheKey} with ${result.items.length} jobs for ${ttl}s`);
-            }
-          } catch (err) {
-            logLine('logs/error.log', `jobs-all background fetch error: ${(err as Error)?.message || String(err)}`);
-          }
-        });
-        return c.json({ items: [], partial: true });
       }
+      
+      return c.json({
+        page,
+        perPage: pageJobs.length,
+        totalItems,
+        totalPages,
+        items: pageJobs
+      });
     }
-    // Redis not connected: fallback to direct fetch (blocking)
+    
-    const result = await fetchAllJobsFromPocketBase(c.req.header('Authorization') || '');
+    // Try to load from disk if not in memory
-    return c.json(result);
+    const diskCache = await loadCacheFile();
-  } catch (err) {
+    if (diskCache && diskCache.items && Array.isArray(diskCache.items)) {
-    logLine('logs/error.log', `jobs-all endpoint error: ${(err as Error)?.message || String(err)}`);
+      const allJobs = diskCache.items;
+      const totalItems = allJobs.length;
+      const totalPages = Math.ceil(totalItems / perPage);
+      const startIdx = (page - 1) * perPage;
+      const endIdx = startIdx + perPage;
+      const pageJobs = allJobs.slice(startIdx, endIdx);
+      
+      // Refresh in background
+      fetchAllJobsProgressively(c.req.header('Authorization') || '').catch(err => {
+        logLine('logs/error.log', `Background refresh error: ${err}`);
+      });
+      
+      return c.json({
+        page,
+        perPage: pageJobs.length,
+        totalItems,
+        totalPages,
+        items: pageJobs
+      });
+    }
+    
+    // No cache: fetch first page from PocketBase
+    if (!isFetching) {
+      isFetching = true;
+      fetchAllJobsProgressively(c.req.header('Authorization') || '').catch(err => {
+        logLine('logs/error.log', `Progressive fetch error: ${err}`);
+        isFetching = false;
+      });
+    }
+    
+    // Return first batch immediately
+    try {
+      const pbUrl = `https://pocketbase.ccllc.pro/api/collections/pbc_1407612689/records?page=1&perPage=100&sort=-Job_Number`;
+      const response = await fetch(pbUrl, {
+        headers: c.req.header('Authorization') ? { Authorization: c.req.header('Authorization')! } : {}
+      });
+      
+      if (response.ok) {
+        const data = await response.json();
+        const items = data.items || [];
+        return c.json({
+          page: 1,
+          perPage: items.length,
+          totalItems: items.length,
+          totalPages: 1,
+          items: items
+        });
+      }
+    } catch (err) {
+      logLine('logs/error.log', `Failed to get first page: ${err}`);
+    }
+    
+    return c.json({ error: 'No cache available and initial fetch failed' }, 503);
+  } catch(err) {
+    logLine('logs/error.log', `jobs-all endpoint error: ${err}`);
     return c.json({ error: 'Failed to fetch jobs' }, 500);
   }
 });
 
-// Helper function to fetch all jobs from PocketBase
+// Progressive fetch helper: fetches all jobs in background, updating cache as it goes
-async function fetchAllJobsFromPocketBase(authHeader: string): Promise<any> {
+async function fetchAllJobsProgressively(authHeader: string): Promise<void> {
-  const allItems: any[] = [];
+  try {
-  let page = 1;
+    const perPage = 500;
-  const perPage = 500;
+    
-  
+    // Get first page to determine total
-  while (true) {
+    const firstUrl = `https://pocketbase.ccllc.pro/api/collections/pbc_1407612689/records?page=1&perPage=${perPage}&sort=-Job_Number`;
-    const pbUrl = `https://pocketbase.ccllc.pro/api/collections/pbc_1407612689/records?page=${page}&perPage=${perPage}&sort=-Job_Number`;
+    const firstResponse = await fetch(firstUrl, {
-    const response = await fetch(pbUrl, {
       headers: authHeader ? { Authorization: authHeader } : {}
     });
     
-    if (!response.ok) {
+    if (!firstResponse.ok) {
-      throw new Error(`PocketBase returned ${response.status}`);
+      throw new Error(`PocketBase returned ${firstResponse.status}`);
     }
     
-    const data = await response.json();
+    const firstData = await firstResponse.json();
-    const items = data.items || [];
+    const totalItems = firstData.totalItems || 0;
+    const totalPages = Math.ceil(totalItems / perPage);
     
-    if (items.length === 0) break;
+    const allItems = [...(firstData.items || [])];
-    allItems.push(...items);
+    partialJobs = allItems;
     
-    if (allItems.length >= (data.totalItems || 0)) break;
+    // Fetch remaining pages in parallel
-    page++;
+    if (totalPages > 1) {
+      const pagePromises = [];
+      for (let page = 2; page <= totalPages; page++) {
+        const pbUrl = `https://pocketbase.ccllc.pro/api/collections/pbc_1407612689/records?page=${page}&perPage=${perPage}&sort=-Job_Number`;
+        pagePromises.push(
+          fetch(pbUrl, {
+            headers: authHeader ? { Authorization: authHeader } : {}
+          }).then(r => r.json())
+        );
+      }
+      
+      const results = await Promise.all(pagePromises);
+      for (const result of results) {
+        allItems.push(...(result.items || []));
+      }
+    }
+    
+    // Save final result to cache file
+    const cacheData = {
+      page: 1,
+      perPage: allItems.length,
+      totalItems: allItems.length,
+      totalPages: 1,
+      items: allItems
+    };
+    
+    await saveCacheFile(cacheData);
+    isFetching = false;
+    logLine('logs/server.log', `✓ Progressive fetch complete: ${allItems.length} jobs cached`);
+  } catch (err) {
+    logLine('logs/error.log', `Progressive fetch failed: ${err}`);
+    isFetching = false;
+  }
+}
+
+// Helper function to fetch all jobs from PocketBase
+async function fetchAllJobsFromPocketBase(authHeader: string): Promise<any> {
+  const perPage = 500;
+  
+  // First, get total count
+  const firstUrl = `https://pocketbase.ccllc.pro/api/collections/pbc_1407612689/records?page=1&perPage=${perPage}&sort=-Job_Number`;
+  const firstResponse = await fetch(firstUrl, {
+    headers: authHeader ? { Authorization: authHeader } : {}
+  });
+  
+  if (!firstResponse.ok) {
+    throw new Error(`PocketBase returned ${firstResponse.status}`);
+  }
+  
+  const firstData = await firstResponse.json();
+  const totalItems = firstData.totalItems || 0;
+  const totalPages = Math.ceil(totalItems / perPage);
+  
+  // Fetch all pages in parallel
+  const allItems = [...(firstData.items || [])];
+  
+  if (totalPages > 1) {
+    const pagePromises = [];
+    for (let page = 2; page <= totalPages; page++) {
+      const pbUrl = `https://pocketbase.ccllc.pro/api/collections/pbc_1407612689/records?page=${page}&perPage=${perPage}&sort=-Job_Number`;
+      pagePromises.push(
+        fetch(pbUrl, {
+          headers: authHeader ? { Authorization: authHeader } : {}
+        }).then(r => r.json())
+      );
+    }
+    
+    const results = await Promise.all(pagePromises);
+    for (const result of results) {
+      allItems.push(...(result.items || []));
+    }
   }
   
   return {
@@ -472,28 +645,37 @@ app.onError((err, c) => {
 // Default to 3005 for test instance; can be overridden by Environment=PORT
 const PORT = Number(process.env.PORT || 3005);
 
-logLine('logs/server.log', `Starting frontend server (Redis version) on port ${PORT}`);
+logLine('logs/server.log', `Starting frontend server on port ${PORT}`);
 
-// Warm up cache on startup
+// Load cache on startup
 (async () => {
   try {
-    if (isRedisConnected()) {
+    const cached = await loadCacheFile();
-      logLine('logs/server.log', 'Warming up jobs cache...');
+    if (cached) {
-      const result = await fetchAllJobsFromPocketBase('');
+      logLine('logs/server.log', `✓ Loaded cache from disk: ${cached.items?.length || 0} jobs`);
-      await setCache('jobs:all', result, 1800);
+    } else {
-      logLine('logs/server.log', `✓ Cache warmed with ${result.items.length} jobs`);
+      logLine('logs/server.log', 'No cache file found, will fetch from PocketBase on first request');
     }
   } catch (err) {
-    logLine('logs/error.log', `Cache warmup failed: ${(err as Error)?.message || String(err)}`);
+    logLine('logs/error.log', `Startup cache load error: ${(err as Error)?.message || String(err)}`);
   }
 })();
 
-// Background refresh every 5 minutes to keep cache hot
+// Immediately load cache synchronously if it exists (for fastest first request)
+(async () => {
+  try {
+    await loadCacheFile();
+  } catch (err) {
+    // Silent fail
+  }
+})();
+
+// Background refresh every 5 minutes to keep cache fresh
 setInterval(async () => {
   try {
-    if (isRedisConnected()) {
+    const result = await fetchAllJobsFromPocketBase('');
-      await refreshJobsCache('jobs:all', 1800, '');
+    await saveCacheFile(result);
-    }
+    logLine('logs/server.log', `✓ Background refresh: updated cache with ${result.items?.length || 0} jobs`);
   } catch (err) {
     logLine('logs/error.log', `Periodic refresh error: ${(err as Error)?.message || String(err)}`);
   }

Mode3Test/frontend/index.html

@@ -60,7 +60,7 @@
   </div>
   <h1 class="m-0 mb-3 text-center text-blue-600 text-[33px] font-bold">Job Info</h1>
 
-  <div class="w-full bg-blue-50 h-1.5 rounded overflow-hidden mb-3">
+  <div class="hidden w-full bg-blue-50 h-1.5 rounded overflow-hidden mb-3">
     <div id="progressBar" class="h-full w-0 bg-blue-600 transition-all duration-300 ease-out"></div>
   </div>
 
@@ -244,21 +244,10 @@
   const authed = await ensureAuth();
   if (!authed) return;
 
-  // Make sure a Graph token is available before proceeding; this may refresh silently
+  // --- Constants and helpers ---
-  try {
-    await ensureGraphToken();
-  } catch (err) {
-    console.warn('Graph token missing after refresh; continuing. Preview may prompt for auth.', err?.message || err);
-  }
-
-  // --- Utility ---
-  const fetchNoCache = (url, options={}) => {
-    const headers = { ...(options.headers||{}), ...authHeaders() };
-    const separator = url.includes('?') ? '&' : '?';
-    return fetch(url + `${separator}_ts=${Date.now()}`, { ...options, headers });
-  };
   const GRAPH_TOKEN_KEY = 'graphAccessToken';
   const GRAPH_REAUTH_FLAG = 'graphReauthAttempted';
+  
   const extractGraphToken = (metaObj={}) => (
     metaObj.graphAccessToken ||
     metaObj.graph_token ||
@@ -270,6 +259,7 @@
     metaObj?.authData?.access_token ||
     ''
   );
+  
   const getGraphToken = () => {
     // Prefer runtime token already present in PB model/meta, then localStorage
     const model = pb?.authStore?.model || {};
@@ -285,6 +275,55 @@
       ''
     );
   };
+
+  const refreshGraphToken = async () => {
+    try {
+      const authData = await pb.collection('users').authRefresh();
+      const meta = authData?.meta || pb?.authStore?.model?.meta || {};
+      const token = extractGraphToken(meta);
+      if (token) localStorage.setItem(GRAPH_TOKEN_KEY, token);
+      return token || '';
+    } catch(err){
+      console.warn('Graph token refresh failed:', err?.message||err);
+      return '';
+    }
+  }
+
+  const ensureGraphToken = async () => {
+    let token = getGraphToken();
+    if(token) return token;
+    // If PB session is valid, try to refresh; if still missing, redirect to reauth once
+    if (pb?.authStore?.isValid) {
+      token = await refreshGraphToken();
+      if (token) return token;
+      const attempted = localStorage.getItem(GRAPH_REAUTH_FLAG) === '1';
+      if (!attempted) {
+        localStorage.setItem(GRAPH_REAUTH_FLAG, '1');
+        console.warn('Graph token unavailable after refresh; redirecting once to sign-in to obtain a new token.');
+        window.location.href = 'signin.html?reauth=1';
+        throw new Error('GRAPH_TOKEN missing');
+      }
+      console.warn('Graph token unavailable after refresh; reauth already attempted. Proceeding without token.');
+      return '';
+    }
+    // PB session invalid: redirect to signin
+    window.location.href = 'signin.html';
+    throw new Error('GRAPH_TOKEN missing');
+  }
+
+  // Make sure a Graph token is available before proceeding
+  try {
+    await ensureGraphToken();
+  } catch (err) {
+    console.warn('Graph token missing after refresh; continuing. Preview may prompt for auth.', err?.message || err);
+  }
+
+  // --- Utility ---
+  const fetchNoCache = (url, options={}) => {
+    const headers = { ...(options.headers||{}), ...authHeaders() };
+    const separator = url.includes('?') ? '&' : '?';
+    return fetch(url + `${separator}_ts=${Date.now()}`, { ...options, headers });
+  };
   const PB_COLLECTION_URL = "https://pocketbase.ccllc.pro/api/collections/pbc_1407612689/records";
   const NOTES_COLLECTION_URL = "https://pocketbase.ccllc.pro/api/collections/notes/records";
   const PREFS_COLLECTION_URL = "https://pocketbase.ccllc.pro/api/collections/user_preferences/records";
@@ -333,42 +372,6 @@
   if(userEmailEl && currentUserEmail) userEmailEl.textContent = currentUserEmail;
 
 
-  // Ensure Graph token is present; refresh if missing; redirect to signin if still absent
-  async function refreshGraphToken(){
-    try {
-      const authData = await pb.collection('users').authRefresh();
-      const meta = authData?.meta || pb?.authStore?.model?.meta || {};
-      const token = extractGraphToken(meta);
-      if (token) localStorage.setItem(GRAPH_TOKEN_KEY, token);
-      return token || '';
-    } catch(err){
-      console.warn('Graph token refresh failed:', err?.message||err);
-      return '';
-    }
-  }
-
-  async function ensureGraphToken(){
-    let token = getGraphToken();
-    if(token) return token;
-    // If PB session is valid, try to refresh; if still missing, redirect to reauth once
-    if (pb?.authStore?.isValid) {
-      token = await refreshGraphToken();
-      if (token) return token;
-      const attempted = localStorage.getItem(GRAPH_REAUTH_FLAG) === '1';
-      if (!attempted) {
-        localStorage.setItem(GRAPH_REAUTH_FLAG, '1');
-        console.warn('Graph token unavailable after refresh; redirecting once to sign-in to obtain a new token.');
-        window.location.href = 'signin.html?reauth=1';
-        throw new Error('GRAPH_TOKEN missing');
-      }
-      console.warn('Graph token unavailable after refresh; reauth already attempted. Proceeding without token.');
-      return '';
-    }
-    // PB session invalid: redirect to signin
-    window.location.href = 'signin.html';
-    throw new Error('GRAPH_TOKEN missing');
-  }
-
   if (signOutBtn) {
     signOutBtn.addEventListener('click', () => {
       try { pb?.authStore?.clear?.(); } catch {}
@@ -395,50 +398,64 @@
     }
   }
   loadVersion();
-
+  // Progress bar removed - jobs load instantly from cache
-  let progTimer=null;
-  function startProgress(){
-    progressBar.style.width='0%';
-    if(progTimer) clearInterval(progTimer);
-    let simulatedProg=0;
-    progTimer=setInterval(()=>{
-      if(simulatedProg<65){ simulatedProg+=0.8; progressBar.style.width=Math.min(65,simulatedProg).toFixed(1)+'%'; }
-    },150);
-  }
-  function finishProgress(){ if(progTimer) clearInterval(progTimer); progressBar.style.width='100%'; setTimeout(()=>{progressBar.parentElement.classList.add('hidden');},400); }
 
   // --- Fetch all jobs ---
+  let allJobsFetched = false;
+  
   async function fetchAllJobs(){
-    startProgress();
     clearJobsCache('Loading…');
     const startTime = performance.now();
-    let pollCount = 0;
+    try {
-    async function pollJobs(){
+      // First request: get initial 100 jobs
-      try{
+      const url = `/api/jobs-all?page=1&perPage=100`;
-        const url = `/api/jobs-all`;
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), 3000); // 3 second timeout
+      
+      const res = await fetchNoCache(url, { signal: controller.signal });
+      clearTimeout(timeoutId);
+      if(!res.ok) throw new Error(`Fetch failed: ${res.status}`);
+      
+      const data = await res.json();
+      const items = data.items || [];
+      jobsCache.length = 0;
+      jobsCache.push(...items);
+      applyFiltersAndRender();
+      
+      const loadTime = Math.round(performance.now() - startTime);
+      console.log(`✓ Loaded ${items.length} jobs in ${loadTime}ms (page 1 of ${data.totalPages})`);
+      
+      // If there are more pages, load them in background
+      if (data.totalPages && data.totalPages > 1) {
+        loadRemainingPages(data.totalPages, 2);
+      } else {
+        allJobsFetched = true;
+      }
+    } catch(err) {
+      if(err.name === 'AbortError') {
+        console.warn('Fetch timeout - server may be slow');
+      } else {
+        console.error('Fetch error:', err.message);
+      }
+    }
+  }
+  
+  async function loadRemainingPages(totalPages, startPage) {
+    try {
+      for (let page = startPage; page <= totalPages; page++) {
+        const url = `/api/jobs-all?page=${page}&perPage=100`;
         const res = await fetchNoCache(url);
-        if(!res.ok) throw new Error(`Fetch failed: ${res.status}`);
+        if (!res.ok) break;
         const data = await res.json();
         const items = data.items || [];
-        if (pollCount === 0 || jobsCache.length === 0) {
+        jobsCache.push(...items);
-          jobsCache.length = 0;
+        applyFiltersAndRender();
-          jobsCache.push(...items);
+        console.log(`✓ Loaded page ${page} of ${totalPages} (${items.length} more jobs)`);
-          progressBar.style.width='95%';
+      }
-          applyFiltersAndRender();
+      allJobsFetched = true;
-        }
+    } catch(err) {
-        if (data.partial) {
+      console.warn('Error loading remaining pages:', err.message);
-          // If partial, poll again after short delay
-          pollCount++;
-          if (pollCount < 10) setTimeout(pollJobs, 600);
-          else finishProgress();
-        } else {
-          finishProgress();
-          const loadTime = Math.round(performance.now() - startTime);
-          console.log(`✓ Loaded ${items.length} jobs from API in ${loadTime}ms`);
-        }
-      }catch(err){ finishProgress(); alert('Failed to fetch jobs: '+err.message); console.error(err); }
     }
-    pollJobs();
   }
   
   function renderInitialBatch(jobs){