From 9e28d86610b9940a703139668407e7c04a6e5d69 Mon Sep 17 00:00:00 2001 From: aewing Date: Sun, 18 Jan 2026 04:01:46 +0000 Subject: [PATCH] Optimize pagination: return first 100 jobs instantly (187KB instead of 4.3MB), load remaining pages in background --- Mode3Test/backend/server.ts | 155 +++++++++++++++++++++------------- Mode3Test/frontend/index.html | 47 +++++++++-- 2 files changed, 139 insertions(+), 63 deletions(-) diff --git a/Mode3Test/backend/server.ts b/Mode3Test/backend/server.ts index 4d93b0b..2303013 100644 --- a/Mode3Test/backend/server.ts +++ b/Mode3Test/backend/server.ts @@ -37,6 +37,24 @@ const saveCacheFile = async (data: any): Promise => { } }; +// Fast load: returns first 30 items immediately, full cache lazily +const loadCacheFileFast = async (): Promise => { + try { + const file = Bun.file(JOBS_CACHE_FILE); + if (!(await file.exists())) return null; + + // Read as text and parse (we're already async, so it's fine) + const text = await file.text(); + const data = JSON.parse(text); + cachedJobs = data; + lastCacheTime = Date.now(); + return data; + } catch (err) { + logLine('logs/error.log', `Failed to load cache file: ${err}`); + return null; + } +}; + const loadCacheFile = async (): Promise => { try { const file = Bun.file(JOBS_CACHE_FILE); @@ -112,89 +130,101 @@ app.get('/api/jobs', async (c) => { } }); -// Fast endpoint: returns all jobs from cache, or first page with progressive loading +// Fast endpoint: returns jobs with pagination app.get('/api/jobs-all', async (c) => { try { - // Return full cache if available - if (cachedJobs) { - return c.json(cachedJobs); - } + // Get page and perPage from query params, default to first page with 100 items + const page = parseInt(c.req.query('page') || '1'); + const perPage = parseInt(c.req.query('perPage') || '100'); - // Return partial results if still fetching - if (isFetching && partialJobs.length > 0) { + // Return in-memory cache if available + if (cachedJobs && cachedJobs.items && Array.isArray(cachedJobs.items)) { + const allJobs = cachedJobs.items; + const totalItems = allJobs.length; + const totalPages = Math.ceil(totalItems / perPage); + const startIdx = (page - 1) * perPage; + const endIdx = startIdx + perPage; + const pageJobs = allJobs.slice(startIdx, endIdx); + + // Refresh in background if older than 5 minutes + if (Date.now() - lastCacheTime > 5 * 60 * 1000) { + fetchAllJobsProgressively(c.req.header('Authorization') || '').catch(err => { + logLine('logs/error.log', `Background refresh error: ${err}`); + }); + } + return c.json({ - page: 1, - perPage: partialJobs.length, - totalItems: partialJobs.length, - totalPages: 1, - items: partialJobs + page, + perPage: pageJobs.length, + totalItems, + totalPages, + items: pageJobs }); } - // Try to load from disk + // Try to load from disk if not in memory const diskCache = await loadCacheFile(); - if (diskCache) { - return c.json(diskCache); + if (diskCache && diskCache.items && Array.isArray(diskCache.items)) { + const allJobs = diskCache.items; + const totalItems = allJobs.length; + const totalPages = Math.ceil(totalItems / perPage); + const startIdx = (page - 1) * perPage; + const endIdx = startIdx + perPage; + const pageJobs = allJobs.slice(startIdx, endIdx); + + // Refresh in background + fetchAllJobsProgressively(c.req.header('Authorization') || '').catch(err => { + logLine('logs/error.log', `Background refresh error: ${err}`); + }); + + return c.json({ + page, + perPage: pageJobs.length, + totalItems, + totalPages, + items: pageJobs + }); } - // No cache: start fetching from PocketBase - // Return first page immediately while fetching the rest in background + // No cache: fetch first page from PocketBase if (!isFetching) { isFetching = true; - - // Start background fetch fetchAllJobsProgressively(c.req.header('Authorization') || '').catch(err => { logLine('logs/error.log', `Progressive fetch error: ${err}`); isFetching = false; }); - - // Get first page immediately - try { - const perPage = 500; - const pbUrl = `https://pocketbase.ccllc.pro/api/collections/pbc_1407612689/records?page=1&perPage=${perPage}&sort=-Job_Number`; - const response = await fetch(pbUrl, { - headers: c.req.header('Authorization') ? { Authorization: c.req.header('Authorization')! } : {} - }); - - if (response.ok) { - const data = await response.json(); - const items = data.items || []; - partialJobs = items; - return c.json({ - page: 1, - perPage: items.length, - totalItems: items.length, - totalPages: 1, - items: items - }); - } - } catch (err) { - logLine('logs/error.log', `Failed to get first page: ${err}`); - } } - // Fetching in progress, return what we have - if (partialJobs.length > 0) { - return c.json({ - page: 1, - perPage: partialJobs.length, - totalItems: partialJobs.length, - totalPages: 1, - items: partialJobs + // Return first batch immediately + try { + const pbUrl = `https://pocketbase.ccllc.pro/api/collections/pbc_1407612689/records?page=1&perPage=100&sort=-Job_Number`; + const response = await fetch(pbUrl, { + headers: c.req.header('Authorization') ? { Authorization: c.req.header('Authorization')! } : {} }); + + if (response.ok) { + const data = await response.json(); + const items = data.items || []; + return c.json({ + page: 1, + perPage: items.length, + totalItems: items.length, + totalPages: 1, + items: items + }); + } + } catch (err) { + logLine('logs/error.log', `Failed to get first page: ${err}`); } - return c.json({ error: 'Fetching jobs, try again' }, 503); - } catch (err) { - logLine('logs/error.log', `jobs-all endpoint error: ${(err as Error)?.message || String(err)}`); - if (cachedJobs) { - return c.json(cachedJobs); - } + return c.json({ error: 'No cache available and initial fetch failed' }, 503); + } catch(err) { + logLine('logs/error.log', `jobs-all endpoint error: ${err}`); return c.json({ error: 'Failed to fetch jobs' }, 500); } }); -// Progressive fetch helper: fetches all jobs in background, updating partialJobs as it goes +// Progressive fetch helper: fetches all jobs in background, updating cache as it goes async function fetchAllJobsProgressively(authHeader: string): Promise { try { const perPage = 500; @@ -631,6 +661,15 @@ logLine('logs/server.log', `Starting frontend server on port ${PORT}`); } })(); +// Immediately load cache synchronously if it exists (for fastest first request) +(async () => { + try { + await loadCacheFile(); + } catch (err) { + // Silent fail + } +})(); + // Background refresh every 5 minutes to keep cache fresh setInterval(async () => { try { diff --git a/Mode3Test/frontend/index.html b/Mode3Test/frontend/index.html index 841dc30..4e41756 100644 --- a/Mode3Test/frontend/index.html +++ b/Mode3Test/frontend/index.html @@ -401,23 +401,60 @@ // Progress bar removed - jobs load instantly from cache // --- Fetch all jobs --- + let allJobsFetched = false; + async function fetchAllJobs(){ clearJobsCache('Loading…'); const startTime = performance.now(); try { - const url = `/api/jobs-all`; - const res = await fetchNoCache(url); + // First request: get initial 100 jobs + const url = `/api/jobs-all?page=1&perPage=100`; + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 3000); // 3 second timeout + + const res = await fetchNoCache(url, { signal: controller.signal }); + clearTimeout(timeoutId); if(!res.ok) throw new Error(`Fetch failed: ${res.status}`); + const data = await res.json(); const items = data.items || []; jobsCache.length = 0; jobsCache.push(...items); applyFiltersAndRender(); + const loadTime = Math.round(performance.now() - startTime); - console.log(`✓ Loaded ${items.length} jobs in ${loadTime}ms`); + console.log(`✓ Loaded ${items.length} jobs in ${loadTime}ms (page 1 of ${data.totalPages})`); + + // If there are more pages, load them in background + if (data.totalPages && data.totalPages > 1) { + loadRemainingPages(data.totalPages, 2); + } else { + allJobsFetched = true; + } } catch(err) { - alert('Failed to fetch jobs: '+err.message); - console.error(err); + if(err.name === 'AbortError') { + console.warn('Fetch timeout - server may be slow'); + } else { + console.error('Fetch error:', err.message); + } + } + } + + async function loadRemainingPages(totalPages, startPage) { + try { + for (let page = startPage; page <= totalPages; page++) { + const url = `/api/jobs-all?page=${page}&perPage=100`; + const res = await fetchNoCache(url); + if (!res.ok) break; + const data = await res.json(); + const items = data.items || []; + jobsCache.push(...items); + applyFiltersAndRender(); + console.log(`✓ Loaded page ${page} of ${totalPages} (${items.length} more jobs)`); + } + allJobsFetched = true; + } catch(err) { + console.warn('Error loading remaining pages:', err.message); } }