diff --git a/backend/src/index.ts b/backend/src/index.ts index 3a0e7da..037c6ef 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -141,16 +141,27 @@ app.get('/api/jobs-all', async (c) => { if (isRedisConnected()) { const cached = await getCache(cacheKey); if (cached) { - logLine('logs/server.log', `Cache HIT for ${cacheKey} (${cached.items?.length || 0} jobs)`); - // Return cached data immediately + // Return first 100 jobs immediately for instant display, rest loads in background + const fastBatch = Array.isArray(cached.items) ? cached.items.slice(0, 100) : []; + const hasMore = cached.items && cached.items.length > 100; + + logLine('logs/server.log', `Cache HIT for ${cacheKey} - sending ${fastBatch.length} jobs (${hasMore ? 'partial' : 'complete'})`); + + // Refresh cache in background setImmediate(async () => { try { await refreshJobsCache(cacheKey, ttl, c.req.header('Authorization') || ''); } catch (err) {} }); - // For ultra-fast display, send only the first 40 jobs in the initial response - const fastBatch = Array.isArray(cached.items) ? cached.items.slice(0, 40) : []; - return c.json({ ...cached, items: fastBatch, partial: cached.items && cached.items.length > 40 }); + + return c.json({ + page: 1, + perPage: fastBatch.length, + totalItems: cached.totalItems || fastBatch.length, + totalPages: 1, + items: fastBatch, + partial: hasMore + }); } else { // No cache: return empty array instantly, trigger background fetch setImmediate(async () => { @@ -176,6 +187,27 @@ app.get('/api/jobs-all', async (c) => { } }); +// Get remaining jobs after initial batch +app.get('/api/jobs-remaining', async (c) => { + const cacheKey = 'jobs:all'; + try { + if (isRedisConnected()) { + const cached = await getCache(cacheKey); + if (cached && cached.items && cached.items.length > 100) { + const remaining = cached.items.slice(100); + return c.json({ + items: remaining, + totalItems: cached.totalItems || cached.items.length + }); + } + } + return c.json({ items: [] }); + } catch (err) { + logLine('logs/error.log', `jobs-remaining error: ${(err as Error)?.message || String(err)}`); + return c.json({ error: 'Failed to fetch remaining jobs' }, 500); + } +}); + // Helper function to fetch all jobs from PocketBase async function fetchAllJobsFromPocketBase(authHeader: string): Promise { const allItems: any[] = []; @@ -510,6 +542,25 @@ app.get('/api/job-file-pdf', async (c) => { const itemId = c.req.query('itemId'); if (!driveId || !itemId) return c.json({ error: 'driveId and itemId required' }, 400); + // Check cache first + const cacheKey = `job-file-pdf:${driveId}:${itemId}`; + if (isRedisConnected()) { + const cached = await getCache(cacheKey); + if (cached && cached.data) { + logLine('logs/server.log', `PDF cache HIT: ${driveId}:${itemId}`); + const buffer = Buffer.from(cached.data, 'base64'); + const headers: Record = { + 'Content-Type': 'application/pdf', + }; + if (cached.filename) { + headers['Content-Disposition'] = `inline; filename="${cached.filename}"`; + } + return new Response(buffer, { status: 200, headers }); + } else { + logLine('logs/server.log', `PDF cache MISS: ${driveId}:${itemId}`); + } + } + // Use Graph API to convert to PDF format const res = await fetch(`https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/content?format=pdf`, { headers: { Authorization: `Bearer ${token}` }, @@ -519,16 +570,30 @@ app.get('/api/job-file-pdf', async (c) => { logLine('logs/error.log', `/api/job-file-pdf conversion failed: ${res.status} - ${text}`); return c.json({ error: `PDF conversion failed: ${res.status}` }, 502); } + + const pdfBuffer = Buffer.from(await res.arrayBuffer()); const headers: Record = { 'Content-Type': 'application/pdf', }; const cd = res.headers.get('content-disposition'); + let filename = 'document.pdf'; if (cd) { // Replace .docx/.doc extension with .pdf in filename headers['Content-Disposition'] = cd.replace(/\.(docx?)/gi, '.pdf'); + const match = cd.match(/filename="?([^"]+)"?/); + if (match) filename = match[1].replace(/\.(docx?)/gi, '.pdf'); } + + // Cache the converted PDF for 30 minutes + if (isRedisConnected()) { + await setCache(cacheKey, { + data: pdfBuffer.toString('base64'), + filename + }, 1800); + } + logLine('logs/server.log', `Successfully converted document to PDF: driveId=${driveId}, itemId=${itemId}`); - return new Response(res.body, { status: 200, headers }); + return new Response(pdfBuffer, { status: 200, headers }); } catch (err) { const message = (err as Error)?.message || String(err); const status = (err as any)?.status || 500; @@ -595,8 +660,8 @@ logLine('logs/server.log', `Starting frontend server (Redis version) on port ${P const token = await getAppOnlyToken(); if (token) { logLine('logs/server.log', 'Service principal authenticated, starting background file caching...'); - // Cache files for first 1000 jobs with folder links - startFileCaching(result.items.slice(0, 1000), token); + // Cache files for ALL jobs with folder links + startFileCaching(result.items, token); } else { logLine('logs/server.log', 'Service principal not configured, file caching will occur on-demand'); } @@ -615,9 +680,9 @@ let fileCachingTimer: NodeJS.Timeout | null = null; async function startFileCaching(jobs: any[], token: string) { if (fileCachingActive) return; - fileCachingJobs = jobs.filter(job => job.Job_Folder_Link); + fileCachingJobs = jobs; if (fileCachingJobs.length === 0) { - logLine('logs/server.log', 'No jobs with folder links to cache'); + logLine('logs/server.log', 'No jobs to cache'); return; } @@ -632,14 +697,16 @@ async function startFileCaching(jobs: any[], token: string) { async function cacheNextFileBatch() { if (!fileCachingActive || !isRedisConnected()) return; - const batchSize = 5; // Cache 5 jobs at a time for faster coverage + const batchSize = 10; // Cache 10 jobs at a time for faster initial coverage const maxJobs = fileCachingJobs.length; if (fileCachingIndex >= maxJobs) { - // Completed one full cycle, refresh job list and restart after 3 minutes - logLine('logs/server.log', `✓ Completed file caching cycle for ${maxJobs} jobs. Restarting in 3 minutes...`); + // Completed initial full cache of all jobs + console.log(`✓ Completed caching all ${maxJobs} jobs. Scheduling refresh in 10 minutes...`); + logLine('logs/server.log', `✓ Completed caching all ${maxJobs} jobs. Scheduling refresh in 10 minutes...`); fileCachingIndex = 0; + // After completing all jobs, wait 10 minutes before refreshing fileCachingTimer = setTimeout(async () => { try { // Refresh the jobs cache first @@ -648,8 +715,9 @@ async function cacheNextFileBatch() { // Get updated job list from cache const cached = await getCache('jobs:all'); if (cached && cached.items) { - fileCachingJobs = cached.items.slice(0, 1000).filter((job: any) => job.Job_Folder_Link); - logLine('logs/server.log', `Updated file caching list with ${fileCachingJobs.length} jobs`); + fileCachingJobs = cached.items; + console.log(`Starting refresh cycle: ${fileCachingJobs.length} jobs to update`); + logLine('logs/server.log', `Starting refresh cycle: ${fileCachingJobs.length} jobs to update`); } // Refresh token if needed @@ -658,18 +726,24 @@ async function cacheNextFileBatch() { fileCachingToken = newToken; cacheNextFileBatch(); } else { + console.error('Failed to refresh app-only token, stopping file caching'); logLine('logs/error.log', 'Failed to refresh app-only token, stopping file caching'); fileCachingActive = false; } } catch (err) { + console.error(`File cache cycle restart error: ${(err as Error)?.message}`); logLine('logs/error.log', `File cache cycle restart error: ${(err as Error)?.message}`); } - }, 3 * 60 * 1000); + }, 10 * 60 * 1000); return; } // Get next batch of jobs to cache const batch = fileCachingJobs.slice(fileCachingIndex, fileCachingIndex + batchSize); + const jobNumbers = batch.map(j => j.Job_Number).join(', '); + const batchMsg = `[Cache Batch] Processing jobs: ${jobNumbers} (${fileCachingIndex}/${maxJobs})`; + console.log(batchMsg); + logLine('logs/server.log', batchMsg); for (const job of batch) { try { @@ -682,13 +756,20 @@ async function cacheNextFileBatch() { fileCachingIndex += batchSize; - // Continue with next batch after a short delay (2 seconds between batches for faster cycling) - fileCachingTimer = setTimeout(() => cacheNextFileBatch(), 2000); + // Continue with next batch after a short delay (1 second between batches for faster throughput) + fileCachingTimer = setTimeout(() => cacheNextFileBatch(), 1000); } async function cacheJobFiles(job: any, token: string) { const link = job.Job_Folder_Link; - if (!link) return; + if (!link) { + // Job has no folder link, just mark it as cached + const cacheKey = `job-files:${job.Job_Number}`; + await setCache(cacheKey, { items: [] }, 1800); + console.log(`✓ No folder for job ${job.Job_Number}`); + logLine('logs/server.log', `✓ No folder for job ${job.Job_Number}`); + return; + } const cacheKey = `job-files:${link}`; const ttl = 1800; // 30 minutes @@ -714,7 +795,93 @@ async function cacheJobFiles(job: any, token: string) { }; await setCache(cacheKey, data, ttl); - logLine('logs/server.log', `✓ Cached ${items.length} files for job ${job.Job_Number}`); + + // Pre-cache PDF conversions for Word/Excel documents AND native PDF files + const officeFiles = items.filter(item => { + const name = (item.name || '').toLowerCase(); + return name.endsWith('.docx') || name.endsWith('.doc') || name.endsWith('.xlsx') || name.endsWith('.xls'); + }); + + const pdfFiles = items.filter(item => { + const name = (item.name || '').toLowerCase(); + return name.endsWith('.pdf'); + }); + + let pdfsCached = 0; + + // Cache converted Office documents + for (const file of officeFiles) { + try { + const fileDriveId = file.parentReference?.driveId || driveId; + const pdfCacheKey = `job-file-pdf:${fileDriveId}:${file.id}`; + + // Check if already cached + const existing = await getCache(pdfCacheKey); + if (existing) { + pdfsCached++; + continue; + } + + // Convert and cache + const res = await fetch(`https://graph.microsoft.com/v1.0/drives/${fileDriveId}/items/${file.id}/content?format=pdf`, { + headers: { Authorization: `Bearer ${token}` }, + }); + + if (res.ok) { + const pdfBuffer = Buffer.from(await res.arrayBuffer()); + const cd = res.headers.get('content-disposition'); + let filename = file.name.replace(/\.(docx?|xlsx?)$/gi, '.pdf'); + if (cd) { + const match = cd.match(/filename="?([^"]+)"?/); + if (match) filename = match[1].replace(/\.(docx?|xlsx?)$/gi, '.pdf'); + } + + await setCache(pdfCacheKey, { + data: pdfBuffer.toString('base64'), + filename + }, ttl); + pdfsCached++; + } + } catch (err) { + // Silently skip PDF caching errors + } + } + + // Cache native PDF files + for (const file of pdfFiles) { + try { + const fileDriveId = file.parentReference?.driveId || driveId; + const pdfCacheKey = `job-file-pdf:${fileDriveId}:${file.id}`; + + // Check if already cached + const existing = await getCache(pdfCacheKey); + if (existing) { + pdfsCached++; + continue; + } + + // Fetch and cache native PDF + const res = await fetch(`https://graph.microsoft.com/v1.0/drives/${fileDriveId}/items/${file.id}/content`, { + headers: { Authorization: `Bearer ${token}` }, + }); + + if (res.ok) { + const pdfBuffer = Buffer.from(await res.arrayBuffer()); + await setCache(pdfCacheKey, { + data: pdfBuffer.toString('base64'), + filename: file.name + }, ttl); + pdfsCached++; + } + } catch (err) { + // Silently skip PDF caching errors + } + } + + const totalPDFs = officeFiles.length + pdfFiles.length; + const msg = `✓ Cached ${items.length} files${totalPDFs > 0 ? ` + ${pdfsCached}/${totalPDFs} PDFs` : ''} for job ${job.Job_Number}`; + console.log(msg); + logLine('logs/server.log', msg); } catch (err: any) { // Only log if not a token expiration issue if (!err.message?.includes('401') && !err.message?.includes('token')) { diff --git a/frontend/public/index.html b/frontend/public/index.html index 4bafb19..001ff50 100644 --- a/frontend/public/index.html +++ b/frontend/public/index.html @@ -219,7 +219,7 @@ -
+
@@ -338,30 +338,56 @@ // --- Fetch all jobs --- async function fetchAllJobs(){ - clearJobsCache('Loading…'); const startTime = performance.now(); let pollCount = 0; + async function pollJobs(){ try{ - const url = `/api/jobs-all${pollCount > 0 ? '?offset=1' : ''}`; + const url = `/api/jobs-all`; const res = await fetchNoCache(url); if(!res.ok) throw new Error(`Fetch failed: ${res.status}`); const data = await res.json(); const items = data.items || []; + if (pollCount === 0 || jobsCache.length === 0) { jobsCache.length = 0; jobsCache.push(...items); + // Render immediately as soon as we have first batch applyFiltersAndRender(); - } - if (data.partial) { - // If partial, poll again after short delay - pollCount++; - if (pollCount < 10) setTimeout(pollJobs, 600); - } else { const loadTime = Math.round(performance.now() - startTime); - console.log(`✓ Loaded ${items.length} jobs from API in ${loadTime}ms`); + console.log(`✓ Displayed ${items.length} jobs in ${loadTime}ms`); } - }catch(err){ alert('Failed to fetch jobs: '+err.message); console.error(err); } + + if (data.partial) { + // Load remaining jobs in background + pollCount++; + if (pollCount === 1) { + // After first batch, get remaining jobs + setTimeout(async () => { + try { + const remainingRes = await fetchNoCache('/api/jobs-remaining'); + if (remainingRes.ok) { + const remainingData = await remainingRes.json(); + const remainingItems = remainingData.items || []; + if (remainingItems.length > 0) { + jobsCache.push(...remainingItems); + console.log(`✓ Loaded ${remainingItems.length} additional jobs in background`); + // Re-render if no filters are active to show all jobs + if (!searchBox.value && document.querySelectorAll('.filter-division:checked, .filter-active:checked, .filter-status:checked, .filter-estimator:checked').length === 0) { + applyFiltersAndRender(); + } + } + } + } catch (err) { + console.error('Failed to load remaining jobs:', err); + } + }, 100); + } + } + }catch(err){ + resultsEl.innerHTML = '
Failed to load jobs. Please refresh the page.
'; + console.error(err); + } } pollJobs(); } @@ -481,24 +507,53 @@ sortByRecent, resultsCount: visibleJobs.length }); + + // Reset virtual scroll state for new filter results + virtualScrollData.visibleStartIndex = 0; + virtualScrollData.visibleCount = 30; + renderCards(visibleJobs); } + // Virtual scroll state + let virtualScrollData = { + allJobs: [], + visibleStartIndex: 0, + visibleCount: 30, + itemHeight: 100, // Approximate card height + scrolling: false + }; + function renderCards(list){ - resultsEl.innerHTML=''; - if(!list.length){ resultsEl.innerHTML = `
No results
`; return; } + virtualScrollData.allJobs = list; + + if(!list.length){ + resultsEl.innerHTML = `
No results
`; + return; + } - const firstChunk = list.slice(0,20); - const remaining = list.slice(20); + // Render only visible cards using virtual scrolling + renderVirtualCards(); + } - for(const job of firstChunk){ - const card=document.createElement('div'); card.className='bg-white rounded-lg p-2.5 border border-blue-100 shadow-sm cursor-pointer relative flex flex-col justify-center gap-1'; + function renderVirtualCards() { + const { allJobs, visibleStartIndex, visibleCount } = virtualScrollData; + const endIndex = Math.min(visibleStartIndex + visibleCount, allJobs.length); + const visibleJobs = allJobs.slice(visibleStartIndex, endIndex); + + // Clear and render visible cards only + resultsEl.innerHTML = ''; + + for(const job of visibleJobs){ + const card=document.createElement('div'); + card.className='bg-white rounded-lg p-2.5 border border-blue-100 shadow-sm cursor-pointer relative flex flex-col justify-center gap-1'; card.innerHTML=`
Job#
${escapeHtml(job.Job_Number||'')}
Job Name:
${escapeHtml(job.Job_Full_Name||'')}
Contact:
${escapeHtml(job.Contact_Person||'')}
`; - const dot=document.createElement('div'); dot.className='absolute top-2.5 right-2.5 w-3.5 h-3.5 rounded-full border-2 border-white shadow-sm'; + const dot=document.createElement('div'); + dot.className='absolute top-2.5 right-2.5 w-3.5 h-3.5 rounded-full border-2 border-white shadow-sm'; { const b = toBool(job.Active); dot.style.background = b===true ? 'green' : (b===false ? 'red' : '#cbd5e1'); @@ -507,26 +562,17 @@ card.addEventListener('click',()=>openDetail(job)); resultsEl.appendChild(card); } - - if(remaining.length){ - setTimeout(()=>{ - for(const job of remaining){ - const card=document.createElement('div'); card.className='bg-white rounded-lg p-2.5 border border-blue-100 shadow-sm cursor-pointer relative flex flex-col justify-center gap-1'; - card.innerHTML=` -
Job#
${escapeHtml(job.Job_Number||'')}
-
Job Name:
${escapeHtml(job.Job_Full_Name||'')}
-
Contact:
${escapeHtml(job.Contact_Person||'')}
- `; - const dot=document.createElement('div'); dot.className='absolute top-2.5 right-2.5 w-3.5 h-3.5 rounded-full border-2 border-white shadow-sm'; - { - const b = toBool(job.Active); - dot.style.background = b===true ? 'green' : (b===false ? 'red' : '#cbd5e1'); - } - card.appendChild(dot); - card.addEventListener('click',()=>openDetail(job)); - resultsEl.appendChild(card); - } - }, 50); + + // Add "Load More" button if there are more results + if (endIndex < allJobs.length) { + const loadMoreBtn = document.createElement('button'); + loadMoreBtn.className = 'w-full py-3 mt-2 bg-blue-600 hover:bg-blue-700 text-white rounded-lg font-medium'; + loadMoreBtn.textContent = `Load More (${allJobs.length - endIndex} remaining)`; + loadMoreBtn.onclick = () => { + virtualScrollData.visibleCount += 30; + renderVirtualCards(); + }; + resultsEl.appendChild(loadMoreBtn); } } function escapeHtml(str){ return String(str).replaceAll('&','&').replaceAll('<','<').replaceAll('>','>'); } @@ -2174,7 +2220,7 @@ submittals: ['submittal', 'submittals', 'submit'], }; - const DEFAULT_PDF_SCALE = 0.45; + const DEFAULT_PDF_SCALE = 1.4; const MIN_PDF_SCALE = 0.3; const MAX_PDF_SCALE = 3.0; const fileListState = { items: [], filter: '', jobNumber: '', jobName: '' }; @@ -2183,6 +2229,7 @@ let pdfPan = { x: 0, y: 0, dragging: false, startX: 0, startY: 0, baseX: 0, baseY: 0 }; const pdfViewer = document.getElementById('pdfViewer'); const pdfCanvas = document.getElementById('pdfCanvas'); + const pdfViewport = document.getElementById('pdfViewport'); const pdfControls = document.getElementById('pdfControls'); const pdfPageNum = document.getElementById('pdfPageNum'); const pdfPageCount = document.getElementById('pdfPageCount'); @@ -2556,12 +2603,23 @@ const data = await blob.arrayBuffer(); pdfjsLib.GlobalWorkerOptions.workerSrc = 'https://cdnjs.cloudflare.com/ajax/libs/pdf.js/3.11.174/pdf.worker.min.js'; const doc = await pdfjsLib.getDocument({ data }).promise; - const initialScale = isWordDoc ? 1.4 : DEFAULT_PDF_SCALE; - currentPdf = { doc, page: 1, pages: doc.numPages, scale: initialScale }; + const targetScale = DEFAULT_PDF_SCALE; + // Start at 100% to ensure proper positioning + currentPdf = { doc, page: 1, pages: doc.numPages, scale: 1.0 }; pdfPan = { x: 0, y: 0, dragging: false, startX: 0, startY: 0, baseX: 0, baseY: 0 }; pdfCanvas.style.transform = 'translate(0px, 0px)'; - await renderPdfPage(); showPdfViewer(); + await renderPdfPage(); + // Set scroll with small offset (48px right, 36px down) + if (pdfViewport) { + pdfViewport.scrollLeft = 48; + pdfViewport.scrollTop = 36; + } + // Now scale to target zoom if different + if (targetScale !== 1.0) { + currentPdf.scale = targetScale; + await renderPdfPage(); + } } else if (isImage) { // Show images in the PDF viewer area for consistency pdfViewer.classList.remove('hidden');