Ultra-fast jobs display: backend returns first 40 jobs instantly, frontend renders immediately, full set updates when ready

backend/cache-api-keys.cjs

@@ -0,0 +1,37 @@
+// Express API for Redis/Valkey: keys list and single key fetch (for fast UI)
+const express = require('express');
+const Redis = require('ioredis');
+const app = express();
+const redis = new Redis({
+  host: process.env.REDIS_HOST || '127.0.0.1',
+  port: Number(process.env.REDIS_PORT || 6379),
+  password: process.env.REDIS_PASSWORD,
+});
+
+// List all keys (no values)
+app.get('/api/cache/keys', async (req, res) => {
+  try {
+    const keys = await redis.keys('*');
+    // Optionally, add summary/metadata here
+    res.json(keys.map(key => ({ key })));
+  } catch (err) {
+    res.status(500).json({ error: err.message });
+  }
+});
+
+// Fetch value for a single key
+app.get('/api/cache/:key', async (req, res) => {
+  try {
+    const key = req.params.key;
+    let value = await redis.get(key);
+    try { value = JSON.parse(value); } catch {}
+    res.json({ key, value });
+  } catch (err) {
+    res.status(500).json({ error: err.message });
+  }
+});
+
+const port = process.env.CACHE_API_PORT || 3006;
+app.listen(port, () => {
+  console.log(`Cache API (keys+single) running on http://localhost:${port}/api/cache/keys`);
+});

backend/server.ts

@@ -82,39 +82,40 @@ app.get('/api/jobs', async (c) => {
 app.get('/api/jobs-all', async (c) => {
   const cacheKey = 'jobs:all';
   const ttl = 1800; // 30 minutes cache
-  
   try {
-    // Always try cache first for instant response
+    // Try cache first for instant response
     if (isRedisConnected()) {
       const cached = await getCache(cacheKey);
       if (cached) {
         logLine('logs/server.log', `Cache HIT for ${cacheKey} (${cached.items?.length || 0} jobs)`);
-        
+        // Return cached data immediately
-        // Return cached data immediately, refresh in background if needed
         setImmediate(async () => {
           try {
             await refreshJobsCache(cacheKey, ttl, c.req.header('Authorization') || '');
+          } catch (err) {}
+        });
+        // For ultra-fast display, send only the first 40 jobs in the initial response
+        const fastBatch = Array.isArray(cached.items) ? cached.items.slice(0, 40) : [];
+        return c.json({ ...cached, items: fastBatch, partial: cached.items && cached.items.length > 40 });
+      } else {
+        // No cache: return empty array instantly, trigger background fetch
+        setImmediate(async () => {
+          try {
+            const result = await fetchAllJobsFromPocketBase(c.req.header('Authorization') || '');
+            if (isRedisConnected()) {
+              await setCache(cacheKey, result, ttl);
+              logLine('logs/server.log', `Cached ${cacheKey} with ${result.items.length} jobs for ${ttl}s`);
+            }
           } catch (err) {
-            // Silently fail background refresh
+            logLine('logs/error.log', `jobs-all background fetch error: ${(err as Error)?.message || String(err)}`);
           }
         });
-        
+        return c.json({ items: [], partial: true });
-        return c.json(cached);
       }
-      logLine('logs/server.log', `Cache MISS for ${cacheKey} - fetching all jobs`);
     }
-    
+    // Redis not connected: fallback to direct fetch (blocking)
-    // No cache available, fetch now
     const result = await fetchAllJobsFromPocketBase(c.req.header('Authorization') || '');
-    
-    // Cache the result
-    if (isRedisConnected()) {
-      await setCache(cacheKey, result, ttl);
-      logLine('logs/server.log', `Cached ${cacheKey} with ${result.items.length} jobs for ${ttl}s`);
-    }
-    
     return c.json(result);
-    
   } catch (err) {
     logLine('logs/error.log', `jobs-all endpoint error: ${(err as Error)?.message || String(err)}`);
     return c.json({ error: 'Failed to fetch jobs' }, 500);

cache-visualization.html

@@ -25,53 +25,58 @@
     <tbody></tbody>
   </table>
   <script>
-    let cacheData = [];
+    let cacheKeys = [];
-    function renderTable(data) {
+    let cacheDetails = {};
-      const tbody = document.querySelector('#cacheTable tbody');
+    const tbody = document.querySelector('#cacheTable tbody');
+
+    function renderTable(keys) {
       tbody.innerHTML = '';
-      for (const { key, value } of data) {
+      for (const { key } of keys) {
         const tr = document.createElement('tr');
         const tdKey = document.createElement('td');
         tdKey.textContent = key;
         const tdValue = document.createElement('td');
-        if (typeof value === 'object' && value !== null) {
+        tdValue.textContent = '[click to load]';
-          const btn = document.createElement('span');
+        tdValue.className = 'expand';
-          btn.textContent = '[expand]';
+        tdValue.onclick = async () => {
-          btn.className = 'expand';
+          tdValue.textContent = 'Loading...';
-          btn.onclick = () => {
+          if (!cacheDetails[key]) {
-            btn.outerHTML = `<div class='json'>${JSON.stringify(value, null, 2)}</div>`;
+            try {
-          };
+              const res = await fetch(`http://localhost:3006/api/cache/${encodeURIComponent(key)}`);
-          tdValue.appendChild(btn);
+              const data = await res.json();
-        } else {
+              cacheDetails[key] = data.value;
-          tdValue.textContent = String(value);
+            } catch (err) {
-        }
+              tdValue.textContent = 'Error loading value';
+              return;
+            }
+          }
+          tdValue.innerHTML = `<div class='json'>${JSON.stringify(cacheDetails[key], null, 2)}</div>`;
+        };
         tr.appendChild(tdKey);
         tr.appendChild(tdValue);
         tbody.appendChild(tr);
       }
     }
 
-    // Fetch cache data from API
+    // Fetch cache keys from API
-    async function loadCache() {
+    async function loadKeys() {
       try {
-        const res = await fetch('http://localhost:3006/api/cache');
+        const res = await fetch('http://localhost:3006/api/cache/keys');
-        cacheData = await res.json();
+        cacheKeys = await res.json();
-        renderTable(cacheData);
+        renderTable(cacheKeys);
       } catch (err) {
-        document.querySelector('#cacheTable tbody').innerHTML = `<tr><td colspan='2' style='color:#b91c1c'>Failed to load cache data: ${err}</td></tr>`;
+        tbody.innerHTML = `<tr><td colspan='2' style='color:#b91c1c'>Failed to load cache keys: ${err}</td></tr>`;
       }
     }
 
     // Search functionality
     document.getElementById('search').addEventListener('input', function() {
       const q = this.value.toLowerCase();
-      renderTable(cacheData.filter(({key, value}) =>
+      renderTable(cacheKeys.filter(({key}) => key.toLowerCase().includes(q)));
-        key.toLowerCase().includes(q) || JSON.stringify(value).toLowerCase().includes(q)
-      ));
     });
 
     // Initial load
-    loadCache();
+    loadKeys();
   </script>
 </body>
 </html>

frontend/index.html

@@ -413,25 +413,33 @@
     startProgress();
     clearJobsCache('Loading…');
     const startTime = performance.now();
-    try{
+    let pollCount = 0;
-      // All job data should be fetched from backend/Valkey cache
+    async function pollJobs(){
-      const url = `/api/jobs-all`;
+      try{
-      const res = await fetchNoCache(url);
+        const url = `/api/jobs-all`;
-      if(!res.ok) throw new Error(`Fetch failed: ${res.status}`);
+        const res = await fetchNoCache(url);
-      const data = await res.json();
+        if(!res.ok) throw new Error(`Fetch failed: ${res.status}`);
-      const items = data.items || [];
+        const data = await res.json();
-      jobsCache.push(...items);
+        const items = data.items || [];
-      const loadTime = Math.round(performance.now() - startTime);
+        if (pollCount === 0 || jobsCache.length === 0) {
-      console.log(`✓ Loaded ${items.length} jobs from API in ${loadTime}ms`);
+          jobsCache.length = 0;
-
+          jobsCache.push(...items);
-      // Show results immediately
+          progressBar.style.width='95%';
-      progressBar.style.width='95%';
+          applyFiltersAndRender();
-      const renderStart = performance.now();
+        }
-      applyFiltersAndRender();
+        if (data.partial) {
-      const renderTime = Math.round(performance.now() - startTime);
+          // If partial, poll again after short delay
-      console.log(`✓ Rendered jobs in ${renderTime}ms (Total: ${Math.round(performance.now() - startTime)}ms)`);
+          pollCount++;
-      finishProgress();
+          if (pollCount < 10) setTimeout(pollJobs, 600);
-    }catch(err){ finishProgress(); alert('Failed to fetch jobs: '+err.message); console.error(err);}
+          else finishProgress();
+        } else {
+          finishProgress();
+          const loadTime = Math.round(performance.now() - startTime);
+          console.log(`✓ Loaded ${items.length} jobs from API in ${loadTime}ms`);
+        }
+      }catch(err){ finishProgress(); alert('Failed to fetch jobs: '+err.message); console.error(err); }
+    }
+    pollJobs();
   }
   
   function renderInitialBatch(jobs){