Ultra-fast jobs display: backend returns first 40 jobs instantly, frontend renders immediately, full set updates when ready

2026-01-01 07:13:09 +00:00
parent 6dea66b0cf
commit eaf5149d15
4 changed files with 113 additions and 62 deletions
@@ -0,0 +1,37 @@
+// Express API for Redis/Valkey: keys list and single key fetch (for fast UI)
+const express = require('express');
+const Redis = require('ioredis');
+const app = express();
+const redis = new Redis({
+  host: process.env.REDIS_HOST || '127.0.0.1',
+  port: Number(process.env.REDIS_PORT || 6379),
+  password: process.env.REDIS_PASSWORD,
+});
+
+// List all keys (no values)
+app.get('/api/cache/keys', async (req, res) => {
+  try {
+    const keys = await redis.keys('*');
+    // Optionally, add summary/metadata here
+    res.json(keys.map(key => ({ key })));
+  } catch (err) {
+    res.status(500).json({ error: err.message });
+  }
+});
+
+// Fetch value for a single key
+app.get('/api/cache/:key', async (req, res) => {
+  try {
+    const key = req.params.key;
+    let value = await redis.get(key);
+    try { value = JSON.parse(value); } catch {}
+    res.json({ key, value });
+  } catch (err) {
+    res.status(500).json({ error: err.message });
+  }
+});
+
+const port = process.env.CACHE_API_PORT || 3006;
+app.listen(port, () => {
+  console.log(`Cache API (keys+single) running on http://localhost:${port}/api/cache/keys`);
+});
@@ -82,39 +82,40 @@ app.get('/api/jobs', async (c) => {
 app.get('/api/jobs-all', async (c) => {
  const cacheKey = 'jobs:all';
  const ttl = 1800; // 30 minutes cache
-  
  try {
-    // Always try cache first for instant response
+    // Try cache first for instant response
    if (isRedisConnected()) {
      const cached = await getCache(cacheKey);
      if (cached) {
        logLine('logs/server.log', `Cache HIT for ${cacheKey} (${cached.items?.length || 0} jobs)`);
-        
-        // Return cached data immediately, refresh in background if needed
+        // Return cached data immediately
        setImmediate(async () => {
          try {
            await refreshJobsCache(cacheKey, ttl, c.req.header('Authorization') || '');
+          } catch (err) {}
+        });
+        // For ultra-fast display, send only the first 40 jobs in the initial response
+        const fastBatch = Array.isArray(cached.items) ? cached.items.slice(0, 40) : [];
+        return c.json({ ...cached, items: fastBatch, partial: cached.items && cached.items.length > 40 });
+      } else {
+        // No cache: return empty array instantly, trigger background fetch
+        setImmediate(async () => {
+          try {
+            const result = await fetchAllJobsFromPocketBase(c.req.header('Authorization') || '');
+            if (isRedisConnected()) {
+              await setCache(cacheKey, result, ttl);
+              logLine('logs/server.log', `Cached ${cacheKey} with ${result.items.length} jobs for ${ttl}s`);
+            }
          } catch (err) {
-            // Silently fail background refresh
+            logLine('logs/error.log', `jobs-all background fetch error: ${(err as Error)?.message || String(err)}`);
          }
        });
-        
-        return c.json(cached);
+        return c.json({ items: [], partial: true });
      }
-      logLine('logs/server.log', `Cache MISS for ${cacheKey} - fetching all jobs`);
    }
-    
-    // No cache available, fetch now
+    // Redis not connected: fallback to direct fetch (blocking)
    const result = await fetchAllJobsFromPocketBase(c.req.header('Authorization') || '');
-    
-    // Cache the result
-    if (isRedisConnected()) {
-      await setCache(cacheKey, result, ttl);
-      logLine('logs/server.log', `Cached ${cacheKey} with ${result.items.length} jobs for ${ttl}s`);
-    }
-    
    return c.json(result);
-    
  } catch (err) {
    logLine('logs/error.log', `jobs-all endpoint error: ${(err as Error)?.message || String(err)}`);
    return c.json({ error: 'Failed to fetch jobs' }, 500);
@@ -25,53 +25,58 @@
    <tbody></tbody>
  </table>
  <script>
-    let cacheData = [];
-    function renderTable(data) {
-      const tbody = document.querySelector('#cacheTable tbody');
+    let cacheKeys = [];
+    let cacheDetails = {};
+    const tbody = document.querySelector('#cacheTable tbody');
+
+    function renderTable(keys) {
      tbody.innerHTML = '';
-      for (const { key, value } of data) {
+      for (const { key } of keys) {
        const tr = document.createElement('tr');
        const tdKey = document.createElement('td');
        tdKey.textContent = key;
        const tdValue = document.createElement('td');
-        if (typeof value === 'object' && value !== null) {
-          const btn = document.createElement('span');
-          btn.textContent = '[expand]';
-          btn.className = 'expand';
-          btn.onclick = () => {
-            btn.outerHTML = `<div class='json'>${JSON.stringify(value, null, 2)}</div>`;
-          };
-          tdValue.appendChild(btn);
-        } else {
-          tdValue.textContent = String(value);
-        }
+        tdValue.textContent = '[click to load]';
+        tdValue.className = 'expand';
+        tdValue.onclick = async () => {
+          tdValue.textContent = 'Loading...';
+          if (!cacheDetails[key]) {
+            try {
+              const res = await fetch(`http://localhost:3006/api/cache/${encodeURIComponent(key)}`);
+              const data = await res.json();
+              cacheDetails[key] = data.value;
+            } catch (err) {
+              tdValue.textContent = 'Error loading value';
+              return;
+            }
+          }
+          tdValue.innerHTML = `<div class='json'>${JSON.stringify(cacheDetails[key], null, 2)}</div>`;
+        };
        tr.appendChild(tdKey);
        tr.appendChild(tdValue);
        tbody.appendChild(tr);
      }
    }

-    // Fetch cache data from API
-    async function loadCache() {
+    // Fetch cache keys from API
+    async function loadKeys() {
      try {
-        const res = await fetch('http://localhost:3006/api/cache');
-        cacheData = await res.json();
-        renderTable(cacheData);
+        const res = await fetch('http://localhost:3006/api/cache/keys');
+        cacheKeys = await res.json();
+        renderTable(cacheKeys);
      } catch (err) {
-        document.querySelector('#cacheTable tbody').innerHTML = `<tr><td colspan='2' style='color:#b91c1c'>Failed to load cache data: ${err}</td></tr>`;
+        tbody.innerHTML = `<tr><td colspan='2' style='color:#b91c1c'>Failed to load cache keys: ${err}</td></tr>`;
      }
    }

    // Search functionality
    document.getElementById('search').addEventListener('input', function() {
      const q = this.value.toLowerCase();
-      renderTable(cacheData.filter(({key, value}) =>
-        key.toLowerCase().includes(q) || JSON.stringify(value).toLowerCase().includes(q)
-      ));
+      renderTable(cacheKeys.filter(({key}) => key.toLowerCase().includes(q)));
    });

    // Initial load
-    loadCache();
+    loadKeys();
  </script>
 </body>
 </html>
@@ -413,25 +413,33 @@
    startProgress();
    clearJobsCache('Loading…');
    const startTime = performance.now();
-    try{
-      // All job data should be fetched from backend/Valkey cache
-      const url = `/api/jobs-all`;
-      const res = await fetchNoCache(url);
-      if(!res.ok) throw new Error(`Fetch failed: ${res.status}`);
-      const data = await res.json();
-      const items = data.items || [];
-      jobsCache.push(...items);
-      const loadTime = Math.round(performance.now() - startTime);
-      console.log(`✓ Loaded ${items.length} jobs from API in ${loadTime}ms`);
-
-      // Show results immediately
-      progressBar.style.width='95%';
-      const renderStart = performance.now();
-      applyFiltersAndRender();
-      const renderTime = Math.round(performance.now() - startTime);
-      console.log(`✓ Rendered jobs in ${renderTime}ms (Total: ${Math.round(performance.now() - startTime)}ms)`);
-      finishProgress();
-    }catch(err){ finishProgress(); alert('Failed to fetch jobs: '+err.message); console.error(err);}
+    let pollCount = 0;
+    async function pollJobs(){
+      try{
+        const url = `/api/jobs-all`;
+        const res = await fetchNoCache(url);
+        if(!res.ok) throw new Error(`Fetch failed: ${res.status}`);
+        const data = await res.json();
+        const items = data.items || [];
+        if (pollCount === 0 || jobsCache.length === 0) {
+          jobsCache.length = 0;
+          jobsCache.push(...items);
+          progressBar.style.width='95%';
+          applyFiltersAndRender();
+        }
+        if (data.partial) {
+          // If partial, poll again after short delay
+          pollCount++;
+          if (pollCount < 10) setTimeout(pollJobs, 600);
+          else finishProgress();
+        } else {
+          finishProgress();
+          const loadTime = Math.round(performance.now() - startTime);
+          console.log(`✓ Loaded ${items.length} jobs from API in ${loadTime}ms`);
+        }
+      }catch(err){ finishProgress(); alert('Failed to fetch jobs: '+err.message); console.error(err); }
+    }
+    pollJobs();
  }
  
  function renderInitialBatch(jobs){