diff --git a/backend/cache-api-keys.cjs b/backend/cache-api-keys.cjs new file mode 100644 index 0000000..640f3b5 --- /dev/null +++ b/backend/cache-api-keys.cjs @@ -0,0 +1,37 @@ +// Express API for Redis/Valkey: keys list and single key fetch (for fast UI) +const express = require('express'); +const Redis = require('ioredis'); +const app = express(); +const redis = new Redis({ + host: process.env.REDIS_HOST || '127.0.0.1', + port: Number(process.env.REDIS_PORT || 6379), + password: process.env.REDIS_PASSWORD, +}); + +// List all keys (no values) +app.get('/api/cache/keys', async (req, res) => { + try { + const keys = await redis.keys('*'); + // Optionally, add summary/metadata here + res.json(keys.map(key => ({ key }))); + } catch (err) { + res.status(500).json({ error: err.message }); + } +}); + +// Fetch value for a single key +app.get('/api/cache/:key', async (req, res) => { + try { + const key = req.params.key; + let value = await redis.get(key); + try { value = JSON.parse(value); } catch {} + res.json({ key, value }); + } catch (err) { + res.status(500).json({ error: err.message }); + } +}); + +const port = process.env.CACHE_API_PORT || 3006; +app.listen(port, () => { + console.log(`Cache API (keys+single) running on http://localhost:${port}/api/cache/keys`); +}); diff --git a/backend/server.ts b/backend/server.ts index e9de023..63b6113 100644 --- a/backend/server.ts +++ b/backend/server.ts @@ -82,39 +82,40 @@ app.get('/api/jobs', async (c) => { app.get('/api/jobs-all', async (c) => { const cacheKey = 'jobs:all'; const ttl = 1800; // 30 minutes cache - try { - // Always try cache first for instant response + // Try cache first for instant response if (isRedisConnected()) { const cached = await getCache(cacheKey); if (cached) { logLine('logs/server.log', `Cache HIT for ${cacheKey} (${cached.items?.length || 0} jobs)`); - - // Return cached data immediately, refresh in background if needed + // Return cached data immediately setImmediate(async () => { try { await refreshJobsCache(cacheKey, ttl, c.req.header('Authorization') || ''); + } catch (err) {} + }); + // For ultra-fast display, send only the first 40 jobs in the initial response + const fastBatch = Array.isArray(cached.items) ? cached.items.slice(0, 40) : []; + return c.json({ ...cached, items: fastBatch, partial: cached.items && cached.items.length > 40 }); + } else { + // No cache: return empty array instantly, trigger background fetch + setImmediate(async () => { + try { + const result = await fetchAllJobsFromPocketBase(c.req.header('Authorization') || ''); + if (isRedisConnected()) { + await setCache(cacheKey, result, ttl); + logLine('logs/server.log', `Cached ${cacheKey} with ${result.items.length} jobs for ${ttl}s`); + } } catch (err) { - // Silently fail background refresh + logLine('logs/error.log', `jobs-all background fetch error: ${(err as Error)?.message || String(err)}`); } }); - - return c.json(cached); + return c.json({ items: [], partial: true }); } - logLine('logs/server.log', `Cache MISS for ${cacheKey} - fetching all jobs`); } - - // No cache available, fetch now + // Redis not connected: fallback to direct fetch (blocking) const result = await fetchAllJobsFromPocketBase(c.req.header('Authorization') || ''); - - // Cache the result - if (isRedisConnected()) { - await setCache(cacheKey, result, ttl); - logLine('logs/server.log', `Cached ${cacheKey} with ${result.items.length} jobs for ${ttl}s`); - } - return c.json(result); - } catch (err) { logLine('logs/error.log', `jobs-all endpoint error: ${(err as Error)?.message || String(err)}`); return c.json({ error: 'Failed to fetch jobs' }, 500); diff --git a/cache-visualization.html b/cache-visualization.html index 26f7fa6..60a7de5 100644 --- a/cache-visualization.html +++ b/cache-visualization.html @@ -25,53 +25,58 @@ diff --git a/frontend/index.html b/frontend/index.html index 80af9e5..f85a158 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -413,25 +413,33 @@ startProgress(); clearJobsCache('Loading…'); const startTime = performance.now(); - try{ - // All job data should be fetched from backend/Valkey cache - const url = `/api/jobs-all`; - const res = await fetchNoCache(url); - if(!res.ok) throw new Error(`Fetch failed: ${res.status}`); - const data = await res.json(); - const items = data.items || []; - jobsCache.push(...items); - const loadTime = Math.round(performance.now() - startTime); - console.log(`✓ Loaded ${items.length} jobs from API in ${loadTime}ms`); - - // Show results immediately - progressBar.style.width='95%'; - const renderStart = performance.now(); - applyFiltersAndRender(); - const renderTime = Math.round(performance.now() - startTime); - console.log(`✓ Rendered jobs in ${renderTime}ms (Total: ${Math.round(performance.now() - startTime)}ms)`); - finishProgress(); - }catch(err){ finishProgress(); alert('Failed to fetch jobs: '+err.message); console.error(err);} + let pollCount = 0; + async function pollJobs(){ + try{ + const url = `/api/jobs-all`; + const res = await fetchNoCache(url); + if(!res.ok) throw new Error(`Fetch failed: ${res.status}`); + const data = await res.json(); + const items = data.items || []; + if (pollCount === 0 || jobsCache.length === 0) { + jobsCache.length = 0; + jobsCache.push(...items); + progressBar.style.width='95%'; + applyFiltersAndRender(); + } + if (data.partial) { + // If partial, poll again after short delay + pollCount++; + if (pollCount < 10) setTimeout(pollJobs, 600); + else finishProgress(); + } else { + finishProgress(); + const loadTime = Math.round(performance.now() - startTime); + console.log(`✓ Loaded ${items.length} jobs from API in ${loadTime}ms`); + } + }catch(err){ finishProgress(); alert('Failed to fetch jobs: '+err.message); console.error(err); } + } + pollJobs(); } function renderInitialBatch(jobs){