Fix Graph token initialization order and progressive job loading

2026-01-18 03:37:19 +00:00
parent 09fc949a06
commit 3ed6c888ee
3 changed files with 156671 additions and 149 deletions
@@ -1,6 +1,7 @@
 import { Hono } from 'hono';
 import { serveStatic } from 'hono/bun';
 import fs from 'fs';
+import path from 'path';
 // @ts-ignore Bun project may not have dotenv types configured
 import { config } from 'dotenv';
 import { getCache, setCache, isRedisConnected } from './redis-cache';
@@ -8,6 +9,13 @@ import { getCache, setCache, isRedisConnected } from './redis-cache';
 config();

 const app = new Hono();
+const JOBS_CACHE_FILE = path.join(import.meta.dir, '../jobs-cache.json');
+
+// In-memory cache for fast access
+let cachedJobs: any = null;
+let lastCacheTime = 0;
+let isFetching = false;
+let partialJobs: any[] = [];

 // Minimal log helpers
 const logLine = (path: string, line: string) => {
@@ -18,6 +26,32 @@ const logLine = (path: string, line: string) => {
  }
 };

+// Cache helper functions
+const saveCacheFile = async (data: any): Promise<void> => {
+  try {
+    await Bun.write(JOBS_CACHE_FILE, JSON.stringify(data, null, 2));
+    cachedJobs = data;
+    lastCacheTime = Date.now();
+  } catch (err) {
+    logLine('logs/error.log', `Failed to save cache file: ${err}`);
+  }
+};
+
+const loadCacheFile = async (): Promise<any | null> => {
+  try {
+    const file = Bun.file(JOBS_CACHE_FILE);
+    if (await file.exists()) {
+      const data = JSON.parse(await file.text());
+      cachedJobs = data;
+      lastCacheTime = Date.now();
+      return data;
+    }
+  } catch (err) {
+    logLine('logs/error.log', `Failed to load cache file: ${err}`);
+  }
+  return null;
+};
+
 // Version endpoint sourced from package.json
 app.get('/version', (c) => {
  try {
@@ -78,73 +112,182 @@ app.get('/api/jobs', async (c) => {
  }
 });

-// Fast endpoint: returns all jobs in a single cached request
+// Fast endpoint: returns all jobs from cache, or first page with progressive loading
 app.get('/api/jobs-all', async (c) => {
-  const cacheKey = 'jobs:all';
-  const ttl = 1800; // 30 minutes cache
  try {
-    // Try cache first for instant response
-    if (isRedisConnected()) {
-      const cached = await getCache(cacheKey);
-      if (cached) {
-        logLine('logs/server.log', `Cache HIT for ${cacheKey} (${cached.items?.length || 0} jobs)`);
-        // Return cached data immediately
-        setImmediate(async () => {
-          try {
-            await refreshJobsCache(cacheKey, ttl, c.req.header('Authorization') || '');
-          } catch (err) {}
+    // Return full cache if available
+    if (cachedJobs) {
+      return c.json(cachedJobs);
+    }
+    
+    // Return partial results if still fetching
+    if (isFetching && partialJobs.length > 0) {
+      return c.json({
+        page: 1,
+        perPage: partialJobs.length,
+        totalItems: partialJobs.length,
+        totalPages: 1,
+        items: partialJobs
+      });
+    }
+    
+    // Try to load from disk
+    const diskCache = await loadCacheFile();
+    if (diskCache) {
+      return c.json(diskCache);
+    }
+    
+    // No cache: start fetching from PocketBase
+    // Return first page immediately while fetching the rest in background
+    if (!isFetching) {
+      isFetching = true;
+      
+      // Start background fetch
+      fetchAllJobsProgressively(c.req.header('Authorization') || '').catch(err => {
+        logLine('logs/error.log', `Progressive fetch error: ${err}`);
+        isFetching = false;
+      });
+      
+      // Get first page immediately
+      try {
+        const perPage = 500;
+        const pbUrl = `https://pocketbase.ccllc.pro/api/collections/pbc_1407612689/records?page=1&perPage=${perPage}&sort=-Job_Number`;
+        const response = await fetch(pbUrl, {
+          headers: c.req.header('Authorization') ? { Authorization: c.req.header('Authorization')! } : {}
        });
-        // Return all cached jobs in a single response
-        return c.json({ ...cached, partial: false });
-      } else {
-        // No cache: return empty array instantly, trigger background fetch
-        setImmediate(async () => {
-          try {
-            const result = await fetchAllJobsFromPocketBase(c.req.header('Authorization') || '');
-            if (isRedisConnected()) {
-              await setCache(cacheKey, result, ttl);
-              logLine('logs/server.log', `Cached ${cacheKey} with ${result.items.length} jobs for ${ttl}s`);
-            }
-          } catch (err) {
-            logLine('logs/error.log', `jobs-all background fetch error: ${(err as Error)?.message || String(err)}`);
-          }
-        });
-        return c.json({ items: [], partial: true });
+        
+        if (response.ok) {
+          const data = await response.json();
+          const items = data.items || [];
+          partialJobs = items;
+          return c.json({
+            page: 1,
+            perPage: items.length,
+            totalItems: items.length,
+            totalPages: 1,
+            items: items
+          });
+        }
+      } catch (err) {
+        logLine('logs/error.log', `Failed to get first page: ${err}`);
      }
    }
-    // Redis not connected: fallback to direct fetch (blocking)
-    const result = await fetchAllJobsFromPocketBase(c.req.header('Authorization') || '');
-    return c.json(result);
+    
+    // Fetching in progress, return what we have
+    if (partialJobs.length > 0) {
+      return c.json({
+        page: 1,
+        perPage: partialJobs.length,
+        totalItems: partialJobs.length,
+        totalPages: 1,
+        items: partialJobs
+      });
+    }
+    
+    return c.json({ error: 'Fetching jobs, try again' }, 503);
  } catch (err) {
    logLine('logs/error.log', `jobs-all endpoint error: ${(err as Error)?.message || String(err)}`);
+    if (cachedJobs) {
+      return c.json(cachedJobs);
+    }
    return c.json({ error: 'Failed to fetch jobs' }, 500);
  }
 });

-// Helper function to fetch all jobs from PocketBase
-async function fetchAllJobsFromPocketBase(authHeader: string): Promise<any> {
-  const allItems: any[] = [];
-  let page = 1;
-  const perPage = 500;
-  
-  while (true) {
-    const pbUrl = `https://pocketbase.ccllc.pro/api/collections/pbc_1407612689/records?page=${page}&perPage=${perPage}&sort=-Job_Number`;
-    const response = await fetch(pbUrl, {
+// Progressive fetch helper: fetches all jobs in background, updating partialJobs as it goes
+async function fetchAllJobsProgressively(authHeader: string): Promise<void> {
+  try {
+    const perPage = 500;
+    
+    // Get first page to determine total
+    const firstUrl = `https://pocketbase.ccllc.pro/api/collections/pbc_1407612689/records?page=1&perPage=${perPage}&sort=-Job_Number`;
+    const firstResponse = await fetch(firstUrl, {
      headers: authHeader ? { Authorization: authHeader } : {}
    });
    
-    if (!response.ok) {
-      throw new Error(`PocketBase returned ${response.status}`);
+    if (!firstResponse.ok) {
+      throw new Error(`PocketBase returned ${firstResponse.status}`);
    }
    
-    const data = await response.json();
-    const items = data.items || [];
+    const firstData = await firstResponse.json();
+    const totalItems = firstData.totalItems || 0;
+    const totalPages = Math.ceil(totalItems / perPage);
    
-    if (items.length === 0) break;
-    allItems.push(...items);
+    const allItems = [...(firstData.items || [])];
+    partialJobs = allItems;
    
-    if (allItems.length >= (data.totalItems || 0)) break;
-    page++;
+    // Fetch remaining pages in parallel
+    if (totalPages > 1) {
+      const pagePromises = [];
+      for (let page = 2; page <= totalPages; page++) {
+        const pbUrl = `https://pocketbase.ccllc.pro/api/collections/pbc_1407612689/records?page=${page}&perPage=${perPage}&sort=-Job_Number`;
+        pagePromises.push(
+          fetch(pbUrl, {
+            headers: authHeader ? { Authorization: authHeader } : {}
+          }).then(r => r.json())
+        );
+      }
+      
+      const results = await Promise.all(pagePromises);
+      for (const result of results) {
+        allItems.push(...(result.items || []));
+      }
+    }
+    
+    // Save final result to cache file
+    const cacheData = {
+      page: 1,
+      perPage: allItems.length,
+      totalItems: allItems.length,
+      totalPages: 1,
+      items: allItems
+    };
+    
+    await saveCacheFile(cacheData);
+    isFetching = false;
+    logLine('logs/server.log', `✓ Progressive fetch complete: ${allItems.length} jobs cached`);
+  } catch (err) {
+    logLine('logs/error.log', `Progressive fetch failed: ${err}`);
+    isFetching = false;
+  }
+}
+
+// Helper function to fetch all jobs from PocketBase
+async function fetchAllJobsFromPocketBase(authHeader: string): Promise<any> {
+  const perPage = 500;
+  
+  // First, get total count
+  const firstUrl = `https://pocketbase.ccllc.pro/api/collections/pbc_1407612689/records?page=1&perPage=${perPage}&sort=-Job_Number`;
+  const firstResponse = await fetch(firstUrl, {
+    headers: authHeader ? { Authorization: authHeader } : {}
+  });
+  
+  if (!firstResponse.ok) {
+    throw new Error(`PocketBase returned ${firstResponse.status}`);
+  }
+  
+  const firstData = await firstResponse.json();
+  const totalItems = firstData.totalItems || 0;
+  const totalPages = Math.ceil(totalItems / perPage);
+  
+  // Fetch all pages in parallel
+  const allItems = [...(firstData.items || [])];
+  
+  if (totalPages > 1) {
+    const pagePromises = [];
+    for (let page = 2; page <= totalPages; page++) {
+      const pbUrl = `https://pocketbase.ccllc.pro/api/collections/pbc_1407612689/records?page=${page}&perPage=${perPage}&sort=-Job_Number`;
+      pagePromises.push(
+        fetch(pbUrl, {
+          headers: authHeader ? { Authorization: authHeader } : {}
+        }).then(r => r.json())
+      );
+    }
+    
+    const results = await Promise.all(pagePromises);
+    for (const result of results) {
+      allItems.push(...(result.items || []));
+    }
  }
  
  return {
@@ -472,28 +615,28 @@ app.onError((err, c) => {
 // Default to 3005 for test instance; can be overridden by Environment=PORT
 const PORT = Number(process.env.PORT || 3005);

-logLine('logs/server.log', `Starting frontend server (Redis version) on port ${PORT}`);
+logLine('logs/server.log', `Starting frontend server on port ${PORT}`);

-// Warm up cache on startup
+// Load cache on startup
 (async () => {
  try {
-    if (isRedisConnected()) {
-      logLine('logs/server.log', 'Warming up jobs cache...');
-      const result = await fetchAllJobsFromPocketBase('');
-      await setCache('jobs:all', result, 1800);
-      logLine('logs/server.log', `✓ Cache warmed with ${result.items.length} jobs`);
+    const cached = await loadCacheFile();
+    if (cached) {
+      logLine('logs/server.log', `✓ Loaded cache from disk: ${cached.items?.length || 0} jobs`);
+    } else {
+      logLine('logs/server.log', 'No cache file found, will fetch from PocketBase on first request');
    }
  } catch (err) {
-    logLine('logs/error.log', `Cache warmup failed: ${(err as Error)?.message || String(err)}`);
+    logLine('logs/error.log', `Startup cache load error: ${(err as Error)?.message || String(err)}`);
  }
 })();

-// Background refresh every 5 minutes to keep cache hot
+// Background refresh every 5 minutes to keep cache fresh
 setInterval(async () => {
  try {
-    if (isRedisConnected()) {
-      await refreshJobsCache('jobs:all', 1800, '');
-    }
+    const result = await fetchAllJobsFromPocketBase('');
+    await saveCacheFile(result);
+    logLine('logs/server.log', `✓ Background refresh: updated cache with ${result.items?.length || 0} jobs`);
  } catch (err) {
    logLine('logs/error.log', `Periodic refresh error: ${(err as Error)?.message || String(err)}`);
  }