// excelsync.js import { FIELD_CONFIG } from "./fieldConfig.js"; import fs from "fs"; import axios from "axios"; import qs from "qs"; import dotenv from "dotenv"; import PocketBase from "pocketbase"; dotenv.config(); // ----------------------------- // EXCEL PARSER // ----------------------------- const HEADER_ROW_INDEX = 2; // 0-index → Row 3 actual const DATA_START_ROW_INDEX = 3; // 0-index → Row 4 actual // ----------------------------- // CONFIG // ----------------------------- const CONFIG = JSON.parse(fs.readFileSync("config.json", "utf8")); const tenantId = CONFIG.tenantId; const clientId = CONFIG.clientId; const pbUrl = CONFIG.pbUrl; const pbCollection = CONFIG.pbCollection || "Job_Info_TestEnv"; const pbEmail = CONFIG.pbEmail; const clientSecret = process.env.CLIENT_SECRET; const pbPassword = process.env.PB_PASSWORD; // ----------------------------- // LOGGING // ----------------------------- function log(message) { console.log(message); fs.appendFileSync("excelsync.log", message + "\n"); } // ----------------------------- // GRAPH TOKEN // ----------------------------- async function getGraphToken() { log("🔵 Requesting Graph token..."); const tokenUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`; const body = qs.stringify({ client_id: clientId, client_secret: clientSecret, scope: "https://graph.microsoft.com/.default", grant_type: "client_credentials", }); const res = await axios.post(tokenUrl, body, { headers: { "Content-Type": "application/x-www-form-urlencoded" }, }); log("✅ Graph token acquired"); return res.data.access_token; } // ----------------------------- // POCKETBASE LOGIN // ----------------------------- async function getPocketBaseClient() { log("🔵 Logging into PocketBase..."); const pb = new PocketBase(pbUrl); // ⚠️ CHANGE "users" if your auth collection is different await pb.collection("users").authWithPassword(pbEmail, pbPassword); log("✅ PocketBase token acquired"); return pb; } // ----------------------------- // CLEAN + COERCE // ----------------------------- function cleanString(value) { if (value == null) return ""; return String(value).replace(/\u00A0/g, " ").trim(); } function coerceValue(fieldConfig, rawValue) { switch (fieldConfig.type) { case "string": return cleanString(rawValue); case "bool": return ["true","1","yes","y"].includes(cleanString(rawValue).toLowerCase()); case "date": if (typeof rawValue === "number") { const origin = new Date(Date.UTC(1899, 11, 30)); return new Date(origin.getTime() + rawValue * 86400000).toISOString(); } const d = new Date(cleanString(rawValue)); return isNaN(d.getTime()) ? null : d.toISOString(); case "number": return Number(cleanString(rawValue)); case "json": try { return JSON.parse(cleanString(rawValue)); } catch { return null; } case "relation": return cleanString(rawValue); default: return cleanString(rawValue); } } // ----------------------------- // PAYLOAD BUILDER // ----------------------------- export function buildPayload(headers, row) { const payload = {}; headers.forEach((excelHeader, i) => { const fieldConfig = Object.values(FIELD_CONFIG).find(f => f.excelHeader === excelHeader); if (!fieldConfig) return; payload[fieldConfig.pbField] = coerceValue(fieldConfig, row[i]); }); return payload; } // ----------------------------- // DIFFERENCE LOGGER + UPDATE CHECK // ----------------------------- function logDifferences(pbRecord, headers, row) { const diffs = []; headers.forEach((excelHeader, i) => { const fieldConfig = Object.values(FIELD_CONFIG).find(f => f.excelHeader === excelHeader); if (!fieldConfig) return; const newValue = coerceValue(fieldConfig, row[i]); const currentValue = coerceValue(fieldConfig, pbRecord[fieldConfig.pbField]); if (newValue !== currentValue) { diffs.push({ field: fieldConfig.pbField, excelHeader, current: currentValue, incoming: newValue }); } }); if (diffs.length > 0) { log(`Differences detected for record: ${pbRecord.id}`); diffs.forEach(d => { log(`Field: ${d.field} (Excel: ${d.excelHeader})\n Current: ${JSON.stringify(d.current)}\n Incoming: ${JSON.stringify(d.incoming)}\n`); }); } } export function needsUpdate(pbRecord, headers, row) { let hasDiff = false; headers.forEach((excelHeader, i) => { const fieldConfig = Object.values(FIELD_CONFIG).find(f => f.excelHeader === excelHeader); if (!fieldConfig) return; const newValue = coerceValue(fieldConfig, row[i]); const currentValue = coerceValue(fieldConfig, pbRecord[fieldConfig.pbField]); if (newValue !== currentValue) hasDiff = true; }); if (hasDiff) logDifferences(pbRecord, headers, row); return hasDiff; } // ----------------------------- // GRAPH HELPERS // ----------------------------- async function getUsedRangeInfo(graphToken, driveId, itemId, worksheetName) { log(`🔵 Requesting usedRange for ${worksheetName}`); const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/worksheets('${worksheetName}')/usedRange`; const res = await axios.get(url, { headers: { Authorization: `Bearer ${graphToken}` } }); log(`✅ Used range address: ${res.data.address}`); return res.data.address; } function extractLastColumn(address) { const match = address.match(/:([A-Z]+)\d+$/); if (!match) { log("⚠️ Falling back to column Z, check sheet width"); return "Z"; } return match[1]; } async function fetchExcelRange(graphToken, driveId, itemId, worksheetName, range) { log(`🔵 Fetching range ${range} from worksheet ${worksheetName}`); const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/worksheets('${worksheetName}')/range(address='${range}')`; const res = await axios.get(url, { headers: { Authorization: `Bearer ${graphToken}` } }); log(`✅ Range ${range} returned ${res.data.values?.length || 0} rows`); return res.data.values || []; } async function fetchAllRows(graphToken, driveId, itemId, worksheetName, blockSize = 500) { const address = await getUsedRangeInfo(graphToken, driveId, itemId, worksheetName); const lastCol = extractLastColumn(address); let allRows = []; let startRow = 1; while (true) { const endRow = startRow + blockSize - 1; const range = `A${startRow}:${lastCol}${endRow}`; const rows = await fetchExcelRange(graphToken, driveId, itemId, worksheetName, range); if (!rows.length) break; allRows = allRows.concat(rows); if (rows.length < blockSize) break; startRow = endRow + 1; } return allRows; } // ----------------------------- // MAIN // ----------------------------- (async () => { try { log("🔵 Checking token acquisition and excel readiness"); const graphToken = await getGraphToken(); const pb = await getPocketBaseClient(); const driveId = "b!9YOqqr2xM0G2DFBwMEJYY4UzQgocFddEtpLYWuS9_AtkCKl2q8yjQJteQ7Ti4QSx"; const itemId = "01SPNXLDSQTMG4F6BDANCLJU2FZKEDQLVU"; const worksheetName = "Job Sheet"; const rows = await fetchAllRows(graphToken, driveId, itemId, worksheetName); log(`✅ Total rows fetched: ${rows.length}`); if (!rows.length) { log("⚠️ No rows returned from Graph. Check driveId, itemId, and worksheetName."); return; } const headers = rows[HEADER_ROW_INDEX]; const dataRows = rows.slice(DATA_START_ROW_INDEX); if (!headers || !Array.isArray(headers)) { log("❌ Header row not found at expected index. Verify HEADER_ROW_INDEX alignment."); return; } log(`✅ Headers detected: ${JSON.stringify(headers)}`); log(`✅ Data rows to process: ${dataRows.length}`); for (const row of dataRows) { if (!row || row.length === 0) continue; // skip empty rows const payload = buildPayload(headers, row); // Ensure primary key exists before querying PocketBase const jobNumber = payload.Job_Number; if (!jobNumber || String(jobNumber).trim() === "") { log("⚠️ Skipping row with missing Job_Number."); continue; } let existing = null; try { existing = await pb.collection(pbCollection).getFirstListItem(`Job_Number="${jobNumber}"`); } catch { existing = null; } if (!existing) { log(`➕ Creating new record: ${jobNumber}`); await pb.collection(pbCollection).create(payload); } else { if (needsUpdate(existing, headers, row)) { log(`✏️ Updating record: ${existing.id} (${jobNumber})`); await pb.collection(pbCollection).update(existing.id, payload); } else { log(`✅ No changes for record: ${existing.id} (${jobNumber})`); } } } log("✅ Sync complete, PocketBase updated!"); } catch (err) { log("❌ Error, not ready for updating. " + (err?.message || err)); process.exit(1); } })();