261 lines
8.8 KiB
JavaScript
261 lines
8.8 KiB
JavaScript
// excelsync.js
|
||
import { FIELD_CONFIG } from "./fieldConfig.js";
|
||
import fs from "fs";
|
||
import axios from "axios";
|
||
import qs from "qs";
|
||
import dotenv from "dotenv";
|
||
import PocketBase from "pocketbase";
|
||
|
||
dotenv.config();
|
||
|
||
// -----------------------------
|
||
// EXCEL PARSER
|
||
// -----------------------------
|
||
const HEADER_ROW_INDEX = 2; // 0-index → Row 3 actual
|
||
const DATA_START_ROW_INDEX = 3; // 0-index → Row 4 actual
|
||
|
||
// -----------------------------
|
||
// CONFIG
|
||
// -----------------------------
|
||
const CONFIG = JSON.parse(fs.readFileSync("config.json", "utf8"));
|
||
const tenantId = CONFIG.tenantId;
|
||
const clientId = CONFIG.clientId;
|
||
const pbUrl = CONFIG.pbUrl;
|
||
const pbCollection = CONFIG.pbCollection || "Job_Info_TestEnv";
|
||
const pbEmail = CONFIG.pbEmail;
|
||
const clientSecret = process.env.CLIENT_SECRET;
|
||
const pbPassword = process.env.PB_PASSWORD;
|
||
|
||
// -----------------------------
|
||
// LOGGING
|
||
// -----------------------------
|
||
function log(message) {
|
||
console.log(message);
|
||
fs.appendFileSync("excelsync.log", message + "\n");
|
||
}
|
||
|
||
// -----------------------------
|
||
// GRAPH TOKEN
|
||
// -----------------------------
|
||
async function getGraphToken() {
|
||
log("🔵 Requesting Graph token...");
|
||
const tokenUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`;
|
||
const body = qs.stringify({
|
||
client_id: clientId,
|
||
client_secret: clientSecret,
|
||
scope: "https://graph.microsoft.com/.default",
|
||
grant_type: "client_credentials",
|
||
});
|
||
const res = await axios.post(tokenUrl, body, {
|
||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||
});
|
||
log("✅ Graph token acquired");
|
||
return res.data.access_token;
|
||
}
|
||
|
||
// -----------------------------
|
||
// POCKETBASE LOGIN
|
||
// -----------------------------
|
||
async function getPocketBaseClient() {
|
||
log("🔵 Logging into PocketBase...");
|
||
const pb = new PocketBase(pbUrl);
|
||
// ⚠️ CHANGE "users" if your auth collection is different
|
||
await pb.collection("users").authWithPassword(pbEmail, pbPassword);
|
||
log("✅ PocketBase token acquired");
|
||
return pb;
|
||
}
|
||
// -----------------------------
|
||
// CLEAN + COERCE
|
||
// -----------------------------
|
||
function cleanString(value) {
|
||
if (value == null) return "";
|
||
return String(value).replace(/\u00A0/g, " ").trim();
|
||
}
|
||
|
||
function coerceValue(fieldConfig, rawValue) {
|
||
switch (fieldConfig.type) {
|
||
case "string": return cleanString(rawValue);
|
||
case "bool": return ["true","1","yes","y"].includes(cleanString(rawValue).toLowerCase());
|
||
case "date":
|
||
if (typeof rawValue === "number") {
|
||
const origin = new Date(Date.UTC(1899, 11, 30));
|
||
return new Date(origin.getTime() + rawValue * 86400000).toISOString();
|
||
}
|
||
const d = new Date(cleanString(rawValue));
|
||
return isNaN(d.getTime()) ? null : d.toISOString();
|
||
case "number": return Number(cleanString(rawValue));
|
||
case "json": try { return JSON.parse(cleanString(rawValue)); } catch { return null; }
|
||
case "relation": return cleanString(rawValue);
|
||
default: return cleanString(rawValue);
|
||
}
|
||
}
|
||
|
||
// -----------------------------
|
||
// PAYLOAD BUILDER
|
||
// -----------------------------
|
||
export function buildPayload(headers, row) {
|
||
const payload = {};
|
||
headers.forEach((excelHeader, i) => {
|
||
const fieldConfig = Object.values(FIELD_CONFIG).find(f => f.excelHeader === excelHeader);
|
||
if (!fieldConfig) return;
|
||
payload[fieldConfig.pbField] = coerceValue(fieldConfig, row[i]);
|
||
});
|
||
return payload;
|
||
}
|
||
|
||
// -----------------------------
|
||
// DIFFERENCE LOGGER + UPDATE CHECK
|
||
// -----------------------------
|
||
function logDifferences(pbRecord, headers, row) {
|
||
const diffs = [];
|
||
headers.forEach((excelHeader, i) => {
|
||
const fieldConfig = Object.values(FIELD_CONFIG).find(f => f.excelHeader === excelHeader);
|
||
if (!fieldConfig) return;
|
||
const newValue = coerceValue(fieldConfig, row[i]);
|
||
const currentValue = coerceValue(fieldConfig, pbRecord[fieldConfig.pbField]);
|
||
if (newValue !== currentValue) {
|
||
diffs.push({ field: fieldConfig.pbField, excelHeader, current: currentValue, incoming: newValue });
|
||
}
|
||
});
|
||
if (diffs.length > 0) {
|
||
log(`Differences detected for record: ${pbRecord.id}`);
|
||
diffs.forEach(d => {
|
||
log(`Field: ${d.field} (Excel: ${d.excelHeader})\n Current: ${JSON.stringify(d.current)}\n Incoming: ${JSON.stringify(d.incoming)}\n`);
|
||
});
|
||
}
|
||
}
|
||
|
||
export function needsUpdate(pbRecord, headers, row) {
|
||
let hasDiff = false;
|
||
headers.forEach((excelHeader, i) => {
|
||
const fieldConfig = Object.values(FIELD_CONFIG).find(f => f.excelHeader === excelHeader);
|
||
if (!fieldConfig) return;
|
||
const newValue = coerceValue(fieldConfig, row[i]);
|
||
const currentValue = coerceValue(fieldConfig, pbRecord[fieldConfig.pbField]);
|
||
if (newValue !== currentValue) hasDiff = true;
|
||
});
|
||
if (hasDiff) logDifferences(pbRecord, headers, row);
|
||
return hasDiff;
|
||
}
|
||
// -----------------------------
|
||
// GRAPH HELPERS
|
||
// -----------------------------
|
||
async function getUsedRangeInfo(graphToken, driveId, itemId, worksheetName) {
|
||
log(`🔵 Requesting usedRange for ${worksheetName}`);
|
||
const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/worksheets('${worksheetName}')/usedRange`;
|
||
const res = await axios.get(url, { headers: { Authorization: `Bearer ${graphToken}` } });
|
||
log(`✅ Used range address: ${res.data.address}`);
|
||
return res.data.address;
|
||
}
|
||
|
||
function extractLastColumn(address) {
|
||
const match = address.match(/:([A-Z]+)\d+$/);
|
||
if (!match) {
|
||
log("⚠️ Falling back to column Z, check sheet width");
|
||
return "Z";
|
||
}
|
||
return match[1];
|
||
}
|
||
|
||
async function fetchExcelRange(graphToken, driveId, itemId, worksheetName, range) {
|
||
log(`🔵 Fetching range ${range} from worksheet ${worksheetName}`);
|
||
const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/worksheets('${worksheetName}')/range(address='${range}')`;
|
||
const res = await axios.get(url, { headers: { Authorization: `Bearer ${graphToken}` } });
|
||
log(`✅ Range ${range} returned ${res.data.values?.length || 0} rows`);
|
||
return res.data.values || [];
|
||
}
|
||
|
||
async function fetchAllRows(graphToken, driveId, itemId, worksheetName, blockSize = 500) {
|
||
const address = await getUsedRangeInfo(graphToken, driveId, itemId, worksheetName);
|
||
const lastCol = extractLastColumn(address);
|
||
|
||
let allRows = [];
|
||
let startRow = 1;
|
||
|
||
while (true) {
|
||
const endRow = startRow + blockSize - 1;
|
||
const range = `A${startRow}:${lastCol}${endRow}`;
|
||
const rows = await fetchExcelRange(graphToken, driveId, itemId, worksheetName, range);
|
||
|
||
if (!rows.length) break;
|
||
allRows = allRows.concat(rows);
|
||
|
||
if (rows.length < blockSize) break;
|
||
startRow = endRow + 1;
|
||
}
|
||
|
||
return allRows;
|
||
}
|
||
|
||
// -----------------------------
|
||
// MAIN
|
||
// -----------------------------
|
||
(async () => {
|
||
try {
|
||
log("🔵 Checking token acquisition and excel readiness");
|
||
|
||
const graphToken = await getGraphToken();
|
||
const pb = await getPocketBaseClient();
|
||
|
||
const driveId = "b!9YOqqr2xM0G2DFBwMEJYY4UzQgocFddEtpLYWuS9_AtkCKl2q8yjQJteQ7Ti4QSx";
|
||
const itemId = "01SPNXLDSQTMG4F6BDANCLJU2FZKEDQLVU";
|
||
const worksheetName = "Job Sheet";
|
||
|
||
const rows = await fetchAllRows(graphToken, driveId, itemId, worksheetName);
|
||
log(`✅ Total rows fetched: ${rows.length}`);
|
||
|
||
if (!rows.length) {
|
||
log("⚠️ No rows returned from Graph. Check driveId, itemId, and worksheetName.");
|
||
return;
|
||
}
|
||
|
||
const headers = rows[HEADER_ROW_INDEX];
|
||
const dataRows = rows.slice(DATA_START_ROW_INDEX);
|
||
|
||
if (!headers || !Array.isArray(headers)) {
|
||
log("❌ Header row not found at expected index. Verify HEADER_ROW_INDEX alignment.");
|
||
return;
|
||
}
|
||
|
||
log(`✅ Headers detected: ${JSON.stringify(headers)}`);
|
||
log(`✅ Data rows to process: ${dataRows.length}`);
|
||
|
||
for (const row of dataRows) {
|
||
if (!row || row.length === 0) continue; // skip empty rows
|
||
|
||
const payload = buildPayload(headers, row);
|
||
|
||
// Ensure primary key exists before querying PocketBase
|
||
const jobNumber = payload.Job_Number;
|
||
if (!jobNumber || String(jobNumber).trim() === "") {
|
||
log("⚠️ Skipping row with missing Job_Number.");
|
||
continue;
|
||
}
|
||
|
||
let existing = null;
|
||
try {
|
||
existing = await pb.collection(pbCollection).getFirstListItem(`Job_Number="${jobNumber}"`);
|
||
} catch {
|
||
existing = null;
|
||
}
|
||
|
||
if (!existing) {
|
||
log(`➕ Creating new record: ${jobNumber}`);
|
||
await pb.collection(pbCollection).create(payload);
|
||
} else {
|
||
if (needsUpdate(existing, headers, row)) {
|
||
log(`✏️ Updating record: ${existing.id} (${jobNumber})`);
|
||
await pb.collection(pbCollection).update(existing.id, payload);
|
||
} else {
|
||
log(`✅ No changes for record: ${existing.id} (${jobNumber})`);
|
||
}
|
||
}
|
||
}
|
||
|
||
log("✅ Sync complete, PocketBase updated!");
|
||
} catch (err) {
|
||
log("❌ Error, not ready for updating. " + (err?.message || err));
|
||
process.exit(1);
|
||
}
|
||
})();
|