INIT Backend
This commit is contained in:
@@ -0,0 +1,260 @@
|
||||
// excelsync.js
|
||||
import { FIELD_CONFIG } from "./fieldConfig.js";
|
||||
import fs from "fs";
|
||||
import axios from "axios";
|
||||
import qs from "qs";
|
||||
import dotenv from "dotenv";
|
||||
import PocketBase from "pocketbase";
|
||||
|
||||
dotenv.config();
|
||||
|
||||
// -----------------------------
|
||||
// EXCEL PARSER
|
||||
// -----------------------------
|
||||
const HEADER_ROW_INDEX = 2; // 0-index → Row 3 actual
|
||||
const DATA_START_ROW_INDEX = 3; // 0-index → Row 4 actual
|
||||
|
||||
// -----------------------------
|
||||
// CONFIG
|
||||
// -----------------------------
|
||||
const CONFIG = JSON.parse(fs.readFileSync("config.json", "utf8"));
|
||||
const tenantId = CONFIG.tenantId;
|
||||
const clientId = CONFIG.clientId;
|
||||
const pbUrl = CONFIG.pbUrl;
|
||||
const pbCollection = CONFIG.pbCollection || "Job_Info_TestEnv";
|
||||
const pbEmail = CONFIG.pbEmail;
|
||||
const clientSecret = process.env.CLIENT_SECRET;
|
||||
const pbPassword = process.env.PB_PASSWORD;
|
||||
|
||||
// -----------------------------
|
||||
// LOGGING
|
||||
// -----------------------------
|
||||
function log(message) {
|
||||
console.log(message);
|
||||
fs.appendFileSync("excelsync.log", message + "\n");
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// GRAPH TOKEN
|
||||
// -----------------------------
|
||||
async function getGraphToken() {
|
||||
log("🔵 Requesting Graph token...");
|
||||
const tokenUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`;
|
||||
const body = qs.stringify({
|
||||
client_id: clientId,
|
||||
client_secret: clientSecret,
|
||||
scope: "https://graph.microsoft.com/.default",
|
||||
grant_type: "client_credentials",
|
||||
});
|
||||
const res = await axios.post(tokenUrl, body, {
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||
});
|
||||
log("✅ Graph token acquired");
|
||||
return res.data.access_token;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// POCKETBASE LOGIN
|
||||
// -----------------------------
|
||||
async function getPocketBaseClient() {
|
||||
log("🔵 Logging into PocketBase...");
|
||||
const pb = new PocketBase(pbUrl);
|
||||
// ⚠️ CHANGE "users" if your auth collection is different
|
||||
await pb.collection("users").authWithPassword(pbEmail, pbPassword);
|
||||
log("✅ PocketBase token acquired");
|
||||
return pb;
|
||||
}
|
||||
// -----------------------------
|
||||
// CLEAN + COERCE
|
||||
// -----------------------------
|
||||
function cleanString(value) {
|
||||
if (value == null) return "";
|
||||
return String(value).replace(/\u00A0/g, " ").trim();
|
||||
}
|
||||
|
||||
function coerceValue(fieldConfig, rawValue) {
|
||||
switch (fieldConfig.type) {
|
||||
case "string": return cleanString(rawValue);
|
||||
case "bool": return ["true","1","yes","y"].includes(cleanString(rawValue).toLowerCase());
|
||||
case "date":
|
||||
if (typeof rawValue === "number") {
|
||||
const origin = new Date(Date.UTC(1899, 11, 30));
|
||||
return new Date(origin.getTime() + rawValue * 86400000).toISOString();
|
||||
}
|
||||
const d = new Date(cleanString(rawValue));
|
||||
return isNaN(d.getTime()) ? null : d.toISOString();
|
||||
case "number": return Number(cleanString(rawValue));
|
||||
case "json": try { return JSON.parse(cleanString(rawValue)); } catch { return null; }
|
||||
case "relation": return cleanString(rawValue);
|
||||
default: return cleanString(rawValue);
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// PAYLOAD BUILDER
|
||||
// -----------------------------
|
||||
export function buildPayload(headers, row) {
|
||||
const payload = {};
|
||||
headers.forEach((excelHeader, i) => {
|
||||
const fieldConfig = Object.values(FIELD_CONFIG).find(f => f.excelHeader === excelHeader);
|
||||
if (!fieldConfig) return;
|
||||
payload[fieldConfig.pbField] = coerceValue(fieldConfig, row[i]);
|
||||
});
|
||||
return payload;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// DIFFERENCE LOGGER + UPDATE CHECK
|
||||
// -----------------------------
|
||||
function logDifferences(pbRecord, headers, row) {
|
||||
const diffs = [];
|
||||
headers.forEach((excelHeader, i) => {
|
||||
const fieldConfig = Object.values(FIELD_CONFIG).find(f => f.excelHeader === excelHeader);
|
||||
if (!fieldConfig) return;
|
||||
const newValue = coerceValue(fieldConfig, row[i]);
|
||||
const currentValue = coerceValue(fieldConfig, pbRecord[fieldConfig.pbField]);
|
||||
if (newValue !== currentValue) {
|
||||
diffs.push({ field: fieldConfig.pbField, excelHeader, current: currentValue, incoming: newValue });
|
||||
}
|
||||
});
|
||||
if (diffs.length > 0) {
|
||||
log(`Differences detected for record: ${pbRecord.id}`);
|
||||
diffs.forEach(d => {
|
||||
log(`Field: ${d.field} (Excel: ${d.excelHeader})\n Current: ${JSON.stringify(d.current)}\n Incoming: ${JSON.stringify(d.incoming)}\n`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function needsUpdate(pbRecord, headers, row) {
|
||||
let hasDiff = false;
|
||||
headers.forEach((excelHeader, i) => {
|
||||
const fieldConfig = Object.values(FIELD_CONFIG).find(f => f.excelHeader === excelHeader);
|
||||
if (!fieldConfig) return;
|
||||
const newValue = coerceValue(fieldConfig, row[i]);
|
||||
const currentValue = coerceValue(fieldConfig, pbRecord[fieldConfig.pbField]);
|
||||
if (newValue !== currentValue) hasDiff = true;
|
||||
});
|
||||
if (hasDiff) logDifferences(pbRecord, headers, row);
|
||||
return hasDiff;
|
||||
}
|
||||
// -----------------------------
|
||||
// GRAPH HELPERS
|
||||
// -----------------------------
|
||||
async function getUsedRangeInfo(graphToken, driveId, itemId, worksheetName) {
|
||||
log(`🔵 Requesting usedRange for ${worksheetName}`);
|
||||
const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/worksheets('${worksheetName}')/usedRange`;
|
||||
const res = await axios.get(url, { headers: { Authorization: `Bearer ${graphToken}` } });
|
||||
log(`✅ Used range address: ${res.data.address}`);
|
||||
return res.data.address;
|
||||
}
|
||||
|
||||
function extractLastColumn(address) {
|
||||
const match = address.match(/:([A-Z]+)\d+$/);
|
||||
if (!match) {
|
||||
log("⚠️ Falling back to column Z, check sheet width");
|
||||
return "Z";
|
||||
}
|
||||
return match[1];
|
||||
}
|
||||
|
||||
async function fetchExcelRange(graphToken, driveId, itemId, worksheetName, range) {
|
||||
log(`🔵 Fetching range ${range} from worksheet ${worksheetName}`);
|
||||
const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/worksheets('${worksheetName}')/range(address='${range}')`;
|
||||
const res = await axios.get(url, { headers: { Authorization: `Bearer ${graphToken}` } });
|
||||
log(`✅ Range ${range} returned ${res.data.values?.length || 0} rows`);
|
||||
return res.data.values || [];
|
||||
}
|
||||
|
||||
async function fetchAllRows(graphToken, driveId, itemId, worksheetName, blockSize = 500) {
|
||||
const address = await getUsedRangeInfo(graphToken, driveId, itemId, worksheetName);
|
||||
const lastCol = extractLastColumn(address);
|
||||
|
||||
let allRows = [];
|
||||
let startRow = 1;
|
||||
|
||||
while (true) {
|
||||
const endRow = startRow + blockSize - 1;
|
||||
const range = `A${startRow}:${lastCol}${endRow}`;
|
||||
const rows = await fetchExcelRange(graphToken, driveId, itemId, worksheetName, range);
|
||||
|
||||
if (!rows.length) break;
|
||||
allRows = allRows.concat(rows);
|
||||
|
||||
if (rows.length < blockSize) break;
|
||||
startRow = endRow + 1;
|
||||
}
|
||||
|
||||
return allRows;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// MAIN
|
||||
// -----------------------------
|
||||
(async () => {
|
||||
try {
|
||||
log("🔵 Checking token acquisition and excel readiness");
|
||||
|
||||
const graphToken = await getGraphToken();
|
||||
const pb = await getPocketBaseClient();
|
||||
|
||||
const driveId = "b!9YOqqr2xM0G2DFBwMEJYY4UzQgocFddEtpLYWuS9_AtkCKl2q8yjQJteQ7Ti4QSx";
|
||||
const itemId = "01SPNXLDSQTMG4F6BDANCLJU2FZKEDQLVU";
|
||||
const worksheetName = "Job Sheet";
|
||||
|
||||
const rows = await fetchAllRows(graphToken, driveId, itemId, worksheetName);
|
||||
log(`✅ Total rows fetched: ${rows.length}`);
|
||||
|
||||
if (!rows.length) {
|
||||
log("⚠️ No rows returned from Graph. Check driveId, itemId, and worksheetName.");
|
||||
return;
|
||||
}
|
||||
|
||||
const headers = rows[HEADER_ROW_INDEX];
|
||||
const dataRows = rows.slice(DATA_START_ROW_INDEX);
|
||||
|
||||
if (!headers || !Array.isArray(headers)) {
|
||||
log("❌ Header row not found at expected index. Verify HEADER_ROW_INDEX alignment.");
|
||||
return;
|
||||
}
|
||||
|
||||
log(`✅ Headers detected: ${JSON.stringify(headers)}`);
|
||||
log(`✅ Data rows to process: ${dataRows.length}`);
|
||||
|
||||
for (const row of dataRows) {
|
||||
if (!row || row.length === 0) continue; // skip empty rows
|
||||
|
||||
const payload = buildPayload(headers, row);
|
||||
|
||||
// Ensure primary key exists before querying PocketBase
|
||||
const jobNumber = payload.Job_Number;
|
||||
if (!jobNumber || String(jobNumber).trim() === "") {
|
||||
log("⚠️ Skipping row with missing Job_Number.");
|
||||
continue;
|
||||
}
|
||||
|
||||
let existing = null;
|
||||
try {
|
||||
existing = await pb.collection(pbCollection).getFirstListItem(`Job_Number="${jobNumber}"`);
|
||||
} catch {
|
||||
existing = null;
|
||||
}
|
||||
|
||||
if (!existing) {
|
||||
log(`➕ Creating new record: ${jobNumber}`);
|
||||
await pb.collection(pbCollection).create(payload);
|
||||
} else {
|
||||
if (needsUpdate(existing, headers, row)) {
|
||||
log(`✏️ Updating record: ${existing.id} (${jobNumber})`);
|
||||
await pb.collection(pbCollection).update(existing.id, payload);
|
||||
} else {
|
||||
log(`✅ No changes for record: ${existing.id} (${jobNumber})`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log("✅ Sync complete, PocketBase updated!");
|
||||
} catch (err) {
|
||||
log("❌ Error, not ready for updating. " + (err?.message || err));
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
Reference in New Issue
Block a user