WIP
This commit is contained in:
+46289
File diff suppressed because it is too large
Load Diff
@@ -6,6 +6,7 @@
|
||||
"scripts": {
|
||||
"dev": "bun --watch src/server.ts",
|
||||
"sharelink": "bun run utilities/sharepointShareLinktoGraph.html",
|
||||
"getpb": "bun run utilities/pbjobitempull.js",
|
||||
"sync": "bun run /src/excelsync.js",
|
||||
"server": "bun run src/server.ts"
|
||||
},
|
||||
|
||||
+3739
File diff suppressed because one or more lines are too long
+198116
File diff suppressed because it is too large
Load Diff
+213
-315
@@ -1,21 +1,20 @@
|
||||
//12/1/2025 Allows Excel download and conversion of headers mapped to pb headers.
|
||||
//coersion added to match pb field types. however, may still have difficulty in assimilating certain date formats from excel to pb date fields.
|
||||
//due to Excel storing as all caps for boolean fields
|
||||
// excelsync.js
|
||||
// Excel → CSV → PocketBase sync by Job_Number.
|
||||
// Raw strings only (true CSV), no coercion. Progress logs + robust error logging.
|
||||
|
||||
import { FIELD_CONFIG } from "./fieldConfig.js";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import axios from "axios";
|
||||
import qs from "qs";
|
||||
import * as XLSX from "xlsx";
|
||||
import ExcelJS from "exceljs";
|
||||
import dotenv from "dotenv";
|
||||
import { serve } from "bun";
|
||||
import PocketBase from "pocketbase"
|
||||
import PocketBase from "pocketbase";
|
||||
|
||||
dotenv.config();
|
||||
|
||||
// -----------------------------
|
||||
// CONFIG FOR TOKEN ACQUISITION GRAPH API + POCKETBASE
|
||||
// CONFIG
|
||||
// -----------------------------
|
||||
const CONFIG = JSON.parse(fs.readFileSync("config.json", "utf8"));
|
||||
const tenantId = CONFIG.tenantId;
|
||||
@@ -31,24 +30,16 @@ const pbPassword = process.env.PB_PASSWORD;
|
||||
// -----------------------------
|
||||
function log(message) {
|
||||
console.log(message);
|
||||
fs.appendFileSync("excelsync.log", message + "\n");
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// HEADER DETECTOR
|
||||
// -----------------------------
|
||||
//Gets the last column letter in the table converting from 0 index to excel actual column in letter form.
|
||||
function columnLetter(index) {
|
||||
let letter = "";
|
||||
while (index >= 0) {
|
||||
letter = String.fromCharCode((index % 26) + 65) + letter;
|
||||
index = Math.floor(index / 26) - 1;
|
||||
const logfile = path.resolve(process.cwd(), "excelsync.log");
|
||||
try {
|
||||
fs.appendFileSync(logfile, message + "\n", "utf8");
|
||||
} catch (e) {
|
||||
console.error("Log write failed:", e?.message || e);
|
||||
}
|
||||
return letter;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// GRAPH TOKEN ACQUISITION
|
||||
// GRAPH TOKEN + DOWNLOAD
|
||||
// -----------------------------
|
||||
async function getGraphToken() {
|
||||
const tokenUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`;
|
||||
@@ -65,20 +56,17 @@ async function getGraphToken() {
|
||||
return res.data.access_token;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// GRAPH DOWNLOADER HELPER
|
||||
// -----------------------------
|
||||
async function downloadExcelFromGraph(graphToken, driveId, itemId) {
|
||||
const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/content`;
|
||||
const res = await axios.get(url, {
|
||||
headers: { Authorization: `Bearer ${graphToken}` },
|
||||
responseType: "arraybuffer", // important: get raw binary
|
||||
responseType: "arraybuffer",
|
||||
});
|
||||
return Buffer.from(res.data); // convert to Buffer for XLSX
|
||||
return Buffer.from(res.data);
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// POCKETBASE LOGIN AND TOKEN ACQUISITION
|
||||
// POCKETBASE LOGIN
|
||||
// -----------------------------
|
||||
async function getPocketBaseClient() {
|
||||
const pb = new PocketBase(pbUrl);
|
||||
@@ -86,98 +74,43 @@ async function getPocketBaseClient() {
|
||||
log("✅ PocketBase token acquired");
|
||||
return pb;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// Get Data From Excel File
|
||||
// Header mapping
|
||||
// -----------------------------
|
||||
|
||||
const HEADER_ROW_INDEX = 2; // Row 3 in Excel
|
||||
const DATA_START_ROW_INDEX = 3; // Row 4 in Excel → first data row
|
||||
|
||||
//clean and coerce Excel values to match PocketBase schema expectations
|
||||
function cleanString(val) {
|
||||
if (val == null) return "";
|
||||
return String(val)
|
||||
.replace(/\u00A0/g, " ") // normalize non-breaking spaces
|
||||
.replace(/[\r\n\t]/g, " ") // strip control chars
|
||||
.replace(/\s+/g, " ") // collapse whitespace
|
||||
.trim();
|
||||
function buildHeaderMapping(rawHeaders) {
|
||||
const excelHeaderToConfig = Object.values(FIELD_CONFIG).reduce((map, cfg) => {
|
||||
if (cfg.excelHeader) {
|
||||
const key = String(cfg.excelHeader).trim();
|
||||
map[key] = cfg;
|
||||
map[key.toLowerCase()] = cfg;
|
||||
}
|
||||
return map;
|
||||
}, {});
|
||||
return rawHeaders.map(raw => {
|
||||
const clean = String(raw || "").trim();
|
||||
const cfg = excelHeaderToConfig[clean] || excelHeaderToConfig[clean.toLowerCase()];
|
||||
return cfg ? { pbField: cfg.pbField, isMapped: true } : { pbField: "", isMapped: false };
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeForPB(val, type) {
|
||||
const cleaned = cleanString(val);
|
||||
// -----------------------------
|
||||
// Excel → CSV string
|
||||
// -----------------------------
|
||||
const HEADER_ROW_INDEX = 2;
|
||||
const DATA_START_ROW_INDEX = 3;
|
||||
|
||||
// Treat common placeholders as empty
|
||||
const emptyTokens = ["n/a", "na", "null", "none"];
|
||||
if (emptyTokens.includes(cleaned.toLowerCase())) {
|
||||
switch (type) {
|
||||
case "number": return 0; // PocketBase defaults to 0
|
||||
case "bool": return null;
|
||||
case "string": return null;
|
||||
case "email": return null; // email empty
|
||||
case "json": return null;
|
||||
case "date": return null; // date empty
|
||||
default: return null;
|
||||
}
|
||||
}
|
||||
|
||||
if (cleaned === "") {
|
||||
switch (type) {
|
||||
case "number": return 0;
|
||||
case "bool": return null;
|
||||
case "string": return null;
|
||||
case "email": return null;
|
||||
case "json": return null;
|
||||
case "date": return null;
|
||||
default: return null;
|
||||
}
|
||||
}
|
||||
|
||||
switch (type) {
|
||||
case "bool":
|
||||
case "boolean": {
|
||||
const lower = cleaned.toLowerCase();
|
||||
if (["true","yes","y","1"].includes(lower)) return true; // boolean true
|
||||
if (["false","no","n","0"].includes(lower)) return false; // boolean false
|
||||
return null;
|
||||
}
|
||||
case "number": {
|
||||
const num = Number(cleaned);
|
||||
return Number.isNaN(num) ? 0 : num;
|
||||
}
|
||||
case "date": {
|
||||
const d = new Date(cleaned);
|
||||
return isNaN(d.getTime()) ? null : d.toISOString();
|
||||
}
|
||||
case "string":
|
||||
default:
|
||||
return cleaned;
|
||||
}
|
||||
}
|
||||
|
||||
export function parseExcelFile(input) {
|
||||
let data;
|
||||
if (Buffer.isBuffer(input)) {
|
||||
data = input;
|
||||
log("Reading Excel file from Graph buffer");
|
||||
} else {
|
||||
data = fs.readFileSync(input);
|
||||
log(`Reading Excel file: ${input}`);
|
||||
}
|
||||
|
||||
const workbook = XLSX.read(data, { type: "buffer", cellDates: true });
|
||||
function excelToCSV(buffer) {
|
||||
const workbook = XLSX.read(buffer, { type: "buffer" });
|
||||
const sheet = workbook.Sheets["Job Sheet"];
|
||||
if (!sheet) throw new Error('Sheet named "Job Sheet" not found');
|
||||
|
||||
const range = XLSX.utils.decode_range(sheet["!ref"]);
|
||||
const merges = sheet["!merges"] || [];
|
||||
|
||||
// Helper to get value from cell or merged region
|
||||
const getValue = (r, c) => {
|
||||
const addr = XLSX.utils.encode_cell({ r, c });
|
||||
const cell = sheet[addr];
|
||||
if (cell) return cell.v ?? cell.w ?? null;
|
||||
|
||||
for (const m of merges) {
|
||||
if (m.s.r <= r && r <= m.e.r && m.s.c <= c && c <= m.e.c) {
|
||||
const master = sheet[XLSX.utils.encode_cell(m.s)];
|
||||
@@ -187,255 +120,220 @@ export function parseExcelFile(input) {
|
||||
return null;
|
||||
};
|
||||
|
||||
// 1. Read headers (Row 3)
|
||||
const rawHeaders = [];
|
||||
for (let c = range.s.c; c <= range.e.c; c++) {
|
||||
const val = getValue(HEADER_ROW_INDEX, c);
|
||||
rawHeaders.push(val ? String(val).trim() : "");
|
||||
}
|
||||
|
||||
// 2. Build lookup map once
|
||||
const excelHeaderToConfig = Object.values(FIELD_CONFIG).reduce((map, cfg) => {
|
||||
if (cfg.excelHeader) {
|
||||
const key = cfg.excelHeader.trim();
|
||||
map[key] = cfg;
|
||||
map[key.toLowerCase()] = cfg;
|
||||
}
|
||||
return map;
|
||||
}, {});
|
||||
const headerMapping = buildHeaderMapping(rawHeaders);
|
||||
const pbHeaders = headerMapping.filter(h => h.isMapped).map(h => h.pbField);
|
||||
|
||||
// 3. Rich header mapping
|
||||
const headerMapping = rawHeaders.map(raw => {
|
||||
const clean = raw.trim();
|
||||
const config = excelHeaderToConfig[clean] || excelHeaderToConfig[clean.toLowerCase()];
|
||||
if (!pbHeaders.includes("Job_Number")) {
|
||||
throw new Error("Job_Number must be present in mapped headers.");
|
||||
}
|
||||
|
||||
return config
|
||||
? { original: clean, pbField: config.pbField, type: config.type, isMapped: true, config }
|
||||
: { original: clean, pbField: "", type: "string", isMapped: false, isUnknown: true };
|
||||
});
|
||||
|
||||
log(`Mapped ${headerMapping.filter(h => h.isMapped).length} columns`);
|
||||
|
||||
// 4. Parse data rows (starting Row 4)
|
||||
const rows = [];
|
||||
for (let r = DATA_START_ROW_INDEX; r <= range.e.r; r++) {
|
||||
const values = [];
|
||||
const rowVals = [];
|
||||
let hasData = false;
|
||||
|
||||
for (let c = range.s.c; c <= range.e.c; c++) {
|
||||
const val = getValue(r, c);
|
||||
values.push(val ?? null);
|
||||
if (val !== null && val !== "") hasData = true;
|
||||
const v = getValue(r, c);
|
||||
if (v !== null && v !== "") hasData = true;
|
||||
rowVals.push(v === null ? "" : String(v));
|
||||
}
|
||||
if (!hasData) continue;
|
||||
|
||||
const rowObj = {};
|
||||
headerMapping.forEach((map, i) => {
|
||||
if (map.isMapped) {
|
||||
rowObj[map.pbField] = normalizeForPB(values[i], map.type);
|
||||
}
|
||||
});
|
||||
|
||||
rows.push(rowObj);
|
||||
rows.push(rowVals);
|
||||
}
|
||||
|
||||
return {
|
||||
data: rows, // parsed row objects
|
||||
headerMapping, // rich info about headers
|
||||
rawHeaders, // original headers for debugging
|
||||
totalRows: rows.length // count of parsed rows
|
||||
const escapeCSV = (s) => {
|
||||
const needsQuotes = /[",\n\r]/.test(s);
|
||||
const withDoubles = s.replace(/"/g, '""');
|
||||
return needsQuotes ? `"${withDoubles}"` : withDoubles;
|
||||
};
|
||||
|
||||
const mappedColIndexes = headerMapping.map((h, idx) => (h.isMapped ? idx : -1)).filter(idx => idx !== -1);
|
||||
const headerLine = pbHeaders.map(h => escapeCSV(h)).join(",");
|
||||
const csvLines = [headerLine];
|
||||
|
||||
for (const row of rows) {
|
||||
const mappedValues = mappedColIndexes.map(idx => escapeCSV(row[idx] || ""));
|
||||
csvLines.push(mappedValues.join(","));
|
||||
}
|
||||
|
||||
const csv = csvLines.join("\n");
|
||||
log(`🧾 Built CSV with ${pbHeaders.length} columns and ${rows.length} data rows`);
|
||||
return { csv, headers: pbHeaders };
|
||||
}
|
||||
|
||||
|
||||
// -----------------------------
|
||||
// MAIN
|
||||
// Parse CSV string → array of objects
|
||||
// -----------------------------
|
||||
let parsedRows = [];
|
||||
let rawHeaders = [];
|
||||
(async () => {
|
||||
try {
|
||||
log("🔵 Starting Excel Sync…");
|
||||
const graphToken = await getGraphToken();
|
||||
const pb = await getPocketBaseClient();
|
||||
// These can be update if the Excel File changes
|
||||
const driveId = "b!9YOqqr2xM0G2DFBwMEJYY4UzQgocFddEtpLYWuS9_AtkCKl2q8yjQJteQ7Ti4QSx";
|
||||
const itemId = "01SPNXLDSQTMG4F6BDANCLJU2FZKEDQLVU";
|
||||
const excelBuffer = await downloadExcelFromGraph(graphToken, driveId, itemId);
|
||||
const parsed = parseExcelFile(excelBuffer);
|
||||
parsedRows = parsed.data;
|
||||
rawHeaders = parsed.rawHeaders;
|
||||
log(rawHeaders)
|
||||
log("✅ First part complete: tokens + Excel parsing ready");
|
||||
} catch (err) {
|
||||
log("❌ Sync failed: " + (err.message || err));
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
function parseCSV(csv) {
|
||||
const lines = csv.split(/\r?\n/).filter(l => l.length > 0);
|
||||
if (lines.length === 0) return { headers: [], rows: [] };
|
||||
|
||||
// -------------------------
|
||||
// Export to xlsx - Excel
|
||||
// ---------------------------
|
||||
|
||||
// Toggle this flag to enable/disable the export server
|
||||
const ENABLE_EXPORT = true; // set to false to bypass download
|
||||
|
||||
function exportExcelforDL(parsedRows) {
|
||||
if (!parsedRows || parsedRows.length === 0) {
|
||||
throw new Error("⚠️ No rows to export");
|
||||
}
|
||||
|
||||
const worksheet = XLSX.utils.json_to_sheet(parsedRows);
|
||||
|
||||
// Ensure Job_Number is the first column
|
||||
const cols = Object.keys(parsedRows[0] || {});
|
||||
if (cols.includes("Job_Number")) {
|
||||
const ordered = ["Job_Number", ...cols.filter(c => c !== "Job_Number")];
|
||||
const remapped = parsedRows.map(r =>
|
||||
ordered.reduce((acc, key) => { acc[key] = r[key]; return acc; }, {})
|
||||
);
|
||||
const ws2 = XLSX.utils.json_to_sheet(remapped);
|
||||
Object.assign(worksheet, ws2);
|
||||
}
|
||||
|
||||
const workbook = XLSX.utils.book_new();
|
||||
XLSX.utils.book_append_sheet(workbook, worksheet, "ParsedData");
|
||||
|
||||
return XLSX.write(workbook, { type: "buffer", bookType: "xlsx" });
|
||||
}
|
||||
|
||||
if (ENABLE_EXPORT) {
|
||||
const PORT = 3000;
|
||||
serve({
|
||||
port: PORT,
|
||||
fetch(req) {
|
||||
const url = new URL(req.url);
|
||||
|
||||
if (url.pathname === "/") {
|
||||
const html = `
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head><meta charset="utf-8"><title>ExcelSync Export</title></head>
|
||||
<body>
|
||||
<h1>ExcelSync Export</h1>
|
||||
<p>Click below to download the parsed Excel file:</p>
|
||||
<button onclick="window.location.href='/download'">Download ParsedOutput.xlsx</button>
|
||||
</body>
|
||||
</html>`;
|
||||
return new Response(html, { headers: { "Content-Type": "text/html" } });
|
||||
const parseLine = (line) => {
|
||||
const out = [];
|
||||
let cur = "";
|
||||
let inQuotes = false;
|
||||
for (let i = 0; i < line.length; i++) {
|
||||
const ch = line[i];
|
||||
if (inQuotes) {
|
||||
if (ch === '"') {
|
||||
if (line[i + 1] === '"') { cur += '"'; i++; }
|
||||
else { inQuotes = false; }
|
||||
} else { cur += ch; }
|
||||
} else {
|
||||
if (ch === '"') inQuotes = true;
|
||||
else if (ch === ",") { out.push(cur); cur = ""; }
|
||||
else { cur += ch; }
|
||||
}
|
||||
|
||||
if (url.pathname === "/download") {
|
||||
try {
|
||||
const buffer = exportExcelforDL(parsedRows);
|
||||
return new Response(buffer, {
|
||||
headers: {
|
||||
"Content-Type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"Content-Disposition": "attachment; filename=ParsedOutput.xlsx"
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
return new Response("❌ Failed to generate Excel file: " + err.message, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
return new Response("Visit / to open the export page.");
|
||||
}
|
||||
});
|
||||
out.push(cur);
|
||||
return out;
|
||||
};
|
||||
|
||||
log(`🚀 ExcelSync export server running at http://localhost:${PORT}/`);
|
||||
log(`👉 Open this URL in your browser to verify and download the parsed Excel file.`);
|
||||
} else {
|
||||
log("⏭️ Export server bypassed (ENABLE_EXPORT=false)");
|
||||
const headers = parseLine(lines[0]).map(h => h.trim());
|
||||
const rows = [];
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const cols = parseLine(lines[i]);
|
||||
const obj = {};
|
||||
for (let c = 0; c < headers.length; c++) {
|
||||
obj[headers[c]] = cols[c] ?? "";
|
||||
}
|
||||
rows.push(obj);
|
||||
}
|
||||
return { headers, rows };
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// Error logging helper
|
||||
// -----------------------------
|
||||
// -----------------------------
|
||||
// Error logging helper
|
||||
// -----------------------------
|
||||
function logPocketBaseError(err, context) {
|
||||
log(`❌ ${context}`);
|
||||
log(`Error message: ${err.message || "no message"}`);
|
||||
|
||||
// Dump everything we can from the error object
|
||||
try {
|
||||
log(`Error dump: ${JSON.stringify(err, Object.getOwnPropertyNames(err), 2)}`);
|
||||
} catch {
|
||||
log("⚠️ Could not stringify error object");
|
||||
}
|
||||
|
||||
// Axios attaches the server's JSON body here
|
||||
if (err.response) {
|
||||
log(`Response status: ${err.response.status} ${err.response.statusText || ""}`);
|
||||
if (err.response.data) {
|
||||
try {
|
||||
log(`Server response data: ${JSON.stringify(err.response.data, null, 2)}`);
|
||||
} catch {
|
||||
log("⚠️ Could not stringify response.data");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// PocketBase client sometimes attaches .data separately
|
||||
if (err.data) {
|
||||
try {
|
||||
log(`Server details: ${JSON.stringify(err.data, null, 2)}`);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// -----------------------------
|
||||
// UPDATE NOTES CELLS IN SHAREPOINT (REAL-TIME)
|
||||
// RESYNC: by Job_Number
|
||||
// -----------------------------
|
||||
//function getNotesColumnLetterFromParsedHeaders(rawHeaders) {
|
||||
// const notesIndex = rawHeaders.findIndex(h => h.trim().toLowerCase() === "notes");
|
||||
// if (notesIndex === -1) {
|
||||
// log(`❌ 'Notes' header not found in HEADER_MAP keys: ${rawHeaders.join(", ")}`);
|
||||
// throw new Error("Notes column not found");
|
||||
// }
|
||||
// return columnLetter(notesIndex);
|
||||
//}
|
||||
//await updateNotesInSharePoint(graphToken, driveId, itemId, "Job Sheet", parsedRows, pb, excelBuffer);
|
||||
// Update only changed Notes cells using workbook session + PATCH.
|
||||
// This version relies solely on DATA_START_ROW_INDEX for row mapping.
|
||||
//async function updateNotesInSharePoint(graphToken, driveId, itemId, worksheetName, parsedRows, pb, workbookBuffer) {
|
||||
// const wb = XLSX.read(workbookBuffer, { type: "buffer", cellDates: true });
|
||||
// const sheet = wb.Sheets[worksheetName];
|
||||
// if (!sheet) throw new Error(`Worksheet "${worksheetName}" not found`);
|
||||
// -----------------------------
|
||||
// RESYNC: by Job_Number
|
||||
// -----------------------------
|
||||
async function resyncPocketBase(pb, csvRows) {
|
||||
log(`🔄 Starting resync for ${csvRows.length} rows...`);
|
||||
|
||||
// Column detection aligned to HEADER_MAP (same mapping used by parseExcelFile)
|
||||
//const notesColumnLetter = getNotesColumnLetterFromParsedHeaders(rawHeaders);
|
||||
//og(`✅ Notes column resolved to ${notesColumnLetter} via rawHeaders`);
|
||||
let existing = [];
|
||||
try {
|
||||
existing = await pb.collection(pbCollection).getFullList();
|
||||
} catch (err) {
|
||||
logPocketBaseError(err, "Failed to load existing records");
|
||||
}
|
||||
|
||||
// Create a workbook session that persists changes
|
||||
//const sessionRes = await axios.post(
|
||||
// `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/createSession`,
|
||||
// { persistChanges: true },
|
||||
// { headers: { Authorization: `Bearer ${graphToken}` } }
|
||||
// );
|
||||
// const sessionId = sessionRes.data.id;
|
||||
const jobMap = {};
|
||||
for (const rec of existing) {
|
||||
jobMap[rec.Job_Number] = rec.id;
|
||||
}
|
||||
|
||||
// let changes = 0;
|
||||
const allowedFields = new Set(
|
||||
Object.values(FIELD_CONFIG).map(cfg => cfg.pbField).filter(Boolean)
|
||||
);
|
||||
|
||||
// Loop parsed rows; map i to Excel row using DATA_START_ROW_INDEX (zero-based)
|
||||
// for (let i = 0; i < parsedRows.length; i++) {
|
||||
// const row = parsedRows[i];
|
||||
// const jobNumber = row.Job_Number;
|
||||
// if (jobNumber == null || jobNumber === "") continue;
|
||||
for (let i = 0; i < csvRows.length; i++) {
|
||||
const row = csvRows[i];
|
||||
|
||||
// try {
|
||||
// const records = await pb.collection(pbCollection).getFullList({
|
||||
// filter: Number.isFinite(jobNumber) ? `Job_Number=${jobNumber}` : `Job_Number="${jobNumber}"`,
|
||||
// batchSize: 1
|
||||
// });
|
||||
// if (records.length === 0) continue;
|
||||
// Progress every 250 records
|
||||
if (i > 0 && i % 250 === 0) {
|
||||
log(`⏳ Progress: processed ${i} of ${csvRows.length} records...`);
|
||||
}
|
||||
|
||||
// Normalize notes for fair comparison
|
||||
// const normalize = v => (v ?? "").toString().replace(/\s+/g, " ").trim();
|
||||
// const pbNotes = normalize(records[0].Notes);
|
||||
// const excelNotes = normalize(row.Notes);
|
||||
const jobNumber = row.Job_Number;
|
||||
if (!jobNumber || jobNumber === "") continue;
|
||||
|
||||
// if (pbNotes !== excelNotes) {
|
||||
// Excel uses 1-based row numbers; DATA_START_ROW_INDEX is zero-based
|
||||
// const excelRowNumber = DATA_START_ROW_INDEX + i + 1;
|
||||
// const cellAddress = `${notesColumnLetter}${excelRowNumber}`;
|
||||
// Build payload with only allowed fields
|
||||
const payload = {};
|
||||
for (const [k, v] of Object.entries(row)) {
|
||||
if (k === "Job_Number") continue;
|
||||
if (allowedFields.has(k)) payload[k] = v;
|
||||
}
|
||||
|
||||
// log(`✏️ Notes change for Job_Number=${jobNumber}`);
|
||||
// log(` Before: "${excelNotes}"`);
|
||||
// log(` After : "${pbNotes}"`);
|
||||
// log(`➡️ Writing to ${worksheetName}!${cellAddress}`);
|
||||
// Helper to log errors with server response
|
||||
const logPocketBaseError = (err, context) => {
|
||||
log(`❌ ${context}`);
|
||||
log(`Error message: ${err.message || "no message"}`);
|
||||
|
||||
// await axios.patch(
|
||||
// `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/worksheets('${worksheetName}')/range(address='${cellAddress}')`,
|
||||
// { values: [[pbNotes]] },
|
||||
// {
|
||||
// headers: {
|
||||
// Authorization: `Bearer ${graphToken}`,
|
||||
// "Content-Type": "application/json",
|
||||
// "workbook-session-id": sessionId
|
||||
// }
|
||||
// }
|
||||
// );
|
||||
try {
|
||||
log(`Error dump: ${JSON.stringify(err, Object.getOwnPropertyNames(err), 2)}`);
|
||||
} catch {}
|
||||
|
||||
// row.Notes = pbNotes;
|
||||
// changes++;
|
||||
// }
|
||||
// } catch (err) {
|
||||
// log(`❌ Failed for Job_Number=${jobNumber}: ${err.message}`);
|
||||
// }
|
||||
// }
|
||||
if (err.response) {
|
||||
log(`Response status: ${err.response.status} ${err.response.statusText || ""}`);
|
||||
if (err.response.data) {
|
||||
try {
|
||||
log(`Server response data: ${JSON.stringify(err.response.data, null, 2)}`);
|
||||
} catch {
|
||||
log("⚠️ Could not stringify response.data");
|
||||
}
|
||||
} else {
|
||||
log("⚠️ No response.data present");
|
||||
}
|
||||
} else {
|
||||
log("⚠️ No err.response present");
|
||||
}
|
||||
|
||||
// await axios.post(
|
||||
// `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/closeSession`,
|
||||
// {},
|
||||
// { headers: { Authorization: `Bearer ${graphToken}`, "workbook-session-id": sessionId } }
|
||||
// );
|
||||
if (err.data) {
|
||||
try {
|
||||
log(`Server details: ${JSON.stringify(err.data, null, 2)}`);
|
||||
} catch {}
|
||||
}
|
||||
};
|
||||
|
||||
// log(`✅ Notes-only sync complete. Changes applied: ${changes}`);
|
||||
//}
|
||||
if (jobMap[jobNumber]) {
|
||||
const id = jobMap[jobNumber];
|
||||
try {
|
||||
await pb.collection(pbCollection).update(id, payload);
|
||||
} catch (err) {
|
||||
logPocketBaseError(err, `Update failed for id=${id} Job_Number=${jobNumber}`);
|
||||
}
|
||||
} else {
|
||||
const createPayload = { Job_Number: jobNumber, ...payload };
|
||||
try {
|
||||
await pb.collection(pbCollection).create(createPayload);
|
||||
} catch (err) {
|
||||
logPocketBaseError(err, `Create failed for Job_Number=${jobNumber}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log("✅ Resync complete.");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,521 @@
|
||||
//12/1/2025 Allows Excel download and conversion of headers mapped to pb headers.
|
||||
//coersion added to match pb field types. however, may still have difficulty in assimilating certain date formats from excel to pb date fields.
|
||||
//due to Excel storing as all caps for boolean fields
|
||||
// excelsync.js
|
||||
import { FIELD_CONFIG } from "./fieldConfig.js";
|
||||
import fs from "fs";
|
||||
import axios from "axios";
|
||||
import qs from "qs";
|
||||
import * as XLSX from "xlsx";
|
||||
import ExcelJS from "exceljs";
|
||||
import dotenv from "dotenv";
|
||||
import { serve } from "bun";
|
||||
import PocketBase from "pocketbase"
|
||||
|
||||
dotenv.config();
|
||||
|
||||
// -----------------------------
|
||||
// CONFIG FOR TOKEN ACQUISITION GRAPH API + POCKETBASE
|
||||
// -----------------------------
|
||||
const CONFIG = JSON.parse(fs.readFileSync("config.json", "utf8"));
|
||||
const tenantId = CONFIG.tenantId;
|
||||
const clientId = CONFIG.clientId;
|
||||
const pbUrl = CONFIG.pbUrl;
|
||||
const pbCollection = CONFIG.pbCollection;
|
||||
const pbEmail = CONFIG.pbEmail;
|
||||
const clientSecret = process.env.CLIENT_SECRET;
|
||||
const pbPassword = process.env.PB_PASSWORD;
|
||||
|
||||
// -----------------------------
|
||||
// LOGGING
|
||||
// -----------------------------
|
||||
function log(message) {
|
||||
console.log(message);
|
||||
fs.appendFileSync("excelsync.log", message + "\n");
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// HEADER DETECTOR
|
||||
// -----------------------------
|
||||
//Gets the last column letter in the table converting from 0 index to excel actual column in letter form.
|
||||
function columnLetter(index) {
|
||||
let letter = "";
|
||||
while (index >= 0) {
|
||||
letter = String.fromCharCode((index % 26) + 65) + letter;
|
||||
index = Math.floor(index / 26) - 1;
|
||||
}
|
||||
return letter;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// GRAPH TOKEN ACQUISITION
|
||||
// -----------------------------
|
||||
async function getGraphToken() {
|
||||
const tokenUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`;
|
||||
const body = qs.stringify({
|
||||
client_id: clientId,
|
||||
client_secret: clientSecret,
|
||||
scope: "https://graph.microsoft.com/.default",
|
||||
grant_type: "client_credentials",
|
||||
});
|
||||
const res = await axios.post(tokenUrl, body, {
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||
});
|
||||
log("✅ Graph token acquired");
|
||||
return res.data.access_token;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// GRAPH DOWNLOADER HELPER
|
||||
// -----------------------------
|
||||
async function downloadExcelFromGraph(graphToken, driveId, itemId) {
|
||||
const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/content`;
|
||||
const res = await axios.get(url, {
|
||||
headers: { Authorization: `Bearer ${graphToken}` },
|
||||
responseType: "arraybuffer", // important: get raw binary
|
||||
});
|
||||
return Buffer.from(res.data); // convert to Buffer for XLSX
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// POCKETBASE LOGIN AND TOKEN ACQUISITION
|
||||
// -----------------------------
|
||||
async function getPocketBaseClient() {
|
||||
const pb = new PocketBase(pbUrl);
|
||||
await pb.collection("users").authWithPassword(pbEmail, pbPassword);
|
||||
log("✅ PocketBase token acquired");
|
||||
return pb;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// Get Data From Excel File
|
||||
// -----------------------------
|
||||
|
||||
const HEADER_ROW_INDEX = 2; // Row 3 in Excel
|
||||
const DATA_START_ROW_INDEX = 3; // Row 4 in Excel → first data row
|
||||
|
||||
//clean and coerce Excel values to match PocketBase schema expectations
|
||||
function cleanString(val) {
|
||||
if (val == null) return "";
|
||||
return String(val)
|
||||
.replace(/\u00A0/g, " ") // normalize non-breaking spaces
|
||||
.replace(/[\r\n\t]/g, " ") // strip control chars
|
||||
.replace(/\s+/g, " ") // collapse whitespace
|
||||
.trim();
|
||||
}
|
||||
|
||||
function normalizeForPB(val, type) {
|
||||
const cleaned = cleanString(val);
|
||||
|
||||
// Treat common placeholders as empty
|
||||
const emptyTokens = ["n/a", "na", "null", "none"];
|
||||
if (emptyTokens.includes(cleaned.toLowerCase())) {
|
||||
switch (type) {
|
||||
case "number": return 0; // PocketBase defaults to 0
|
||||
case "bool": return null;
|
||||
case "string": return null;
|
||||
case "email": return null; // email empty
|
||||
case "json": return null;
|
||||
case "date": return null; // date empty
|
||||
default: return null;
|
||||
}
|
||||
}
|
||||
|
||||
if (cleaned === "") {
|
||||
switch (type) {
|
||||
case "number": return 0;
|
||||
case "bool": return null;
|
||||
case "string": return null;
|
||||
case "email": return null;
|
||||
case "json": return null;
|
||||
case "date": return null;
|
||||
default: return null;
|
||||
}
|
||||
}
|
||||
|
||||
switch (type) {
|
||||
case "bool":
|
||||
case "boolean": {
|
||||
const lower = cleaned.toLowerCase();
|
||||
if (["true","yes","y","1"].includes(lower)) return true; // boolean true
|
||||
if (["false","no","n","0"].includes(lower)) return false; // boolean false
|
||||
return null;
|
||||
}
|
||||
case "number": {
|
||||
const num = Number(cleaned);
|
||||
return Number.isNaN(num) ? 0 : num;
|
||||
}
|
||||
case "date": {
|
||||
const d = new Date(cleaned);
|
||||
return isNaN(d.getTime()) ? null : d.toISOString();
|
||||
}
|
||||
case "string":
|
||||
default:
|
||||
return cleaned;
|
||||
}
|
||||
}
|
||||
|
||||
export function parseExcelFile(input) {
|
||||
let data;
|
||||
if (Buffer.isBuffer(input)) {
|
||||
data = input;
|
||||
log("Reading Excel file from Graph buffer");
|
||||
} else {
|
||||
data = fs.readFileSync(input);
|
||||
log(`Reading Excel file: ${input}`);
|
||||
}
|
||||
|
||||
const workbook = XLSX.read(data, { type: "buffer", cellDates: true });
|
||||
const sheet = workbook.Sheets["Job Sheet"];
|
||||
if (!sheet) throw new Error('Sheet named "Job Sheet" not found');
|
||||
|
||||
const range = XLSX.utils.decode_range(sheet["!ref"]);
|
||||
const merges = sheet["!merges"] || [];
|
||||
|
||||
// Helper to get value from cell or merged region
|
||||
const getValue = (r, c) => {
|
||||
const addr = XLSX.utils.encode_cell({ r, c });
|
||||
const cell = sheet[addr];
|
||||
if (cell) return cell.v ?? cell.w ?? null;
|
||||
|
||||
for (const m of merges) {
|
||||
if (m.s.r <= r && r <= m.e.r && m.s.c <= c && c <= m.e.c) {
|
||||
const master = sheet[XLSX.utils.encode_cell(m.s)];
|
||||
return master ? (master.v ?? master.w ?? null) : null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
// 1. Read headers (Row 3)
|
||||
const rawHeaders = [];
|
||||
for (let c = range.s.c; c <= range.e.c; c++) {
|
||||
const val = getValue(HEADER_ROW_INDEX, c);
|
||||
rawHeaders.push(val ? String(val).trim() : "");
|
||||
}
|
||||
|
||||
// 2. Build lookup map once
|
||||
const excelHeaderToConfig = Object.values(FIELD_CONFIG).reduce((map, cfg) => {
|
||||
if (cfg.excelHeader) {
|
||||
const key = cfg.excelHeader.trim();
|
||||
map[key] = cfg;
|
||||
map[key.toLowerCase()] = cfg;
|
||||
}
|
||||
return map;
|
||||
}, {});
|
||||
|
||||
// 3. Rich header mapping
|
||||
const headerMapping = rawHeaders.map(raw => {
|
||||
const clean = raw.trim();
|
||||
const config = excelHeaderToConfig[clean] || excelHeaderToConfig[clean.toLowerCase()];
|
||||
|
||||
return config
|
||||
? { original: clean, pbField: config.pbField, type: config.type, isMapped: true, config }
|
||||
: { original: clean, pbField: "", type: "string", isMapped: false, isUnknown: true };
|
||||
});
|
||||
|
||||
log(`Mapped ${headerMapping.filter(h => h.isMapped).length} columns`);
|
||||
|
||||
// 4. Parse data rows (starting Row 4)
|
||||
const rows = [];
|
||||
for (let r = DATA_START_ROW_INDEX; r <= range.e.r; r++) {
|
||||
const values = [];
|
||||
let hasData = false;
|
||||
|
||||
for (let c = range.s.c; c <= range.e.c; c++) {
|
||||
const val = getValue(r, c);
|
||||
values.push(val ?? null);
|
||||
if (val !== null && val !== "") hasData = true;
|
||||
}
|
||||
if (!hasData) continue;
|
||||
|
||||
const rowObj = {};
|
||||
headerMapping.forEach((map, i) => {
|
||||
if (map.isMapped) {
|
||||
rowObj[map.pbField] = normalizeForPB(values[i], map.type);
|
||||
}
|
||||
});
|
||||
|
||||
rows.push(rowObj);
|
||||
}
|
||||
|
||||
return {
|
||||
data: rows, // parsed row objects
|
||||
headerMapping, // rich info about headers
|
||||
rawHeaders, // original headers for debugging
|
||||
totalRows: rows.length // count of parsed rows
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
// -----------------------------
|
||||
// MAIN
|
||||
// -----------------------------
|
||||
let parsedRows = [];
|
||||
let rawHeaders = [];
|
||||
(async () => {
|
||||
try {
|
||||
log("🔵 Starting Excel Sync…");
|
||||
const graphToken = await getGraphToken();
|
||||
const pb = await getPocketBaseClient();
|
||||
// These can be update if the Excel File changes
|
||||
const driveId = "b!9YOqqr2xM0G2DFBwMEJYY4UzQgocFddEtpLYWuS9_AtkCKl2q8yjQJteQ7Ti4QSx";
|
||||
const itemId = "01SPNXLDSQTMG4F6BDANCLJU2FZKEDQLVU";
|
||||
const excelBuffer = await downloadExcelFromGraph(graphToken, driveId, itemId);
|
||||
const parsed = parseExcelFile(excelBuffer);
|
||||
parsedRows = parsed.data;
|
||||
rawHeaders = parsed.rawHeaders;
|
||||
// UPDATE AND LOG CHANGES TO POCKETBASE
|
||||
await syncRowsToPocketBase(pb, parsedRows);
|
||||
log(rawHeaders)
|
||||
log("✅ First part complete: tokens + Excel parsing ready");
|
||||
} catch (err) {
|
||||
log("❌ Sync failed: " + (err.message || err));
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
|
||||
// -------------------------
|
||||
// Export to xlsx - Excel
|
||||
// ---------------------------
|
||||
|
||||
// Toggle this flag to enable/disable the export server
|
||||
const ENABLE_EXPORT = false; // set to false to bypass download
|
||||
|
||||
function exportExcelforDL(parsedRows) {
|
||||
if (!parsedRows || parsedRows.length === 0) {
|
||||
throw new Error("⚠️ No rows to export");
|
||||
}
|
||||
|
||||
const worksheet = XLSX.utils.json_to_sheet(parsedRows);
|
||||
|
||||
// Ensure Job_Number is the first column
|
||||
const cols = Object.keys(parsedRows[0] || {});
|
||||
if (cols.includes("Job_Number")) {
|
||||
const ordered = ["Job_Number", ...cols.filter(c => c !== "Job_Number")];
|
||||
const remapped = parsedRows.map(r =>
|
||||
ordered.reduce((acc, key) => { acc[key] = r[key]; return acc; }, {})
|
||||
);
|
||||
const ws2 = XLSX.utils.json_to_sheet(remapped);
|
||||
Object.assign(worksheet, ws2);
|
||||
}
|
||||
|
||||
const workbook = XLSX.utils.book_new();
|
||||
XLSX.utils.book_append_sheet(workbook, worksheet, "ParsedData");
|
||||
|
||||
return XLSX.write(workbook, { type: "buffer", bookType: "xlsx" });
|
||||
}
|
||||
|
||||
if (ENABLE_EXPORT) {
|
||||
const PORT = 3000;
|
||||
serve({
|
||||
port: PORT,
|
||||
fetch(req) {
|
||||
const url = new URL(req.url);
|
||||
|
||||
if (url.pathname === "/") {
|
||||
const html = `
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head><meta charset="utf-8"><title>ExcelSync Export</title></head>
|
||||
<body>
|
||||
<h1>ExcelSync Export</h1>
|
||||
<p>Click below to download the parsed Excel file:</p>
|
||||
<button onclick="window.location.href='/download'">Download ParsedOutput.xlsx</button>
|
||||
</body>
|
||||
</html>`;
|
||||
return new Response(html, { headers: { "Content-Type": "text/html" } });
|
||||
}
|
||||
|
||||
if (url.pathname === "/download") {
|
||||
try {
|
||||
const buffer = exportExcelforDL(parsedRows);
|
||||
return new Response(buffer, {
|
||||
headers: {
|
||||
"Content-Type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"Content-Disposition": "attachment; filename=ParsedOutput.xlsx"
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
return new Response("❌ Failed to generate Excel file: " + err.message, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
return new Response("Visit / to open the export page.");
|
||||
}
|
||||
});
|
||||
|
||||
log(`🚀 ExcelSync export server running at http://localhost:${PORT}/`);
|
||||
log(`👉 Open this URL in your browser to verify and download the parsed Excel file.`);
|
||||
} else {
|
||||
log("⏭️ Export server bypassed (ENABLE_EXPORT=false)");
|
||||
}
|
||||
|
||||
async function syncRowsToPocketBase(pb, parsedRows) {
|
||||
let changes = 0;
|
||||
|
||||
for (const row of parsedRows) {
|
||||
const jobNumber = row.Job_Number;
|
||||
if (!jobNumber) continue;
|
||||
|
||||
try {
|
||||
// Find existing record by Job_Number
|
||||
const records = await pb.collection(pbCollection).getFullList({
|
||||
filter: Number.isFinite(jobNumber)
|
||||
? `Job_Number=${jobNumber}`
|
||||
: `Job_Number="${jobNumber}"`,
|
||||
batchSize: 1
|
||||
});
|
||||
|
||||
if (records.length > 0) {
|
||||
const record = records[0];
|
||||
|
||||
// Compare field-by-field
|
||||
const updatePayload = {};
|
||||
const changedFields = [];
|
||||
|
||||
for (const [key, value] of Object.entries(row)) {
|
||||
// Look up the expected type from your header mapping
|
||||
const type = headerMapping.find(h => h.pbField === key)?.type || "string";
|
||||
|
||||
if (!valuesEqual(record[key], value, type)) {
|
||||
updatePayload[key] = value;
|
||||
changedFields.push({
|
||||
field: key,
|
||||
before: record[key],
|
||||
after: value
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (changedFields.length > 0) {
|
||||
await pb.collection(pbCollection).update(record.id, updatePayload);
|
||||
|
||||
// Log detailed changes
|
||||
const details = changedFields
|
||||
.map(f => `${f.field}: "${f.before}" → "${f.after}"`)
|
||||
.join("; ");
|
||||
log(`🔄 Updated Job_Number=${jobNumber}, changes: ${details}`);
|
||||
|
||||
changes++;
|
||||
}
|
||||
} else {
|
||||
await pb.collection(pbCollection).create(row);
|
||||
log(`➕ Created new record for Job_Number=${jobNumber}`);
|
||||
changes++;
|
||||
}
|
||||
} catch (err) {
|
||||
log(`❌ Sync failed for Job_Number=${jobNumber}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
log(`✅ Sync complete. Records added/updated: ${changes}`);
|
||||
}
|
||||
function valuesEqual(pbVal, excelVal, type) {
|
||||
if (pbVal == null && excelVal == null) return true;
|
||||
|
||||
switch (type) {
|
||||
case "date": {
|
||||
const pbDate = new Date(pbVal);
|
||||
const excelDate = new Date(excelVal);
|
||||
return !isNaN(pbDate) && !isNaN(excelDate) && pbDate.getTime() === excelDate.getTime();
|
||||
}
|
||||
case "bool":
|
||||
return Boolean(pbVal) === Boolean(excelVal);
|
||||
case "number":
|
||||
return Number(pbVal) === Number(excelVal);
|
||||
default:
|
||||
const normalize = v => (v ?? "").toString().replace(/\s+/g, " ").trim();
|
||||
return normalize(pbVal) === normalize(excelVal);
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// UPDATE NOTES CELLS IN SHAREPOINT (REAL-TIME)
|
||||
// -----------------------------
|
||||
//function getNotesColumnLetterFromParsedHeaders(rawHeaders) {
|
||||
// const notesIndex = rawHeaders.findIndex(h => h.trim().toLowerCase() === "notes");
|
||||
// if (notesIndex === -1) {
|
||||
// log(`❌ 'Notes' header not found in HEADER_MAP keys: ${rawHeaders.join(", ")}`);
|
||||
// throw new Error("Notes column not found");
|
||||
// }
|
||||
// return columnLetter(notesIndex);
|
||||
//}
|
||||
//await updateNotesInSharePoint(graphToken, driveId, itemId, "Job Sheet", parsedRows, pb, excelBuffer);
|
||||
// Update only changed Notes cells using workbook session + PATCH.
|
||||
// This version relies solely on DATA_START_ROW_INDEX for row mapping.
|
||||
//async function updateNotesInSharePoint(graphToken, driveId, itemId, worksheetName, parsedRows, pb, workbookBuffer) {
|
||||
// const wb = XLSX.read(workbookBuffer, { type: "buffer", cellDates: true });
|
||||
// const sheet = wb.Sheets[worksheetName];
|
||||
// if (!sheet) throw new Error(`Worksheet "${worksheetName}" not found`);
|
||||
|
||||
// Column detection aligned to HEADER_MAP (same mapping used by parseExcelFile)
|
||||
//const notesColumnLetter = getNotesColumnLetterFromParsedHeaders(rawHeaders);
|
||||
//og(`✅ Notes column resolved to ${notesColumnLetter} via rawHeaders`);
|
||||
|
||||
// Create a workbook session that persists changes
|
||||
//const sessionRes = await axios.post(
|
||||
// `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/createSession`,
|
||||
// { persistChanges: true },
|
||||
// { headers: { Authorization: `Bearer ${graphToken}` } }
|
||||
// );
|
||||
// const sessionId = sessionRes.data.id;
|
||||
|
||||
// let changes = 0;
|
||||
|
||||
// Loop parsed rows; map i to Excel row using DATA_START_ROW_INDEX (zero-based)
|
||||
// for (let i = 0; i < parsedRows.length; i++) {
|
||||
// const row = parsedRows[i];
|
||||
// const jobNumber = row.Job_Number;
|
||||
// if (jobNumber == null || jobNumber === "") continue;
|
||||
|
||||
// try {
|
||||
// const records = await pb.collection(pbCollection).getFullList({
|
||||
// filter: Number.isFinite(jobNumber) ? `Job_Number=${jobNumber}` : `Job_Number="${jobNumber}"`,
|
||||
// batchSize: 1
|
||||
// });
|
||||
// if (records.length === 0) continue;
|
||||
|
||||
// Normalize notes for fair comparison
|
||||
// const normalize = v => (v ?? "").toString().replace(/\s+/g, " ").trim();
|
||||
// const pbNotes = normalize(records[0].Notes);
|
||||
// const excelNotes = normalize(row.Notes);
|
||||
|
||||
// if (pbNotes !== excelNotes) {
|
||||
// Excel uses 1-based row numbers; DATA_START_ROW_INDEX is zero-based
|
||||
// const excelRowNumber = DATA_START_ROW_INDEX + i + 1;
|
||||
// const cellAddress = `${notesColumnLetter}${excelRowNumber}`;
|
||||
|
||||
// log(`✏️ Notes change for Job_Number=${jobNumber}`);
|
||||
// log(` Before: "${excelNotes}"`);
|
||||
// log(` After : "${pbNotes}"`);
|
||||
// log(`➡️ Writing to ${worksheetName}!${cellAddress}`);
|
||||
|
||||
// await axios.patch(
|
||||
// `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/worksheets('${worksheetName}')/range(address='${cellAddress}')`,
|
||||
// { values: [[pbNotes]] },
|
||||
// {
|
||||
// headers: {
|
||||
// Authorization: `Bearer ${graphToken}`,
|
||||
// "Content-Type": "application/json",
|
||||
// "workbook-session-id": sessionId
|
||||
// }
|
||||
// }
|
||||
// );
|
||||
|
||||
// row.Notes = pbNotes;
|
||||
// changes++;
|
||||
// }
|
||||
// } catch (err) {
|
||||
// log(`❌ Failed for Job_Number=${jobNumber}: ${err.message}`);
|
||||
// }
|
||||
// }
|
||||
|
||||
// await axios.post(
|
||||
// `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/closeSession`,
|
||||
// {},
|
||||
// { headers: { Authorization: `Bearer ${graphToken}`, "workbook-session-id": sessionId } }
|
||||
// );
|
||||
|
||||
// log(`✅ Notes-only sync complete. Changes applied: ${changes}`);
|
||||
//}
|
||||
@@ -0,0 +1,545 @@
|
||||
[
|
||||
{
|
||||
autogeneratePattern: "[a-z0-9]{15}",
|
||||
hidden: false,
|
||||
id: "text3208210256",
|
||||
max: 15,
|
||||
min: 15,
|
||||
name: "id",
|
||||
pattern: "^[a-z0-9]+$",
|
||||
presentable: false,
|
||||
primaryKey: true,
|
||||
required: true,
|
||||
system: true,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text2388997171",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Job_Number",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text2378820285",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Job_Full_Name",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text2158740783",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Job_Name",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text3732958801",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Job_Address",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text1381428224",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Job_Type",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text1677020795",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Job_Status",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text2518113217",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Job_Division",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text2325867089",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Estimator",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text3229868610",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Office_Rep",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text899226661",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Project_Manager",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text806559182",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Company_Client",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text1854315289",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Contact_Person",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text1956405909",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Phone_Number",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text642995056",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Email",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text4112652156",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Due_Date_Source",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text1362567553",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Due_Date_Counter",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text2050867814",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Due_Time",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text811897623",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Schedule_Confidence",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text3235547528",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Notes",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text1270095775",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Job_QB_Link",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text957490280",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Voxer_Link",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text2814497178",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Job_Codes",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text3795106522",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Job_Folder_Link",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "bool545115032",
|
||||
name: "Has_Attachments",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "bool",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "json2488040839",
|
||||
maxSize: 0,
|
||||
name: "Attachment_List",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "json",
|
||||
}, {
|
||||
cascadeDelete: false,
|
||||
collectionId: "pbc_194787171",
|
||||
hidden: false,
|
||||
id: "relation2779200245",
|
||||
maxSelect: 999,
|
||||
minSelect: 0,
|
||||
name: "Note_Ref",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "relation",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "json4080242417",
|
||||
maxSize: 0,
|
||||
name: "latest_note_summary",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "json",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "json2575510732",
|
||||
maxSize: 0,
|
||||
name: "note_thread_text",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "json",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "number3771328437",
|
||||
max: null,
|
||||
min: null,
|
||||
name: "note_count",
|
||||
onlyInt: false,
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "number",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "date2500832881",
|
||||
max: "",
|
||||
min: "",
|
||||
name: "last_note_at",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "date",
|
||||
}, {
|
||||
autogeneratePattern: "",
|
||||
hidden: false,
|
||||
id: "text2569355707",
|
||||
max: 0,
|
||||
min: 0,
|
||||
name: "Job_Number_Parent",
|
||||
pattern: "",
|
||||
presentable: false,
|
||||
primaryKey: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "text",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "date2051602006",
|
||||
max: "",
|
||||
min: "",
|
||||
name: "Start_Date",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "date",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "date275762613",
|
||||
max: "",
|
||||
min: "",
|
||||
name: "Submission_Date",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "date",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "bool1908819108",
|
||||
name: "Flag",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "bool",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "bool282225560",
|
||||
name: "Tax_Exempt",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "bool",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "bool1286797620",
|
||||
name: "Active",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "bool",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "date3208080729",
|
||||
max: "",
|
||||
min: "",
|
||||
name: "Due_Date",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "date",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "bool1523500494",
|
||||
name: "Docs_Uploaded",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "bool",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "bool3561832989",
|
||||
name: "Tax_Exempt_Docs_Uploaded",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "bool",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "bool4143394575",
|
||||
name: "Estimate_Approved",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "bool",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "bool701636273",
|
||||
name: "QB_Created",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "bool",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "bool159074814",
|
||||
name: "Added_To_Calendar",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "bool",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "bool2443123173",
|
||||
name: "PSwift_Uploaded",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "bool",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "bool1028487862",
|
||||
name: "Voxer_Created",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "bool",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "bool2654168435",
|
||||
name: "Estimate_Reviewed",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "bool",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "bool1033433223",
|
||||
name: "Estimate_Draft",
|
||||
presentable: false,
|
||||
required: false,
|
||||
system: false,
|
||||
type: "bool",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "autodate2990389176",
|
||||
name: "created",
|
||||
onCreate: true,
|
||||
onUpdate: false,
|
||||
presentable: false,
|
||||
system: false,
|
||||
type: "autodate",
|
||||
}, {
|
||||
hidden: false,
|
||||
id: "autodate3332085495",
|
||||
name: "updated",
|
||||
onCreate: true,
|
||||
onUpdate: true,
|
||||
presentable: false,
|
||||
system: false,
|
||||
type: "autodate",
|
||||
}
|
||||
],
|
||||
indexes: [],
|
||||
created: "2025-11-14 21:39:05.100Z",
|
||||
updated: "2025-11-26 02:40:36.242Z",
|
||||
system: false,
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import dotenv from "dotenv";
|
||||
import PocketBase from "pocketbase";
|
||||
import fs from "fs";
|
||||
|
||||
dotenv.config();
|
||||
|
||||
// -----------------------------
|
||||
// CONFIG
|
||||
// -----------------------------
|
||||
const CONFIG = JSON.parse(fs.readFileSync("config.json", "utf8"));
|
||||
const pbUrl = CONFIG.pbUrl;
|
||||
const pbCollection = CONFIG.pbCollection;
|
||||
const superEmail = process.env.Super_Email;
|
||||
const superPassword = process.env.Super_Password;
|
||||
|
||||
// -----------------------------
|
||||
// ADMIN LOGIN
|
||||
// -----------------------------
|
||||
async function getPocketBaseClient() {
|
||||
const pb = new PocketBase(pbUrl);
|
||||
await pb.admins.authWithPassword(superEmail, superPassword);
|
||||
console.log("✅ Admin token acquired");
|
||||
return pb;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// FETCH SCHEMA & SAVE TO FILE
|
||||
// -----------------------------
|
||||
async function getAndSaveSchema(pb) {
|
||||
const collection = await pb.collections.getOne(pbCollection);
|
||||
|
||||
// Debug: log the full collection object
|
||||
console.log("📦 Collection object:", JSON.stringify(collection, null, 2));
|
||||
|
||||
// Log raw schema directly
|
||||
console.log("📑 Raw schema:", collection.schema);
|
||||
|
||||
if (collection.schema && Array.isArray(collection.schema)) {
|
||||
if (collection.schema.length === 0) {
|
||||
console.warn("⚠️ Schema array is empty for collection:", pbCollection);
|
||||
return {};
|
||||
}
|
||||
|
||||
// Build schema map (fieldName → type)
|
||||
const schemaMap = {};
|
||||
for (const field of collection.schema) {
|
||||
schemaMap[field.name] = field.type;
|
||||
}
|
||||
|
||||
// Save full schema JSON for later use
|
||||
fs.writeFileSync("schema.json", JSON.stringify(collection.schema, null, 2), "utf8");
|
||||
|
||||
console.log("✅ Schema saved to schema.json");
|
||||
console.log("Schema map:", schemaMap);
|
||||
|
||||
return schemaMap;
|
||||
} else {
|
||||
console.error("❌ Schema property missing or not an array. Collection object:", collection);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// MAIN
|
||||
// -----------------------------
|
||||
(async () => {
|
||||
try {
|
||||
const pb = await getPocketBaseClient();
|
||||
await getAndSaveSchema(pb);
|
||||
} catch (err) {
|
||||
console.error("❌ Error fetching schema:", err);
|
||||
}
|
||||
})();
|
||||
@@ -0,0 +1,65 @@
|
||||
import dotenv from "dotenv";
|
||||
import PocketBase from "pocketbase";
|
||||
import fs from "fs";
|
||||
|
||||
dotenv.config();
|
||||
|
||||
// -----------------------------
|
||||
// CONFIG
|
||||
// -----------------------------
|
||||
const CONFIG = JSON.parse(fs.readFileSync("config.json", "utf8"));
|
||||
const pbUrl = CONFIG.pbUrl;
|
||||
const pbCollection = CONFIG.pbCollection;
|
||||
const superEmail = process.env.Super_Email;
|
||||
const superPassword = process.env.Super_Password;
|
||||
|
||||
// -----------------------------
|
||||
// ADMIN LOGIN
|
||||
// -----------------------------
|
||||
async function getPocketBaseClient() {
|
||||
const pb = new PocketBase(pbUrl);
|
||||
await pb.admins.authWithPassword(superEmail, superPassword);
|
||||
console.log("✅ Admin token acquired");
|
||||
return pb;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// FETCH SCHEMA & SAVE TO FILE
|
||||
// -----------------------------
|
||||
async function getAndSaveSchema(pb) {
|
||||
const collection = await pb.collections.getOne(pbCollection);
|
||||
|
||||
// Debug: log the full collection object so you can inspect it
|
||||
console.log("📦 Collection object:", JSON.stringify(collection, null, 2));
|
||||
|
||||
if (!collection.schema) {
|
||||
console.error("❌ No schema property found on collection object.");
|
||||
return {};
|
||||
}
|
||||
|
||||
// Build schema map (fieldName → type)
|
||||
const schemaMap = {};
|
||||
for (const field of collection.schema) {
|
||||
schemaMap[field.name] = field.type;
|
||||
}
|
||||
|
||||
// Save full schema JSON for later use
|
||||
fs.writeFileSync("schema.json", JSON.stringify(collection.schema, null, 2), "utf8");
|
||||
|
||||
console.log("✅ Schema saved to schema.json");
|
||||
console.log("Schema map:", schemaMap);
|
||||
|
||||
return schemaMap;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// MAIN
|
||||
// -----------------------------
|
||||
(async () => {
|
||||
try {
|
||||
const pb = await getPocketBaseClient();
|
||||
await getAndSaveSchema(pb);
|
||||
} catch (err) {
|
||||
console.error("❌ Error fetching schema:", err);
|
||||
}
|
||||
})();
|
||||
@@ -0,0 +1,145 @@
|
||||
//works to update pb field
|
||||
import dotenv from "dotenv";
|
||||
import PocketBase from "pocketbase";
|
||||
import fs from "fs";
|
||||
|
||||
dotenv.config();
|
||||
|
||||
// -----------------------------
|
||||
// CONFIG
|
||||
// -----------------------------
|
||||
const CONFIG = JSON.parse(fs.readFileSync("config.json", "utf8"));
|
||||
const pbUrl = CONFIG.pbUrl;
|
||||
const pbCollection = CONFIG.pbCollection;
|
||||
const pbEmail = CONFIG.pbEmail;
|
||||
const pbPassword = process.env.PB_PASSWORD;
|
||||
|
||||
// -----------------------------
|
||||
// PB LOGIN
|
||||
// -----------------------------
|
||||
async function getPocketBaseClient() {
|
||||
const pb = new PocketBase(pbUrl);
|
||||
await pb.collection("users").authWithPassword(pbEmail, pbPassword);
|
||||
console.log("✅ PocketBase token acquired");
|
||||
return pb;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// UPDATE FROM JSON EXPORT
|
||||
// -----------------------------
|
||||
async function updateFromJSON(pb) {
|
||||
const data = JSON.parse(fs.readFileSync("records.json", "utf8"));
|
||||
const errors = [];
|
||||
|
||||
for (const record of data) {
|
||||
try {
|
||||
// Strip system fields from payload
|
||||
const { id, created, updated, expand, ...payload } = record;
|
||||
|
||||
// Update by id
|
||||
await pb.collection(pbCollection).update(id, payload);
|
||||
} catch (err) {
|
||||
errors.push({ id: record.id, error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length) {
|
||||
console.error("❌ Errors during update:", errors);
|
||||
} else {
|
||||
console.log("✅ All records updated successfully");
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// FETCH CURRENT RECORDS
|
||||
// -----------------------------
|
||||
async function getPocketBaseRecords(pb) {
|
||||
return await pb.collection(pbCollection).getFullList();
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// NORMALIZATION HELPERS
|
||||
// -----------------------------
|
||||
function normalizeValue(field, value) {
|
||||
// Adjust based on common PB schema rules
|
||||
if (value === undefined) return null;
|
||||
|
||||
// Example: treat text fields like Note_Ref as "" when empty
|
||||
if (field.toLowerCase().includes("note") || field.toLowerCase().includes("ref")) {
|
||||
return value === null ? "" : String(value);
|
||||
}
|
||||
|
||||
// Numbers
|
||||
if (typeof value === "number") return value;
|
||||
if (value === null || value === "") return null;
|
||||
|
||||
// Booleans
|
||||
if (typeof value === "boolean") return value;
|
||||
|
||||
// Dates
|
||||
if (value instanceof Date) return value.toISOString();
|
||||
if (Date.parse(value)) return new Date(value).toISOString();
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// COMPARE ORIGINAL VS CURRENT
|
||||
// -----------------------------
|
||||
function compareRecords(original, current) {
|
||||
const diffs = [];
|
||||
for (const orig of original) {
|
||||
const curr = current.find(r => r.id === orig.id);
|
||||
if (!curr) {
|
||||
diffs.push({ id: orig.id, issue: "Missing in current DB" });
|
||||
continue;
|
||||
}
|
||||
|
||||
const fields = Object.keys(orig).filter(
|
||||
k => !["id", "created", "updated", "expand"].includes(k)
|
||||
);
|
||||
|
||||
for (const f of fields) {
|
||||
const origVal = normalizeValue(f, orig[f]);
|
||||
const currVal = normalizeValue(f, curr[f]);
|
||||
if (origVal !== currVal) {
|
||||
diffs.push({
|
||||
id: orig.id,
|
||||
field: f,
|
||||
original: origVal,
|
||||
current: currVal
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return diffs;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// MAIN TEST
|
||||
// -----------------------------
|
||||
(async () => {
|
||||
const pb = await getPocketBaseClient();
|
||||
|
||||
// Step 1: Update from your existing export
|
||||
const original = await updateFromJSON(pb);
|
||||
|
||||
// Step 2: Fetch current records
|
||||
const current = await getPocketBaseRecords(pb);
|
||||
|
||||
// Step 3: Compare
|
||||
const diffs = compareRecords(original, current);
|
||||
|
||||
if (diffs.length) {
|
||||
console.error("❌ Differences found:");
|
||||
for (const d of diffs) {
|
||||
console.log(
|
||||
`Record ${d.id}, field ${d.field}\n original: ${d.original}\n current : ${d.current}\n`
|
||||
);
|
||||
}
|
||||
} else {
|
||||
console.log("✅ No differences — round‑trip succeeded");
|
||||
}
|
||||
})();
|
||||
Reference in New Issue
Block a user