excelcync.js mostly working
This commit is contained in:
+145
-46328
File diff suppressed because it is too large
Load Diff
+92
-260
@@ -1,15 +1,11 @@
|
||||
// excelsync.js
|
||||
// Excel → CSV → PocketBase sync by Job_Number.
|
||||
// Raw strings only (true CSV), no coercion. Progress logs + robust error logging.
|
||||
|
||||
import { FIELD_CONFIG } from "./fieldConfig.js";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import axios from "axios";
|
||||
import qs from "qs";
|
||||
import * as XLSX from "xlsx";
|
||||
import dotenv from "dotenv";
|
||||
import PocketBase from "pocketbase";
|
||||
import XLSX from "xlsx";
|
||||
|
||||
dotenv.config();
|
||||
|
||||
@@ -25,6 +21,13 @@ const pbEmail = CONFIG.pbEmail;
|
||||
const clientSecret = process.env.CLIENT_SECRET;
|
||||
const pbPassword = process.env.PB_PASSWORD;
|
||||
|
||||
// -----------------------------
|
||||
// EXCEL HEADER AND DATA ROW PARAMETERS
|
||||
// -----------------------------
|
||||
// Zero-based: headers on row 2 (index 1), data starts on row 3 (index 2)
|
||||
const HEADER_ROW_INDEX = 1;
|
||||
const DATA_START_ROW_INDEX = 2;
|
||||
|
||||
// -----------------------------
|
||||
// LOGGING
|
||||
// -----------------------------
|
||||
@@ -39,7 +42,35 @@ function log(message) {
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// GRAPH TOKEN + DOWNLOAD
|
||||
// DATE CONVERSION HELPER
|
||||
// -----------------------------
|
||||
function convertExcelDateToUTC(value) {
|
||||
if (typeof value === "number") {
|
||||
// Excel serial date
|
||||
const dateObj = XLSX.SSF.parse_date_code(value);
|
||||
if (!dateObj) return null;
|
||||
const jsDate = new Date(Date.UTC(dateObj.y, dateObj.m - 1, dateObj.d, dateObj.H, dateObj.M, dateObj.S));
|
||||
return jsDate.toISOString(); // UTC ISO format
|
||||
} else if (typeof value === "string") {
|
||||
// Try parsing string date
|
||||
const jsDate = new Date(value);
|
||||
return isNaN(jsDate.getTime()) ? value : jsDate.toISOString();
|
||||
}
|
||||
return value; // Return as-is if not a date
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// POCKETBASE TOKEN ACQUISITION
|
||||
// -----------------------------
|
||||
async function getPocketBaseClient() {
|
||||
const pb = new PocketBase(pbUrl);
|
||||
await pb.collection("users").authWithPassword(pbEmail, pbPassword);
|
||||
log("✅ PocketBase token acquired");
|
||||
return pb;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// GRAPH TOKEN ACQUISITION
|
||||
// -----------------------------
|
||||
async function getGraphToken() {
|
||||
const tokenUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`;
|
||||
@@ -56,6 +87,9 @@ async function getGraphToken() {
|
||||
return res.data.access_token;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// GRAPH EXCEL DOWNLOAD HELPER
|
||||
// -----------------------------
|
||||
async function downloadExcelFromGraph(graphToken, driveId, itemId) {
|
||||
const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/content`;
|
||||
const res = await axios.get(url, {
|
||||
@@ -66,274 +100,72 @@ async function downloadExcelFromGraph(graphToken, driveId, itemId) {
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// POCKETBASE LOGIN
|
||||
// MAIN SYNC LOGIC
|
||||
// -----------------------------
|
||||
async function getPocketBaseClient() {
|
||||
const pb = new PocketBase(pbUrl);
|
||||
await pb.collection("users").authWithPassword(pbEmail, pbPassword);
|
||||
log("✅ PocketBase token acquired");
|
||||
return pb;
|
||||
}
|
||||
// -----------------------------
|
||||
// Header mapping
|
||||
// -----------------------------
|
||||
function buildHeaderMapping(rawHeaders) {
|
||||
const excelHeaderToConfig = Object.values(FIELD_CONFIG).reduce((map, cfg) => {
|
||||
if (cfg.excelHeader) {
|
||||
const key = String(cfg.excelHeader).trim();
|
||||
map[key] = cfg;
|
||||
map[key.toLowerCase()] = cfg;
|
||||
}
|
||||
return map;
|
||||
}, {});
|
||||
return rawHeaders.map(raw => {
|
||||
const clean = String(raw || "").trim();
|
||||
const cfg = excelHeaderToConfig[clean] || excelHeaderToConfig[clean.toLowerCase()];
|
||||
return cfg ? { pbField: cfg.pbField, isMapped: true } : { pbField: "", isMapped: false };
|
||||
});
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// Excel → CSV string
|
||||
// -----------------------------
|
||||
const HEADER_ROW_INDEX = 2;
|
||||
const DATA_START_ROW_INDEX = 3;
|
||||
|
||||
function excelToCSV(buffer) {
|
||||
const workbook = XLSX.read(buffer, { type: "buffer" });
|
||||
const sheet = workbook.Sheets["Job Sheet"];
|
||||
if (!sheet) throw new Error('Sheet named "Job Sheet" not found');
|
||||
|
||||
const range = XLSX.utils.decode_range(sheet["!ref"]);
|
||||
const merges = sheet["!merges"] || [];
|
||||
|
||||
const getValue = (r, c) => {
|
||||
const addr = XLSX.utils.encode_cell({ r, c });
|
||||
const cell = sheet[addr];
|
||||
if (cell) return cell.v ?? cell.w ?? null;
|
||||
for (const m of merges) {
|
||||
if (m.s.r <= r && r <= m.e.r && m.s.c <= c && c <= m.e.c) {
|
||||
const master = sheet[XLSX.utils.encode_cell(m.s)];
|
||||
return master ? (master.v ?? master.w ?? null) : null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const rawHeaders = [];
|
||||
for (let c = range.s.c; c <= range.e.c; c++) {
|
||||
const val = getValue(HEADER_ROW_INDEX, c);
|
||||
rawHeaders.push(val ? String(val).trim() : "");
|
||||
}
|
||||
|
||||
const headerMapping = buildHeaderMapping(rawHeaders);
|
||||
const pbHeaders = headerMapping.filter(h => h.isMapped).map(h => h.pbField);
|
||||
|
||||
if (!pbHeaders.includes("Job_Number")) {
|
||||
throw new Error("Job_Number must be present in mapped headers.");
|
||||
}
|
||||
|
||||
const rows = [];
|
||||
for (let r = DATA_START_ROW_INDEX; r <= range.e.r; r++) {
|
||||
const rowVals = [];
|
||||
let hasData = false;
|
||||
for (let c = range.s.c; c <= range.e.c; c++) {
|
||||
const v = getValue(r, c);
|
||||
if (v !== null && v !== "") hasData = true;
|
||||
rowVals.push(v === null ? "" : String(v));
|
||||
}
|
||||
if (!hasData) continue;
|
||||
rows.push(rowVals);
|
||||
}
|
||||
|
||||
const escapeCSV = (s) => {
|
||||
const needsQuotes = /[",\n\r]/.test(s);
|
||||
const withDoubles = s.replace(/"/g, '""');
|
||||
return needsQuotes ? `"${withDoubles}"` : withDoubles;
|
||||
};
|
||||
|
||||
const mappedColIndexes = headerMapping.map((h, idx) => (h.isMapped ? idx : -1)).filter(idx => idx !== -1);
|
||||
const headerLine = pbHeaders.map(h => escapeCSV(h)).join(",");
|
||||
const csvLines = [headerLine];
|
||||
|
||||
for (const row of rows) {
|
||||
const mappedValues = mappedColIndexes.map(idx => escapeCSV(row[idx] || ""));
|
||||
csvLines.push(mappedValues.join(","));
|
||||
}
|
||||
|
||||
const csv = csvLines.join("\n");
|
||||
log(`🧾 Built CSV with ${pbHeaders.length} columns and ${rows.length} data rows`);
|
||||
return { csv, headers: pbHeaders };
|
||||
}
|
||||
// -----------------------------
|
||||
// Parse CSV string → array of objects
|
||||
// -----------------------------
|
||||
function parseCSV(csv) {
|
||||
const lines = csv.split(/\r?\n/).filter(l => l.length > 0);
|
||||
if (lines.length === 0) return { headers: [], rows: [] };
|
||||
|
||||
const parseLine = (line) => {
|
||||
const out = [];
|
||||
let cur = "";
|
||||
let inQuotes = false;
|
||||
for (let i = 0; i < line.length; i++) {
|
||||
const ch = line[i];
|
||||
if (inQuotes) {
|
||||
if (ch === '"') {
|
||||
if (line[i + 1] === '"') { cur += '"'; i++; }
|
||||
else { inQuotes = false; }
|
||||
} else { cur += ch; }
|
||||
} else {
|
||||
if (ch === '"') inQuotes = true;
|
||||
else if (ch === ",") { out.push(cur); cur = ""; }
|
||||
else { cur += ch; }
|
||||
}
|
||||
}
|
||||
out.push(cur);
|
||||
return out;
|
||||
};
|
||||
|
||||
const headers = parseLine(lines[0]).map(h => h.trim());
|
||||
const rows = [];
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const cols = parseLine(lines[i]);
|
||||
const obj = {};
|
||||
for (let c = 0; c < headers.length; c++) {
|
||||
obj[headers[c]] = cols[c] ?? "";
|
||||
}
|
||||
rows.push(obj);
|
||||
}
|
||||
return { headers, rows };
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// Error logging helper
|
||||
// -----------------------------
|
||||
// -----------------------------
|
||||
// Error logging helper
|
||||
// -----------------------------
|
||||
function logPocketBaseError(err, context) {
|
||||
log(`❌ ${context}`);
|
||||
log(`Error message: ${err.message || "no message"}`);
|
||||
|
||||
// Dump everything we can from the error object
|
||||
(async () => {
|
||||
try {
|
||||
log(`Error dump: ${JSON.stringify(err, Object.getOwnPropertyNames(err), 2)}`);
|
||||
} catch {
|
||||
log("⚠️ Could not stringify error object");
|
||||
}
|
||||
const graphToken = await getGraphToken();
|
||||
const pb = await getPocketBaseClient();
|
||||
|
||||
// Axios attaches the server's JSON body here
|
||||
if (err.response) {
|
||||
log(`Response status: ${err.response.status} ${err.response.statusText || ""}`);
|
||||
if (err.response.data) {
|
||||
try {
|
||||
log(`Server response data: ${JSON.stringify(err.response.data, null, 2)}`);
|
||||
} catch {
|
||||
log("⚠️ Could not stringify response.data");
|
||||
}
|
||||
}
|
||||
}
|
||||
// These IDs point to your Excel file in OneDrive/SharePoint
|
||||
const driveId = "b!9YOqqr2xM0G2DFBwMEJYY4UzQgocFddEtpLYWuS9_AtkCKl2q8yjQJteQ7Ti4QSx";
|
||||
const itemId = "01SPNXLDSQTMG4F6BDANCLJU2FZKEDQLVU";
|
||||
|
||||
// PocketBase client sometimes attaches .data separately
|
||||
if (err.data) {
|
||||
try {
|
||||
log(`Server details: ${JSON.stringify(err.data, null, 2)}`);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
const excelBuffer = await downloadExcelFromGraph(graphToken, driveId, itemId);
|
||||
|
||||
// Parse Excel
|
||||
const workbook = XLSX.read(excelBuffer, { type: "buffer" });
|
||||
const worksheet = workbook.Sheets["Job_List_Range"];
|
||||
const rows = XLSX.utils.sheet_to_json(worksheet, { header: 1 });
|
||||
|
||||
// -----------------------------
|
||||
// RESYNC: by Job_Number
|
||||
// -----------------------------
|
||||
// -----------------------------
|
||||
// RESYNC: by Job_Number
|
||||
// -----------------------------
|
||||
async function resyncPocketBase(pb, csvRows) {
|
||||
log(`🔄 Starting resync for ${csvRows.length} rows...`);
|
||||
const headers = rows[HEADER_ROW_INDEX]; // Row 2 (index 1)
|
||||
const dataRows = rows.slice(DATA_START_ROW_INDEX); // Row 3 onward
|
||||
|
||||
let existing = [];
|
||||
try {
|
||||
existing = await pb.collection(pbCollection).getFullList();
|
||||
} catch (err) {
|
||||
logPocketBaseError(err, "Failed to load existing records");
|
||||
}
|
||||
log(`✅ Found ${dataRows.length} data rows`);
|
||||
|
||||
const jobMap = {};
|
||||
for (const rec of existing) {
|
||||
jobMap[rec.Job_Number] = rec.id;
|
||||
}
|
||||
for (const row of dataRows) {
|
||||
if (!row || row.length === 0) continue; // Skip empty rows
|
||||
|
||||
const allowedFields = new Set(
|
||||
Object.values(FIELD_CONFIG).map(cfg => cfg.pbField).filter(Boolean)
|
||||
);
|
||||
|
||||
for (let i = 0; i < csvRows.length; i++) {
|
||||
const row = csvRows[i];
|
||||
|
||||
// Progress every 250 records
|
||||
if (i > 0 && i % 250 === 0) {
|
||||
log(`⏳ Progress: processed ${i} of ${csvRows.length} records...`);
|
||||
}
|
||||
|
||||
const jobNumber = row.Job_Number;
|
||||
if (!jobNumber || jobNumber === "") continue;
|
||||
|
||||
// Build payload with only allowed fields
|
||||
const payload = {};
|
||||
for (const [k, v] of Object.entries(row)) {
|
||||
if (k === "Job_Number") continue;
|
||||
if (allowedFields.has(k)) payload[k] = v;
|
||||
}
|
||||
|
||||
// Helper to log errors with server response
|
||||
const logPocketBaseError = (err, context) => {
|
||||
log(`❌ ${context}`);
|
||||
log(`Error message: ${err.message || "no message"}`);
|
||||
|
||||
try {
|
||||
log(`Error dump: ${JSON.stringify(err, Object.getOwnPropertyNames(err), 2)}`);
|
||||
} catch {}
|
||||
|
||||
if (err.response) {
|
||||
log(`Response status: ${err.response.status} ${err.response.statusText || ""}`);
|
||||
if (err.response.data) {
|
||||
try {
|
||||
log(`Server response data: ${JSON.stringify(err.response.data, null, 2)}`);
|
||||
} catch {
|
||||
log("⚠️ Could not stringify response.data");
|
||||
}
|
||||
} else {
|
||||
log("⚠️ No response.data present");
|
||||
const record = {};
|
||||
headers.forEach((header, i) => {
|
||||
let cellValue = row[i];
|
||||
// ✅ Convert all date fields to UTC ISO format
|
||||
if (header.toLowerCase().includes("date")) {
|
||||
cellValue = convertExcelDateToUTC(cellValue);
|
||||
}
|
||||
} else {
|
||||
log("⚠️ No err.response present");
|
||||
record[header] = cellValue;
|
||||
});
|
||||
|
||||
const jobNumber = record["Job_Number"];
|
||||
if (!jobNumber) {
|
||||
log("⚠️ Skipped row with no Job_Number");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (err.data) {
|
||||
try {
|
||||
log(`Server details: ${JSON.stringify(err.data, null, 2)}`);
|
||||
} catch {}
|
||||
}
|
||||
};
|
||||
try {
|
||||
// Check if record exists
|
||||
const existing = await pb.collection(pbCollection).getList(1, 1, {
|
||||
filter: `Job_Number="${jobNumber}"`,
|
||||
});
|
||||
|
||||
if (jobMap[jobNumber]) {
|
||||
const id = jobMap[jobNumber];
|
||||
try {
|
||||
await pb.collection(pbCollection).update(id, payload);
|
||||
if (existing.items.length > 0) {
|
||||
const id = existing.items[0].id;
|
||||
// Clear all fields except Job_Number
|
||||
const clearedData = { Job_Number: jobNumber };
|
||||
Object.assign(clearedData, record);
|
||||
await pb.collection(pbCollection).update(id, clearedData);
|
||||
log(`✅ Updated record for Job_Number: ${jobNumber}`);
|
||||
} else {
|
||||
await pb.collection(pbCollection).create(record);
|
||||
log(`✅ Created new record for Job_Number: ${jobNumber}`);
|
||||
}
|
||||
} catch (err) {
|
||||
logPocketBaseError(err, `Update failed for id=${id} Job_Number=${jobNumber}`);
|
||||
}
|
||||
} else {
|
||||
const createPayload = { Job_Number: jobNumber, ...payload };
|
||||
try {
|
||||
await pb.collection(pbCollection).create(createPayload);
|
||||
} catch (err) {
|
||||
logPocketBaseError(err, `Create failed for Job_Number=${jobNumber}`);
|
||||
log(`❌ Error processing Job_Number ${jobNumber}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log("✅ Resync complete.");
|
||||
}
|
||||
log("✅ Sync complete");
|
||||
} catch (err) {
|
||||
log(`❌ Fatal error: ${err.message}`);
|
||||
}
|
||||
})();
|
||||
|
||||
@@ -0,0 +1,339 @@
|
||||
// excelsync.js
|
||||
// Excel → CSV → PocketBase sync by Job_Number.
|
||||
// Raw strings only (true CSV), no coercion. Progress logs + robust error logging.
|
||||
|
||||
import { FIELD_CONFIG } from "./fieldConfig.js";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import axios from "axios";
|
||||
import qs from "qs";
|
||||
import * as XLSX from "xlsx";
|
||||
import dotenv from "dotenv";
|
||||
import PocketBase from "pocketbase";
|
||||
|
||||
dotenv.config();
|
||||
|
||||
// -----------------------------
|
||||
// CONFIG
|
||||
// -----------------------------
|
||||
const CONFIG = JSON.parse(fs.readFileSync("config.json", "utf8"));
|
||||
const tenantId = CONFIG.tenantId;
|
||||
const clientId = CONFIG.clientId;
|
||||
const pbUrl = CONFIG.pbUrl;
|
||||
const pbCollection = CONFIG.pbCollection;
|
||||
const pbEmail = CONFIG.pbEmail;
|
||||
const clientSecret = process.env.CLIENT_SECRET;
|
||||
const pbPassword = process.env.PB_PASSWORD;
|
||||
|
||||
// -----------------------------
|
||||
// LOGGING
|
||||
// -----------------------------
|
||||
function log(message) {
|
||||
console.log(message);
|
||||
const logfile = path.resolve(process.cwd(), "excelsync.log");
|
||||
try {
|
||||
fs.appendFileSync(logfile, message + "\n", "utf8");
|
||||
} catch (e) {
|
||||
console.error("Log write failed:", e?.message || e);
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// GRAPH TOKEN + DOWNLOAD
|
||||
// -----------------------------
|
||||
async function getGraphToken() {
|
||||
const tokenUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`;
|
||||
const body = qs.stringify({
|
||||
client_id: clientId,
|
||||
client_secret: clientSecret,
|
||||
scope: "https://graph.microsoft.com/.default",
|
||||
grant_type: "client_credentials",
|
||||
});
|
||||
const res = await axios.post(tokenUrl, body, {
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||
});
|
||||
log("✅ Graph token acquired");
|
||||
return res.data.access_token;
|
||||
}
|
||||
|
||||
async function downloadExcelFromGraph(graphToken, driveId, itemId) {
|
||||
const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/content`;
|
||||
const res = await axios.get(url, {
|
||||
headers: { Authorization: `Bearer ${graphToken}` },
|
||||
responseType: "arraybuffer",
|
||||
});
|
||||
return Buffer.from(res.data);
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// POCKETBASE LOGIN
|
||||
// -----------------------------
|
||||
async function getPocketBaseClient() {
|
||||
const pb = new PocketBase(pbUrl);
|
||||
await pb.collection("users").authWithPassword(pbEmail, pbPassword);
|
||||
log("✅ PocketBase token acquired");
|
||||
return pb;
|
||||
}
|
||||
// -----------------------------
|
||||
// Header mapping
|
||||
// -----------------------------
|
||||
function buildHeaderMapping(rawHeaders) {
|
||||
const excelHeaderToConfig = Object.values(FIELD_CONFIG).reduce((map, cfg) => {
|
||||
if (cfg.excelHeader) {
|
||||
const key = String(cfg.excelHeader).trim();
|
||||
map[key] = cfg;
|
||||
map[key.toLowerCase()] = cfg;
|
||||
}
|
||||
return map;
|
||||
}, {});
|
||||
return rawHeaders.map(raw => {
|
||||
const clean = String(raw || "").trim();
|
||||
const cfg = excelHeaderToConfig[clean] || excelHeaderToConfig[clean.toLowerCase()];
|
||||
return cfg ? { pbField: cfg.pbField, isMapped: true } : { pbField: "", isMapped: false };
|
||||
});
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// Excel → CSV string
|
||||
// -----------------------------
|
||||
const HEADER_ROW_INDEX = 2;
|
||||
const DATA_START_ROW_INDEX = 3;
|
||||
|
||||
function excelToCSV(buffer) {
|
||||
const workbook = XLSX.read(buffer, { type: "buffer" });
|
||||
const sheet = workbook.Sheets["Job Sheet"];
|
||||
if (!sheet) throw new Error('Sheet named "Job Sheet" not found');
|
||||
|
||||
const range = XLSX.utils.decode_range(sheet["!ref"]);
|
||||
const merges = sheet["!merges"] || [];
|
||||
|
||||
const getValue = (r, c) => {
|
||||
const addr = XLSX.utils.encode_cell({ r, c });
|
||||
const cell = sheet[addr];
|
||||
if (cell) return cell.v ?? cell.w ?? null;
|
||||
for (const m of merges) {
|
||||
if (m.s.r <= r && r <= m.e.r && m.s.c <= c && c <= m.e.c) {
|
||||
const master = sheet[XLSX.utils.encode_cell(m.s)];
|
||||
return master ? (master.v ?? master.w ?? null) : null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const rawHeaders = [];
|
||||
for (let c = range.s.c; c <= range.e.c; c++) {
|
||||
const val = getValue(HEADER_ROW_INDEX, c);
|
||||
rawHeaders.push(val ? String(val).trim() : "");
|
||||
}
|
||||
|
||||
const headerMapping = buildHeaderMapping(rawHeaders);
|
||||
const pbHeaders = headerMapping.filter(h => h.isMapped).map(h => h.pbField);
|
||||
|
||||
if (!pbHeaders.includes("Job_Number")) {
|
||||
throw new Error("Job_Number must be present in mapped headers.");
|
||||
}
|
||||
|
||||
const rows = [];
|
||||
for (let r = DATA_START_ROW_INDEX; r <= range.e.r; r++) {
|
||||
const rowVals = [];
|
||||
let hasData = false;
|
||||
for (let c = range.s.c; c <= range.e.c; c++) {
|
||||
const v = getValue(r, c);
|
||||
if (v !== null && v !== "") hasData = true;
|
||||
rowVals.push(v === null ? "" : String(v));
|
||||
}
|
||||
if (!hasData) continue;
|
||||
rows.push(rowVals);
|
||||
}
|
||||
|
||||
const escapeCSV = (s) => {
|
||||
const needsQuotes = /[",\n\r]/.test(s);
|
||||
const withDoubles = s.replace(/"/g, '""');
|
||||
return needsQuotes ? `"${withDoubles}"` : withDoubles;
|
||||
};
|
||||
|
||||
const mappedColIndexes = headerMapping.map((h, idx) => (h.isMapped ? idx : -1)).filter(idx => idx !== -1);
|
||||
const headerLine = pbHeaders.map(h => escapeCSV(h)).join(",");
|
||||
const csvLines = [headerLine];
|
||||
|
||||
for (const row of rows) {
|
||||
const mappedValues = mappedColIndexes.map(idx => escapeCSV(row[idx] || ""));
|
||||
csvLines.push(mappedValues.join(","));
|
||||
}
|
||||
|
||||
const csv = csvLines.join("\n");
|
||||
log(`🧾 Built CSV with ${pbHeaders.length} columns and ${rows.length} data rows`);
|
||||
return { csv, headers: pbHeaders };
|
||||
}
|
||||
// -----------------------------
|
||||
// Parse CSV string → array of objects
|
||||
// -----------------------------
|
||||
function parseCSV(csv) {
|
||||
const lines = csv.split(/\r?\n/).filter(l => l.length > 0);
|
||||
if (lines.length === 0) return { headers: [], rows: [] };
|
||||
|
||||
const parseLine = (line) => {
|
||||
const out = [];
|
||||
let cur = "";
|
||||
let inQuotes = false;
|
||||
for (let i = 0; i < line.length; i++) {
|
||||
const ch = line[i];
|
||||
if (inQuotes) {
|
||||
if (ch === '"') {
|
||||
if (line[i + 1] === '"') { cur += '"'; i++; }
|
||||
else { inQuotes = false; }
|
||||
} else { cur += ch; }
|
||||
} else {
|
||||
if (ch === '"') inQuotes = true;
|
||||
else if (ch === ",") { out.push(cur); cur = ""; }
|
||||
else { cur += ch; }
|
||||
}
|
||||
}
|
||||
out.push(cur);
|
||||
return out;
|
||||
};
|
||||
|
||||
const headers = parseLine(lines[0]).map(h => h.trim());
|
||||
const rows = [];
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const cols = parseLine(lines[i]);
|
||||
const obj = {};
|
||||
for (let c = 0; c < headers.length; c++) {
|
||||
obj[headers[c]] = cols[c] ?? "";
|
||||
}
|
||||
rows.push(obj);
|
||||
}
|
||||
return { headers, rows };
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// Error logging helper
|
||||
// -----------------------------
|
||||
// -----------------------------
|
||||
// Error logging helper
|
||||
// -----------------------------
|
||||
function logPocketBaseError(err, context) {
|
||||
log(`❌ ${context}`);
|
||||
log(`Error message: ${err.message || "no message"}`);
|
||||
|
||||
// Dump everything we can from the error object
|
||||
try {
|
||||
log(`Error dump: ${JSON.stringify(err, Object.getOwnPropertyNames(err), 2)}`);
|
||||
} catch {
|
||||
log("⚠️ Could not stringify error object");
|
||||
}
|
||||
|
||||
// Axios attaches the server's JSON body here
|
||||
if (err.response) {
|
||||
log(`Response status: ${err.response.status} ${err.response.statusText || ""}`);
|
||||
if (err.response.data) {
|
||||
try {
|
||||
log(`Server response data: ${JSON.stringify(err.response.data, null, 2)}`);
|
||||
} catch {
|
||||
log("⚠️ Could not stringify response.data");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// PocketBase client sometimes attaches .data separately
|
||||
if (err.data) {
|
||||
try {
|
||||
log(`Server details: ${JSON.stringify(err.data, null, 2)}`);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// -----------------------------
|
||||
// RESYNC: by Job_Number
|
||||
// -----------------------------
|
||||
// -----------------------------
|
||||
// RESYNC: by Job_Number
|
||||
// -----------------------------
|
||||
async function resyncPocketBase(pb, csvRows) {
|
||||
log(`🔄 Starting resync for ${csvRows.length} rows...`);
|
||||
|
||||
let existing = [];
|
||||
try {
|
||||
existing = await pb.collection(pbCollection).getFullList();
|
||||
} catch (err) {
|
||||
logPocketBaseError(err, "Failed to load existing records");
|
||||
}
|
||||
|
||||
const jobMap = {};
|
||||
for (const rec of existing) {
|
||||
jobMap[rec.Job_Number] = rec.id;
|
||||
}
|
||||
|
||||
const allowedFields = new Set(
|
||||
Object.values(FIELD_CONFIG).map(cfg => cfg.pbField).filter(Boolean)
|
||||
);
|
||||
|
||||
for (let i = 0; i < csvRows.length; i++) {
|
||||
const row = csvRows[i];
|
||||
|
||||
// Progress every 250 records
|
||||
if (i > 0 && i % 250 === 0) {
|
||||
log(`⏳ Progress: processed ${i} of ${csvRows.length} records...`);
|
||||
}
|
||||
|
||||
const jobNumber = row.Job_Number;
|
||||
if (!jobNumber || jobNumber === "") continue;
|
||||
|
||||
// Build payload with only allowed fields
|
||||
const payload = {};
|
||||
for (const [k, v] of Object.entries(row)) {
|
||||
if (k === "Job_Number") continue;
|
||||
if (allowedFields.has(k)) payload[k] = v;
|
||||
}
|
||||
|
||||
// Helper to log errors with server response
|
||||
const logPocketBaseError = (err, context) => {
|
||||
log(`❌ ${context}`);
|
||||
log(`Error message: ${err.message || "no message"}`);
|
||||
|
||||
try {
|
||||
log(`Error dump: ${JSON.stringify(err, Object.getOwnPropertyNames(err), 2)}`);
|
||||
} catch {}
|
||||
|
||||
if (err.response) {
|
||||
log(`Response status: ${err.response.status} ${err.response.statusText || ""}`);
|
||||
if (err.response.data) {
|
||||
try {
|
||||
log(`Server response data: ${JSON.stringify(err.response.data, null, 2)}`);
|
||||
} catch {
|
||||
log("⚠️ Could not stringify response.data");
|
||||
}
|
||||
} else {
|
||||
log("⚠️ No response.data present");
|
||||
}
|
||||
} else {
|
||||
log("⚠️ No err.response present");
|
||||
}
|
||||
|
||||
if (err.data) {
|
||||
try {
|
||||
log(`Server details: ${JSON.stringify(err.data, null, 2)}`);
|
||||
} catch {}
|
||||
}
|
||||
};
|
||||
|
||||
if (jobMap[jobNumber]) {
|
||||
const id = jobMap[jobNumber];
|
||||
try {
|
||||
await pb.collection(pbCollection).update(id, payload);
|
||||
} catch (err) {
|
||||
logPocketBaseError(err, `Update failed for id=${id} Job_Number=${jobNumber}`);
|
||||
}
|
||||
} else {
|
||||
const createPayload = { Job_Number: jobNumber, ...payload };
|
||||
try {
|
||||
await pb.collection(pbCollection).create(createPayload);
|
||||
} catch (err) {
|
||||
logPocketBaseError(err, `Create failed for Job_Number=${jobNumber}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log("✅ Resync complete.");
|
||||
}
|
||||
Reference in New Issue
Block a user