Update excel export and value coersion to pb field expectations
This commit is contained in:
@@ -1,7 +1,6 @@
|
|||||||
{
|
{
|
||||||
"tenantId": "3fd97ea7-b124-41f1-855f-52d8ac3b16c7",
|
"tenantId": "3fd97ea7-b124-41f1-855f-52d8ac3b16c7",
|
||||||
"clientId": "3c846e71-9609-40e1-b458-0eb805e21b9f",
|
"clientId": "3c846e71-9609-40e1-b458-0eb805e21b9f",
|
||||||
"sharepointFileUrl": "https://czflex.sharepoint.com/:x:/r/sites/Team/Shared%20Documents/General/JOBS%201000-9999.xls",
|
|
||||||
"pbUrl": "http://10.10.1.109:8080",
|
"pbUrl": "http://10.10.1.109:8080",
|
||||||
"pbCollection": "Job_Info_TestEnv",
|
"pbCollection": "Job_Info_TestEnv",
|
||||||
"pbEmail": "pbserviceupdate@cardoza.construction"
|
"pbEmail": "pbserviceupdate@cardoza.construction"
|
||||||
|
|||||||
+60
-1915
File diff suppressed because it is too large
Load Diff
+203
-275
@@ -1,9 +1,13 @@
|
|||||||
|
//12/1/2025 Allows Excel download and conversion of headers mapped to pb headers.
|
||||||
|
//coersion added to match pb field types. however, may still have difficulty in assimilating certain date formats from excel to pb date fields.
|
||||||
|
//due to Excel storing as all caps for boolean fields
|
||||||
// excelsync.js
|
// excelsync.js
|
||||||
import { FIELD_CONFIG } from "./fieldConfig.js";
|
import { FIELD_CONFIG } from "./fieldConfig.js";
|
||||||
import fs from "fs";
|
import fs from "fs";
|
||||||
import axios from "axios";
|
import axios from "axios";
|
||||||
import qs from "qs";
|
import qs from "qs";
|
||||||
import * as XLSX from "xlsx";
|
import * as XLSX from "xlsx";
|
||||||
|
import ExcelJS from "exceljs";
|
||||||
import dotenv from "dotenv";
|
import dotenv from "dotenv";
|
||||||
import { serve } from "bun";
|
import { serve } from "bun";
|
||||||
import PocketBase from "pocketbase"
|
import PocketBase from "pocketbase"
|
||||||
@@ -11,38 +15,17 @@ import PocketBase from "pocketbase"
|
|||||||
dotenv.config();
|
dotenv.config();
|
||||||
|
|
||||||
// -----------------------------
|
// -----------------------------
|
||||||
// EXCEL PARSER
|
// CONFIG FOR TOKEN ACQUISITION GRAPH API + POCKETBASE
|
||||||
// -----------------------------
|
|
||||||
const HEADER_ROW_INDEX = 2; // Row 3
|
|
||||||
const DATA_START_ROW_INDEX = 3; // Row 4
|
|
||||||
|
|
||||||
// -----------------------------
|
|
||||||
// CONFIG
|
|
||||||
// -----------------------------
|
// -----------------------------
|
||||||
const CONFIG = JSON.parse(fs.readFileSync("config.json", "utf8"));
|
const CONFIG = JSON.parse(fs.readFileSync("config.json", "utf8"));
|
||||||
const tenantId = CONFIG.tenantId;
|
const tenantId = CONFIG.tenantId;
|
||||||
const clientId = CONFIG.clientId;
|
const clientId = CONFIG.clientId;
|
||||||
const sharepointFileUrl = CONFIG.sharepointFileUrl; // production Excel file URL
|
|
||||||
const pbUrl = CONFIG.pbUrl;
|
const pbUrl = CONFIG.pbUrl;
|
||||||
const pbCollection = CONFIG.pbCollection || "Job_Info_TestEnv";
|
const pbCollection = CONFIG.pbCollection;
|
||||||
const pbEmail = CONFIG.pbEmail;
|
const pbEmail = CONFIG.pbEmail;
|
||||||
|
|
||||||
const clientSecret = process.env.CLIENT_SECRET;
|
const clientSecret = process.env.CLIENT_SECRET;
|
||||||
const pbPassword = process.env.PB_PASSWORD;
|
const pbPassword = process.env.PB_PASSWORD;
|
||||||
|
|
||||||
if (!tenantId || !clientId || !clientSecret || !sharepointFileUrl || !pbUrl || !pbEmail || !pbPassword) {
|
|
||||||
console.error("❌ Missing required config/env values");
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
console.log({
|
|
||||||
tenantId,
|
|
||||||
clientId,
|
|
||||||
clientSecret: clientSecret ? clientSecret.slice(0,4) + "..." : null,
|
|
||||||
sharepointFileUrl,
|
|
||||||
pbUrl,
|
|
||||||
pbEmail,
|
|
||||||
pbPassword: pbPassword ? "***" : null
|
|
||||||
});
|
|
||||||
// -----------------------------
|
// -----------------------------
|
||||||
// LOGGING
|
// LOGGING
|
||||||
// -----------------------------
|
// -----------------------------
|
||||||
@@ -52,7 +35,7 @@ function log(message) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// -----------------------------
|
// -----------------------------
|
||||||
// HEADER + ROW DETECTION HELPERS
|
// HEADER DETECTOR
|
||||||
// -----------------------------
|
// -----------------------------
|
||||||
//Gets the last column letter in the table converting from 0 index to excel actual column in letter form.
|
//Gets the last column letter in the table converting from 0 index to excel actual column in letter form.
|
||||||
function columnLetter(index) {
|
function columnLetter(index) {
|
||||||
@@ -64,30 +47,8 @@ function columnLetter(index) {
|
|||||||
return letter;
|
return letter;
|
||||||
}
|
}
|
||||||
|
|
||||||
const headerRowIndex = HEADER_ROW_INDEX + 1;
|
|
||||||
|
|
||||||
function getColumnLetterForHeader(headerName, sheet, headerRowIndex) {
|
|
||||||
const headerRow = XLSX.utils.sheet_to_json(sheet, { header: 1, range: `${headerRowIndex}:${headerRowIndex}` })[0]
|
|
||||||
.map(h => (h || "").toString().trim().toLowerCase());
|
|
||||||
const idx = headerRow.findIndex(h => h === headerName.toLowerCase());
|
|
||||||
if (idx === -1) {
|
|
||||||
log(`❌ Header "${headerName}" not found. Row contents: ${JSON.stringify(headerRow)}`);
|
|
||||||
throw new Error(`Header "${headerName}" not found on row ${headerRowIndex}`);
|
|
||||||
}
|
|
||||||
log(`✅ Header "${headerName}" found at column index ${idx}`);
|
|
||||||
return columnLetter(idx);
|
|
||||||
}
|
|
||||||
function getNotesColumnLetterFromParsedHeaders(headerMap) {
|
|
||||||
const headers = Object.keys(headerMap);
|
|
||||||
const notesIndex = headers.findIndex(h => h.trim().toLowerCase() === "notes");
|
|
||||||
if (notesIndex === -1) {
|
|
||||||
log(`❌ 'Notes' header not found in HEADER_MAP keys: ${headers.join(", ")}`);
|
|
||||||
throw new Error("Notes column not found");
|
|
||||||
}
|
|
||||||
return columnLetter(notesIndex);
|
|
||||||
}
|
|
||||||
// -----------------------------
|
// -----------------------------
|
||||||
// GRAPH TOKEN
|
// GRAPH TOKEN ACQUISITION
|
||||||
// -----------------------------
|
// -----------------------------
|
||||||
async function getGraphToken() {
|
async function getGraphToken() {
|
||||||
const tokenUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`;
|
const tokenUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`;
|
||||||
@@ -117,22 +78,7 @@ async function downloadExcelFromGraph(graphToken, driveId, itemId) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// -----------------------------
|
// -----------------------------
|
||||||
// GRAPH UPLOADER HELPER
|
// POCKETBASE LOGIN AND TOKEN ACQUISITION
|
||||||
// -----------------------------
|
|
||||||
async function uploadExcelToGraph(graphToken, driveId, itemId, buffer) {
|
|
||||||
const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/content`;
|
|
||||||
const res = await axios.put(url, buffer, {
|
|
||||||
headers: {
|
|
||||||
Authorization: `Bearer ${graphToken}`,
|
|
||||||
"Content-Type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
|
||||||
}
|
|
||||||
});
|
|
||||||
log(`✅ Excel file updated in SharePoint: ${res.status} ${res.statusText}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// -----------------------------
|
|
||||||
// POCKETBASE LOGIN
|
|
||||||
// -----------------------------
|
// -----------------------------
|
||||||
async function getPocketBaseClient() {
|
async function getPocketBaseClient() {
|
||||||
const pb = new PocketBase(pbUrl);
|
const pb = new PocketBase(pbUrl);
|
||||||
@@ -141,215 +87,198 @@ async function getPocketBaseClient() {
|
|||||||
return pb;
|
return pb;
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseExcelFile(input, headerMap) {
|
// -----------------------------
|
||||||
|
// Get Data From Excel File
|
||||||
|
// -----------------------------
|
||||||
|
|
||||||
|
const HEADER_ROW_INDEX = 2; // Row 3 in Excel
|
||||||
|
const DATA_START_ROW_INDEX = 3; // Row 4 in Excel → first data row
|
||||||
|
|
||||||
|
//clean and coerce Excel values to match PocketBase schema expectations
|
||||||
|
function cleanString(val) {
|
||||||
|
if (val == null) return "";
|
||||||
|
return String(val)
|
||||||
|
.replace(/\u00A0/g, " ") // normalize non-breaking spaces
|
||||||
|
.replace(/[\r\n\t]/g, " ") // strip control chars
|
||||||
|
.replace(/\s+/g, " ") // collapse whitespace
|
||||||
|
.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeForPB(val, type) {
|
||||||
|
const cleaned = cleanString(val);
|
||||||
|
|
||||||
|
// Treat common placeholders as empty
|
||||||
|
const emptyTokens = ["n/a", "na", "null", "none"];
|
||||||
|
if (emptyTokens.includes(cleaned.toLowerCase())) {
|
||||||
|
switch (type) {
|
||||||
|
case "number": return 0; // PocketBase defaults to 0
|
||||||
|
case "bool": return null;
|
||||||
|
case "string": return null;
|
||||||
|
case "email": return null; // email empty
|
||||||
|
case "json": return null;
|
||||||
|
case "date": return null; // date empty
|
||||||
|
default: return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cleaned === "") {
|
||||||
|
switch (type) {
|
||||||
|
case "number": return 0;
|
||||||
|
case "bool": return null;
|
||||||
|
case "string": return null;
|
||||||
|
case "email": return null;
|
||||||
|
case "json": return null;
|
||||||
|
case "date": return null;
|
||||||
|
default: return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (type) {
|
||||||
|
case "bool":
|
||||||
|
case "boolean": {
|
||||||
|
const lower = cleaned.toLowerCase();
|
||||||
|
if (["true","yes","y","1"].includes(lower)) return true; // boolean true
|
||||||
|
if (["false","no","n","0"].includes(lower)) return false; // boolean false
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
case "number": {
|
||||||
|
const num = Number(cleaned);
|
||||||
|
return Number.isNaN(num) ? 0 : num;
|
||||||
|
}
|
||||||
|
case "date": {
|
||||||
|
const d = new Date(cleaned);
|
||||||
|
return isNaN(d.getTime()) ? null : d.toISOString();
|
||||||
|
}
|
||||||
|
case "string":
|
||||||
|
default:
|
||||||
|
return cleaned;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseExcelFile(input) {
|
||||||
let data;
|
let data;
|
||||||
// In case of data from graph
|
|
||||||
if (Buffer.isBuffer(input)) {
|
if (Buffer.isBuffer(input)) {
|
||||||
data = input;
|
data = input;
|
||||||
log(`Reading Excel file from Graph buffer`);
|
log("Reading Excel file from Graph buffer");
|
||||||
// In case of file path (Not used at this time)
|
|
||||||
} else {
|
} else {
|
||||||
data = fs.readFileSync(input);
|
data = fs.readFileSync(input);
|
||||||
log(`Reading Excel file: ${input}`);
|
log(`Reading Excel file: ${input}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const workbook = XLSX.read(data, { type: "buffer", cellDates: true });
|
const workbook = XLSX.read(data, { type: "buffer", cellDates: true });
|
||||||
const sheet = workbook.Sheets["Job Sheet"]; // explicitly use Job Sheet
|
const sheet = workbook.Sheets["Job Sheet"];
|
||||||
const range = XLSX.utils.decode_range(sheet["!ref"]);
|
if (!sheet) throw new Error('Sheet named "Job Sheet" not found');
|
||||||
|
|
||||||
|
const range = XLSX.utils.decode_range(sheet["!ref"]);
|
||||||
const merges = sheet["!merges"] || [];
|
const merges = sheet["!merges"] || [];
|
||||||
|
|
||||||
|
// Helper to get value from cell or merged region
|
||||||
const getValue = (r, c) => {
|
const getValue = (r, c) => {
|
||||||
const addr = XLSX.utils.encode_cell({ r, c });
|
const addr = XLSX.utils.encode_cell({ r, c });
|
||||||
const cell = sheet[addr];
|
const cell = sheet[addr];
|
||||||
if (cell) return cell.v ?? cell.w ?? null;
|
if (cell) return cell.v ?? cell.w ?? null;
|
||||||
|
|
||||||
for (const m of merges) {
|
for (const m of merges) {
|
||||||
if (m.s.r <= r && r <= m.e.r && m.s.c <= c && c <= m.e.c) {
|
if (m.s.r <= r && r <= m.e.r && m.s.c <= c && c <= m.e.c) {
|
||||||
const master = sheet[XLSX.utils.encode_cell({ r: m.s.r, c: m.s.c })];
|
const master = sheet[XLSX.utils.encode_cell(m.s)];
|
||||||
return master ? (master.v ?? master.w ?? null) : null;
|
return master ? (master.v ?? master.w ?? null) : null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
};
|
};
|
||||||
|
|
||||||
// Read headers
|
// 1. Read headers (Row 3)
|
||||||
const rawHeaders = [];
|
const rawHeaders = [];
|
||||||
for (let c = range.s.c; c <= range.e.c; c++) {
|
for (let c = range.s.c; c <= range.e.c; c++) {
|
||||||
let h = getValue(HEADER_ROW_INDEX, c);
|
const val = getValue(HEADER_ROW_INDEX, c);
|
||||||
rawHeaders.push(h ? String(h).trim() : "");
|
rawHeaders.push(val ? String(val).trim() : "");
|
||||||
}
|
}
|
||||||
log(`Found ${rawHeaders.length} headers in Row 3`);
|
|
||||||
log(`Headers: ${rawHeaders.join(" | ")}`);
|
// 2. Build lookup map once
|
||||||
// Compare Excel headers against HEADER_MAP
|
const excelHeaderToConfig = Object.values(FIELD_CONFIG).reduce((map, cfg) => {
|
||||||
const mappedHeaders = [];
|
if (cfg.excelHeader) {
|
||||||
const unmappedHeaders = [];
|
const key = cfg.excelHeader.trim();
|
||||||
for (const h of rawHeaders) {
|
map[key] = cfg;
|
||||||
if (headerMap[h]) {
|
map[key.toLowerCase()] = cfg;
|
||||||
mappedHeaders.push([h, headerMap[h]]);
|
|
||||||
} else {
|
|
||||||
unmappedHeaders.push([h, ""]);
|
|
||||||
}
|
}
|
||||||
}
|
return map;
|
||||||
|
}, {});
|
||||||
|
|
||||||
// Output as CSV-style rows
|
// 3. Rich header mapping
|
||||||
console.log("\n📑 Header mapping preview (CSV style):");
|
const headerMapping = rawHeaders.map(raw => {
|
||||||
console.log("ExcelHeader,PocketBaseField");
|
const clean = raw.trim();
|
||||||
for (const [excel, pb] of mappedHeaders) {
|
const config = excelHeaderToConfig[clean] || excelHeaderToConfig[clean.toLowerCase()];
|
||||||
console.log(`${excel},${pb}`);
|
|
||||||
}
|
|
||||||
for (const [excel, pb] of unmappedHeaders) {
|
|
||||||
console.log(`${excel},${pb}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse data rows
|
return config
|
||||||
|
? { original: clean, pbField: config.pbField, type: config.type, isMapped: true, config }
|
||||||
|
: { original: clean, pbField: "", type: "string", isMapped: false, isUnknown: true };
|
||||||
|
});
|
||||||
|
|
||||||
|
log(`Mapped ${headerMapping.filter(h => h.isMapped).length} columns`);
|
||||||
|
|
||||||
|
// 4. Parse data rows (starting Row 4)
|
||||||
const rows = [];
|
const rows = [];
|
||||||
for (let r = DATA_START_ROW_INDEX; r <= range.e.r; r++) {
|
for (let r = DATA_START_ROW_INDEX; r <= range.e.r; r++) {
|
||||||
const row = {};
|
const values = [];
|
||||||
let hasData = false;
|
let hasData = false;
|
||||||
|
|
||||||
for (let c = range.s.c; c <= range.e.c; c++) {
|
for (let c = range.s.c; c <= range.e.c; c++) {
|
||||||
const excelHeader = rawHeaders[c - range.s.c];
|
const val = getValue(r, c);
|
||||||
const pbField = headerMap[excelHeader];
|
values.push(val ?? null);
|
||||||
if (!pbField) continue;
|
|
||||||
let val = getValue(r, c);
|
|
||||||
if (val !== null && val !== undefined) {
|
|
||||||
const s = String(val).trim();
|
|
||||||
if (s === "" || /^n\/?a$/i.test(s) || s === "#N/A") val = null;
|
|
||||||
}
|
|
||||||
row[pbField] = val;
|
|
||||||
if (val !== null && val !== "") hasData = true;
|
if (val !== null && val !== "") hasData = true;
|
||||||
}
|
}
|
||||||
if (hasData) rows.push(row);
|
if (!hasData) continue;
|
||||||
|
|
||||||
|
const rowObj = {};
|
||||||
|
headerMapping.forEach((map, i) => {
|
||||||
|
if (map.isMapped) {
|
||||||
|
rowObj[map.pbField] = normalizeForPB(values[i], map.type);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
rows.push(rowObj);
|
||||||
}
|
}
|
||||||
|
|
||||||
log(`Loaded ${rows.length} real jobs`);
|
return {
|
||||||
if (rows[0]) {
|
data: rows, // parsed row objects
|
||||||
log(`First job preview: Job_Number=${rows[0].Job_Number}, pb_id=${rows[0].id}`);
|
headerMapping, // rich info about headers
|
||||||
|
rawHeaders, // original headers for debugging
|
||||||
}
|
totalRows: rows.length // count of parsed rows
|
||||||
return rows;
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// -----------------------------
|
|
||||||
// HEADER MAP
|
|
||||||
// -----------------------------
|
|
||||||
const HEADER_MAP = {
|
|
||||||
"Job_Number": "Job_Number",
|
|
||||||
"Job_Full_Name": "Job_Full_Name",
|
|
||||||
"Job_Name": "Job_Name",
|
|
||||||
"Job Address": "Job_Address",
|
|
||||||
"Job Type": "Job_Type",
|
|
||||||
"Flag": "Flag",
|
|
||||||
"Tax_Exempt": "Tax_Exempt",
|
|
||||||
"Job Status": "Job_Status",
|
|
||||||
"Job Division": "Job_Division",
|
|
||||||
"Estimator": "Estimator",
|
|
||||||
"Office Rep": "Office_Rep",
|
|
||||||
"Manager": "Project_Manager",
|
|
||||||
"Company/Client": "Company_Client",
|
|
||||||
"Contact Person": "Contact_Person",
|
|
||||||
"Phone Number": "Phone_Number",
|
|
||||||
"Email": "Email",
|
|
||||||
"Due Date": "Due_Date",
|
|
||||||
"Due Date Source": "Due_Date_Source",
|
|
||||||
"Due Date Counter": "Due_Date_Counter",
|
|
||||||
"Due Time": "Due_Time",
|
|
||||||
"Tax_Exempt_Docs_Uploaded": "Tax_Exempt_Docs_Uploaded",
|
|
||||||
"Docs Uploaded": "Docs_Uploaded",
|
|
||||||
"QB Created": "QB_Created",
|
|
||||||
"Added to Calendar": "Added_To_Calendar",
|
|
||||||
"PSwift Uploaded": "PSwift_Uploaded",
|
|
||||||
"Voxer Created": "Voxer_Created",
|
|
||||||
"Estimate Reviewed": "Estimate_Reviewed",
|
|
||||||
"Estimate Draft": "Estimate_Draft",
|
|
||||||
"Estimate Approved": "Estimate_Approved",
|
|
||||||
"Start Date": "Start_Date",
|
|
||||||
"Schedule_Confidence": "Schedule_Confidence",
|
|
||||||
"Notes": "Notes",
|
|
||||||
"Job QB Link": "Job_QB_Link",
|
|
||||||
"Voxer Link": "Voxer_Link",
|
|
||||||
"Job_Codes": "Job_Codes",
|
|
||||||
"Submission Date": "Submission_Date",
|
|
||||||
"Active": "Active",
|
|
||||||
"Job Folder Link": "Job_Folder_Link",
|
|
||||||
"pb_id": "id"
|
|
||||||
};
|
|
||||||
|
|
||||||
// -----------------------------
|
// -----------------------------
|
||||||
// MAIN
|
// MAIN
|
||||||
// -----------------------------
|
// -----------------------------
|
||||||
let parsedRows = [];
|
let parsedRows = [];
|
||||||
|
let rawHeaders = [];
|
||||||
(async () => {
|
(async () => {
|
||||||
try {
|
try {
|
||||||
log("🔵 Starting Excel Sync…");
|
log("🔵 Starting Excel Sync…");
|
||||||
|
|
||||||
const graphToken = await getGraphToken();
|
const graphToken = await getGraphToken();
|
||||||
const pb = await getPocketBaseClient();
|
const pb = await getPocketBaseClient();
|
||||||
// These can be update if the Excel File changes
|
// These can be update if the Excel File changes
|
||||||
const driveId = "b!9YOqqr2xM0G2DFBwMEJYY4UzQgocFddEtpLYWuS9_AtkCKl2q8yjQJteQ7Ti4QSx";
|
const driveId = "b!9YOqqr2xM0G2DFBwMEJYY4UzQgocFddEtpLYWuS9_AtkCKl2q8yjQJteQ7Ti4QSx";
|
||||||
const itemId = "01SPNXLDSQTMG4F6BDANCLJU2FZKEDQLVU";
|
const itemId = "01SPNXLDSQTMG4F6BDANCLJU2FZKEDQLVU";
|
||||||
const excelBuffer = await downloadExcelFromGraph(graphToken, driveId, itemId);
|
const excelBuffer = await downloadExcelFromGraph(graphToken, driveId, itemId);
|
||||||
parsedRows = parseExcelFile(excelBuffer, HEADER_MAP);
|
const parsed = parseExcelFile(excelBuffer);
|
||||||
|
parsedRows = parsed.data;
|
||||||
// Run enhanced test (safe, no writes)
|
rawHeaders = parsed.rawHeaders;
|
||||||
runEnhancedTest(parsedRows, graphToken, pb);
|
log(rawHeaders)
|
||||||
// At this stage you have:
|
|
||||||
// - graphToken (for SharePoint updates later)
|
|
||||||
// - pb client (for PB updates)
|
|
||||||
// - rows[] mapped to PB fields
|
|
||||||
|
|
||||||
const updatedBuffer = exportExcelforDL(parsedRows);
|
|
||||||
await updateNotesInSharePoint(graphToken, driveId, itemId, "Job Sheet", parsedRows, pb, excelBuffer);
|
|
||||||
// Sync to PocketBase
|
|
||||||
//await syncRowsToPocketBase(parsedRows, pb);
|
|
||||||
|
|
||||||
log("✅ First part complete: tokens + Excel parsing ready");
|
log("✅ First part complete: tokens + Excel parsing ready");
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
log("❌ Sync failed: " + (err.message || err));
|
log("❌ Sync failed: " + (err.message || err));
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
// -----------------------------
|
|
||||||
// ENHANCED TEST HARNESS
|
|
||||||
// -----------------------------
|
|
||||||
function runEnhancedTest(rows, graphToken, pb) {
|
|
||||||
log("🔎 Running enhanced test (no writes)…");
|
|
||||||
|
|
||||||
// 1. Verify tokens
|
|
||||||
log(`Graph token length: ${graphToken.length}`);
|
|
||||||
log(`PocketBase token length: ${pb.authStore.token.length}`);
|
|
||||||
|
|
||||||
// 2. Verify Excel parsing
|
|
||||||
log(`Parsed ${rows.length} rows from Excel`);
|
|
||||||
if (rows.length > 0) {
|
|
||||||
const sample = rows[0];
|
|
||||||
log("Sample row preview:");
|
|
||||||
log(` Job_Number: ${sample.Job_Number}`);
|
|
||||||
log(` Job_Full_Name: ${sample.Job_Full_Name}`);
|
|
||||||
log(` Active: ${sample.Active}`);
|
|
||||||
log(` pb_id: ${sample.id}`);
|
|
||||||
log(` Notes: ${sample.Notes}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. Verify header mapping
|
|
||||||
const mappedFields = Object.values(HEADER_MAP);
|
|
||||||
const sampleKeys = Object.keys(rows[0] || {});
|
|
||||||
const missingFields = mappedFields.filter(f => !sampleKeys.includes(f));
|
|
||||||
if (missingFields.length > 0) {
|
|
||||||
log(`⚠️ Missing mapped fields in sample row: ${missingFields.join(", ")}`);
|
|
||||||
} else {
|
|
||||||
log("✅ All mapped fields present in sample row");
|
|
||||||
}
|
|
||||||
|
|
||||||
log("✅ Enhanced test complete — no writes performed");
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// -------------------------
|
// -------------------------
|
||||||
// Export to xlsx - Excel
|
// Export to xlsx - Excel
|
||||||
// ---------------------------
|
// ---------------------------
|
||||||
|
|
||||||
// Toggle this flag to enable/disable the export server
|
// Toggle this flag to enable/disable the export server
|
||||||
const ENABLE_EXPORT = false; // set to false to bypass download
|
const ENABLE_EXPORT = true; // set to false to bypass download
|
||||||
|
|
||||||
function exportExcelforDL(parsedRows) {
|
function exportExcelforDL(parsedRows) {
|
||||||
if (!parsedRows || parsedRows.length === 0) {
|
if (!parsedRows || parsedRows.length === 0) {
|
||||||
@@ -424,90 +353,89 @@ if (ENABLE_EXPORT) {
|
|||||||
// -----------------------------
|
// -----------------------------
|
||||||
// UPDATE NOTES CELLS IN SHAREPOINT (REAL-TIME)
|
// UPDATE NOTES CELLS IN SHAREPOINT (REAL-TIME)
|
||||||
// -----------------------------
|
// -----------------------------
|
||||||
function getNotesColumnLetterFromParsedHeaders(headerMap) {
|
//function getNotesColumnLetterFromParsedHeaders(rawHeaders) {
|
||||||
const headers = Object.keys(headerMap);
|
// const notesIndex = rawHeaders.findIndex(h => h.trim().toLowerCase() === "notes");
|
||||||
const notesIndex = headers.findIndex(h => h.trim().toLowerCase() === "notes");
|
// if (notesIndex === -1) {
|
||||||
if (notesIndex === -1) {
|
// log(`❌ 'Notes' header not found in HEADER_MAP keys: ${rawHeaders.join(", ")}`);
|
||||||
log(`❌ 'Notes' header not found in HEADER_MAP keys: ${headers.join(", ")}`);
|
// throw new Error("Notes column not found");
|
||||||
throw new Error("Notes column not found");
|
// }
|
||||||
}
|
// return columnLetter(notesIndex);
|
||||||
return columnLetter(notesIndex);
|
//}
|
||||||
}
|
//await updateNotesInSharePoint(graphToken, driveId, itemId, "Job Sheet", parsedRows, pb, excelBuffer);
|
||||||
|
|
||||||
// Update only changed Notes cells using workbook session + PATCH.
|
// Update only changed Notes cells using workbook session + PATCH.
|
||||||
// This version relies solely on DATA_START_ROW_INDEX for row mapping.
|
// This version relies solely on DATA_START_ROW_INDEX for row mapping.
|
||||||
async function updateNotesInSharePoint(graphToken, driveId, itemId, worksheetName, parsedRows, pb, workbookBuffer) {
|
//async function updateNotesInSharePoint(graphToken, driveId, itemId, worksheetName, parsedRows, pb, workbookBuffer) {
|
||||||
const wb = XLSX.read(workbookBuffer, { type: "buffer", cellDates: true });
|
// const wb = XLSX.read(workbookBuffer, { type: "buffer", cellDates: true });
|
||||||
const sheet = wb.Sheets[worksheetName];
|
// const sheet = wb.Sheets[worksheetName];
|
||||||
if (!sheet) throw new Error(`Worksheet "${worksheetName}" not found`);
|
// if (!sheet) throw new Error(`Worksheet "${worksheetName}" not found`);
|
||||||
|
|
||||||
// Column detection aligned to HEADER_MAP (same mapping used by parseExcelFile)
|
// Column detection aligned to HEADER_MAP (same mapping used by parseExcelFile)
|
||||||
const notesColumnLetter = getNotesColumnLetterFromParsedHeaders(HEADER_MAP);
|
//const notesColumnLetter = getNotesColumnLetterFromParsedHeaders(rawHeaders);
|
||||||
log(`✅ Notes column resolved to ${notesColumnLetter} via HEADER_MAP`);
|
//og(`✅ Notes column resolved to ${notesColumnLetter} via rawHeaders`);
|
||||||
|
|
||||||
// Create a workbook session that persists changes
|
// Create a workbook session that persists changes
|
||||||
const sessionRes = await axios.post(
|
//const sessionRes = await axios.post(
|
||||||
`https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/createSession`,
|
// `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/createSession`,
|
||||||
{ persistChanges: true },
|
// { persistChanges: true },
|
||||||
{ headers: { Authorization: `Bearer ${graphToken}` } }
|
// { headers: { Authorization: `Bearer ${graphToken}` } }
|
||||||
);
|
// );
|
||||||
const sessionId = sessionRes.data.id;
|
// const sessionId = sessionRes.data.id;
|
||||||
|
|
||||||
let changes = 0;
|
// let changes = 0;
|
||||||
|
|
||||||
// Loop parsed rows; map i to Excel row using DATA_START_ROW_INDEX (zero-based)
|
// Loop parsed rows; map i to Excel row using DATA_START_ROW_INDEX (zero-based)
|
||||||
for (let i = 0; i < parsedRows.length; i++) {
|
// for (let i = 0; i < parsedRows.length; i++) {
|
||||||
const row = parsedRows[i];
|
// const row = parsedRows[i];
|
||||||
const jobNumber = row.Job_Number;
|
// const jobNumber = row.Job_Number;
|
||||||
if (jobNumber == null || jobNumber === "") continue;
|
// if (jobNumber == null || jobNumber === "") continue;
|
||||||
|
|
||||||
try {
|
// try {
|
||||||
const records = await pb.collection(pbCollection).getFullList({
|
// const records = await pb.collection(pbCollection).getFullList({
|
||||||
filter: Number.isFinite(jobNumber) ? `Job_Number=${jobNumber}` : `Job_Number="${jobNumber}"`,
|
// filter: Number.isFinite(jobNumber) ? `Job_Number=${jobNumber}` : `Job_Number="${jobNumber}"`,
|
||||||
batchSize: 1
|
// batchSize: 1
|
||||||
});
|
// });
|
||||||
if (records.length === 0) continue;
|
// if (records.length === 0) continue;
|
||||||
|
|
||||||
// Normalize notes for fair comparison
|
// Normalize notes for fair comparison
|
||||||
const normalize = v => (v ?? "").toString().replace(/\s+/g, " ").trim();
|
// const normalize = v => (v ?? "").toString().replace(/\s+/g, " ").trim();
|
||||||
const pbNotes = normalize(records[0].Notes);
|
// const pbNotes = normalize(records[0].Notes);
|
||||||
const excelNotes = normalize(row.Notes);
|
// const excelNotes = normalize(row.Notes);
|
||||||
|
|
||||||
if (pbNotes !== excelNotes) {
|
// if (pbNotes !== excelNotes) {
|
||||||
// Excel uses 1-based row numbers; DATA_START_ROW_INDEX is zero-based
|
// Excel uses 1-based row numbers; DATA_START_ROW_INDEX is zero-based
|
||||||
const excelRowNumber = DATA_START_ROW_INDEX + i + 1;
|
// const excelRowNumber = DATA_START_ROW_INDEX + i + 1;
|
||||||
const cellAddress = `${notesColumnLetter}${excelRowNumber}`;
|
// const cellAddress = `${notesColumnLetter}${excelRowNumber}`;
|
||||||
|
|
||||||
log(`✏️ Notes change for Job_Number=${jobNumber}`);
|
// log(`✏️ Notes change for Job_Number=${jobNumber}`);
|
||||||
log(` Before: "${excelNotes}"`);
|
// log(` Before: "${excelNotes}"`);
|
||||||
log(` After : "${pbNotes}"`);
|
// log(` After : "${pbNotes}"`);
|
||||||
log(`➡️ Writing to ${worksheetName}!${cellAddress}`);
|
// log(`➡️ Writing to ${worksheetName}!${cellAddress}`);
|
||||||
|
|
||||||
await axios.patch(
|
// await axios.patch(
|
||||||
`https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/worksheets('${worksheetName}')/range(address='${cellAddress}')`,
|
// `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/worksheets('${worksheetName}')/range(address='${cellAddress}')`,
|
||||||
{ values: [[pbNotes]] },
|
// { values: [[pbNotes]] },
|
||||||
{
|
// {
|
||||||
headers: {
|
// headers: {
|
||||||
Authorization: `Bearer ${graphToken}`,
|
// Authorization: `Bearer ${graphToken}`,
|
||||||
"Content-Type": "application/json",
|
// "Content-Type": "application/json",
|
||||||
"workbook-session-id": sessionId
|
// "workbook-session-id": sessionId
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
);
|
// );
|
||||||
|
|
||||||
row.Notes = pbNotes;
|
// row.Notes = pbNotes;
|
||||||
changes++;
|
// changes++;
|
||||||
}
|
// }
|
||||||
} catch (err) {
|
// } catch (err) {
|
||||||
log(`❌ Failed for Job_Number=${jobNumber}: ${err.message}`);
|
// log(`❌ Failed for Job_Number=${jobNumber}: ${err.message}`);
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
await axios.post(
|
// await axios.post(
|
||||||
`https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/closeSession`,
|
// `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/closeSession`,
|
||||||
{},
|
// {},
|
||||||
{ headers: { Authorization: `Bearer ${graphToken}`, "workbook-session-id": sessionId } }
|
// { headers: { Authorization: `Bearer ${graphToken}`, "workbook-session-id": sessionId } }
|
||||||
);
|
// );
|
||||||
|
|
||||||
log(`✅ Notes-only sync complete. Changes applied: ${changes}`);
|
// log(`✅ Notes-only sync complete. Changes applied: ${changes}`);
|
||||||
}
|
//}
|
||||||
@@ -2,6 +2,7 @@
|
|||||||
<html lang="en">
|
<html lang="en">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8" />
|
<meta charset="UTF-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>SharePoint Link Encoder</title>
|
<title>SharePoint Link Encoder</title>
|
||||||
<style>
|
<style>
|
||||||
body { font-family: sans-serif; margin: 2rem; }
|
body { font-family: sans-serif; margin: 2rem; }
|
||||||
|
|||||||
Reference in New Issue
Block a user