INIT Backend
This commit is contained in:
@@ -0,0 +1,513 @@
|
||||
// excelsync.js
|
||||
import { FIELD_CONFIG } from "./fieldConfig.js";
|
||||
import fs from "fs";
|
||||
import axios from "axios";
|
||||
import qs from "qs";
|
||||
import * as XLSX from "xlsx";
|
||||
import dotenv from "dotenv";
|
||||
import { serve } from "bun";
|
||||
import PocketBase from "pocketbase"
|
||||
|
||||
dotenv.config();
|
||||
|
||||
// -----------------------------
|
||||
// EXCEL PARSER
|
||||
// -----------------------------
|
||||
const HEADER_ROW_INDEX = 2; // Row 3
|
||||
const DATA_START_ROW_INDEX = 3; // Row 4
|
||||
|
||||
// -----------------------------
|
||||
// CONFIG
|
||||
// -----------------------------
|
||||
const CONFIG = JSON.parse(fs.readFileSync("config.json", "utf8"));
|
||||
const tenantId = CONFIG.tenantId;
|
||||
const clientId = CONFIG.clientId;
|
||||
const sharepointFileUrl = CONFIG.sharepointFileUrl; // production Excel file URL
|
||||
const pbUrl = CONFIG.pbUrl;
|
||||
const pbCollection = CONFIG.pbCollection || "Job_Info_TestEnv";
|
||||
const pbEmail = CONFIG.pbEmail;
|
||||
|
||||
const clientSecret = process.env.CLIENT_SECRET;
|
||||
const pbPassword = process.env.PB_PASSWORD;
|
||||
|
||||
if (!tenantId || !clientId || !clientSecret || !sharepointFileUrl || !pbUrl || !pbEmail || !pbPassword) {
|
||||
console.error("❌ Missing required config/env values");
|
||||
process.exit(1);
|
||||
}
|
||||
console.log({
|
||||
tenantId,
|
||||
clientId,
|
||||
clientSecret: clientSecret ? clientSecret.slice(0,4) + "..." : null,
|
||||
sharepointFileUrl,
|
||||
pbUrl,
|
||||
pbEmail,
|
||||
pbPassword: pbPassword ? "***" : null
|
||||
});
|
||||
// -----------------------------
|
||||
// LOGGING
|
||||
// -----------------------------
|
||||
function log(message) {
|
||||
console.log(message);
|
||||
fs.appendFileSync("excelsync.log", message + "\n");
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// HEADER + ROW DETECTION HELPERS
|
||||
// -----------------------------
|
||||
//Gets the last column letter in the table converting from 0 index to excel actual column in letter form.
|
||||
function columnLetter(index) {
|
||||
let letter = "";
|
||||
while (index >= 0) {
|
||||
letter = String.fromCharCode((index % 26) + 65) + letter;
|
||||
index = Math.floor(index / 26) - 1;
|
||||
}
|
||||
return letter;
|
||||
}
|
||||
|
||||
const headerRowIndex = HEADER_ROW_INDEX + 1;
|
||||
|
||||
function getColumnLetterForHeader(headerName, sheet, headerRowIndex) {
|
||||
const headerRow = XLSX.utils.sheet_to_json(sheet, { header: 1, range: `${headerRowIndex}:${headerRowIndex}` })[0]
|
||||
.map(h => (h || "").toString().trim().toLowerCase());
|
||||
const idx = headerRow.findIndex(h => h === headerName.toLowerCase());
|
||||
if (idx === -1) {
|
||||
log(`❌ Header "${headerName}" not found. Row contents: ${JSON.stringify(headerRow)}`);
|
||||
throw new Error(`Header "${headerName}" not found on row ${headerRowIndex}`);
|
||||
}
|
||||
log(`✅ Header "${headerName}" found at column index ${idx}`);
|
||||
return columnLetter(idx);
|
||||
}
|
||||
function getNotesColumnLetterFromParsedHeaders(headerMap) {
|
||||
const headers = Object.keys(headerMap);
|
||||
const notesIndex = headers.findIndex(h => h.trim().toLowerCase() === "notes");
|
||||
if (notesIndex === -1) {
|
||||
log(`❌ 'Notes' header not found in HEADER_MAP keys: ${headers.join(", ")}`);
|
||||
throw new Error("Notes column not found");
|
||||
}
|
||||
return columnLetter(notesIndex);
|
||||
}
|
||||
// -----------------------------
|
||||
// GRAPH TOKEN
|
||||
// -----------------------------
|
||||
async function getGraphToken() {
|
||||
const tokenUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`;
|
||||
const body = qs.stringify({
|
||||
client_id: clientId,
|
||||
client_secret: clientSecret,
|
||||
scope: "https://graph.microsoft.com/.default",
|
||||
grant_type: "client_credentials",
|
||||
});
|
||||
const res = await axios.post(tokenUrl, body, {
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||
});
|
||||
log("✅ Graph token acquired");
|
||||
return res.data.access_token;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// GRAPH DOWNLOADER HELPER
|
||||
// -----------------------------
|
||||
async function downloadExcelFromGraph(graphToken, driveId, itemId) {
|
||||
const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/content`;
|
||||
const res = await axios.get(url, {
|
||||
headers: { Authorization: `Bearer ${graphToken}` },
|
||||
responseType: "arraybuffer", // important: get raw binary
|
||||
});
|
||||
return Buffer.from(res.data); // convert to Buffer for XLSX
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// GRAPH UPLOADER HELPER
|
||||
// -----------------------------
|
||||
async function uploadExcelToGraph(graphToken, driveId, itemId, buffer) {
|
||||
const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/content`;
|
||||
const res = await axios.put(url, buffer, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${graphToken}`,
|
||||
"Content-Type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||
}
|
||||
});
|
||||
log(`✅ Excel file updated in SharePoint: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
|
||||
|
||||
// -----------------------------
|
||||
// POCKETBASE LOGIN
|
||||
// -----------------------------
|
||||
async function getPocketBaseClient() {
|
||||
const pb = new PocketBase(pbUrl);
|
||||
await pb.collection("users").authWithPassword(pbEmail, pbPassword);
|
||||
log("✅ PocketBase token acquired");
|
||||
return pb;
|
||||
}
|
||||
|
||||
function parseExcelFile(input, headerMap) {
|
||||
let data;
|
||||
// In case of data from graph
|
||||
if (Buffer.isBuffer(input)) {
|
||||
data = input;
|
||||
log(`Reading Excel file from Graph buffer`);
|
||||
// In case of file path (Not used at this time)
|
||||
} else {
|
||||
data = fs.readFileSync(input);
|
||||
log(`Reading Excel file: ${input}`);
|
||||
}
|
||||
|
||||
const workbook = XLSX.read(data, { type: "buffer", cellDates: true });
|
||||
const sheet = workbook.Sheets["Job Sheet"]; // explicitly use Job Sheet
|
||||
const range = XLSX.utils.decode_range(sheet["!ref"]);
|
||||
const merges = sheet["!merges"] || [];
|
||||
|
||||
const getValue = (r, c) => {
|
||||
const addr = XLSX.utils.encode_cell({ r, c });
|
||||
const cell = sheet[addr];
|
||||
if (cell) return cell.v ?? cell.w ?? null;
|
||||
for (const m of merges) {
|
||||
if (m.s.r <= r && r <= m.e.r && m.s.c <= c && c <= m.e.c) {
|
||||
const master = sheet[XLSX.utils.encode_cell({ r: m.s.r, c: m.s.c })];
|
||||
return master ? (master.v ?? master.w ?? null) : null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
// Read headers
|
||||
const rawHeaders = [];
|
||||
for (let c = range.s.c; c <= range.e.c; c++) {
|
||||
let h = getValue(HEADER_ROW_INDEX, c);
|
||||
rawHeaders.push(h ? String(h).trim() : "");
|
||||
}
|
||||
log(`Found ${rawHeaders.length} headers in Row 3`);
|
||||
log(`Headers: ${rawHeaders.join(" | ")}`);
|
||||
// Compare Excel headers against HEADER_MAP
|
||||
const mappedHeaders = [];
|
||||
const unmappedHeaders = [];
|
||||
for (const h of rawHeaders) {
|
||||
if (headerMap[h]) {
|
||||
mappedHeaders.push([h, headerMap[h]]);
|
||||
} else {
|
||||
unmappedHeaders.push([h, ""]);
|
||||
}
|
||||
}
|
||||
|
||||
// Output as CSV-style rows
|
||||
console.log("\n📑 Header mapping preview (CSV style):");
|
||||
console.log("ExcelHeader,PocketBaseField");
|
||||
for (const [excel, pb] of mappedHeaders) {
|
||||
console.log(`${excel},${pb}`);
|
||||
}
|
||||
for (const [excel, pb] of unmappedHeaders) {
|
||||
console.log(`${excel},${pb}`);
|
||||
}
|
||||
|
||||
// Parse data rows
|
||||
const rows = [];
|
||||
for (let r = DATA_START_ROW_INDEX; r <= range.e.r; r++) {
|
||||
const row = {};
|
||||
let hasData = false;
|
||||
for (let c = range.s.c; c <= range.e.c; c++) {
|
||||
const excelHeader = rawHeaders[c - range.s.c];
|
||||
const pbField = headerMap[excelHeader];
|
||||
if (!pbField) continue;
|
||||
let val = getValue(r, c);
|
||||
if (val !== null && val !== undefined) {
|
||||
const s = String(val).trim();
|
||||
if (s === "" || /^n\/?a$/i.test(s) || s === "#N/A") val = null;
|
||||
}
|
||||
row[pbField] = val;
|
||||
if (val !== null && val !== "") hasData = true;
|
||||
}
|
||||
if (hasData) rows.push(row);
|
||||
}
|
||||
|
||||
log(`Loaded ${rows.length} real jobs`);
|
||||
if (rows[0]) {
|
||||
log(`First job preview: Job_Number=${rows[0].Job_Number}, pb_id=${rows[0].id}`);
|
||||
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// HEADER MAP
|
||||
// -----------------------------
|
||||
const HEADER_MAP = {
|
||||
"Job_Number": "Job_Number",
|
||||
"Job_Full_Name": "Job_Full_Name",
|
||||
"Job_Name": "Job_Name",
|
||||
"Job Address": "Job_Address",
|
||||
"Job Type": "Job_Type",
|
||||
"Flag": "Flag",
|
||||
"Tax_Exempt": "Tax_Exempt",
|
||||
"Job Status": "Job_Status",
|
||||
"Job Division": "Job_Division",
|
||||
"Estimator": "Estimator",
|
||||
"Office Rep": "Office_Rep",
|
||||
"Manager": "Project_Manager",
|
||||
"Company/Client": "Company_Client",
|
||||
"Contact Person": "Contact_Person",
|
||||
"Phone Number": "Phone_Number",
|
||||
"Email": "Email",
|
||||
"Due Date": "Due_Date",
|
||||
"Due Date Source": "Due_Date_Source",
|
||||
"Due Date Counter": "Due_Date_Counter",
|
||||
"Due Time": "Due_Time",
|
||||
"Tax_Exempt_Docs_Uploaded": "Tax_Exempt_Docs_Uploaded",
|
||||
"Docs Uploaded": "Docs_Uploaded",
|
||||
"QB Created": "QB_Created",
|
||||
"Added to Calendar": "Added_To_Calendar",
|
||||
"PSwift Uploaded": "PSwift_Uploaded",
|
||||
"Voxer Created": "Voxer_Created",
|
||||
"Estimate Reviewed": "Estimate_Reviewed",
|
||||
"Estimate Draft": "Estimate_Draft",
|
||||
"Estimate Approved": "Estimate_Approved",
|
||||
"Start Date": "Start_Date",
|
||||
"Schedule_Confidence": "Schedule_Confidence",
|
||||
"Notes": "Notes",
|
||||
"Job QB Link": "Job_QB_Link",
|
||||
"Voxer Link": "Voxer_Link",
|
||||
"Job_Codes": "Job_Codes",
|
||||
"Submission Date": "Submission_Date",
|
||||
"Active": "Active",
|
||||
"Job Folder Link": "Job_Folder_Link",
|
||||
"pb_id": "id"
|
||||
};
|
||||
|
||||
// -----------------------------
|
||||
// MAIN
|
||||
// -----------------------------
|
||||
let parsedRows = [];
|
||||
(async () => {
|
||||
try {
|
||||
log("🔵 Starting Excel Sync…");
|
||||
|
||||
const graphToken = await getGraphToken();
|
||||
const pb = await getPocketBaseClient();
|
||||
// These can be update if the Excel File changes
|
||||
const driveId = "b!9YOqqr2xM0G2DFBwMEJYY4UzQgocFddEtpLYWuS9_AtkCKl2q8yjQJteQ7Ti4QSx";
|
||||
const itemId = "01SPNXLDSQTMG4F6BDANCLJU2FZKEDQLVU";
|
||||
const excelBuffer = await downloadExcelFromGraph(graphToken, driveId, itemId);
|
||||
parsedRows = parseExcelFile(excelBuffer, HEADER_MAP);
|
||||
|
||||
// Run enhanced test (safe, no writes)
|
||||
runEnhancedTest(parsedRows, graphToken, pb);
|
||||
// At this stage you have:
|
||||
// - graphToken (for SharePoint updates later)
|
||||
// - pb client (for PB updates)
|
||||
// - rows[] mapped to PB fields
|
||||
|
||||
const updatedBuffer = exportExcelforDL(parsedRows);
|
||||
await updateNotesInSharePoint(graphToken, driveId, itemId, "Job Sheet", parsedRows, pb, excelBuffer);
|
||||
// Sync to PocketBase
|
||||
//await syncRowsToPocketBase(parsedRows, pb);
|
||||
|
||||
log("✅ First part complete: tokens + Excel parsing ready");
|
||||
} catch (err) {
|
||||
log("❌ Sync failed: " + (err.message || err));
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
// -----------------------------
|
||||
// ENHANCED TEST HARNESS
|
||||
// -----------------------------
|
||||
function runEnhancedTest(rows, graphToken, pb) {
|
||||
log("🔎 Running enhanced test (no writes)…");
|
||||
|
||||
// 1. Verify tokens
|
||||
log(`Graph token length: ${graphToken.length}`);
|
||||
log(`PocketBase token length: ${pb.authStore.token.length}`);
|
||||
|
||||
// 2. Verify Excel parsing
|
||||
log(`Parsed ${rows.length} rows from Excel`);
|
||||
if (rows.length > 0) {
|
||||
const sample = rows[0];
|
||||
log("Sample row preview:");
|
||||
log(` Job_Number: ${sample.Job_Number}`);
|
||||
log(` Job_Full_Name: ${sample.Job_Full_Name}`);
|
||||
log(` Active: ${sample.Active}`);
|
||||
log(` pb_id: ${sample.id}`);
|
||||
log(` Notes: ${sample.Notes}`);
|
||||
}
|
||||
|
||||
// 3. Verify header mapping
|
||||
const mappedFields = Object.values(HEADER_MAP);
|
||||
const sampleKeys = Object.keys(rows[0] || {});
|
||||
const missingFields = mappedFields.filter(f => !sampleKeys.includes(f));
|
||||
if (missingFields.length > 0) {
|
||||
log(`⚠️ Missing mapped fields in sample row: ${missingFields.join(", ")}`);
|
||||
} else {
|
||||
log("✅ All mapped fields present in sample row");
|
||||
}
|
||||
|
||||
log("✅ Enhanced test complete — no writes performed");
|
||||
}
|
||||
|
||||
|
||||
|
||||
// -------------------------
|
||||
// Export to xlsx - Excel
|
||||
// ---------------------------
|
||||
|
||||
// Toggle this flag to enable/disable the export server
|
||||
const ENABLE_EXPORT = false; // set to false to bypass download
|
||||
|
||||
function exportExcelforDL(parsedRows) {
|
||||
if (!parsedRows || parsedRows.length === 0) {
|
||||
throw new Error("⚠️ No rows to export");
|
||||
}
|
||||
|
||||
const worksheet = XLSX.utils.json_to_sheet(parsedRows);
|
||||
|
||||
// Ensure Job_Number is the first column
|
||||
const cols = Object.keys(parsedRows[0] || {});
|
||||
if (cols.includes("Job_Number")) {
|
||||
const ordered = ["Job_Number", ...cols.filter(c => c !== "Job_Number")];
|
||||
const remapped = parsedRows.map(r =>
|
||||
ordered.reduce((acc, key) => { acc[key] = r[key]; return acc; }, {})
|
||||
);
|
||||
const ws2 = XLSX.utils.json_to_sheet(remapped);
|
||||
Object.assign(worksheet, ws2);
|
||||
}
|
||||
|
||||
const workbook = XLSX.utils.book_new();
|
||||
XLSX.utils.book_append_sheet(workbook, worksheet, "ParsedData");
|
||||
|
||||
return XLSX.write(workbook, { type: "buffer", bookType: "xlsx" });
|
||||
}
|
||||
|
||||
if (ENABLE_EXPORT) {
|
||||
const PORT = 3000;
|
||||
serve({
|
||||
port: PORT,
|
||||
fetch(req) {
|
||||
const url = new URL(req.url);
|
||||
|
||||
if (url.pathname === "/") {
|
||||
const html = `
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head><meta charset="utf-8"><title>ExcelSync Export</title></head>
|
||||
<body>
|
||||
<h1>ExcelSync Export</h1>
|
||||
<p>Click below to download the parsed Excel file:</p>
|
||||
<button onclick="window.location.href='/download'">Download ParsedOutput.xlsx</button>
|
||||
</body>
|
||||
</html>`;
|
||||
return new Response(html, { headers: { "Content-Type": "text/html" } });
|
||||
}
|
||||
|
||||
if (url.pathname === "/download") {
|
||||
try {
|
||||
const buffer = exportExcelforDL(parsedRows);
|
||||
return new Response(buffer, {
|
||||
headers: {
|
||||
"Content-Type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"Content-Disposition": "attachment; filename=ParsedOutput.xlsx"
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
return new Response("❌ Failed to generate Excel file: " + err.message, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
return new Response("Visit / to open the export page.");
|
||||
}
|
||||
});
|
||||
|
||||
log(`🚀 ExcelSync export server running at http://localhost:${PORT}/`);
|
||||
log(`👉 Open this URL in your browser to verify and download the parsed Excel file.`);
|
||||
} else {
|
||||
log("⏭️ Export server bypassed (ENABLE_EXPORT=false)");
|
||||
}
|
||||
|
||||
|
||||
// -----------------------------
|
||||
// UPDATE NOTES CELLS IN SHAREPOINT (REAL-TIME)
|
||||
// -----------------------------
|
||||
function getNotesColumnLetterFromParsedHeaders(headerMap) {
|
||||
const headers = Object.keys(headerMap);
|
||||
const notesIndex = headers.findIndex(h => h.trim().toLowerCase() === "notes");
|
||||
if (notesIndex === -1) {
|
||||
log(`❌ 'Notes' header not found in HEADER_MAP keys: ${headers.join(", ")}`);
|
||||
throw new Error("Notes column not found");
|
||||
}
|
||||
return columnLetter(notesIndex);
|
||||
}
|
||||
|
||||
// Update only changed Notes cells using workbook session + PATCH.
|
||||
// This version relies solely on DATA_START_ROW_INDEX for row mapping.
|
||||
async function updateNotesInSharePoint(graphToken, driveId, itemId, worksheetName, parsedRows, pb, workbookBuffer) {
|
||||
const wb = XLSX.read(workbookBuffer, { type: "buffer", cellDates: true });
|
||||
const sheet = wb.Sheets[worksheetName];
|
||||
if (!sheet) throw new Error(`Worksheet "${worksheetName}" not found`);
|
||||
|
||||
// Column detection aligned to HEADER_MAP (same mapping used by parseExcelFile)
|
||||
const notesColumnLetter = getNotesColumnLetterFromParsedHeaders(HEADER_MAP);
|
||||
log(`✅ Notes column resolved to ${notesColumnLetter} via HEADER_MAP`);
|
||||
|
||||
// Create a workbook session that persists changes
|
||||
const sessionRes = await axios.post(
|
||||
`https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/createSession`,
|
||||
{ persistChanges: true },
|
||||
{ headers: { Authorization: `Bearer ${graphToken}` } }
|
||||
);
|
||||
const sessionId = sessionRes.data.id;
|
||||
|
||||
let changes = 0;
|
||||
|
||||
// Loop parsed rows; map i to Excel row using DATA_START_ROW_INDEX (zero-based)
|
||||
for (let i = 0; i < parsedRows.length; i++) {
|
||||
const row = parsedRows[i];
|
||||
const jobNumber = row.Job_Number;
|
||||
if (jobNumber == null || jobNumber === "") continue;
|
||||
|
||||
try {
|
||||
const records = await pb.collection(pbCollection).getFullList({
|
||||
filter: Number.isFinite(jobNumber) ? `Job_Number=${jobNumber}` : `Job_Number="${jobNumber}"`,
|
||||
batchSize: 1
|
||||
});
|
||||
if (records.length === 0) continue;
|
||||
|
||||
// Normalize notes for fair comparison
|
||||
const normalize = v => (v ?? "").toString().replace(/\s+/g, " ").trim();
|
||||
const pbNotes = normalize(records[0].Notes);
|
||||
const excelNotes = normalize(row.Notes);
|
||||
|
||||
if (pbNotes !== excelNotes) {
|
||||
// Excel uses 1-based row numbers; DATA_START_ROW_INDEX is zero-based
|
||||
const excelRowNumber = DATA_START_ROW_INDEX + i + 1;
|
||||
const cellAddress = `${notesColumnLetter}${excelRowNumber}`;
|
||||
|
||||
log(`✏️ Notes change for Job_Number=${jobNumber}`);
|
||||
log(` Before: "${excelNotes}"`);
|
||||
log(` After : "${pbNotes}"`);
|
||||
log(`➡️ Writing to ${worksheetName}!${cellAddress}`);
|
||||
|
||||
await axios.patch(
|
||||
`https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/worksheets('${worksheetName}')/range(address='${cellAddress}')`,
|
||||
{ values: [[pbNotes]] },
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${graphToken}`,
|
||||
"Content-Type": "application/json",
|
||||
"workbook-session-id": sessionId
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
row.Notes = pbNotes;
|
||||
changes++;
|
||||
}
|
||||
} catch (err) {
|
||||
log(`❌ Failed for Job_Number=${jobNumber}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
await axios.post(
|
||||
`https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/closeSession`,
|
||||
{},
|
||||
{ headers: { Authorization: `Bearer ${graphToken}`, "workbook-session-id": sessionId } }
|
||||
);
|
||||
|
||||
log(`✅ Notes-only sync complete. Changes applied: ${changes}`);
|
||||
}
|
||||
@@ -0,0 +1,370 @@
|
||||
// excelsync.js
|
||||
import fs from "fs";
|
||||
import axios from "axios";
|
||||
import qs from "qs";
|
||||
import * as XLSX from "xlsx";
|
||||
import dotenv from "dotenv";
|
||||
import { serve } from "bun";
|
||||
import PocketBase from "pocketbase"
|
||||
|
||||
dotenv.config();
|
||||
|
||||
// -----------------------------
|
||||
// CONFIG
|
||||
// -----------------------------
|
||||
const CONFIG = JSON.parse(fs.readFileSync("config.json", "utf8"));
|
||||
const tenantId = CONFIG.tenantId;
|
||||
const clientId = CONFIG.clientId;
|
||||
const sharepointFileUrl = CONFIG.sharepointFileUrl; // production Excel file URL
|
||||
const pbUrl = CONFIG.pbUrl;
|
||||
const pbCollection = CONFIG.pbCollection || "Job_Info_TestEnv";
|
||||
const pbEmail = CONFIG.pbEmail;
|
||||
|
||||
const clientSecret = process.env.CLIENT_SECRET;
|
||||
const pbPassword = process.env.PB_PASSWORD;
|
||||
|
||||
if (!tenantId || !clientId || !clientSecret || !sharepointFileUrl || !pbUrl || !pbEmail || !pbPassword) {
|
||||
console.error("❌ Missing required config/env values");
|
||||
process.exit(1);
|
||||
}
|
||||
console.log({
|
||||
tenantId,
|
||||
clientId,
|
||||
clientSecret: clientSecret ? clientSecret.slice(0,4) + "..." : null,
|
||||
sharepointFileUrl,
|
||||
pbUrl,
|
||||
pbEmail,
|
||||
pbPassword: pbPassword ? "***" : null
|
||||
});
|
||||
// -----------------------------
|
||||
// LOGGING
|
||||
// -----------------------------
|
||||
function log(message) {
|
||||
console.log(message);
|
||||
fs.appendFileSync("src/excelsync.log", message + "\n");
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// GRAPH TOKEN
|
||||
// -----------------------------
|
||||
async function getGraphToken() {
|
||||
const tokenUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`;
|
||||
const body = qs.stringify({
|
||||
client_id: clientId,
|
||||
client_secret: clientSecret,
|
||||
scope: "https://graph.microsoft.com/.default",
|
||||
grant_type: "client_credentials",
|
||||
});
|
||||
const res = await axios.post(tokenUrl, body, {
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||
});
|
||||
log("✅ Graph token acquired");
|
||||
return res.data.access_token;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// GRAPH DOWNLOADER HELPER
|
||||
// -----------------------------
|
||||
async function downloadExcelFromGraph(graphToken, driveId, itemId) {
|
||||
const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/content`;
|
||||
const res = await axios.get(url, {
|
||||
headers: { Authorization: `Bearer ${graphToken}` },
|
||||
responseType: "arraybuffer", // important: get raw binary
|
||||
});
|
||||
return Buffer.from(res.data); // convert to Buffer for XLSX
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// POCKETBASE LOGIN
|
||||
// -----------------------------
|
||||
async function getPocketBaseClient() {
|
||||
const pb = new PocketBase(pbUrl);
|
||||
await pb.collection("users").authWithPassword(pbEmail, pbPassword);
|
||||
log("✅ PocketBase token acquired");
|
||||
return pb;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// EXCEL PARSER
|
||||
// -----------------------------
|
||||
const HEADER_ROW_INDEX = 2; // Row 3
|
||||
const DATA_START_ROW_INDEX = 3; // Row 4
|
||||
|
||||
function parseExcelFile(input, headerMap) {
|
||||
let data;
|
||||
// In case of data from graph
|
||||
if (Buffer.isBuffer(input)) {
|
||||
data = input;
|
||||
log(`Reading Excel file from Graph buffer`);
|
||||
// In case of file path (Not used at this time)
|
||||
} else {
|
||||
data = fs.readFileSync(input);
|
||||
log(`Reading Excel file: ${input}`);
|
||||
}
|
||||
|
||||
const workbook = XLSX.read(data, { type: "buffer", cellDates: true });
|
||||
const sheet = workbook.Sheets["Job Sheet"]; // explicitly use Job Sheet
|
||||
const range = XLSX.utils.decode_range(sheet["!ref"]);
|
||||
const merges = sheet["!merges"] || [];
|
||||
|
||||
const getValue = (r, c) => {
|
||||
const addr = XLSX.utils.encode_cell({ r, c });
|
||||
const cell = sheet[addr];
|
||||
if (cell) return cell.v ?? cell.w ?? null;
|
||||
for (const m of merges) {
|
||||
if (m.s.r <= r && r <= m.e.r && m.s.c <= c && c <= m.e.c) {
|
||||
const master = sheet[XLSX.utils.encode_cell({ r: m.s.r, c: m.s.c })];
|
||||
return master ? (master.v ?? master.w ?? null) : null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
// Read headers
|
||||
const rawHeaders = [];
|
||||
for (let c = range.s.c; c <= range.e.c; c++) {
|
||||
let h = getValue(HEADER_ROW_INDEX, c);
|
||||
rawHeaders.push(h ? String(h).trim() : "");
|
||||
}
|
||||
log(`Found ${rawHeaders.length} headers in Row 3`);
|
||||
log(`Headers: ${rawHeaders.join(" | ")}`);
|
||||
// Compare Excel headers against HEADER_MAP
|
||||
const mappedHeaders = [];
|
||||
const unmappedHeaders = [];
|
||||
for (const h of rawHeaders) {
|
||||
if (headerMap[h]) {
|
||||
mappedHeaders.push([h, headerMap[h]]);
|
||||
} else {
|
||||
unmappedHeaders.push([h, ""]);
|
||||
}
|
||||
}
|
||||
|
||||
// Output as CSV-style rows
|
||||
console.log("\n📑 Header mapping preview (CSV style):");
|
||||
console.log("ExcelHeader,PocketBaseField");
|
||||
for (const [excel, pb] of mappedHeaders) {
|
||||
console.log(`${excel},${pb}`);
|
||||
}
|
||||
for (const [excel, pb] of unmappedHeaders) {
|
||||
console.log(`${excel},${pb}`);
|
||||
}
|
||||
|
||||
// Parse data rows
|
||||
const rows = [];
|
||||
for (let r = DATA_START_ROW_INDEX; r <= range.e.r; r++) {
|
||||
const row = {};
|
||||
let hasData = false;
|
||||
for (let c = range.s.c; c <= range.e.c; c++) {
|
||||
const excelHeader = rawHeaders[c - range.s.c];
|
||||
const pbField = headerMap[excelHeader];
|
||||
if (!pbField) continue;
|
||||
let val = getValue(r, c);
|
||||
if (val !== null && val !== undefined) {
|
||||
const s = String(val).trim();
|
||||
if (s === "" || /^n\/?a$/i.test(s) || s === "#N/A") val = null;
|
||||
}
|
||||
row[pbField] = val;
|
||||
if (val !== null && val !== "") hasData = true;
|
||||
}
|
||||
if (hasData) rows.push(row);
|
||||
}
|
||||
|
||||
log(`Loaded ${rows.length} real jobs`);
|
||||
if (rows[0]) {
|
||||
log(`First job preview: Job_Number=${rows[0].Job_Number}, pb_id=${rows[0].id}`);
|
||||
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// HEADER MAP
|
||||
// -----------------------------
|
||||
const HEADER_MAP = {
|
||||
"Job_Number": "Job_Number",
|
||||
"Job_Full_Name": "Job_Full_Name",
|
||||
"Job_Name": "Job_Name",
|
||||
"Job Address": "Job_Address",
|
||||
"Job Type": "Job_Type",
|
||||
"Flag": "Flag",
|
||||
"Tax_Exempt": "Tax_Exempt",
|
||||
"Job Status": "Job_Status",
|
||||
"Job Division": "Division",
|
||||
"Estimator": "Estimator",
|
||||
"Office Rep": "Office_Rep",
|
||||
"Manager": "Project_Manager",
|
||||
"Company/Client": "Company_Client",
|
||||
"Contact Person": "Contact_Person",
|
||||
"Phone Number": "Phone_Number",
|
||||
"Email": "Email",
|
||||
"Due Date": "Due_Date",
|
||||
"Due Date Source": "Due_Date_Source",
|
||||
"Due Date Counter": "Due_Date_Counter",
|
||||
"Due Time": "Due_Time",
|
||||
"Tax_Exempt_Docs_Uploaded": "Tax_Exempt_Docs_Uploaded",
|
||||
"Docs Uploaded": "Docs_Uploaded",
|
||||
"QB Created": "QB_Created",
|
||||
"Added to Calendar": "Calendar_Added",
|
||||
"PSwift Uploaded": "PSwift_Uploaded",
|
||||
"Voxer Created": "Voxer_Created",
|
||||
"Estimate Reviewed": "Estimate_Reviewed",
|
||||
"Estimate Draft": "Estimate_Draft",
|
||||
"Estimate Approved": "Estimate_Approved",
|
||||
"Start Date": "Start_Date",
|
||||
"Schedule_Confidence": "Schedule_Confidence",
|
||||
"Notes": "Notes",
|
||||
"Job QB Link": "Job_QB_Link",
|
||||
"Voxer Link": "Voxer_Link",
|
||||
"Job_Codes": "Job_Codes",
|
||||
"Submission Date": "Submission_Date",
|
||||
"Active": "Active",
|
||||
"Job Folder Link": "Job_Folder_Link",
|
||||
"pb_id": "id"
|
||||
};
|
||||
|
||||
// -----------------------------
|
||||
// MAIN
|
||||
// -----------------------------
|
||||
let parsedRows = [];
|
||||
(async () => {
|
||||
try {
|
||||
log("🔵 Starting Excel Sync…");
|
||||
|
||||
const graphToken = await getGraphToken();
|
||||
const pb = await getPocketBaseClient();
|
||||
// These can be update if the Excel File changes
|
||||
const driveId = "b!9YOqqr2xM0G2DFBwMEJYY4UzQgocFddEtpLYWuS9_AtkCKl2q8yjQJteQ7Ti4QSx";
|
||||
const itemId = "01SPNXLDSQTMG4F6BDANCLJU2FZKEDQLVU";
|
||||
const excelBuffer = await downloadExcelFromGraph(graphToken, driveId, itemId);
|
||||
parsedRows = parseExcelFile(excelBuffer, HEADER_MAP);
|
||||
|
||||
// Run enhanced test (safe, no writes)
|
||||
runEnhancedTest(parsedRows, graphToken, pb);
|
||||
// At this stage you have:
|
||||
// - graphToken (for SharePoint updates later)
|
||||
// - pb client (for PB updates)
|
||||
// - rows[] mapped to PB fields
|
||||
|
||||
// Sync to PocketBase
|
||||
// await syncRowsToPocketBase(parsedRows, pb);
|
||||
|
||||
log("✅ First part complete: tokens + Excel parsing ready");
|
||||
} catch (err) {
|
||||
log("❌ Sync failed: " + (err.message || err));
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
// -----------------------------
|
||||
// ENHANCED TEST HARNESS
|
||||
// -----------------------------
|
||||
function runEnhancedTest(rows, graphToken, pb) {
|
||||
log("🔎 Running enhanced test (no writes)…");
|
||||
|
||||
// 1. Verify tokens
|
||||
log(`Graph token length: ${graphToken.length}`);
|
||||
log(`PocketBase token length: ${pb.authStore.token.length}`);
|
||||
|
||||
// 2. Verify Excel parsing
|
||||
log(`Parsed ${rows.length} rows from Excel`);
|
||||
if (rows.length > 0) {
|
||||
const sample = rows[0];
|
||||
log("Sample row preview:");
|
||||
log(` Job_Number: ${sample.Job_Number}`);
|
||||
log(` Job_Full_Name: ${sample.Job_Full_Name}`);
|
||||
log(` Active: ${sample.Active}`);
|
||||
log(` pb_id: ${sample.id}`);
|
||||
log(` Notes: ${sample.Notes}`);
|
||||
}
|
||||
|
||||
// 3. Verify header mapping
|
||||
const mappedFields = Object.values(HEADER_MAP);
|
||||
const sampleKeys = Object.keys(rows[0] || {});
|
||||
const missingFields = mappedFields.filter(f => !sampleKeys.includes(f));
|
||||
if (missingFields.length > 0) {
|
||||
log(`⚠️ Missing mapped fields in sample row: ${missingFields.join(", ")}`);
|
||||
} else {
|
||||
log("✅ All mapped fields present in sample row");
|
||||
}
|
||||
|
||||
log("✅ Enhanced test complete — no writes performed");
|
||||
}
|
||||
|
||||
|
||||
|
||||
// -------------------------
|
||||
// Export to xlsx - Excel
|
||||
// ---------------------------
|
||||
|
||||
function exportExcelforDL(parsedRows) {
|
||||
if (!parsedRows || parsedRows.length === 0) {
|
||||
throw new Error("⚠️ No rows to export");
|
||||
}
|
||||
|
||||
const worksheet = XLSX.utils.json_to_sheet(parsedRows);
|
||||
|
||||
// Ensure Job_Number is the first column
|
||||
const cols = Object.keys(parsedRows[0] || {});
|
||||
if (cols.includes("Job_Number")) {
|
||||
const ordered = ["Job_Number", ...cols.filter(c => c !== "Job_Number")];
|
||||
const remapped = parsedRows.map(r =>
|
||||
ordered.reduce((acc, key) => { acc[key] = r[key]; return acc; }, {})
|
||||
);
|
||||
const ws2 = XLSX.utils.json_to_sheet(remapped);
|
||||
Object.assign(worksheet, ws2);
|
||||
}
|
||||
|
||||
const workbook = XLSX.utils.book_new();
|
||||
XLSX.utils.book_append_sheet(workbook, worksheet, "ParsedData");
|
||||
|
||||
return XLSX.write(workbook, { type: "buffer", bookType: "xlsx" });
|
||||
}
|
||||
|
||||
function isParsedReady() {
|
||||
return Array.isArray(parsedRows) && parsedRows.length > 0;
|
||||
}
|
||||
|
||||
const PORT = 3000;
|
||||
|
||||
serve({
|
||||
port: PORT,
|
||||
fetch(req) {
|
||||
const url = new URL(req.url);
|
||||
|
||||
if (url.pathname === "/") {
|
||||
const html = `
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head><meta charset="utf-8"><title>ExcelSync Export</title></head>
|
||||
<body>
|
||||
<h1>ExcelSync Export</h1>
|
||||
<p>Click below to download the parsed Excel file:</p>
|
||||
<button onclick="window.location.href='/download'">Download ParsedOutput.xlsx</button>
|
||||
</body>
|
||||
</html>`;
|
||||
return new Response(html, { headers: { "Content-Type": "text/html" } });
|
||||
}
|
||||
|
||||
if (url.pathname === "/download") {
|
||||
try {
|
||||
if (!isParsedReady()) {
|
||||
return new Response("⚠️ Parsed rows are not ready yet. Try again in a moment.", { status: 409 });
|
||||
}
|
||||
const buffer = exportExcelforDL(parsedRows);
|
||||
return new Response(buffer, {
|
||||
headers: {
|
||||
"Content-Type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"Content-Disposition": "attachment; filename=ParsedOutput.xlsx"
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
return new Response("❌ Failed to generate Excel file: " + err.message, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
return new Response("Visit / to open the export page.");
|
||||
}
|
||||
});
|
||||
|
||||
// Prompt on startup
|
||||
log(`🚀 ExcelSync export server running at http://localhost:${PORT}/`);
|
||||
log(`👉 Open this URL in your browser to verify and download the parsed Excel file.`);
|
||||
@@ -0,0 +1,375 @@
|
||||
//notes - works to get excel downloaded bypass downloader with "false"
|
||||
// excelsync.js
|
||||
import fs from "fs";
|
||||
import axios from "axios";
|
||||
import qs from "qs";
|
||||
import * as XLSX from "xlsx";
|
||||
import dotenv from "dotenv";
|
||||
import { serve } from "bun";
|
||||
import PocketBase from "pocketbase"
|
||||
|
||||
dotenv.config();
|
||||
|
||||
// -----------------------------
|
||||
// CONFIG
|
||||
// -----------------------------
|
||||
const CONFIG = JSON.parse(fs.readFileSync("config.json", "utf8"));
|
||||
const tenantId = CONFIG.tenantId;
|
||||
const clientId = CONFIG.clientId;
|
||||
const sharepointFileUrl = CONFIG.sharepointFileUrl; // production Excel file URL
|
||||
const pbUrl = CONFIG.pbUrl;
|
||||
const pbCollection = CONFIG.pbCollection || "Job_Info_TestEnv";
|
||||
const pbEmail = CONFIG.pbEmail;
|
||||
|
||||
const clientSecret = process.env.CLIENT_SECRET;
|
||||
const pbPassword = process.env.PB_PASSWORD;
|
||||
|
||||
if (!tenantId || !clientId || !clientSecret || !sharepointFileUrl || !pbUrl || !pbEmail || !pbPassword) {
|
||||
console.error("❌ Missing required config/env values");
|
||||
process.exit(1);
|
||||
}
|
||||
console.log({
|
||||
tenantId,
|
||||
clientId,
|
||||
clientSecret: clientSecret ? clientSecret.slice(0,4) + "..." : null,
|
||||
sharepointFileUrl,
|
||||
pbUrl,
|
||||
pbEmail,
|
||||
pbPassword: pbPassword ? "***" : null
|
||||
});
|
||||
// -----------------------------
|
||||
// LOGGING
|
||||
// -----------------------------
|
||||
function log(message) {
|
||||
console.log(message);
|
||||
fs.appendFileSync("src/excelsync.log", message + "\n");
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// GRAPH TOKEN
|
||||
// -----------------------------
|
||||
async function getGraphToken() {
|
||||
const tokenUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`;
|
||||
const body = qs.stringify({
|
||||
client_id: clientId,
|
||||
client_secret: clientSecret,
|
||||
scope: "https://graph.microsoft.com/.default",
|
||||
grant_type: "client_credentials",
|
||||
});
|
||||
const res = await axios.post(tokenUrl, body, {
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||
});
|
||||
log("✅ Graph token acquired");
|
||||
return res.data.access_token;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// GRAPH DOWNLOADER HELPER
|
||||
// -----------------------------
|
||||
async function downloadExcelFromGraph(graphToken, driveId, itemId) {
|
||||
const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/content`;
|
||||
const res = await axios.get(url, {
|
||||
headers: { Authorization: `Bearer ${graphToken}` },
|
||||
responseType: "arraybuffer", // important: get raw binary
|
||||
});
|
||||
return Buffer.from(res.data); // convert to Buffer for XLSX
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// POCKETBASE LOGIN
|
||||
// -----------------------------
|
||||
async function getPocketBaseClient() {
|
||||
const pb = new PocketBase(pbUrl);
|
||||
await pb.collection("users").authWithPassword(pbEmail, pbPassword);
|
||||
log("✅ PocketBase token acquired");
|
||||
return pb;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// EXCEL PARSER
|
||||
// -----------------------------
|
||||
const HEADER_ROW_INDEX = 2; // Row 3
|
||||
const DATA_START_ROW_INDEX = 3; // Row 4
|
||||
|
||||
function parseExcelFile(input, headerMap) {
|
||||
let data;
|
||||
// In case of data from graph
|
||||
if (Buffer.isBuffer(input)) {
|
||||
data = input;
|
||||
log(`Reading Excel file from Graph buffer`);
|
||||
// In case of file path (Not used at this time)
|
||||
} else {
|
||||
data = fs.readFileSync(input);
|
||||
log(`Reading Excel file: ${input}`);
|
||||
}
|
||||
|
||||
const workbook = XLSX.read(data, { type: "buffer", cellDates: true });
|
||||
const sheet = workbook.Sheets["Job Sheet"]; // explicitly use Job Sheet
|
||||
const range = XLSX.utils.decode_range(sheet["!ref"]);
|
||||
const merges = sheet["!merges"] || [];
|
||||
|
||||
const getValue = (r, c) => {
|
||||
const addr = XLSX.utils.encode_cell({ r, c });
|
||||
const cell = sheet[addr];
|
||||
if (cell) return cell.v ?? cell.w ?? null;
|
||||
for (const m of merges) {
|
||||
if (m.s.r <= r && r <= m.e.r && m.s.c <= c && c <= m.e.c) {
|
||||
const master = sheet[XLSX.utils.encode_cell({ r: m.s.r, c: m.s.c })];
|
||||
return master ? (master.v ?? master.w ?? null) : null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
// Read headers
|
||||
const rawHeaders = [];
|
||||
for (let c = range.s.c; c <= range.e.c; c++) {
|
||||
let h = getValue(HEADER_ROW_INDEX, c);
|
||||
rawHeaders.push(h ? String(h).trim() : "");
|
||||
}
|
||||
log(`Found ${rawHeaders.length} headers in Row 3`);
|
||||
log(`Headers: ${rawHeaders.join(" | ")}`);
|
||||
// Compare Excel headers against HEADER_MAP
|
||||
const mappedHeaders = [];
|
||||
const unmappedHeaders = [];
|
||||
for (const h of rawHeaders) {
|
||||
if (headerMap[h]) {
|
||||
mappedHeaders.push([h, headerMap[h]]);
|
||||
} else {
|
||||
unmappedHeaders.push([h, ""]);
|
||||
}
|
||||
}
|
||||
|
||||
// Output as CSV-style rows
|
||||
console.log("\n📑 Header mapping preview (CSV style):");
|
||||
console.log("ExcelHeader,PocketBaseField");
|
||||
for (const [excel, pb] of mappedHeaders) {
|
||||
console.log(`${excel},${pb}`);
|
||||
}
|
||||
for (const [excel, pb] of unmappedHeaders) {
|
||||
console.log(`${excel},${pb}`);
|
||||
}
|
||||
|
||||
// Parse data rows
|
||||
const rows = [];
|
||||
for (let r = DATA_START_ROW_INDEX; r <= range.e.r; r++) {
|
||||
const row = {};
|
||||
let hasData = false;
|
||||
for (let c = range.s.c; c <= range.e.c; c++) {
|
||||
const excelHeader = rawHeaders[c - range.s.c];
|
||||
const pbField = headerMap[excelHeader];
|
||||
if (!pbField) continue;
|
||||
let val = getValue(r, c);
|
||||
if (val !== null && val !== undefined) {
|
||||
const s = String(val).trim();
|
||||
if (s === "" || /^n\/?a$/i.test(s) || s === "#N/A") val = null;
|
||||
}
|
||||
row[pbField] = val;
|
||||
if (val !== null && val !== "") hasData = true;
|
||||
}
|
||||
if (hasData) rows.push(row);
|
||||
}
|
||||
|
||||
log(`Loaded ${rows.length} real jobs`);
|
||||
if (rows[0]) {
|
||||
log(`First job preview: Job_Number=${rows[0].Job_Number}, pb_id=${rows[0].id}`);
|
||||
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// HEADER MAP
|
||||
// -----------------------------
|
||||
const HEADER_MAP = {
|
||||
"Job_Number": "Job_Number",
|
||||
"Job_Full_Name": "Job_Full_Name",
|
||||
"Job_Name": "Job_Name",
|
||||
"Job Address": "Job_Address",
|
||||
"Job Type": "Job_Type",
|
||||
"Flag": "Flag",
|
||||
"Tax_Exempt": "Tax_Exempt",
|
||||
"Job Status": "Job_Status",
|
||||
"Job Division": "Job_Division",
|
||||
"Estimator": "Estimator",
|
||||
"Office Rep": "Office_Rep",
|
||||
"Manager": "Project_Manager",
|
||||
"Company/Client": "Company_Client",
|
||||
"Contact Person": "Contact_Person",
|
||||
"Phone Number": "Phone_Number",
|
||||
"Email": "Email",
|
||||
"Due Date": "Due_Date",
|
||||
"Due Date Source": "Due_Date_Source",
|
||||
"Due Date Counter": "Due_Date_Counter",
|
||||
"Due Time": "Due_Time",
|
||||
"Tax_Exempt_Docs_Uploaded": "Tax_Exempt_Docs_Uploaded",
|
||||
"Docs Uploaded": "Docs_Uploaded",
|
||||
"QB Created": "QB_Created",
|
||||
"Added to Calendar": "Added_To_Calendar",
|
||||
"PSwift Uploaded": "PSwift_Uploaded",
|
||||
"Voxer Created": "Voxer_Created",
|
||||
"Estimate Reviewed": "Estimate_Reviewed",
|
||||
"Estimate Draft": "Estimate_Draft",
|
||||
"Estimate Approved": "Estimate_Approved",
|
||||
"Start Date": "Start_Date",
|
||||
"Schedule_Confidence": "Schedule_Confidence",
|
||||
"Notes": "Notes",
|
||||
"Job QB Link": "Job_QB_Link",
|
||||
"Voxer Link": "Voxer_Link",
|
||||
"Job_Codes": "Job_Codes",
|
||||
"Submission Date": "Submission_Date",
|
||||
"Active": "Active",
|
||||
"Job Folder Link": "Job_Folder_Link",
|
||||
"pb_id": "id"
|
||||
};
|
||||
|
||||
// -----------------------------
|
||||
// MAIN
|
||||
// -----------------------------
|
||||
let parsedRows = [];
|
||||
(async () => {
|
||||
try {
|
||||
log("🔵 Starting Excel Sync…");
|
||||
|
||||
const graphToken = await getGraphToken();
|
||||
const pb = await getPocketBaseClient();
|
||||
// These can be update if the Excel File changes
|
||||
const driveId = "b!9YOqqr2xM0G2DFBwMEJYY4UzQgocFddEtpLYWuS9_AtkCKl2q8yjQJteQ7Ti4QSx";
|
||||
const itemId = "01SPNXLDSQTMG4F6BDANCLJU2FZKEDQLVU";
|
||||
const excelBuffer = await downloadExcelFromGraph(graphToken, driveId, itemId);
|
||||
parsedRows = parseExcelFile(excelBuffer, HEADER_MAP);
|
||||
|
||||
// Run enhanced test (safe, no writes)
|
||||
runEnhancedTest(parsedRows, graphToken, pb);
|
||||
// At this stage you have:
|
||||
// - graphToken (for SharePoint updates later)
|
||||
// - pb client (for PB updates)
|
||||
// - rows[] mapped to PB fields
|
||||
|
||||
// Sync to PocketBase
|
||||
//await syncRowsToPocketBase(parsedRows, pb);
|
||||
|
||||
log("✅ First part complete: tokens + Excel parsing ready");
|
||||
} catch (err) {
|
||||
log("❌ Sync failed: " + (err.message || err));
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
// -----------------------------
|
||||
// ENHANCED TEST HARNESS
|
||||
// -----------------------------
|
||||
function runEnhancedTest(rows, graphToken, pb) {
|
||||
log("🔎 Running enhanced test (no writes)…");
|
||||
|
||||
// 1. Verify tokens
|
||||
log(`Graph token length: ${graphToken.length}`);
|
||||
log(`PocketBase token length: ${pb.authStore.token.length}`);
|
||||
|
||||
// 2. Verify Excel parsing
|
||||
log(`Parsed ${rows.length} rows from Excel`);
|
||||
if (rows.length > 0) {
|
||||
const sample = rows[0];
|
||||
log("Sample row preview:");
|
||||
log(` Job_Number: ${sample.Job_Number}`);
|
||||
log(` Job_Full_Name: ${sample.Job_Full_Name}`);
|
||||
log(` Active: ${sample.Active}`);
|
||||
log(` pb_id: ${sample.id}`);
|
||||
log(` Notes: ${sample.Notes}`);
|
||||
}
|
||||
|
||||
// 3. Verify header mapping
|
||||
const mappedFields = Object.values(HEADER_MAP);
|
||||
const sampleKeys = Object.keys(rows[0] || {});
|
||||
const missingFields = mappedFields.filter(f => !sampleKeys.includes(f));
|
||||
if (missingFields.length > 0) {
|
||||
log(`⚠️ Missing mapped fields in sample row: ${missingFields.join(", ")}`);
|
||||
} else {
|
||||
log("✅ All mapped fields present in sample row");
|
||||
}
|
||||
|
||||
log("✅ Enhanced test complete — no writes performed");
|
||||
}
|
||||
|
||||
|
||||
|
||||
// -------------------------
|
||||
// Export to xlsx - Excel
|
||||
// ---------------------------
|
||||
|
||||
// Toggle this flag to enable/disable the export server
|
||||
const ENABLE_EXPORT = true; // set to false to bypass download
|
||||
|
||||
function exportExcelforDL(parsedRows) {
|
||||
if (!parsedRows || parsedRows.length === 0) {
|
||||
throw new Error("⚠️ No rows to export");
|
||||
}
|
||||
|
||||
const worksheet = XLSX.utils.json_to_sheet(parsedRows);
|
||||
|
||||
// Ensure Job_Number is the first column
|
||||
const cols = Object.keys(parsedRows[0] || {});
|
||||
if (cols.includes("Job_Number")) {
|
||||
const ordered = ["Job_Number", ...cols.filter(c => c !== "Job_Number")];
|
||||
const remapped = parsedRows.map(r =>
|
||||
ordered.reduce((acc, key) => { acc[key] = r[key]; return acc; }, {})
|
||||
);
|
||||
const ws2 = XLSX.utils.json_to_sheet(remapped);
|
||||
Object.assign(worksheet, ws2);
|
||||
}
|
||||
|
||||
const workbook = XLSX.utils.book_new();
|
||||
XLSX.utils.book_append_sheet(workbook, worksheet, "ParsedData");
|
||||
|
||||
return XLSX.write(workbook, { type: "buffer", bookType: "xlsx" });
|
||||
}
|
||||
|
||||
if (ENABLE_EXPORT) {
|
||||
const PORT = 3000;
|
||||
serve({
|
||||
port: PORT,
|
||||
fetch(req) {
|
||||
const url = new URL(req.url);
|
||||
|
||||
if (url.pathname === "/") {
|
||||
const html = `
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head><meta charset="utf-8"><title>ExcelSync Export</title></head>
|
||||
<body>
|
||||
<h1>ExcelSync Export</h1>
|
||||
<p>Click below to download the parsed Excel file:</p>
|
||||
<button onclick="window.location.href='/download'">Download ParsedOutput.xlsx</button>
|
||||
</body>
|
||||
</html>`;
|
||||
return new Response(html, { headers: { "Content-Type": "text/html" } });
|
||||
}
|
||||
|
||||
if (url.pathname === "/download") {
|
||||
try {
|
||||
const buffer = exportExcelforDL(parsedRows);
|
||||
return new Response(buffer, {
|
||||
headers: {
|
||||
"Content-Type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"Content-Disposition": "attachment; filename=ParsedOutput.xlsx"
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
return new Response("❌ Failed to generate Excel file: " + err.message, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
return new Response("Visit / to open the export page.");
|
||||
}
|
||||
});
|
||||
|
||||
log(`🚀 ExcelSync export server running at http://localhost:${PORT}/`);
|
||||
log(`👉 Open this URL in your browser to verify and download the parsed Excel file.`);
|
||||
} else {
|
||||
log("⏭️ Export server bypassed (ENABLE_EXPORT=false)");
|
||||
}
|
||||
|
||||
|
||||
// -----------------------------
|
||||
// FINAL SYNC FUNCTION
|
||||
// -----------------------------
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
// fieldConfig.js
|
||||
export const FIELD_CONFIG = {
|
||||
id: {
|
||||
excelHeader: "pb_id",
|
||||
pbField: "id",
|
||||
type: "string",
|
||||
required: true
|
||||
},
|
||||
Job_Number: {
|
||||
excelHeader: "Job_Number",
|
||||
pbField: "Job_Number",
|
||||
type: "string"
|
||||
},
|
||||
Job_Full_Name: {
|
||||
excelHeader: "Job_Full_Name",
|
||||
pbField: "Job_Full_Name",
|
||||
type: "string"
|
||||
},
|
||||
Job_Name: {
|
||||
excelHeader: "Job_Name",
|
||||
pbField: "Job_Name",
|
||||
type: "string"
|
||||
},
|
||||
Job_Address: {
|
||||
excelHeader: "Job Address",
|
||||
pbField: "Job_Address",
|
||||
type: "string"
|
||||
},
|
||||
Job_Type: {
|
||||
excelHeader: "Job Type",
|
||||
pbField: "Job_Type",
|
||||
type: "string"
|
||||
},
|
||||
Job_Status: {
|
||||
excelHeader: "Job Status",
|
||||
pbField: "Job_Status",
|
||||
type: "string"
|
||||
},
|
||||
Job_Division: {
|
||||
excelHeader: "Job Division",
|
||||
pbField: "Job_Division",
|
||||
type: "string"
|
||||
},
|
||||
Estimator: {
|
||||
excelHeader: "Estimator",
|
||||
pbField: "Estimator",
|
||||
type: "string"
|
||||
},
|
||||
Office_Rep: {
|
||||
excelHeader: "Office Rep",
|
||||
pbField: "Office_Rep",
|
||||
type: "string"
|
||||
},
|
||||
Project_Manager: {
|
||||
excelHeader: "Manager",
|
||||
pbField: "Project_Manager",
|
||||
type: "string"
|
||||
},
|
||||
Company_Client: {
|
||||
excelHeader: "Company/Client",
|
||||
pbField: "Company_Client",
|
||||
type: "string"
|
||||
},
|
||||
Contact_Person: {
|
||||
excelHeader: "Contact Person",
|
||||
pbField: "Contact_Person",
|
||||
type: "string"
|
||||
},
|
||||
Phone_Number: {
|
||||
excelHeader: "Phone Number",
|
||||
pbField: "Phone_Number",
|
||||
type: "string"
|
||||
},
|
||||
Email: {
|
||||
excelHeader: "Email",
|
||||
pbField: "Email",
|
||||
type: "string"
|
||||
},
|
||||
Due_Date: {
|
||||
excelHeader: "Due Date",
|
||||
pbField: "Due_Date",
|
||||
type: "date"
|
||||
},
|
||||
Due_Date_Source: {
|
||||
excelHeader: "Due Date Source",
|
||||
pbField: "Due_Date_Source",
|
||||
type: "string"
|
||||
},
|
||||
Due_Date_Counter: {
|
||||
excelHeader: "Due Date Counter",
|
||||
pbField: "Due_Date_Counter",
|
||||
type: "string"
|
||||
},
|
||||
Due_Time: {
|
||||
excelHeader: "Due Time",
|
||||
pbField: "Due_Time",
|
||||
type: "string"
|
||||
},
|
||||
Docs_Uploaded: {
|
||||
excelHeader: "Docs Uploaded",
|
||||
pbField: "Docs_Uploaded",
|
||||
type: "bool"
|
||||
},
|
||||
QB_Created: {
|
||||
excelHeader: "QB Created",
|
||||
pbField: "QB_Created",
|
||||
type: "bool"
|
||||
},
|
||||
Added_To_Calendar: {
|
||||
excelHeader: "Added to Calendar",
|
||||
pbField: "Added_To_Calendar",
|
||||
type: "bool"
|
||||
},
|
||||
PSwift_Uploaded: {
|
||||
excelHeader: "PSwift Uploaded",
|
||||
pbField: "PSwift_Uploaded",
|
||||
type: "bool"
|
||||
},
|
||||
Voxer_Created: {
|
||||
excelHeader: "Voxer Created",
|
||||
pbField: "Voxer_Created",
|
||||
type: "bool"
|
||||
},
|
||||
Estimate_Reviewed: {
|
||||
excelHeader: "Estimate Reviewed",
|
||||
pbField: "Estimate_Reviewed",
|
||||
type: "bool"
|
||||
},
|
||||
Estimate_Draft: {
|
||||
excelHeader: "Estimate Draft",
|
||||
pbField: "Estimate_Draft",
|
||||
type: "bool"
|
||||
},
|
||||
Estimate_Approved: {
|
||||
excelHeader: "Estimate Approved",
|
||||
pbField: "Estimate_Approved",
|
||||
type: "bool"
|
||||
},
|
||||
Start_Date: {
|
||||
excelHeader: "Start Date",
|
||||
pbField: "Start_Date",
|
||||
type: "date"
|
||||
},
|
||||
Submission_Date: {
|
||||
excelHeader: "Submission Date",
|
||||
pbField: "Submission_Date",
|
||||
type: "date"
|
||||
},
|
||||
Active: {
|
||||
excelHeader: "Active",
|
||||
pbField: "Active",
|
||||
type: "bool"
|
||||
},
|
||||
Tax_Exempt: {
|
||||
excelHeader: "Tax_Exempt",
|
||||
pbField: "Tax_Exempt",
|
||||
type: "bool"
|
||||
},
|
||||
Tax_Exempt_Docs_Uploaded: {
|
||||
excelHeader: "Tax_Exempt_Docs_Uploaded",
|
||||
pbField: "Tax_Exempt_Docs_Uploaded",
|
||||
type: "bool"
|
||||
},
|
||||
Flag: {
|
||||
excelHeader: "Flag",
|
||||
pbField: "Flag",
|
||||
type: "bool"
|
||||
},
|
||||
Notes: {
|
||||
excelHeader: "Notes",
|
||||
pbField: "Notes",
|
||||
type: "string"
|
||||
},
|
||||
Job_QB_Link: {
|
||||
excelHeader: "Job QB Link",
|
||||
pbField: "Job_QB_Link",
|
||||
type: "string"
|
||||
},
|
||||
Voxer_Link: {
|
||||
excelHeader: "Voxer Link",
|
||||
pbField: "Voxer_Link",
|
||||
type: "string"
|
||||
},
|
||||
Job_Codes: {
|
||||
excelHeader: "Job_Codes",
|
||||
pbField: "Job_Codes",
|
||||
type: "string"
|
||||
},
|
||||
Job_Folder_Link: {
|
||||
excelHeader: "Job Folder Link",
|
||||
pbField: "Job_Folder_Link",
|
||||
type: "string"
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,26 @@
|
||||
import dotenv from "dotenv";
|
||||
import PocketBase from "pocketbase";
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const pb = new PocketBase(process.env.PB_URL);
|
||||
|
||||
const email = process.env.PB_EMAIL;
|
||||
const password = process.env.PB_PASSWORD;
|
||||
|
||||
if (!email || !password) {
|
||||
throw new Error("Missing PB_EMAIL or PB_PASSWORD in .env");
|
||||
}
|
||||
try {
|
||||
await pb.collection("users").authWithPassword(email, password);
|
||||
} catch (err: any) {
|
||||
console.error("Error response:", err.response);
|
||||
}
|
||||
|
||||
const authData = await pb.collection("users").authWithPassword(email, password);
|
||||
|
||||
console.log("Valid:", pb.authStore.isValid);
|
||||
console.log("Token:", pb.authStore.token);
|
||||
console.log("User ID:", pb.authStore.record?.id);
|
||||
|
||||
pb.authStore.clear();
|
||||
@@ -0,0 +1,77 @@
|
||||
// sharepointDownload.js
|
||||
import axios from "axios";
|
||||
import qs from "qs";
|
||||
import dotenv from "dotenv";
|
||||
import fs from "fs";
|
||||
|
||||
dotenv.config();
|
||||
|
||||
// -----------------------------
|
||||
// CONFIG
|
||||
// -----------------------------
|
||||
const CONFIG = JSON.parse(fs.readFileSync("config.json", "utf8"));
|
||||
const tenantId = CONFIG.tenantId;
|
||||
const clientId = CONFIG.clientId;
|
||||
|
||||
// Secret stays in .env
|
||||
const clientSecret = process.env.CLIENT_SECRET;
|
||||
|
||||
// -----------------------------
|
||||
// GRAPH TOKEN
|
||||
// -----------------------------
|
||||
async function getGraphToken() {
|
||||
const tokenUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`;
|
||||
const body = qs.stringify({
|
||||
client_id: clientId,
|
||||
client_secret: clientSecret,
|
||||
scope: "https://graph.microsoft.com/.default",
|
||||
grant_type: "client_credentials",
|
||||
});
|
||||
const res = await axios.post(tokenUrl, body, {
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||
});
|
||||
console.log("✅ Graph token acquired");
|
||||
return res.data.access_token;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// DOWNLOAD BY ITEM ID
|
||||
// -----------------------------
|
||||
// Example link: https://czflex.sharepoint.com/sites/Team/Shared%20Documents/General/JOBS%201000-9999.xlsx?d=wc20d9b5023f84403b4d345ca88382eb4
|
||||
// The ?d=... part is the itemId
|
||||
async function downloadFileById(graphToken, sitePath, itemId) {
|
||||
const url = `https://graph.microsoft.com/v1.0/sites/${sitePath}/drive/items/${itemId}/content`;
|
||||
const res = await axios.get(url, {
|
||||
headers: { Authorization: `Bearer ${graphToken}` },
|
||||
responseType: "arraybuffer",
|
||||
});
|
||||
console.log("✅ File downloaded, bytes:", res.data.byteLength);
|
||||
return res.data;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// MAIN
|
||||
// -----------------------------
|
||||
(async () => {
|
||||
try {
|
||||
const graphToken = await getGraphToken();
|
||||
|
||||
// sitePath matches your site
|
||||
const sitePath = "czflex.sharepoint.com:/sites/Team";
|
||||
|
||||
// itemId from your link (?d=...)
|
||||
const itemId = "wc20d9b5023f84403b4d345ca88382eb4";
|
||||
|
||||
const buffer = await downloadFileById(graphToken, sitePath, itemId);
|
||||
|
||||
// At this point you can feed buffer into XLSX.read(buffer, { type: "buffer", cellDates: true })
|
||||
// Example:
|
||||
// const workbook = XLSX.read(buffer, { type: "buffer", cellDates: true });
|
||||
// const sheetNames = workbook.SheetNames;
|
||||
// console.log("Sheets:", sheetNames);
|
||||
|
||||
} catch (err) {
|
||||
console.error("❌ Error:", err.response?.data || err.message || err);
|
||||
}
|
||||
})();
|
||||
"id": "czflex.sharepoint.com,aaaa83f5-b1bd-4133-b60c-507030425863,0a423385-151c-44d7-b692-d85ae4bdfc0b"
|
||||
@@ -0,0 +1,260 @@
|
||||
// excelsync.js
|
||||
import { FIELD_CONFIG } from "./fieldConfig.js";
|
||||
import fs from "fs";
|
||||
import axios from "axios";
|
||||
import qs from "qs";
|
||||
import dotenv from "dotenv";
|
||||
import PocketBase from "pocketbase";
|
||||
|
||||
dotenv.config();
|
||||
|
||||
// -----------------------------
|
||||
// EXCEL PARSER
|
||||
// -----------------------------
|
||||
const HEADER_ROW_INDEX = 2; // 0-index → Row 3 actual
|
||||
const DATA_START_ROW_INDEX = 3; // 0-index → Row 4 actual
|
||||
|
||||
// -----------------------------
|
||||
// CONFIG
|
||||
// -----------------------------
|
||||
const CONFIG = JSON.parse(fs.readFileSync("config.json", "utf8"));
|
||||
const tenantId = CONFIG.tenantId;
|
||||
const clientId = CONFIG.clientId;
|
||||
const pbUrl = CONFIG.pbUrl;
|
||||
const pbCollection = CONFIG.pbCollection || "Job_Info_TestEnv";
|
||||
const pbEmail = CONFIG.pbEmail;
|
||||
const clientSecret = process.env.CLIENT_SECRET;
|
||||
const pbPassword = process.env.PB_PASSWORD;
|
||||
|
||||
// -----------------------------
|
||||
// LOGGING
|
||||
// -----------------------------
|
||||
function log(message) {
|
||||
console.log(message);
|
||||
fs.appendFileSync("excelsync.log", message + "\n");
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// GRAPH TOKEN
|
||||
// -----------------------------
|
||||
async function getGraphToken() {
|
||||
log("🔵 Requesting Graph token...");
|
||||
const tokenUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`;
|
||||
const body = qs.stringify({
|
||||
client_id: clientId,
|
||||
client_secret: clientSecret,
|
||||
scope: "https://graph.microsoft.com/.default",
|
||||
grant_type: "client_credentials",
|
||||
});
|
||||
const res = await axios.post(tokenUrl, body, {
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||
});
|
||||
log("✅ Graph token acquired");
|
||||
return res.data.access_token;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// POCKETBASE LOGIN
|
||||
// -----------------------------
|
||||
async function getPocketBaseClient() {
|
||||
log("🔵 Logging into PocketBase...");
|
||||
const pb = new PocketBase(pbUrl);
|
||||
// ⚠️ CHANGE "users" if your auth collection is different
|
||||
await pb.collection("users").authWithPassword(pbEmail, pbPassword);
|
||||
log("✅ PocketBase token acquired");
|
||||
return pb;
|
||||
}
|
||||
// -----------------------------
|
||||
// CLEAN + COERCE
|
||||
// -----------------------------
|
||||
function cleanString(value) {
|
||||
if (value == null) return "";
|
||||
return String(value).replace(/\u00A0/g, " ").trim();
|
||||
}
|
||||
|
||||
function coerceValue(fieldConfig, rawValue) {
|
||||
switch (fieldConfig.type) {
|
||||
case "string": return cleanString(rawValue);
|
||||
case "bool": return ["true","1","yes","y"].includes(cleanString(rawValue).toLowerCase());
|
||||
case "date":
|
||||
if (typeof rawValue === "number") {
|
||||
const origin = new Date(Date.UTC(1899, 11, 30));
|
||||
return new Date(origin.getTime() + rawValue * 86400000).toISOString();
|
||||
}
|
||||
const d = new Date(cleanString(rawValue));
|
||||
return isNaN(d.getTime()) ? null : d.toISOString();
|
||||
case "number": return Number(cleanString(rawValue));
|
||||
case "json": try { return JSON.parse(cleanString(rawValue)); } catch { return null; }
|
||||
case "relation": return cleanString(rawValue);
|
||||
default: return cleanString(rawValue);
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// PAYLOAD BUILDER
|
||||
// -----------------------------
|
||||
export function buildPayload(headers, row) {
|
||||
const payload = {};
|
||||
headers.forEach((excelHeader, i) => {
|
||||
const fieldConfig = Object.values(FIELD_CONFIG).find(f => f.excelHeader === excelHeader);
|
||||
if (!fieldConfig) return;
|
||||
payload[fieldConfig.pbField] = coerceValue(fieldConfig, row[i]);
|
||||
});
|
||||
return payload;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// DIFFERENCE LOGGER + UPDATE CHECK
|
||||
// -----------------------------
|
||||
function logDifferences(pbRecord, headers, row) {
|
||||
const diffs = [];
|
||||
headers.forEach((excelHeader, i) => {
|
||||
const fieldConfig = Object.values(FIELD_CONFIG).find(f => f.excelHeader === excelHeader);
|
||||
if (!fieldConfig) return;
|
||||
const newValue = coerceValue(fieldConfig, row[i]);
|
||||
const currentValue = coerceValue(fieldConfig, pbRecord[fieldConfig.pbField]);
|
||||
if (newValue !== currentValue) {
|
||||
diffs.push({ field: fieldConfig.pbField, excelHeader, current: currentValue, incoming: newValue });
|
||||
}
|
||||
});
|
||||
if (diffs.length > 0) {
|
||||
log(`Differences detected for record: ${pbRecord.id}`);
|
||||
diffs.forEach(d => {
|
||||
log(`Field: ${d.field} (Excel: ${d.excelHeader})\n Current: ${JSON.stringify(d.current)}\n Incoming: ${JSON.stringify(d.incoming)}\n`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function needsUpdate(pbRecord, headers, row) {
|
||||
let hasDiff = false;
|
||||
headers.forEach((excelHeader, i) => {
|
||||
const fieldConfig = Object.values(FIELD_CONFIG).find(f => f.excelHeader === excelHeader);
|
||||
if (!fieldConfig) return;
|
||||
const newValue = coerceValue(fieldConfig, row[i]);
|
||||
const currentValue = coerceValue(fieldConfig, pbRecord[fieldConfig.pbField]);
|
||||
if (newValue !== currentValue) hasDiff = true;
|
||||
});
|
||||
if (hasDiff) logDifferences(pbRecord, headers, row);
|
||||
return hasDiff;
|
||||
}
|
||||
// -----------------------------
|
||||
// GRAPH HELPERS
|
||||
// -----------------------------
|
||||
async function getUsedRangeInfo(graphToken, driveId, itemId, worksheetName) {
|
||||
log(`🔵 Requesting usedRange for ${worksheetName}`);
|
||||
const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/worksheets('${worksheetName}')/usedRange`;
|
||||
const res = await axios.get(url, { headers: { Authorization: `Bearer ${graphToken}` } });
|
||||
log(`✅ Used range address: ${res.data.address}`);
|
||||
return res.data.address;
|
||||
}
|
||||
|
||||
function extractLastColumn(address) {
|
||||
const match = address.match(/:([A-Z]+)\d+$/);
|
||||
if (!match) {
|
||||
log("⚠️ Falling back to column Z, check sheet width");
|
||||
return "Z";
|
||||
}
|
||||
return match[1];
|
||||
}
|
||||
|
||||
async function fetchExcelRange(graphToken, driveId, itemId, worksheetName, range) {
|
||||
log(`🔵 Fetching range ${range} from worksheet ${worksheetName}`);
|
||||
const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/worksheets('${worksheetName}')/range(address='${range}')`;
|
||||
const res = await axios.get(url, { headers: { Authorization: `Bearer ${graphToken}` } });
|
||||
log(`✅ Range ${range} returned ${res.data.values?.length || 0} rows`);
|
||||
return res.data.values || [];
|
||||
}
|
||||
|
||||
async function fetchAllRows(graphToken, driveId, itemId, worksheetName, blockSize = 500) {
|
||||
const address = await getUsedRangeInfo(graphToken, driveId, itemId, worksheetName);
|
||||
const lastCol = extractLastColumn(address);
|
||||
|
||||
let allRows = [];
|
||||
let startRow = 1;
|
||||
|
||||
while (true) {
|
||||
const endRow = startRow + blockSize - 1;
|
||||
const range = `A${startRow}:${lastCol}${endRow}`;
|
||||
const rows = await fetchExcelRange(graphToken, driveId, itemId, worksheetName, range);
|
||||
|
||||
if (!rows.length) break;
|
||||
allRows = allRows.concat(rows);
|
||||
|
||||
if (rows.length < blockSize) break;
|
||||
startRow = endRow + 1;
|
||||
}
|
||||
|
||||
return allRows;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// MAIN
|
||||
// -----------------------------
|
||||
(async () => {
|
||||
try {
|
||||
log("🔵 Checking token acquisition and excel readiness");
|
||||
|
||||
const graphToken = await getGraphToken();
|
||||
const pb = await getPocketBaseClient();
|
||||
|
||||
const driveId = "b!9YOqqr2xM0G2DFBwMEJYY4UzQgocFddEtpLYWuS9_AtkCKl2q8yjQJteQ7Ti4QSx";
|
||||
const itemId = "01SPNXLDSQTMG4F6BDANCLJU2FZKEDQLVU";
|
||||
const worksheetName = "Job Sheet";
|
||||
|
||||
const rows = await fetchAllRows(graphToken, driveId, itemId, worksheetName);
|
||||
log(`✅ Total rows fetched: ${rows.length}`);
|
||||
|
||||
if (!rows.length) {
|
||||
log("⚠️ No rows returned from Graph. Check driveId, itemId, and worksheetName.");
|
||||
return;
|
||||
}
|
||||
|
||||
const headers = rows[HEADER_ROW_INDEX];
|
||||
const dataRows = rows.slice(DATA_START_ROW_INDEX);
|
||||
|
||||
if (!headers || !Array.isArray(headers)) {
|
||||
log("❌ Header row not found at expected index. Verify HEADER_ROW_INDEX alignment.");
|
||||
return;
|
||||
}
|
||||
|
||||
log(`✅ Headers detected: ${JSON.stringify(headers)}`);
|
||||
log(`✅ Data rows to process: ${dataRows.length}`);
|
||||
|
||||
for (const row of dataRows) {
|
||||
if (!row || row.length === 0) continue; // skip empty rows
|
||||
|
||||
const payload = buildPayload(headers, row);
|
||||
|
||||
// Ensure primary key exists before querying PocketBase
|
||||
const jobNumber = payload.Job_Number;
|
||||
if (!jobNumber || String(jobNumber).trim() === "") {
|
||||
log("⚠️ Skipping row with missing Job_Number.");
|
||||
continue;
|
||||
}
|
||||
|
||||
let existing = null;
|
||||
try {
|
||||
existing = await pb.collection(pbCollection).getFirstListItem(`Job_Number="${jobNumber}"`);
|
||||
} catch {
|
||||
existing = null;
|
||||
}
|
||||
|
||||
if (!existing) {
|
||||
log(`➕ Creating new record: ${jobNumber}`);
|
||||
await pb.collection(pbCollection).create(payload);
|
||||
} else {
|
||||
if (needsUpdate(existing, headers, row)) {
|
||||
log(`✏️ Updating record: ${existing.id} (${jobNumber})`);
|
||||
await pb.collection(pbCollection).update(existing.id, payload);
|
||||
} else {
|
||||
log(`✅ No changes for record: ${existing.id} (${jobNumber})`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log("✅ Sync complete, PocketBase updated!");
|
||||
} catch (err) {
|
||||
log("❌ Error, not ready for updating. " + (err?.message || err));
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
@@ -0,0 +1,136 @@
|
||||
// excelsync.js
|
||||
import fs from "fs";
|
||||
import axios from "axios";
|
||||
import qs from "qs";
|
||||
// may need for reading excel
|
||||
//import * as XLSX from "xlsx";
|
||||
import dotenv from "dotenv";
|
||||
// may need to serve html to port
|
||||
//import { serve } from "bun";
|
||||
import PocketBase from "pocketbase"
|
||||
|
||||
dotenv.config(); // connects to .env for sensitive info
|
||||
|
||||
// -----------------------------
|
||||
// EXCEL PARSER
|
||||
// -----------------------------
|
||||
const HEADER_ROW_INDEX = 2; // O index --> Row 3 Actual
|
||||
const DATA_START_ROW_INDEX = 3; // 0 index --> Row 4 Actual
|
||||
const RANGE_HEADER_ROW = 3 // Excel index headers
|
||||
const RANGE_DATA_ROW = 4 // Excel index data
|
||||
|
||||
// -----------------------------
|
||||
// CONFIG
|
||||
// -----------------------------
|
||||
const CONFIG = JSON.parse(fs.readFileSync("config.json", "utf8"));
|
||||
const tenantId = CONFIG.tenantId;
|
||||
const clientId = CONFIG.clientId;
|
||||
//const sharepointFileUrl = CONFIG.sharepointFileUrl; // production Excel file URL
|
||||
const pbUrl = CONFIG.pbUrl;
|
||||
const pbCollection = CONFIG.pbCollection || "Job_Info_TestEnv";
|
||||
const pbEmail = CONFIG.pbEmail;
|
||||
const clientSecret = process.env.CLIENT_SECRET;
|
||||
const pbPassword = process.env.PB_PASSWORD;
|
||||
|
||||
// -----------------------------
|
||||
// LOGGING
|
||||
// -----------------------------
|
||||
function log(message) {
|
||||
console.log(message);
|
||||
fs.appendFileSync("excelsync.log", message + "\n");
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// GRAPH TOKEN
|
||||
// -----------------------------
|
||||
async function getGraphToken() {
|
||||
const tokenUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`;
|
||||
const body = qs.stringify({
|
||||
client_id: clientId,
|
||||
client_secret: clientSecret,
|
||||
scope: "https://graph.microsoft.com/.default",
|
||||
grant_type: "client_credentials",
|
||||
});
|
||||
const res = await axios.post(tokenUrl, body, {
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||
});
|
||||
log("✅ Graph token acquired");
|
||||
return res.data.access_token;
|
||||
}
|
||||
// -----------------------------
|
||||
// POCKETBASE LOGIN AND TOKEN
|
||||
// -----------------------------
|
||||
async function getPocketBaseClient() {
|
||||
const pb = new PocketBase(pbUrl);
|
||||
await pb.collection("users").authWithPassword(pbEmail, pbPassword);
|
||||
log("✅ PocketBase token acquired");
|
||||
return pb;
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// HEADER MAP
|
||||
// -----------------------------
|
||||
const HEADER_MAP = {
|
||||
"Job_Number": "Job_Number",
|
||||
"Job_Full_Name": "Job_Full_Name",
|
||||
"Job_Name": "Job_Name",
|
||||
"Job Address": "Job_Address",
|
||||
"Job Type": "Job_Type",
|
||||
"Flag": "Flag",
|
||||
"Tax_Exempt": "Tax_Exempt",
|
||||
"Job Status": "Job_Status",
|
||||
"Job Division": "Job_Division",
|
||||
"Estimator": "Estimator",
|
||||
"Office Rep": "Office_Rep",
|
||||
"Manager": "Project_Manager",
|
||||
"Company/Client": "Company_Client",
|
||||
"Contact Person": "Contact_Person",
|
||||
"Phone Number": "Phone_Number",
|
||||
"Email": "Email",
|
||||
"Due Date": "Due_Date",
|
||||
"Due Date Source": "Due_Date_Source",
|
||||
"Due Date Counter": "Due_Date_Counter",
|
||||
"Due Time": "Due_Time",
|
||||
"Tax_Exempt_Docs_Uploaded": "Tax_Exempt_Docs_Uploaded",
|
||||
"Docs Uploaded": "Docs_Uploaded",
|
||||
"QB Created": "QB_Created",
|
||||
"Added to Calendar": "Added_To_Calendar",
|
||||
"PSwift Uploaded": "PSwift_Uploaded",
|
||||
"Voxer Created": "Voxer_Created",
|
||||
"Estimate Reviewed": "Estimate_Reviewed",
|
||||
"Estimate Draft": "Estimate_Draft",
|
||||
"Estimate Approved": "Estimate_Approved",
|
||||
"Start Date": "Start_Date",
|
||||
"Schedule_Confidence": "Schedule_Confidence",
|
||||
"Notes": "Notes",
|
||||
"Job QB Link": "Job_QB_Link",
|
||||
"Voxer Link": "Voxer_Link",
|
||||
"Job_Codes": "Job_Codes",
|
||||
"Submission Date": "Submission_Date",
|
||||
"Active": "Active",
|
||||
"Job Folder Link": "Job_Folder_Link",
|
||||
"pb_id": "id"
|
||||
};
|
||||
|
||||
// -----------------------------
|
||||
// MAIN
|
||||
// -----------------------------
|
||||
(async () => {
|
||||
try {
|
||||
log("🔵 Checking token acquisition and excel readiness");
|
||||
|
||||
const graphToken = await getGraphToken();
|
||||
const pb = await getPocketBaseClient();
|
||||
// These can be update if the Excel File changes
|
||||
//const driveId = "b!9YOqqr2xM0G2DFBwMEJYY4UzQgocFddEtpLYWuS9_AtkCKl2q8yjQJteQ7Ti4QSx";
|
||||
//const itemId = "01SPNXLDSQTMG4F6BDANCLJU2FZKEDQLVU"
|
||||
|
||||
await exceltopb(graphToken, driveId, itemId, "Job Sheet", pb);
|
||||
await pbtoupdate(pbCollection);
|
||||
|
||||
log("✅ Ready for updating!");
|
||||
} catch (err) {
|
||||
log("❌ Error, not ready for updating." + (err.message || err));
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
@@ -0,0 +1,105 @@
|
||||
import axios from "axios";
|
||||
import qs from "qs";
|
||||
import fs from "fs";
|
||||
import dotenv from "dotenv";
|
||||
|
||||
// -----------------------------
|
||||
// CONFIG
|
||||
// -----------------------------
|
||||
dotenv.config();
|
||||
// Load non‑sensitive values from config.json
|
||||
const CONFIG = JSON.parse(fs.readFileSync("config.json", "utf8"));
|
||||
const tenantId = CONFIG.tenantId;
|
||||
const clientId = CONFIG.clientId;
|
||||
const testSharePointFileUrl = CONFIG.testSharePointFileUrl;
|
||||
|
||||
// Load secret only from .env
|
||||
const clientSecret = process.env.CLIENT_SECRET;
|
||||
|
||||
// -----------------------------
|
||||
// Validation
|
||||
// -----------------------------
|
||||
const missing = [];
|
||||
if (!tenantId) missing.push("tenantId");
|
||||
if (!clientId) missing.push("clientId");
|
||||
if (!clientSecret) missing.push("clientSecret");
|
||||
if (!testSharePointFileUrl) missing.push("testSharePointFileUrl");
|
||||
|
||||
if (missing.length > 0) {
|
||||
console.error("❌ Missing values:", missing.join(", "));
|
||||
process.exit(1);
|
||||
} else {
|
||||
console.log("✅ All required values present");
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// 1. Get Graph Token (Client Credentials)
|
||||
// -----------------------------
|
||||
async function getGraphToken() {
|
||||
const tokenUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`;
|
||||
|
||||
const body = qs.stringify({
|
||||
client_id: clientId,
|
||||
client_secret: clientSecret,
|
||||
scope: "https://graph.microsoft.com/.default",
|
||||
grant_type: "client_credentials",
|
||||
});
|
||||
|
||||
try {
|
||||
const res = await axios.post(tokenUrl, body, {
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||
});
|
||||
|
||||
console.log("✅ Token acquired");
|
||||
// can use to test to see the full token as needed, just remove this comment// console.log("🔑 Access Token:", res.data.access_token);
|
||||
return res.data.access_token;
|
||||
|
||||
} catch (err) {
|
||||
console.error("❌ Failed to acquire token");
|
||||
console.error(err.response?.data || err.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// 2. Convert SharePoint File URL → Drive Item ID
|
||||
// -----------------------------
|
||||
function normalizeSharePointUrl(fileUrl) {
|
||||
return fileUrl
|
||||
.replace("guestaccess.aspx", "")
|
||||
.replace(/\?.*$/, ""); // remove ?query
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// 3. Test GET file metadata
|
||||
// -----------------------------
|
||||
async function testFileAccess(token) {
|
||||
const cleanUrl = normalizeSharePointUrl(testSharePointFileUrl);
|
||||
|
||||
console.log("🔎 Normalized URL:", cleanUrl);
|
||||
|
||||
const graphUrl =
|
||||
`https://graph.microsoft.com/v1.0/shares/u!${Buffer.from(cleanUrl).toString("base64")}/driveItem`;
|
||||
|
||||
try {
|
||||
const res = await axios.get(graphUrl, {
|
||||
headers: { Authorization: `Bearer ${token}` },
|
||||
});
|
||||
|
||||
console.log("✅ SharePoint file access test succeeded!");
|
||||
console.log("File name:", res.data.name);
|
||||
console.log("Drive Item ID:", res.data.id);
|
||||
} catch (err) {
|
||||
console.error("❌ SharePoint file access failed");
|
||||
console.error(err.response?.data || err.message);
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
// MAIN
|
||||
// -----------------------------
|
||||
(async () => {
|
||||
console.log("🔵 Testing SharePoint Access…");
|
||||
const token = await getGraphToken();
|
||||
await testFileAccess(token);
|
||||
})();
|
||||
Reference in New Issue
Block a user