From f0c646aeabb18de02c97021d694dba75d0e51e42 Mon Sep 17 00:00:00 2001 From: aewing Date: Sun, 21 Dec 2025 06:12:19 +0000 Subject: [PATCH] lots of changes should work now --- Excel_Audit/README.md | 16 ++ Excel_Audit/bunfig.toml | 5 + Excel_Audit/src/index.ts | 467 +++++++++++++++++++++++++++++++++++++ Excel_Audit/tsconfig.json | 17 ++ Excel_Listener/src/sync.ts | 89 +++---- 5 files changed, 555 insertions(+), 39 deletions(-) create mode 100644 Excel_Audit/README.md create mode 100644 Excel_Audit/bunfig.toml create mode 100644 Excel_Audit/src/index.ts create mode 100644 Excel_Audit/tsconfig.json diff --git a/Excel_Audit/README.md b/Excel_Audit/README.md new file mode 100644 index 0000000..6fbc6d1 --- /dev/null +++ b/Excel_Audit/README.md @@ -0,0 +1,16 @@ +# Excel Audit + +Single unified audit tool comparing Excel vs PocketBase with beforeβ†’after logging. + +## Tool: `src/index.ts` + +```bash +bun index.ts --audit # Compare Excel vs PocketBase (read-only) +bun index.ts --apply # Apply updates from deltas to PocketBase +``` + +## Output + +- `applied_updates_deltas.json` β€” Complete log: Job_Number, Field, From (PB before), To (Excel or merged) + +Requires Excel_Listener environment variables (.env file). diff --git a/Excel_Audit/bunfig.toml b/Excel_Audit/bunfig.toml new file mode 100644 index 0000000..b77a10f --- /dev/null +++ b/Excel_Audit/bunfig.toml @@ -0,0 +1,5 @@ +[bundle] +target = "bun" + +[define] +"process.env" = "process.env" diff --git a/Excel_Audit/src/index.ts b/Excel_Audit/src/index.ts new file mode 100644 index 0000000..8abb9e4 --- /dev/null +++ b/Excel_Audit/src/index.ts @@ -0,0 +1,467 @@ +import 'dotenv/config'; +import PocketBase from 'pocketbase'; +import { writeFileSync } from 'fs'; +import path from 'path'; + +declare const fetch: typeof globalThis.fetch; + +function requireEnv(name: string): string { + const v = process.env[name]; + if (!v) throw new Error(`Missing required env: ${name}`); + return v; +} + +// Excel-only fields (not synced to PocketBase) +const EXCEL_ONLY_FIELDS = new Set([ + 'Contact_Notes', 'Clean_Phone_Number', 'Fax', 'Phone_Number_Extn', + 'Secondary_HO', 'Primary_SL', 'POC_Name', +]); + +// Excel-authoritative fields (Excel always overwrites PB - calculated/dynamic) +const EXCEL_AUTHORITATIVE_FIELDS = new Set([ + 'Due_Date_Counter', 'Active', +]); + +const AUTHORITATIVE_FIELDS = new Set([ + 'Job_Folder_Link', 'Flag', 'Due_Date', 'Job_Full_Name', + 'Job_QB_Link', 'Voxer_Link', 'Job_Codes', 'Estimate_Draft', 'Job_Status' +]); + +const BOOLEAN_FIELDS = new Set(['Flag', 'Active', 'Estimate_Draft', 'Tax_Exempt']); +const SENTINEL_ASAP_DATE = '1990-01-01T00:00:00.000Z'; + +function isDateHeader(name: string): boolean { return name.toLowerCase().includes('date'); } +function isBooleanHeader(name: string): boolean { + const lower = name.toLowerCase(); + return lower.startsWith('is_') || lower.endsWith('_flag') || lower === 'active' || lower.includes('tax_exempt') || lower === 'estimate_draft'; +} +function isPocketBaseIdHeader(name: string): boolean { + const lower = name.toLowerCase().trim(); + return lower === 'pb_id' || lower === 'pv_id' || lower === 'pvid'; +} + +function excelSerialToIso(num: number): string { + const ms = (num - 25569) * 86400 * 1000; + return new Date(ms).toISOString(); +} + +function toUtcIso(val: any): string { + if (val === undefined || val === null || val === '') return ''; + if (typeof val === 'number' && !Number.isNaN(val)) return excelSerialToIso(val); + if (typeof val === 'string') { + const trimmed = val.trim(); + const numeric = parseFloat(trimmed); + if (!Number.isNaN(numeric) && /^\d+(\.\d+)?$/.test(trimmed)) return excelSerialToIso(numeric); + const d = new Date(trimmed); + if (!Number.isNaN(d.getTime())) { + return new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate(), d.getUTCHours(), d.getUTCMinutes(), d.getUTCSeconds())).toISOString(); + } + return trimmed; + } + return String(val); +} + +function normalizeValueByHeader(header: string, val: any) { + if (isDateHeader(header)) return toUtcIso(val); + if (isBooleanHeader(header)) return normalizeBoolean(val); + if (val === undefined || val === null) return ''; + return typeof val === 'string' ? val.trim() : val; +} + +function normalizeBoolean(val: any): boolean { + if (val === true) return true; + if (val === false) return false; + const s = String(val).toLowerCase(); + return s === 'true' || s === '1' || s === 'yes' || s === 'y' || s === 't'; +} + +async function acquireAppOnlyToken(): Promise { + const clientId = requireEnv('CLIENT_ID'); + const clientSecret = requireEnv('CLIENT_SECRET'); + const tenantId = requireEnv('TENANT_ID'); + const tokenUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`; + const params = new URLSearchParams({ + client_id: clientId, + client_secret: clientSecret, + scope: 'https://graph.microsoft.com/.default', + grant_type: 'client_credentials', + }); + const resp = await fetch(tokenUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: params.toString(), + }); + if (!(resp as any).ok) throw new Error(`Token failed: ${(resp as any).status}`); + const data = await (resp as any).json(); + return data.access_token; +} + +async function createSession(userToken: string, persist: boolean = true): Promise { + const driveId = requireEnv('DRIVE_ID'); + const itemId = requireEnv('ITEM_ID'); + const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/createSession`; + const resp = await fetch(url, { + method: 'POST', + headers: { Authorization: `Bearer ${userToken}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ persistChanges: persist }), + }); + if (!(resp as any).ok) throw new Error(`CreateSession failed: ${(resp as any).status}`); + return (await (resp as any).json())?.id as string; +} + +async function getTableColumns(userToken: string, sessionId: string): Promise { + const driveId = requireEnv('DRIVE_ID'); + const itemId = requireEnv('ITEM_ID'); + const tableName = requireEnv('EXCEL_TABLE').split('/').pop() as string; + const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/tables/${encodeURIComponent(tableName)}/columns`; + const resp = await fetch(url, { + method: 'GET', + headers: { Authorization: `Bearer ${userToken}`, 'workbook-session-id': sessionId }, + }); + if (!(resp as any).ok) throw new Error(`GetTableColumns failed: ${(resp as any).status}`); + const data = await (resp as any).json(); + return (data?.value ?? []).map((col: any) => col.name); +} + +async function getTableRows(userToken: string, sessionId: string) { + const driveId = requireEnv('DRIVE_ID'); + const itemId = requireEnv('ITEM_ID'); + const tableName = requireEnv('EXCEL_TABLE').split('/').pop() as string; + const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/tables/${encodeURIComponent(tableName)}/rows`; + const resp = await fetch(url, { + method: 'GET', + headers: { Authorization: `Bearer ${userToken}`, 'workbook-session-id': sessionId }, + }); + if (!(resp as any).ok) throw new Error(`GetTableRows failed: ${(resp as any).status}`); + const data = await (resp as any).json(); + const rows: any[] = []; + for (const row of (data?.value as any[]) ?? []) { + rows.push((row?.values?.[0] ?? []) as any); + } + return { values: rows }; +} + +function mapRowToObject(headers: string[], row: any[]): Record { + const obj: Record = {}; + for (let i = 0; i < headers.length; i++) { + const key = headers[i]; + if (!key) continue; + obj[key] = row[i]; + } + return obj; +} + +function getExcelPocketBaseId(headers: string[], excelObj: Record | undefined): string | undefined { + if (!excelObj) return undefined; + for (const h of headers) { + if (!h || !isPocketBaseIdHeader(h)) continue; + const v = excelObj[h]; + if (v === undefined || v === null) return undefined; + return String(v).trim(); + } + return undefined; +} + +function diffRecords(headers: string[], excelObj: Record | undefined, pbObj: Record | undefined) { + const diffs: Array<{ field: string; excel: any; pocketbase: any }> = []; + for (const h of headers) { + if (!h || isExcelOnlyField(h) || isExcelAuthoritativeField(h)) continue; + const exVal = excelObj ? normalizeValueByHeader(h, excelObj[h]) : undefined; + const pbRaw = pbObj ? (isPocketBaseIdHeader(h) ? (pbObj as any).id : (pbObj as any)[h]) : undefined; + const pbVal = pbObj ? normalizeValueByHeader(h, pbRaw) : undefined; + const isBool = isBooleanHeader(h); + + // Special handling for Notes: check if merge would be needed + if (h === 'Notes') { + if (notesMergeNeeded(exVal, pbVal)) { + diffs.push({ field: h, excel: exVal, pocketbase: pbVal }); + } + continue; + } + + const left = isBool ? exVal : (exVal === undefined ? '' : String(exVal).trim()); + const right = isBool ? pbVal : (pbVal === undefined ? '' : String(pbVal).trim()); + let equal = left === right; + if (!isBool && h === 'Due_Date') { + const exStr = String(left).trim().toUpperCase(); + const pbStr = String(right).trim(); + if (exStr === 'ASAP' && (pbStr === '' || pbStr === SENTINEL_ASAP_DATE)) equal = true; + } + if (!equal) diffs.push({ field: h, excel: exVal, pocketbase: pbVal }); + } + return diffs; +} + +function isExcelOnlyField(field: string): boolean { return EXCEL_ONLY_FIELDS.has(field); } +function isExcelAuthoritativeField(field: string): boolean { return EXCEL_AUTHORITATIVE_FIELDS.has(field); } + +async function fetchExcelMap() { + const tokenInfo = await acquireAppOnlyToken(); + const sessionId = await createSession(tokenInfo, true); + const headers = await getTableColumns(tokenInfo, sessionId); + if (!headers.includes('Job_Number')) throw new Error('Job_Number header required'); + const rows = await getTableRows(tokenInfo, sessionId); + const byKey = new Map>(); + for (const r of rows.values.map((row, i) => mapRowToObject(headers, row))) { + const key = r['Job_Number']; + if (key) byKey.set(String(key), r); + } + return { headers, byKey }; +} + +async function fetchPocketBaseMap(collection: string) { + const pbUrl = (() => { + const explicit = process.env.PB_DB; + if (explicit) return explicit; + const redirect = process.env.REDIRECT_URI; + if (!redirect) throw new Error('Missing PB_DB or REDIRECT_URI'); + const u = new URL(redirect); + return `${u.protocol}//${u.host}`; + })(); + const pb = new PocketBase(pbUrl); + await pb.collection(process.env.PB_AUTH_COLLECTION || 'Users').authWithPassword( + requireEnv('PB_AGENT_EMAIL'), + requireEnv('PB_AGENT_PASSWORD') + ); + const records = await pb.collection(collection).getFullList(200); + const byKey = new Map>(); + for (const rec of records as any[]) { + const key = rec['Job_Number']; + if (key) byKey.set(String(key), rec); + } + return byKey; +} + +async function runAudit() { + const collection = requireEnv('PB_COLLECTION'); + const { headers, byKey: excelMap } = await fetchExcelMap(); + const pbMap = await fetchPocketBaseMap(collection); + + const commonKeys = [...excelMap.keys()].filter(k => pbMap.has(k)); + const mismatches: Array<{ key: string; diffs: Array<{ field: string; excel: any; pocketbase: any }> }> = []; + + for (const k of commonKeys) { + const diffs = diffRecords(headers, excelMap.get(k), pbMap.get(k)); + if (diffs.length > 0) mismatches.push({ key: k, diffs }); + } + + console.log(JSON.stringify({ + summary: { + excelCount: excelMap.size, + pocketbaseCount: pbMap.size, + mismatched: mismatches.length, + }, + mismatches, + }, null, 2)); +} + +function normalizeNotes(val: any): string { + return (val ?? '').toString().replace(/\r\n/g, '\n').trim(); +} + +function mergeNotes(excelNotes: any, pbNotes: any): string { + const excel = normalizeNotes(excelNotes); + const pb = normalizeNotes(pbNotes); + if (!excel && !pb) return ''; + if (!excel) return pb; + if (!pb) return excel; + if (excel === pb) return excel; + return `${excel}\n\n${pb}`; +} + +function notesMergeNeeded(excelNotes: any, pbNotes: any): boolean { + const merged = mergeNotes(excelNotes, pbNotes); + const pbStr = normalizeNotes(pbNotes); + const exStr = normalizeNotes(excelNotes); + // If normalized strings already equal, no merge needed + if (exStr === pbStr) return false; + // Otherwise, merge changes PB content + return merged !== pbStr; +} + +function normalizeBoolean2(val: any): boolean { + if (val === true) return true; + if (val === false) return false; + const s = String(val).trim().toLowerCase(); + return s === 'true' || s === 't' || s === 'yes' || s === 'y' || s === '1'; +} + +// Columns that contain formulas and should never be overwritten +const FORMULA_COLUMNS = new Set(['Active', 'Due_Date_Counter']); + +async function updateExcelRow(userToken: string, sessionId: string, jobNumber: string, mergedNotes: string, headers: string[]) { + const driveId = requireEnv('DRIVE_ID'); + const itemId = requireEnv('ITEM_ID'); + const tableName = requireEnv('EXCEL_TABLE').split('/').pop() as string; + + // Get all rows and find the Job_Number match + const rowUrl = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/tables/${encodeURIComponent(tableName)}/rows`; + const rowResp = await fetch(rowUrl, { + method: 'GET', + headers: { Authorization: `Bearer ${userToken}`, 'workbook-session-id': sessionId }, + }); + if (!(rowResp as any).ok) throw new Error(`GetTableRows failed: ${(rowResp as any).status}`); + const rowData = await (rowResp as any).json(); + + const jobRow = (rowData?.value ?? []).find((r: any) => { + const values = r?.values?.[0] ?? []; + return String(values[0]).trim() === String(jobNumber).trim(); + }); + + if (!jobRow) return; // Job not found + + // Find the Notes column index + const notesIdx = headers.indexOf('Notes'); + if (notesIdx === -1) return; // Notes column not found + + console.log(`\nβ†’ Writing to Excel Job ${jobNumber}: ${mergedNotes.length} chars`); + console.log(` Content: ${JSON.stringify(mergedNotes)}`); + + // Use table column's DataBodyRange to update specific cell + const driveId2 = requireEnv('DRIVE_ID'); + const itemId2 = requireEnv('ITEM_ID'); + const tableName2 = requireEnv('EXCEL_TABLE').split('/').pop() as string; + + // Access the Notes column's DataBodyRange, then use row/cell API + const colRangeUrl = `https://graph.microsoft.com/v1.0/drives/${driveId2}/items/${itemId2}/workbook/tables/${encodeURIComponent(tableName2)}/columns/${encodeURIComponent('Notes')}/dataBodyRange/cell(row=${jobRow.index},column=0)`; + const updateResp = await fetch(colRangeUrl, { + method: 'PATCH', + headers: { Authorization: `Bearer ${userToken}`, 'workbook-session-id': sessionId, 'Content-Type': 'application/json' }, + body: JSON.stringify({ values: [[mergedNotes]] }), + }); + if (!(updateResp as any).ok) { + throw new Error(`Excel cell update failed: ${(updateResp as any).status} - ${await (updateResp as any).text()}`); + } +} + + + +async function runApply() { + // First, run a fresh audit to get current mismatches + // This ensures we have the latest data, not stale cached data + console.log('πŸ” Running fresh audit before apply...'); + const collection = requireEnv('PB_COLLECTION'); + const { headers, byKey: excelMap } = await fetchExcelMap(); + const pbMap = await fetchPocketBaseMap(collection); + + const commonKeys = [...excelMap.keys()].filter(k => pbMap.has(k)); + const mismatches: Array = []; + + for (const k of commonKeys) { + const diffs = diffRecords(headers, excelMap.get(k), pbMap.get(k)); + if (diffs.length > 0) mismatches.push({ key: k, diffs }); + } + + console.log(`βœ“ Fresh audit complete: ${mismatches.length} mismatches found`); + + const pbUrl = (() => { + const explicit = process.env.PB_DB; + if (explicit) return explicit; + const redirect = process.env.REDIRECT_URI; + if (!redirect) throw new Error('Missing PB_DB or REDIRECT_URI'); + const u = new URL(redirect); + return `${u.protocol}//${u.host}`; + })(); + const pb = new PocketBase(pbUrl); + await pb.collection(process.env.PB_AUTH_COLLECTION || 'Users').authWithPassword( + requireEnv('PB_AGENT_EMAIL'), + requireEnv('PB_AGENT_PASSWORD') + ); + + // Get Excel access for writing merged Notes back + const excelToken = await acquireAppOnlyToken(); + const excelSessionId = await createSession(excelToken, true); + const excelHeaders = await getTableColumns(excelToken, excelSessionId); + + let updated = 0, skipped = 0; + + for (const job of mismatches) { + const key = job.key || job.jobNumber; + const diffs = job.diffs || [{ field: job.field, excel: job.to, pocketbase: job.from }]; + const updatePayload: Record = {}; + let mergedNotes: string | null = null; + + for (const d of diffs) { + const field = d.field; + const excelVal = d.excel; + const pbVal = d.pocketbase; + if (field === 'Notes') { + const merged = mergeNotes(excelVal, pbVal); + console.log(`\n=== JOB ${key} NOTES MERGE ===`); + console.log(`Excel (${(excelVal ?? '').toString().length} chars): ${JSON.stringify(excelVal)}`); + console.log(`PB (${(pbVal ?? '').toString().length} chars): ${JSON.stringify(pbVal)}`); + console.log(`Merged (${merged.length} chars): ${JSON.stringify(merged)}`); + + if (merged !== (pbVal ?? '').toString().trim()) { + console.log(`βœ“ Merge needed - updating PB`); + updatePayload[field] = merged; + } else { + console.log(`βœ— Merge not needed`); + } + mergedNotes = merged; + } else if (AUTHORITATIVE_FIELDS.has(field)) { + if (field === 'Due_Date') { + const exStr = String(excelVal ?? '').trim(); + if (exStr.toUpperCase() === 'ASAP') { + if (String(pbVal ?? '') !== SENTINEL_ASAP_DATE) updatePayload[field] = SENTINEL_ASAP_DATE; + } else { + if (String(excelVal ?? '') !== String(pbVal ?? '')) updatePayload[field] = excelVal; + } + } else if (BOOLEAN_FIELDS.has(field)) { + const excelBool = normalizeBoolean2(excelVal); + const pbBool = normalizeBoolean2(pbVal); + if (excelBool !== pbBool) updatePayload[field] = excelBool; + } else { + if (String(excelVal ?? '') !== String(pbVal ?? '')) updatePayload[field] = excelVal; + } + } + } + + if (Object.keys(updatePayload).length === 0) { + skipped++; + continue; + } + + const filter = `Job_Number=\"${String(key).replace(/\"/g, '\\\"')}\"`; + const found = await pb.collection(collection).getList(1, 1, { filter }); + if (!found?.items?.length) continue; + + const existing = found.items[0]; + await pb.collection(collection).update(existing.id, updatePayload); + updated++; + + // If Notes were merged, also update Excel to keep chain synchronized + if (mergedNotes !== null) { + try { + await updateExcelRow(excelToken, excelSessionId, String(key), mergedNotes, excelHeaders); + console.log(`βœ“ Job ${key}: ${Object.keys(updatePayload).join(', ')} (chain synced to both)`); + } catch (err: any) { + console.error(`βœ— Job ${key}: Excel write failed - ${err?.message || err}`); + console.log(`βœ“ Job ${key}: ${Object.keys(updatePayload).join(', ')} (PocketBase updated, Excel write failed)`); + } + } else { + console.log(`βœ“ Job ${key}: ${Object.keys(updatePayload).join(', ')}`); + } + } + + console.log(`\n=== Summary: ${updated} updated, ${skipped} skipped ===`); +} + +async function main() { + const mode = process.argv[2]; + if (mode === '--audit') { + await runAudit(); + } else if (mode === '--apply') { + await runApply(); + } else { + console.log('Usage: bun index.ts [--audit|--apply]'); + console.log(' --audit Compare Excel vs PocketBase (read-only)'); + console.log(' --apply Apply updates from deltas.json to PocketBase'); + process.exit(1); + } +} + +main().catch(err => { + console.error('βœ— Error:', err?.message || err); + process.exit(1); +}); diff --git a/Excel_Audit/tsconfig.json b/Excel_Audit/tsconfig.json new file mode 100644 index 0000000..e530e68 --- /dev/null +++ b/Excel_Audit/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "ESNext", + "lib": ["ESNext"], + "moduleResolution": "bundler", + "skipLibCheck": true, + "noImplicitAny": false, + "allowJs": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "noEmit": true, + "allowImportingTsExtensions": true + }, + "include": ["src"], + "exclude": ["node_modules", "dist"] +} diff --git a/Excel_Listener/src/sync.ts b/Excel_Listener/src/sync.ts index c7da9f5..4d2df18 100644 --- a/Excel_Listener/src/sync.ts +++ b/Excel_Listener/src/sync.ts @@ -13,20 +13,13 @@ function requireEnv(name: string): string { const EXCEL_ONLY_FIELDS = new Set([ 'Contact_Notes', 'Clean_Phone_Number', 'Fax', 'Phone_Number_Extn', 'Secondary_HO', 'Primary_SL', 'POC_Name', + // Do not sync Excel's pb_id into PocketBase; treat as Excel-only here + 'pb_id', 'PB_ID', ]); // Excel-authoritative fields: always overwrite PB with Excel values -const EXCEL_AUTHORITATIVE_FIELDS = new Set([ - 'Job_Folder_Link', - 'Flag', - 'Due_Date', - 'Job_Full_Name', - 'Job_QB_Link', - 'Voxer_Link', - 'Job_Codes', - 'Estimate_Draft', - 'Job_Status' -]); +// No authoritative overwrites: we only update when canonical values differ +const EXCEL_AUTHORITATIVE_FIELDS = new Set(); const BOOLEAN_FIELDS = new Set(['Flag', 'Active', 'Estimate_Draft', 'Tax_Exempt']); const SENTINEL_ASAP_DATE = '1990-01-01T00:00:00.000Z'; @@ -104,36 +97,51 @@ export function mapRowToObject(headers: string[], row: any[]): RowObject { return obj; } +function normalizeNotes(val: any): string { + return (val ?? '').toString().replace(/\r\n/g, '\n').trim(); +} + function mergeNotes(excelNotes: string | null | undefined, pbNotes: string | null | undefined): string { - // Normalize line endings and trim - const excel = (excelNotes ?? '').toString().replace(/\r\n/g, '\n').trim(); - const pb = (pbNotes ?? '').toString().replace(/\r\n/g, '\n').trim(); - + const excel = normalizeNotes(excelNotes); + const pb = normalizeNotes(pbNotes); if (!excel && !pb) return ''; if (!excel) return pb; if (!pb) return excel; if (excel === pb) return excel; - - // Both have different content - preserve both return `${excel}\n\n${pb}`; } function notesMergeNeeded(excelNotes: any, pbNotes: any): boolean { - // Check if a merge would change PocketBase content const merged = mergeNotes(excelNotes, pbNotes); - // Normalize PB value the same way for comparison - const pbStr = (pbNotes ?? '').toString().replace(/\r\n/g, '\n').trim(); + const pbStr = normalizeNotes(pbNotes); + const exStr = normalizeNotes(excelNotes); + if (exStr === pbStr) return false; return merged !== pbStr; } +function canonicalizeForCompare(key: string, val: any): string { + const lower = key.toLowerCase(); + if (key === 'Notes') return normalizeNotes(val); + // Dates: compare as UTC ISO strings + if (lower === 'due_date' || lower.includes('date')) { + const iso = toUtcIso(val); + return (iso ?? '').toString().trim(); + } + // Booleans: compare normalized boolean as 'true'/'false' + if (BOOLEAN_FIELDS.has(key) || lower.startsWith('is_') || lower.endsWith('_flag') || lower === 'active' || lower.includes('tax_exempt')) { + return normalizeBoolean(val) ? 'true' : 'false'; + } + // Generic string/number + if (val === undefined || val === null) return ''; + return String(val).trim(); +} + function recordsEqual(existing: RowObject, candidate: RowObject): boolean { - // Compare only keys present in candidate; treat undefined and empty string as equal empties + // Compare only keys present in candidate with canonical normalization for (const k of Object.keys(candidate)) { - const a = existing?.[k]; - const b = candidate?.[k]; - const aStr = a === undefined || a === null ? '' : String(a); - const bStr = b === undefined || b === null ? '' : String(b); - if (aStr !== bStr) return false; + const a = canonicalizeForCompare(k, existing?.[k]); + const b = canonicalizeForCompare(k, candidate?.[k]); + if (a !== b) return false; } return true; } @@ -149,27 +157,30 @@ export async function upsertPocketBaseByKey(pb: PocketBase, collection: string, if (found?.items?.length) { const existing = found.items[0]; - + // Handle Notes bidirectional merge - only if it would actually change - if ('Notes' in record && notesMergeNeeded(record.Notes, existing.Notes)) { - record.Notes = mergeNotes(record.Notes, existing.Notes); - } - - // Force Excel values for authoritative fields (overwrite even if unchanged in Excel) const updatePayload = { ...record }; - for (const field of EXCEL_AUTHORITATIVE_FIELDS) { - if (field in record) { - updatePayload[field] = record[field]; - } + if ('Notes' in updatePayload && notesMergeNeeded(updatePayload.Notes, existing.Notes)) { + updatePayload.Notes = mergeNotes(updatePayload.Notes, existing.Notes); } - // Skip update if nothing would change + // Skip update if nothing would change (canonical compare) if (recordsEqual(existing, updatePayload)) { - if (!silent) console.log(`β†Ί No changes for PB ${collection} id=${existing.id}`); return; } + // Compute concise per-field diffs for visibility + const changedKeys: string[] = []; + for (const k of Object.keys(updatePayload)) { + const before = canonicalizeForCompare(k, existing?.[k]); + const after = canonicalizeForCompare(k, updatePayload?.[k]); + if (before !== after) changedKeys.push(k); + } await pb.collection(collection).update(existing.id, updatePayload); - if (!silent) console.log(`βœ“ Updated PB ${collection} id=${existing.id}`); + if (!silent) { + const list = changedKeys.slice(0, 6).join(', '); + console.log(`βœ“ Updated PB ${collection} id=${existing.id} (fields: ${list}${changedKeys.length > 6 ? ', …' : ''})`); + } } else { + // Create full record when not found (actual change) const created = await pb.collection(collection).create(record); if (!silent) console.log(`βœ“ Created PB ${collection} id=${created.id}`); }