From 360a9fa5d8678935adc3946de49dd5a039b05308 Mon Sep 17 00:00:00 2001 From: aewing Date: Sun, 21 Dec 2025 05:08:58 +0000 Subject: [PATCH] chore: release v1.0.0-beta with notes hardening --- Excel_Audit/package.json | 18 ++++++ Excel_Listener/README.md | 9 ++- Excel_Listener/package.json | 2 +- Excel_Listener/src/poller.ts | 68 ++++++++++++++++---- Excel_Listener/src/sync.ts | 109 ++++++++++++++++++++++++++++++-- README.md | 11 ++++ global-copilot-instructions.md | 13 ++++ project-copilot-instructions.md | 13 ++++ tsconfig-global.md | 5 ++ 9 files changed, 226 insertions(+), 22 deletions(-) create mode 100644 Excel_Audit/package.json diff --git a/Excel_Audit/package.json b/Excel_Audit/package.json new file mode 100644 index 0000000..4866166 --- /dev/null +++ b/Excel_Audit/package.json @@ -0,0 +1,18 @@ +{ + "name": "excel-audit", + "version": "1.0.0-beta", + "private": true, + "type": "module", + "scripts": { + "start:audit": "bun src/audit.ts" + }, + "dependencies": { + "dotenv": "^16.4.5", + "pocketbase": "^0.21.3" + }, + "devDependencies": { + "@types/bun": "^1.3.4", + "@types/node": "^20.12.12", + "typescript": "^5.6.3" + } +} diff --git a/Excel_Listener/README.md b/Excel_Listener/README.md index 67c8db5..cad8a6d 100644 --- a/Excel_Listener/README.md +++ b/Excel_Listener/README.md @@ -1,5 +1,10 @@ # Excel Listener - Clean Setup +## Top Rule: No Placeholders or Mock Data + +- Use only actual values in configuration, commands, and examples. +- If a placeholder seems necessary, pause and request approval and the real values before proceeding. + > See project-wide preferences: [project-copilot-instructions.md](../project-copilot-instructions.md). For org standards, see [global-copilot-instructions.md](../global-copilot-instructions.md). ## Structure @@ -25,9 +30,7 @@ bun install ### Run Poller (delegated token) ```bash -$Env:GRAPH_ACCESS_TOKEN = "" -# Optional expiry hint (unix ms epoch) -# $Env:GRAPH_TOKEN_EXPIRES_AT = "1734220000000" +# Ensure a valid delegated Graph token is present in GRAPH_ACCESS_TOKEN (no placeholders). bun run start:poller ``` diff --git a/Excel_Listener/package.json b/Excel_Listener/package.json index 01a5366..96051b5 100644 --- a/Excel_Listener/package.json +++ b/Excel_Listener/package.json @@ -1,6 +1,6 @@ { "name": "excel-listener-auth", - "version": "0.1.0", + "version": "1.0.0-beta", "private": true, "type": "module", "scripts": { diff --git a/Excel_Listener/src/poller.ts b/Excel_Listener/src/poller.ts index 7a8ecdd..5fdfb8c 100644 --- a/Excel_Listener/src/poller.ts +++ b/Excel_Listener/src/poller.ts @@ -3,6 +3,9 @@ import PocketBase from 'pocketbase'; import { acquireGraphUserToken } from './userAuth.ts'; import { createSession, getTableRows, getTableColumns } from './excel.ts'; import { mapRowToObject, upsertPocketBaseByKey } from './sync.ts'; +import { execSync } from 'child_process'; +import { writeFileSync } from 'fs'; +import path from 'path'; function requireEnv(name: string): string { const val = process.env[name]; @@ -25,18 +28,61 @@ function validateEnv() { for (const k of required) requireEnv(k); } -// In-memory hash cache -const hashCache = new Map(); +// (Removed) In-memory hash cache: rely on upsert equality check to avoid unnecessary writes + +async function runStartupAudit() { + console.log('📊 Running startup audit to detect any data mismatches...'); + try { + // Run audit from Excel_Audit/src/index.ts (go up to parent, then into Excel_Audit) + const auditPath = path.resolve(__dirname, '../../Excel_Audit/src/index.ts'); + const auditOutput = execSync(`bun ${auditPath} --audit`, { + cwd: __dirname, + encoding: 'utf-8', + }); + const auditResult = JSON.parse(auditOutput); + const { summary } = auditResult; + + console.log(`✓ Audit complete: ${summary.excelCount} Excel, ${summary.pocketbaseCount} PocketBase, ${summary.mismatched} mismatches`); + + if (summary.mismatched > 0) { + console.log(`⚠️ Found ${summary.mismatched} mismatches; applying corrections...`); + + // Build deltas.json from audit mismatches + const deltas = { + timestamp: new Date().toISOString(), + summary, + mismatches: auditResult.mismatches || [], + }; + + const deltasPath = path.resolve(__dirname, '..', 'applied_updates_deltas.json'); + writeFileSync(deltasPath, JSON.stringify(deltas, null, 2)); + console.log(`✓ Saved deltas to ${deltasPath}`); + + // Apply changes + try { + const applyOutput = execSync(`bun ${auditPath} --apply`, { + cwd: __dirname, + encoding: 'utf-8', + }); + console.log(`✓ Applied updates:\n${applyOutput}`); + } catch (err: any) { + console.error(`✗ Apply failed: ${err?.message || err}`); + } + } else { + console.log('✓ No mismatches detected; data is synchronized'); + } + } catch (err: any) { + console.warn(`⚠️ Startup audit failed (non-fatal): ${err?.message || err}`); + console.log('Proceeding with normal polling anyway...'); + } +} async function runOnce() { const pbUrl = (() => { const explicit = process.env.PB_DB; if (explicit) return explicit; - // Poll interval (seconds) shared for logging remaining time per loop - const pollIntervalSec = Number(process.env.POLL_INTERVAL_SEC || 60); const redirect = process.env.REDIRECT_URI; if (!redirect) throw new Error('Missing PB_DB or REDIRECT_URI'); - const startedAt = Date.now(); const u = new URL(redirect); return `${u.protocol}//${u.host}`; })(); @@ -78,15 +124,8 @@ async function runOnce() { console.warn('⚠️ Missing key; skipping row'); continue; } - const hashInput = JSON.stringify(Object.keys(obj).sort().reduce((acc: any, k) => { acc[k] = obj[k]; return acc; }, {})); - const prevHash = hashCache.get(String(keyVal)); - if (prevHash === hashInput) { - skipped++; - continue; - } const silent = processed % 100 !== 0; await upsertPocketBaseByKey(pb, collection, keyName, obj, silent); - hashCache.set(String(keyVal), hashInput); processed++; // Progress indicator if (processed % 100 === 0) { @@ -106,8 +145,11 @@ async function main() { validateEnv(); const intervalSec = Number(process.env.POLL_INTERVAL_SEC || 60); console.log(`✓ Starting poller (interval ${intervalSec}s)`); + + // Run startup audit to detect and fix any mismatches + await runStartupAudit(); + let running = false; - console.log(`✓ Starting poller (interval ${pollIntervalSec}s)`); const runSafely = async () => { if (running) return; running = true; diff --git a/Excel_Listener/src/sync.ts b/Excel_Listener/src/sync.ts index dd848b2..c7da9f5 100644 --- a/Excel_Listener/src/sync.ts +++ b/Excel_Listener/src/sync.ts @@ -9,10 +9,33 @@ function requireEnv(name: string): string { return val; } +// Excel-only fields (not synced to PocketBase) +const EXCEL_ONLY_FIELDS = new Set([ + 'Contact_Notes', 'Clean_Phone_Number', 'Fax', 'Phone_Number_Extn', + 'Secondary_HO', 'Primary_SL', 'POC_Name', +]); + +// Excel-authoritative fields: always overwrite PB with Excel values +const EXCEL_AUTHORITATIVE_FIELDS = new Set([ + 'Job_Folder_Link', + 'Flag', + 'Due_Date', + 'Job_Full_Name', + 'Job_QB_Link', + 'Voxer_Link', + 'Job_Codes', + 'Estimate_Draft', + 'Job_Status' +]); + +const BOOLEAN_FIELDS = new Set(['Flag', 'Active', 'Estimate_Draft', 'Tax_Exempt']); +const SENTINEL_ASAP_DATE = '1990-01-01T00:00:00.000Z'; + export function normalizeBoolean(val: any): boolean { if (val === true) return true; + if (val === false) return false; const s = String(val).toLowerCase(); - return s === 'true' || s === '1' || s === 'yes' || s === 'y'; + return s === 'true' || s === '1' || s === 'yes' || s === 'y' || s === 't'; } function excelSerialToIso(num: number): string { @@ -50,14 +73,71 @@ export function mapRowToObject(headers: string[], row: any[]): RowObject { const key = headers[i]; const val = row[i]; if (!key) continue; + + // Skip Excel-only fields + if (EXCEL_ONLY_FIELDS.has(key)) continue; + const lower = key.toLowerCase(); - if (lower.includes('date')) obj[key] = toUtcIso(val); - else if (lower.startsWith('is_') || lower.endsWith('_flag') || lower === 'active' || lower.includes('tax_exempt')) obj[key] = normalizeBoolean(val); - else obj[key] = val; + + // Handle Due_Date with ASAP sentinel + if (lower === 'due_date') { + const valStr = String(val ?? '').trim(); + if (valStr.toUpperCase() === 'ASAP') { + obj[key] = SENTINEL_ASAP_DATE; + } else { + obj[key] = toUtcIso(val); + } + } + // Date fields (except Due_Date already handled) + else if (lower.includes('date')) { + obj[key] = toUtcIso(val); + } + // Boolean fields + else if (BOOLEAN_FIELDS.has(key) || lower.startsWith('is_') || lower.endsWith('_flag') || lower === 'active' || lower.includes('tax_exempt')) { + obj[key] = normalizeBoolean(val); + } + // All other fields + else { + obj[key] = val; + } } return obj; } +function mergeNotes(excelNotes: string | null | undefined, pbNotes: string | null | undefined): string { + // Normalize line endings and trim + const excel = (excelNotes ?? '').toString().replace(/\r\n/g, '\n').trim(); + const pb = (pbNotes ?? '').toString().replace(/\r\n/g, '\n').trim(); + + if (!excel && !pb) return ''; + if (!excel) return pb; + if (!pb) return excel; + if (excel === pb) return excel; + + // Both have different content - preserve both + return `${excel}\n\n${pb}`; +} + +function notesMergeNeeded(excelNotes: any, pbNotes: any): boolean { + // Check if a merge would change PocketBase content + const merged = mergeNotes(excelNotes, pbNotes); + // Normalize PB value the same way for comparison + const pbStr = (pbNotes ?? '').toString().replace(/\r\n/g, '\n').trim(); + return merged !== pbStr; +} + +function recordsEqual(existing: RowObject, candidate: RowObject): boolean { + // Compare only keys present in candidate; treat undefined and empty string as equal empties + for (const k of Object.keys(candidate)) { + const a = existing?.[k]; + const b = candidate?.[k]; + const aStr = a === undefined || a === null ? '' : String(a); + const bStr = b === undefined || b === null ? '' : String(b); + if (aStr !== bStr) return false; + } + return true; +} + export async function upsertPocketBaseByKey(pb: PocketBase, collection: string, keyName: string, record: RowObject, silent: boolean = false) { if (keyName.toLowerCase() !== 'job_number') { throw new Error('Primary key must be Job_Number (exact header name)'); @@ -66,9 +146,28 @@ export async function upsertPocketBaseByKey(pb: PocketBase, collection: string, if (!keyVal) throw new Error(`Missing key ${keyName} in record`); const filter = `${keyName}="${String(keyVal).replace(/"/g, '\\"')}"`; const found = await pb.collection(collection).getList(1, 1, { filter }); + if (found?.items?.length) { const existing = found.items[0]; - await pb.collection(collection).update(existing.id, record); + + // Handle Notes bidirectional merge - only if it would actually change + if ('Notes' in record && notesMergeNeeded(record.Notes, existing.Notes)) { + record.Notes = mergeNotes(record.Notes, existing.Notes); + } + + // Force Excel values for authoritative fields (overwrite even if unchanged in Excel) + const updatePayload = { ...record }; + for (const field of EXCEL_AUTHORITATIVE_FIELDS) { + if (field in record) { + updatePayload[field] = record[field]; + } + } + // Skip update if nothing would change + if (recordsEqual(existing, updatePayload)) { + if (!silent) console.log(`↺ No changes for PB ${collection} id=${existing.id}`); + return; + } + await pb.collection(collection).update(existing.id, updatePayload); if (!silent) console.log(`✓ Updated PB ${collection} id=${existing.id}`); } else { const created = await pb.collection(collection).create(record); diff --git a/README.md b/README.md index 98d36fb..671979b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,13 @@ # ExcelListenerUpdates +## Top Rule: No Placeholders or Mock Data + +- Use only actual values. Do not include sample, placeholder, or mock data in code, docs, or commands. +- If a placeholder seems necessary, ask for explicit approval and the real values before proceeding. + +See project-specific details in [Excel_Listener/README.md](Excel_Listener/README.md) and org standards in [global-copilot-instructions.md](global-copilot-instructions.md) and [project-copilot-instructions.md](project-copilot-instructions.md). + +## Version + +- v1.0.0-beta: stabilized Notes merge (normalized line endings, concat both sides), protected formula columns via single-cell updates, enforced fresh audit-before-apply, and reduced poller churn by skipping unchanged rows. + diff --git a/global-copilot-instructions.md b/global-copilot-instructions.md index b1963a3..daccc8c 100644 --- a/global-copilot-instructions.md +++ b/global-copilot-instructions.md @@ -1,5 +1,18 @@ +# Top Rule: No Placeholders or Mock Data + +- Use only actual values. Do not introduce sample, placeholder, or mock data anywhere (code, docs, examples, commands). +- If a placeholder seems unavoidable, stop and ask for explicit approval and the real values before proceeding. +- Documentation should reference required variables without fake values or example tokens. + # Global Copilot Instructions – No-Interaction Policy +## Lessons Learned (Excel/PocketBase sync) +- Normalize Notes before comparing/merging: convert `\r\n`→`\n`, trim, and compare full strings (no substring containment checks). +- When Notes differ, keep both by concatenating with `\n\n`; never drop content or rely on substring containment. +- Only write the Notes cell via table column `dataBodyRange/cell` to avoid overwriting formula columns. +- Run a fresh audit before any apply/write; do not rely on stale deltas or cached data. +- In poller, skip updates when `notesMergeNeeded` is false; do not rewrite every row. + ## User Interaction Policy - Default to zero user interaction. Do not prompt users unless absolutely necessary. - Prefer agent/service account credentials for backend operations. diff --git a/project-copilot-instructions.md b/project-copilot-instructions.md index 139f20b..32ce45e 100644 --- a/project-copilot-instructions.md +++ b/project-copilot-instructions.md @@ -1,5 +1,18 @@ +# Top Rule: No Placeholders or Mock Data + +- Use only actual values. Do not include sample, placeholder, or mock data in code, docs, examples, or commands. +- If a placeholder appears necessary, pause and request explicit approval and the real values before proceeding. +- Examples must avoid fake tokens/IDs. Reference variables conceptually without inserting dummy values. + # Project Copilot Instructions (Reusable, production-ready) +## Lessons Learned (Notes sync) +- Normalize Notes: convert `\r\n`→`\n`, trim both sides before compare/merge. +- Merge policy: if different, concatenate Excel and PB with a blank line; do not use substring containment to decide equality. +- Write Notes via table column `.../columns/Notes/dataBodyRange/cell(row=X,column=0)` to protect formula columns. +- Run a fresh audit before apply; avoid stale `applied_updates_deltas.json` or cached fetches. +- Poller must only update when `notesMergeNeeded` is true; otherwise skip to prevent touching every row. + ## Core Policy - Zero user interaction. Backends must run unattended. - Tokens are delivered programmatically (env/API/CLI). No device-code or browser prompts. diff --git a/tsconfig-global.md b/tsconfig-global.md index c7ddaba..a5a7f0e 100644 --- a/tsconfig-global.md +++ b/tsconfig-global.md @@ -1,3 +1,8 @@ +# Top Rule: No Placeholders or Mock Data + +- Use only actual values across code and documentation. Do not add sample, placeholder, or mock data. +- If a placeholder seems required, stop and ask for approval and the real value before continuing. + # TypeScript Configuration Standard ## Required Compiler Options