lots of changes should work now

.gitignore

@@ -0,0 +1,20 @@
+node_modules/
+bun.lock
+*.log
+*.env
+.env
+Excel_Listener/.env
+Excel_Audit/.env
+tmp_compare*.ts
+Excel_Listener/applied_updates_deltas.json
+Excel_Listener/audit_output.json
+Excel_Listener/audit_final_check.json
+Excel_Listener/poller_output.txt
+Excel_Listener/poller_test.log
+Excel_Listener/debug_notes.ts
+Excel_Audit/applied_updates_deltas.json
+Excel_Audit/audit_output.json
+Excel_Audit/audit_final_check.json
+Excel_Audit/poller_output.txt
+Excel_Audit/poller_test.log
+Excel_Audit/debug_notes.ts

Excel_Audit/README.md

@@ -0,0 +1,16 @@
+# Excel Audit
+
+Single unified audit tool comparing Excel vs PocketBase with before→after logging.
+
+## Tool: `src/index.ts`
+
+```bash
+bun index.ts --audit      # Compare Excel vs PocketBase (read-only)
+bun index.ts --apply      # Apply updates from deltas to PocketBase
+```
+
+## Output
+
+- `applied_updates_deltas.json` — Complete log: Job_Number, Field, From (PB before), To (Excel or merged)
+
+Requires Excel_Listener environment variables (.env file).

Excel_Audit/bunfig.toml

@@ -0,0 +1,5 @@
+[bundle]
+target = "bun"
+
+[define]
+"process.env" = "process.env"

Excel_Audit/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "excel-audit",
+  "version": "1.0.0-beta",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "start:audit": "bun src/audit.ts"
+  },
+  "dependencies": {
+    "dotenv": "^16.4.5",
+    "pocketbase": "^0.21.3"
+  },
+  "devDependencies": {
+    "@types/bun": "^1.3.4",
+    "@types/node": "^20.12.12",
+    "typescript": "^5.6.3"
+  }
+}

Excel_Audit/src/index.ts

@@ -0,0 +1,467 @@
+import 'dotenv/config';
+import PocketBase from 'pocketbase';
+import { writeFileSync } from 'fs';
+import path from 'path';
+
+declare const fetch: typeof globalThis.fetch;
+
+function requireEnv(name: string): string {
+  const v = process.env[name];
+  if (!v) throw new Error(`Missing required env: ${name}`);
+  return v;
+}
+
+// Excel-only fields (not synced to PocketBase)
+const EXCEL_ONLY_FIELDS = new Set([
+  'Contact_Notes', 'Clean_Phone_Number', 'Fax', 'Phone_Number_Extn',
+  'Secondary_HO', 'Primary_SL', 'POC_Name',
+]);
+
+// Excel-authoritative fields (Excel always overwrites PB - calculated/dynamic)
+const EXCEL_AUTHORITATIVE_FIELDS = new Set([
+  'Due_Date_Counter', 'Active',
+]);
+
+const AUTHORITATIVE_FIELDS = new Set([
+  'Job_Folder_Link', 'Flag', 'Due_Date', 'Job_Full_Name',
+  'Job_QB_Link', 'Voxer_Link', 'Job_Codes', 'Estimate_Draft', 'Job_Status'
+]);
+
+const BOOLEAN_FIELDS = new Set(['Flag', 'Active', 'Estimate_Draft', 'Tax_Exempt']);
+const SENTINEL_ASAP_DATE = '1990-01-01T00:00:00.000Z';
+
+function isDateHeader(name: string): boolean { return name.toLowerCase().includes('date'); }
+function isBooleanHeader(name: string): boolean {
+  const lower = name.toLowerCase();
+  return lower.startsWith('is_') || lower.endsWith('_flag') || lower === 'active' || lower.includes('tax_exempt') || lower === 'estimate_draft';
+}
+function isPocketBaseIdHeader(name: string): boolean {
+  const lower = name.toLowerCase().trim();
+  return lower === 'pb_id' || lower === 'pv_id' || lower === 'pvid';
+}
+
+function excelSerialToIso(num: number): string {
+  const ms = (num - 25569) * 86400 * 1000;
+  return new Date(ms).toISOString();
+}
+
+function toUtcIso(val: any): string {
+  if (val === undefined || val === null || val === '') return '';
+  if (typeof val === 'number' && !Number.isNaN(val)) return excelSerialToIso(val);
+  if (typeof val === 'string') {
+    const trimmed = val.trim();
+    const numeric = parseFloat(trimmed);
+    if (!Number.isNaN(numeric) && /^\d+(\.\d+)?$/.test(trimmed)) return excelSerialToIso(numeric);
+    const d = new Date(trimmed);
+    if (!Number.isNaN(d.getTime())) {
+      return new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate(), d.getUTCHours(), d.getUTCMinutes(), d.getUTCSeconds())).toISOString();
+    }
+    return trimmed;
+  }
+  return String(val);
+}
+
+function normalizeValueByHeader(header: string, val: any) {
+  if (isDateHeader(header)) return toUtcIso(val);
+  if (isBooleanHeader(header)) return normalizeBoolean(val);
+  if (val === undefined || val === null) return '';
+  return typeof val === 'string' ? val.trim() : val;
+}
+
+function normalizeBoolean(val: any): boolean {
+  if (val === true) return true;
+  if (val === false) return false;
+  const s = String(val).toLowerCase();
+  return s === 'true' || s === '1' || s === 'yes' || s === 'y' || s === 't';
+}
+
+async function acquireAppOnlyToken(): Promise<string> {
+  const clientId = requireEnv('CLIENT_ID');
+  const clientSecret = requireEnv('CLIENT_SECRET');
+  const tenantId = requireEnv('TENANT_ID');
+  const tokenUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`;
+  const params = new URLSearchParams({
+    client_id: clientId,
+    client_secret: clientSecret,
+    scope: 'https://graph.microsoft.com/.default',
+    grant_type: 'client_credentials',
+  });
+  const resp = await fetch(tokenUrl, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+    body: params.toString(),
+  });
+  if (!(resp as any).ok) throw new Error(`Token failed: ${(resp as any).status}`);
+  const data = await (resp as any).json();
+  return data.access_token;
+}
+
+async function createSession(userToken: string, persist: boolean = true): Promise<string> {
+  const driveId = requireEnv('DRIVE_ID');
+  const itemId = requireEnv('ITEM_ID');
+  const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/createSession`;
+  const resp = await fetch(url, {
+    method: 'POST',
+    headers: { Authorization: `Bearer ${userToken}`, 'Content-Type': 'application/json' },
+    body: JSON.stringify({ persistChanges: persist }),
+  });
+  if (!(resp as any).ok) throw new Error(`CreateSession failed: ${(resp as any).status}`);
+  return (await (resp as any).json())?.id as string;
+}
+
+async function getTableColumns(userToken: string, sessionId: string): Promise<string[]> {
+  const driveId = requireEnv('DRIVE_ID');
+  const itemId = requireEnv('ITEM_ID');
+  const tableName = requireEnv('EXCEL_TABLE').split('/').pop() as string;
+  const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/tables/${encodeURIComponent(tableName)}/columns`;
+  const resp = await fetch(url, {
+    method: 'GET',
+    headers: { Authorization: `Bearer ${userToken}`, 'workbook-session-id': sessionId },
+  });
+  if (!(resp as any).ok) throw new Error(`GetTableColumns failed: ${(resp as any).status}`);
+  const data = await (resp as any).json();
+  return (data?.value ?? []).map((col: any) => col.name);
+}
+
+async function getTableRows(userToken: string, sessionId: string) {
+  const driveId = requireEnv('DRIVE_ID');
+  const itemId = requireEnv('ITEM_ID');
+  const tableName = requireEnv('EXCEL_TABLE').split('/').pop() as string;
+  const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/tables/${encodeURIComponent(tableName)}/rows`;
+  const resp = await fetch(url, {
+    method: 'GET',
+    headers: { Authorization: `Bearer ${userToken}`, 'workbook-session-id': sessionId },
+  });
+  if (!(resp as any).ok) throw new Error(`GetTableRows failed: ${(resp as any).status}`);
+  const data = await (resp as any).json();
+  const rows: any[] = [];
+  for (const row of (data?.value as any[]) ?? []) {
+    rows.push((row?.values?.[0] ?? []) as any);
+  }
+  return { values: rows };
+}
+
+function mapRowToObject(headers: string[], row: any[]): Record<string, any> {
+  const obj: Record<string, any> = {};
+  for (let i = 0; i < headers.length; i++) {
+    const key = headers[i];
+    if (!key) continue;
+    obj[key] = row[i];
+  }
+  return obj;
+}
+
+function getExcelPocketBaseId(headers: string[], excelObj: Record<string, any> | undefined): string | undefined {
+  if (!excelObj) return undefined;
+  for (const h of headers) {
+    if (!h || !isPocketBaseIdHeader(h)) continue;
+    const v = excelObj[h];
+    if (v === undefined || v === null) return undefined;
+    return String(v).trim();
+  }
+  return undefined;
+}
+
+function diffRecords(headers: string[], excelObj: Record<string, any> | undefined, pbObj: Record<string, any> | undefined) {
+  const diffs: Array<{ field: string; excel: any; pocketbase: any }> = [];
+  for (const h of headers) {
+    if (!h || isExcelOnlyField(h) || isExcelAuthoritativeField(h)) continue;
+    const exVal = excelObj ? normalizeValueByHeader(h, excelObj[h]) : undefined;
+    const pbRaw = pbObj ? (isPocketBaseIdHeader(h) ? (pbObj as any).id : (pbObj as any)[h]) : undefined;
+    const pbVal = pbObj ? normalizeValueByHeader(h, pbRaw) : undefined;
+    const isBool = isBooleanHeader(h);
+    
+    // Special handling for Notes: check if merge would be needed
+    if (h === 'Notes') {
+      if (notesMergeNeeded(exVal, pbVal)) {
+        diffs.push({ field: h, excel: exVal, pocketbase: pbVal });
+      }
+      continue;
+    }
+    
+    const left = isBool ? exVal : (exVal === undefined ? '' : String(exVal).trim());
+    const right = isBool ? pbVal : (pbVal === undefined ? '' : String(pbVal).trim());
+    let equal = left === right;
+    if (!isBool && h === 'Due_Date') {
+      const exStr = String(left).trim().toUpperCase();
+      const pbStr = String(right).trim();
+      if (exStr === 'ASAP' && (pbStr === '' || pbStr === SENTINEL_ASAP_DATE)) equal = true;
+    }
+    if (!equal) diffs.push({ field: h, excel: exVal, pocketbase: pbVal });
+  }
+  return diffs;
+}
+
+function isExcelOnlyField(field: string): boolean { return EXCEL_ONLY_FIELDS.has(field); }
+function isExcelAuthoritativeField(field: string): boolean { return EXCEL_AUTHORITATIVE_FIELDS.has(field); }
+
+async function fetchExcelMap() {
+  const tokenInfo = await acquireAppOnlyToken();
+  const sessionId = await createSession(tokenInfo, true);
+  const headers = await getTableColumns(tokenInfo, sessionId);
+  if (!headers.includes('Job_Number')) throw new Error('Job_Number header required');
+  const rows = await getTableRows(tokenInfo, sessionId);
+  const byKey = new Map<string, Record<string, any>>();
+  for (const r of rows.values.map((row, i) => mapRowToObject(headers, row))) {
+    const key = r['Job_Number'];
+    if (key) byKey.set(String(key), r);
+  }
+  return { headers, byKey };
+}
+
+async function fetchPocketBaseMap(collection: string) {
+  const pbUrl = (() => {
+    const explicit = process.env.PB_DB;
+    if (explicit) return explicit;
+    const redirect = process.env.REDIRECT_URI;
+    if (!redirect) throw new Error('Missing PB_DB or REDIRECT_URI');
+    const u = new URL(redirect);
+    return `${u.protocol}//${u.host}`;
+  })();
+  const pb = new PocketBase(pbUrl);
+  await pb.collection(process.env.PB_AUTH_COLLECTION || 'Users').authWithPassword(
+    requireEnv('PB_AGENT_EMAIL'),
+    requireEnv('PB_AGENT_PASSWORD')
+  );
+  const records = await pb.collection(collection).getFullList(200);
+  const byKey = new Map<string, Record<string, any>>();
+  for (const rec of records as any[]) {
+    const key = rec['Job_Number'];
+    if (key) byKey.set(String(key), rec);
+  }
+  return byKey;
+}
+
+async function runAudit() {
+  const collection = requireEnv('PB_COLLECTION');
+  const { headers, byKey: excelMap } = await fetchExcelMap();
+  const pbMap = await fetchPocketBaseMap(collection);
+
+  const commonKeys = [...excelMap.keys()].filter(k => pbMap.has(k));
+  const mismatches: Array<{ key: string; diffs: Array<{ field: string; excel: any; pocketbase: any }> }> = [];
+
+  for (const k of commonKeys) {
+    const diffs = diffRecords(headers, excelMap.get(k), pbMap.get(k));
+    if (diffs.length > 0) mismatches.push({ key: k, diffs });
+  }
+
+  console.log(JSON.stringify({
+    summary: {
+      excelCount: excelMap.size,
+      pocketbaseCount: pbMap.size,
+      mismatched: mismatches.length,
+    },
+    mismatches,
+  }, null, 2));
+}
+
+function normalizeNotes(val: any): string {
+  return (val ?? '').toString().replace(/\r\n/g, '\n').trim();
+}
+
+function mergeNotes(excelNotes: any, pbNotes: any): string {
+  const excel = normalizeNotes(excelNotes);
+  const pb = normalizeNotes(pbNotes);
+  if (!excel && !pb) return '';
+  if (!excel) return pb;
+  if (!pb) return excel;
+  if (excel === pb) return excel;
+  return `${excel}\n\n${pb}`;
+}
+
+function notesMergeNeeded(excelNotes: any, pbNotes: any): boolean {
+  const merged = mergeNotes(excelNotes, pbNotes);
+  const pbStr = normalizeNotes(pbNotes);
+  const exStr = normalizeNotes(excelNotes);
+  // If normalized strings already equal, no merge needed
+  if (exStr === pbStr) return false;
+  // Otherwise, merge changes PB content
+  return merged !== pbStr;
+}
+
+function normalizeBoolean2(val: any): boolean {
+  if (val === true) return true;
+  if (val === false) return false;
+  const s = String(val).trim().toLowerCase();
+  return s === 'true' || s === 't' || s === 'yes' || s === 'y' || s === '1';
+}
+
+// Columns that contain formulas and should never be overwritten
+const FORMULA_COLUMNS = new Set(['Active', 'Due_Date_Counter']);
+
+async function updateExcelRow(userToken: string, sessionId: string, jobNumber: string, mergedNotes: string, headers: string[]) {
+  const driveId = requireEnv('DRIVE_ID');
+  const itemId = requireEnv('ITEM_ID');
+  const tableName = requireEnv('EXCEL_TABLE').split('/').pop() as string;
+  
+  // Get all rows and find the Job_Number match
+  const rowUrl = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}/workbook/tables/${encodeURIComponent(tableName)}/rows`;
+  const rowResp = await fetch(rowUrl, {
+    method: 'GET',
+    headers: { Authorization: `Bearer ${userToken}`, 'workbook-session-id': sessionId },
+  });
+  if (!(rowResp as any).ok) throw new Error(`GetTableRows failed: ${(rowResp as any).status}`);
+  const rowData = await (rowResp as any).json();
+  
+  const jobRow = (rowData?.value ?? []).find((r: any) => {
+    const values = r?.values?.[0] ?? [];
+    return String(values[0]).trim() === String(jobNumber).trim();
+  });
+  
+  if (!jobRow) return; // Job not found
+  
+  // Find the Notes column index
+  const notesIdx = headers.indexOf('Notes');
+  if (notesIdx === -1) return; // Notes column not found
+  
+  console.log(`\n→ Writing to Excel Job ${jobNumber}: ${mergedNotes.length} chars`);
+  console.log(`  Content: ${JSON.stringify(mergedNotes)}`);
+  
+  // Use table column's DataBodyRange to update specific cell
+  const driveId2 = requireEnv('DRIVE_ID');
+  const itemId2 = requireEnv('ITEM_ID');
+  const tableName2 = requireEnv('EXCEL_TABLE').split('/').pop() as string;
+  
+  // Access the Notes column's DataBodyRange, then use row/cell API
+  const colRangeUrl = `https://graph.microsoft.com/v1.0/drives/${driveId2}/items/${itemId2}/workbook/tables/${encodeURIComponent(tableName2)}/columns/${encodeURIComponent('Notes')}/dataBodyRange/cell(row=${jobRow.index},column=0)`;
+  const updateResp = await fetch(colRangeUrl, {
+    method: 'PATCH',
+    headers: { Authorization: `Bearer ${userToken}`, 'workbook-session-id': sessionId, 'Content-Type': 'application/json' },
+    body: JSON.stringify({ values: [[mergedNotes]] }),
+  });
+  if (!(updateResp as any).ok) {
+    throw new Error(`Excel cell update failed: ${(updateResp as any).status} - ${await (updateResp as any).text()}`);
+  }
+}
+
+
+
+async function runApply() {
+  // First, run a fresh audit to get current mismatches
+  // This ensures we have the latest data, not stale cached data
+  console.log('🔍 Running fresh audit before apply...');
+  const collection = requireEnv('PB_COLLECTION');
+  const { headers, byKey: excelMap } = await fetchExcelMap();
+  const pbMap = await fetchPocketBaseMap(collection);
+
+  const commonKeys = [...excelMap.keys()].filter(k => pbMap.has(k));
+  const mismatches: Array<any> = [];
+
+  for (const k of commonKeys) {
+    const diffs = diffRecords(headers, excelMap.get(k), pbMap.get(k));
+    if (diffs.length > 0) mismatches.push({ key: k, diffs });
+  }
+
+  console.log(`✓ Fresh audit complete: ${mismatches.length} mismatches found`);
+
+  const pbUrl = (() => {
+    const explicit = process.env.PB_DB;
+    if (explicit) return explicit;
+    const redirect = process.env.REDIRECT_URI;
+    if (!redirect) throw new Error('Missing PB_DB or REDIRECT_URI');
+    const u = new URL(redirect);
+    return `${u.protocol}//${u.host}`;
+  })();
+  const pb = new PocketBase(pbUrl);
+  await pb.collection(process.env.PB_AUTH_COLLECTION || 'Users').authWithPassword(
+    requireEnv('PB_AGENT_EMAIL'),
+    requireEnv('PB_AGENT_PASSWORD')
+  );
+
+  // Get Excel access for writing merged Notes back
+  const excelToken = await acquireAppOnlyToken();
+  const excelSessionId = await createSession(excelToken, true);
+  const excelHeaders = await getTableColumns(excelToken, excelSessionId);
+
+  let updated = 0, skipped = 0;
+
+  for (const job of mismatches) {
+    const key = job.key || job.jobNumber;
+    const diffs = job.diffs || [{ field: job.field, excel: job.to, pocketbase: job.from }];
+    const updatePayload: Record<string, any> = {};
+    let mergedNotes: string | null = null;
+
+    for (const d of diffs) {
+      const field = d.field;
+      const excelVal = d.excel;
+      const pbVal = d.pocketbase;
+      if (field === 'Notes') {
+        const merged = mergeNotes(excelVal, pbVal);
+        console.log(`\n=== JOB ${key} NOTES MERGE ===`);
+        console.log(`Excel (${(excelVal ?? '').toString().length} chars): ${JSON.stringify(excelVal)}`);
+        console.log(`PB (${(pbVal ?? '').toString().length} chars): ${JSON.stringify(pbVal)}`);
+        console.log(`Merged (${merged.length} chars): ${JSON.stringify(merged)}`);
+        
+        if (merged !== (pbVal ?? '').toString().trim()) {
+          console.log(`✓ Merge needed - updating PB`);
+          updatePayload[field] = merged;
+        } else {
+          console.log(`✗ Merge not needed`);
+        }
+        mergedNotes = merged;
+      } else if (AUTHORITATIVE_FIELDS.has(field)) {
+        if (field === 'Due_Date') {
+          const exStr = String(excelVal ?? '').trim();
+          if (exStr.toUpperCase() === 'ASAP') {
+            if (String(pbVal ?? '') !== SENTINEL_ASAP_DATE) updatePayload[field] = SENTINEL_ASAP_DATE;
+          } else {
+            if (String(excelVal ?? '') !== String(pbVal ?? '')) updatePayload[field] = excelVal;
+          }
+        } else if (BOOLEAN_FIELDS.has(field)) {
+          const excelBool = normalizeBoolean2(excelVal);
+          const pbBool = normalizeBoolean2(pbVal);
+          if (excelBool !== pbBool) updatePayload[field] = excelBool;
+        } else {
+          if (String(excelVal ?? '') !== String(pbVal ?? '')) updatePayload[field] = excelVal;
+        }
+      }
+    }
+
+    if (Object.keys(updatePayload).length === 0) {
+      skipped++;
+      continue;
+    }
+
+    const filter = `Job_Number=\"${String(key).replace(/\"/g, '\\\"')}\"`;
+    const found = await pb.collection(collection).getList(1, 1, { filter });
+    if (!found?.items?.length) continue;
+
+    const existing = found.items[0];
+    await pb.collection(collection).update(existing.id, updatePayload);
+    updated++;
+    
+    // If Notes were merged, also update Excel to keep chain synchronized
+    if (mergedNotes !== null) {
+      try {
+        await updateExcelRow(excelToken, excelSessionId, String(key), mergedNotes, excelHeaders);
+        console.log(`✓ Job ${key}: ${Object.keys(updatePayload).join(', ')} (chain synced to both)`);
+      } catch (err: any) {
+        console.error(`✗ Job ${key}: Excel write failed - ${err?.message || err}`);
+        console.log(`✓ Job ${key}: ${Object.keys(updatePayload).join(', ')} (PocketBase updated, Excel write failed)`);
+      }
+    } else {
+      console.log(`✓ Job ${key}: ${Object.keys(updatePayload).join(', ')}`);
+    }
+  }
+
+  console.log(`\n=== Summary: ${updated} updated, ${skipped} skipped ===`);
+}
+
+async function main() {
+  const mode = process.argv[2];
+  if (mode === '--audit') {
+    await runAudit();
+  } else if (mode === '--apply') {
+    await runApply();
+  } else {
+    console.log('Usage: bun index.ts [--audit|--apply]');
+    console.log('  --audit  Compare Excel vs PocketBase (read-only)');
+    console.log('  --apply  Apply updates from deltas.json to PocketBase');
+    process.exit(1);
+  }
+}
+
+main().catch(err => {
+  console.error('✗ Error:', err?.message || err);
+  process.exit(1);
+});

Excel_Audit/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "ESNext",
+    "lib": ["ESNext"],
+    "moduleResolution": "bundler",
+    "skipLibCheck": true,
+    "noImplicitAny": false,
+    "allowJs": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "noEmit": true,
+    "allowImportingTsExtensions": true
+  },
+  "include": ["src"],
+  "exclude": ["node_modules", "dist"]
+}

Excel_Listener/README.md

@@ -1,5 +1,10 @@
 # Excel Listener - Clean Setup
 
+## Top Rule: No Placeholders or Mock Data
+
+- Use only actual values in configuration, commands, and examples.
+- If a placeholder seems necessary, pause and request approval and the real values before proceeding.
+
 > See project-wide preferences: [project-copilot-instructions.md](../project-copilot-instructions.md). For org standards, see [global-copilot-instructions.md](../global-copilot-instructions.md).
 
 ## Structure
@@ -25,9 +30,7 @@ bun install
 
 ### Run Poller (delegated token)
 ```bash
-$Env:GRAPH_ACCESS_TOKEN = "<paste-token>"
-# Optional expiry hint (unix ms epoch)
-# $Env:GRAPH_TOKEN_EXPIRES_AT = "1734220000000"
+# Ensure a valid delegated Graph token is present in GRAPH_ACCESS_TOKEN (no placeholders).
 bun run start:poller
 ```
 

Excel_Listener/package.json

@@ -1,6 +1,6 @@
 {
   "name": "excel-listener-auth",
-  "version": "0.1.0",
+  "version": "1.0.0-beta",
   "private": true,
   "type": "module",
   "scripts": {

Excel_Listener/src/poller.ts

@@ -3,6 +3,9 @@ import PocketBase from 'pocketbase';
 import { acquireGraphUserToken } from './userAuth.ts';
 import { createSession, getTableRows, getTableColumns } from './excel.ts';
 import { mapRowToObject, upsertPocketBaseByKey } from './sync.ts';
+import { execSync } from 'child_process';
+import { writeFileSync } from 'fs';
+import path from 'path';
 
 function requireEnv(name: string): string {
   const val = process.env[name];
@@ -25,18 +28,61 @@ function validateEnv() {
   for (const k of required) requireEnv(k);
 }
 
-// In-memory hash cache
-const hashCache = new Map<string, string>();
+// (Removed) In-memory hash cache: rely on upsert equality check to avoid unnecessary writes
+
+async function runStartupAudit() {
+  console.log('📊 Running startup audit to detect any data mismatches...');
+  try {
+    // Run audit from Excel_Audit/src/index.ts (go up to parent, then into Excel_Audit)
+    const auditPath = path.resolve(__dirname, '../../Excel_Audit/src/index.ts');
+    const auditOutput = execSync(`bun ${auditPath} --audit`, {
+      cwd: __dirname,
+      encoding: 'utf-8',
+    });
+    const auditResult = JSON.parse(auditOutput);
+    const { summary } = auditResult;
+    
+    console.log(`✓ Audit complete: ${summary.excelCount} Excel, ${summary.pocketbaseCount} PocketBase, ${summary.mismatched} mismatches`);
+    
+    if (summary.mismatched > 0) {
+      console.log(`⚠️ Found ${summary.mismatched} mismatches; applying corrections...`);
+      
+      // Build deltas.json from audit mismatches
+      const deltas = {
+        timestamp: new Date().toISOString(),
+        summary,
+        mismatches: auditResult.mismatches || [],
+      };
+      
+      const deltasPath = path.resolve(__dirname, '..', 'applied_updates_deltas.json');
+      writeFileSync(deltasPath, JSON.stringify(deltas, null, 2));
+      console.log(`✓ Saved deltas to ${deltasPath}`);
+      
+      // Apply changes
+      try {
+        const applyOutput = execSync(`bun ${auditPath} --apply`, {
+          cwd: __dirname,
+          encoding: 'utf-8',
+        });
+        console.log(`✓ Applied updates:\n${applyOutput}`);
+      } catch (err: any) {
+        console.error(`✗ Apply failed: ${err?.message || err}`);
+      }
+    } else {
+      console.log('✓ No mismatches detected; data is synchronized');
+    }
+  } catch (err: any) {
+    console.warn(`⚠️ Startup audit failed (non-fatal): ${err?.message || err}`);
+    console.log('Proceeding with normal polling anyway...');
+  }
+}
 
 async function runOnce() {
   const pbUrl = (() => {
     const explicit = process.env.PB_DB;
     if (explicit) return explicit;
-    // Poll interval (seconds) shared for logging remaining time per loop
-    const pollIntervalSec = Number(process.env.POLL_INTERVAL_SEC || 60);
     const redirect = process.env.REDIRECT_URI;
     if (!redirect) throw new Error('Missing PB_DB or REDIRECT_URI');
-      const startedAt = Date.now();
     const u = new URL(redirect);
     return `${u.protocol}//${u.host}`;
   })();
@@ -78,15 +124,8 @@ async function runOnce() {
         console.warn('⚠️ Missing key; skipping row');
         continue;
       }
-      const hashInput = JSON.stringify(Object.keys(obj).sort().reduce((acc: any, k) => { acc[k] = obj[k]; return acc; }, {}));
-      const prevHash = hashCache.get(String(keyVal));
-      if (prevHash === hashInput) {
-        skipped++;
-        continue;
-      }
       const silent = processed % 100 !== 0;
       await upsertPocketBaseByKey(pb, collection, keyName, obj, silent);
-      hashCache.set(String(keyVal), hashInput);
       processed++;
       // Progress indicator
       if (processed % 100 === 0) {
@@ -106,8 +145,11 @@ async function main() {
   validateEnv();
   const intervalSec = Number(process.env.POLL_INTERVAL_SEC || 60);
   console.log(`✓ Starting poller (interval ${intervalSec}s)`);
+  
+  // Run startup audit to detect and fix any mismatches
+  await runStartupAudit();
+  
   let running = false;
-      console.log(`✓ Starting poller (interval ${pollIntervalSec}s)`);
   const runSafely = async () => {
     if (running) return;
     running = true;

Excel_Listener/src/sync.ts

@@ -9,10 +9,26 @@ function requireEnv(name: string): string {
   return val;
 }
 
+// Excel-only fields (not synced to PocketBase)
+const EXCEL_ONLY_FIELDS = new Set([
+  'Contact_Notes', 'Clean_Phone_Number', 'Fax', 'Phone_Number_Extn',
+  'Secondary_HO', 'Primary_SL', 'POC_Name',
+  // Do not sync Excel's pb_id into PocketBase; treat as Excel-only here
+  'pb_id', 'PB_ID',
+]);
+
+// Excel-authoritative fields: always overwrite PB with Excel values
+// No authoritative overwrites: we only update when canonical values differ
+const EXCEL_AUTHORITATIVE_FIELDS = new Set<string>();
+
+const BOOLEAN_FIELDS = new Set(['Flag', 'Active', 'Estimate_Draft', 'Tax_Exempt']);
+const SENTINEL_ASAP_DATE = '1990-01-01T00:00:00.000Z';
+
 export function normalizeBoolean(val: any): boolean {
   if (val === true) return true;
+  if (val === false) return false;
   const s = String(val).toLowerCase();
-  return s === 'true' || s === '1' || s === 'yes' || s === 'y';
+  return s === 'true' || s === '1' || s === 'yes' || s === 'y' || s === 't';
 }
 
 function excelSerialToIso(num: number): string {
@@ -50,14 +66,86 @@ export function mapRowToObject(headers: string[], row: any[]): RowObject {
     const key = headers[i];
     const val = row[i];
     if (!key) continue;
+    
+    // Skip Excel-only fields
+    if (EXCEL_ONLY_FIELDS.has(key)) continue;
+    
     const lower = key.toLowerCase();
-    if (lower.includes('date')) obj[key] = toUtcIso(val);
-    else if (lower.startsWith('is_') || lower.endsWith('_flag') || lower === 'active' || lower.includes('tax_exempt')) obj[key] = normalizeBoolean(val);
-    else obj[key] = val;
+    
+    // Handle Due_Date with ASAP sentinel
+    if (lower === 'due_date') {
+      const valStr = String(val ?? '').trim();
+      if (valStr.toUpperCase() === 'ASAP') {
+        obj[key] = SENTINEL_ASAP_DATE;
+      } else {
+        obj[key] = toUtcIso(val);
+      }
+    }
+    // Date fields (except Due_Date already handled)
+    else if (lower.includes('date')) {
+      obj[key] = toUtcIso(val);
+    }
+    // Boolean fields
+    else if (BOOLEAN_FIELDS.has(key) || lower.startsWith('is_') || lower.endsWith('_flag') || lower === 'active' || lower.includes('tax_exempt')) {
+      obj[key] = normalizeBoolean(val);
+    }
+    // All other fields
+    else {
+      obj[key] = val;
+    }
   }
   return obj;
 }
 
+function normalizeNotes(val: any): string {
+  return (val ?? '').toString().replace(/\r\n/g, '\n').trim();
+}
+
+function mergeNotes(excelNotes: string | null | undefined, pbNotes: string | null | undefined): string {
+  const excel = normalizeNotes(excelNotes);
+  const pb = normalizeNotes(pbNotes);
+  if (!excel && !pb) return '';
+  if (!excel) return pb;
+  if (!pb) return excel;
+  if (excel === pb) return excel;
+  return `${excel}\n\n${pb}`;
+}
+
+function notesMergeNeeded(excelNotes: any, pbNotes: any): boolean {
+  const merged = mergeNotes(excelNotes, pbNotes);
+  const pbStr = normalizeNotes(pbNotes);
+  const exStr = normalizeNotes(excelNotes);
+  if (exStr === pbStr) return false;
+  return merged !== pbStr;
+}
+
+function canonicalizeForCompare(key: string, val: any): string {
+  const lower = key.toLowerCase();
+  if (key === 'Notes') return normalizeNotes(val);
+  // Dates: compare as UTC ISO strings
+  if (lower === 'due_date' || lower.includes('date')) {
+    const iso = toUtcIso(val);
+    return (iso ?? '').toString().trim();
+  }
+  // Booleans: compare normalized boolean as 'true'/'false'
+  if (BOOLEAN_FIELDS.has(key) || lower.startsWith('is_') || lower.endsWith('_flag') || lower === 'active' || lower.includes('tax_exempt')) {
+    return normalizeBoolean(val) ? 'true' : 'false';
+  }
+  // Generic string/number
+  if (val === undefined || val === null) return '';
+  return String(val).trim();
+}
+
+function recordsEqual(existing: RowObject, candidate: RowObject): boolean {
+  // Compare only keys present in candidate with canonical normalization
+  for (const k of Object.keys(candidate)) {
+    const a = canonicalizeForCompare(k, existing?.[k]);
+    const b = canonicalizeForCompare(k, candidate?.[k]);
+    if (a !== b) return false;
+  }
+  return true;
+}
+
 export async function upsertPocketBaseByKey(pb: PocketBase, collection: string, keyName: string, record: RowObject, silent: boolean = false) {
   if (keyName.toLowerCase() !== 'job_number') {
     throw new Error('Primary key must be Job_Number (exact header name)');
@@ -66,11 +154,33 @@ export async function upsertPocketBaseByKey(pb: PocketBase, collection: string,
   if (!keyVal) throw new Error(`Missing key ${keyName} in record`);
   const filter = `${keyName}="${String(keyVal).replace(/"/g, '\\"')}"`;
   const found = await pb.collection(collection).getList(1, 1, { filter });
+  
   if (found?.items?.length) {
     const existing = found.items[0];
-    await pb.collection(collection).update(existing.id, record);
-    if (!silent) console.log(`✓ Updated PB ${collection} id=${existing.id}`);
+
+    // Handle Notes bidirectional merge - only if it would actually change
+    const updatePayload = { ...record };
+    if ('Notes' in updatePayload && notesMergeNeeded(updatePayload.Notes, existing.Notes)) {
+      updatePayload.Notes = mergeNotes(updatePayload.Notes, existing.Notes);
+    }
+    // Skip update if nothing would change (canonical compare)
+    if (recordsEqual(existing, updatePayload)) {
+      return;
+    }
+    // Compute concise per-field diffs for visibility
+    const changedKeys: string[] = [];
+    for (const k of Object.keys(updatePayload)) {
+      const before = canonicalizeForCompare(k, existing?.[k]);
+      const after = canonicalizeForCompare(k, updatePayload?.[k]);
+      if (before !== after) changedKeys.push(k);
+    }
+    await pb.collection(collection).update(existing.id, updatePayload);
+    if (!silent) {
+      const list = changedKeys.slice(0, 6).join(', ');
+      console.log(`✓ Updated PB ${collection} id=${existing.id} (fields: ${list}${changedKeys.length > 6 ? ', …' : ''})`);
+    }
   } else {
+    // Create full record when not found (actual change)
     const created = await pb.collection(collection).create(record);
     if (!silent) console.log(`✓ Created PB ${collection} id=${created.id}`);
   }

README.md

@@ -1,2 +1,13 @@
 # ExcelListenerUpdates
 
+## Top Rule: No Placeholders or Mock Data
+
+- Use only actual values. Do not include sample, placeholder, or mock data in code, docs, or commands.
+- If a placeholder seems necessary, ask for explicit approval and the real values before proceeding.
+
+See project-specific details in [Excel_Listener/README.md](Excel_Listener/README.md) and org standards in [global-copilot-instructions.md](global-copilot-instructions.md) and [project-copilot-instructions.md](project-copilot-instructions.md).
+
+## Version
+
+- v1.0.0-beta: stabilized Notes merge (normalized line endings, concat both sides), protected formula columns via single-cell updates, enforced fresh audit-before-apply, and reduced poller churn by skipping unchanged rows.
+

global-copilot-instructions.md

@@ -1,5 +1,18 @@
+# Top Rule: No Placeholders or Mock Data
+
+- Use only actual values. Do not introduce sample, placeholder, or mock data anywhere (code, docs, examples, commands).
+- If a placeholder seems unavoidable, stop and ask for explicit approval and the real values before proceeding.
+- Documentation should reference required variables without fake values or example tokens.
+
 # Global Copilot Instructions – No-Interaction Policy
 
+## Lessons Learned (Excel/PocketBase sync)
+- Normalize Notes before comparing/merging: convert `\r\n`→`\n`, trim, and compare full strings (no substring containment checks).
+- When Notes differ, keep both by concatenating with `\n\n`; never drop content or rely on substring containment.
+- Only write the Notes cell via table column `dataBodyRange/cell` to avoid overwriting formula columns.
+- Run a fresh audit before any apply/write; do not rely on stale deltas or cached data.
+- In poller, skip updates when `notesMergeNeeded` is false; do not rewrite every row.
+
 ## User Interaction Policy
 - Default to zero user interaction. Do not prompt users unless absolutely necessary.
 - Prefer agent/service account credentials for backend operations.

project-copilot-instructions.md

@@ -1,5 +1,18 @@
+# Top Rule: No Placeholders or Mock Data
+
+- Use only actual values. Do not include sample, placeholder, or mock data in code, docs, examples, or commands.
+- If a placeholder appears necessary, pause and request explicit approval and the real values before proceeding.
+- Examples must avoid fake tokens/IDs. Reference variables conceptually without inserting dummy values.
+
 # Project Copilot Instructions (Reusable, production-ready)
 
+## Lessons Learned (Notes sync)
+- Normalize Notes: convert `\r\n`→`\n`, trim both sides before compare/merge.
+- Merge policy: if different, concatenate Excel and PB with a blank line; do not use substring containment to decide equality.
+- Write Notes via table column `.../columns/Notes/dataBodyRange/cell(row=X,column=0)` to protect formula columns.
+- Run a fresh audit before apply; avoid stale `applied_updates_deltas.json` or cached fetches.
+- Poller must only update when `notesMergeNeeded` is true; otherwise skip to prevent touching every row.
+
 ## Core Policy
 - Zero user interaction. Backends must run unattended.
 - Tokens are delivered programmatically (env/API/CLI). No device-code or browser prompts.

tsconfig-global.md

@@ -1,3 +1,8 @@
+# Top Rule: No Placeholders or Mock Data
+
+- Use only actual values across code and documentation. Do not add sample, placeholder, or mock data.
+- If a placeholder seems required, stop and ask for approval and the real value before continuing.
+
 # TypeScript Configuration Standard
 
 ## Required Compiler Options