chore: release v1.0.0-beta with notes hardening

Excel_Audit/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "excel-audit",
+  "version": "1.0.0-beta",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "start:audit": "bun src/audit.ts"
+  },
+  "dependencies": {
+    "dotenv": "^16.4.5",
+    "pocketbase": "^0.21.3"
+  },
+  "devDependencies": {
+    "@types/bun": "^1.3.4",
+    "@types/node": "^20.12.12",
+    "typescript": "^5.6.3"
+  }
+}

Excel_Listener/README.md

@@ -1,5 +1,10 @@
 # Excel Listener - Clean Setup
 
+## Top Rule: No Placeholders or Mock Data
+
+- Use only actual values in configuration, commands, and examples.
+- If a placeholder seems necessary, pause and request approval and the real values before proceeding.
+
 > See project-wide preferences: [project-copilot-instructions.md](../project-copilot-instructions.md). For org standards, see [global-copilot-instructions.md](../global-copilot-instructions.md).
 
 ## Structure
@@ -25,9 +30,7 @@ bun install
 
 ### Run Poller (delegated token)
 ```bash
-$Env:GRAPH_ACCESS_TOKEN = "<paste-token>"
-# Optional expiry hint (unix ms epoch)
-# $Env:GRAPH_TOKEN_EXPIRES_AT = "1734220000000"
+# Ensure a valid delegated Graph token is present in GRAPH_ACCESS_TOKEN (no placeholders).
 bun run start:poller
 ```
 

Excel_Listener/package.json

@@ -1,6 +1,6 @@
 {
   "name": "excel-listener-auth",
-  "version": "0.1.0",
+  "version": "1.0.0-beta",
   "private": true,
   "type": "module",
   "scripts": {

Excel_Listener/src/poller.ts

@@ -3,6 +3,9 @@ import PocketBase from 'pocketbase';
 import { acquireGraphUserToken } from './userAuth.ts';
 import { createSession, getTableRows, getTableColumns } from './excel.ts';
 import { mapRowToObject, upsertPocketBaseByKey } from './sync.ts';
+import { execSync } from 'child_process';
+import { writeFileSync } from 'fs';
+import path from 'path';
 
 function requireEnv(name: string): string {
   const val = process.env[name];
@@ -25,18 +28,61 @@ function validateEnv() {
   for (const k of required) requireEnv(k);
 }
 
-// In-memory hash cache
-const hashCache = new Map<string, string>();
+// (Removed) In-memory hash cache: rely on upsert equality check to avoid unnecessary writes
+
+async function runStartupAudit() {
+  console.log('📊 Running startup audit to detect any data mismatches...');
+  try {
+    // Run audit from Excel_Audit/src/index.ts (go up to parent, then into Excel_Audit)
+    const auditPath = path.resolve(__dirname, '../../Excel_Audit/src/index.ts');
+    const auditOutput = execSync(`bun ${auditPath} --audit`, {
+      cwd: __dirname,
+      encoding: 'utf-8',
+    });
+    const auditResult = JSON.parse(auditOutput);
+    const { summary } = auditResult;
+    
+    console.log(`✓ Audit complete: ${summary.excelCount} Excel, ${summary.pocketbaseCount} PocketBase, ${summary.mismatched} mismatches`);
+    
+    if (summary.mismatched > 0) {
+      console.log(`⚠️ Found ${summary.mismatched} mismatches; applying corrections...`);
+      
+      // Build deltas.json from audit mismatches
+      const deltas = {
+        timestamp: new Date().toISOString(),
+        summary,
+        mismatches: auditResult.mismatches || [],
+      };
+      
+      const deltasPath = path.resolve(__dirname, '..', 'applied_updates_deltas.json');
+      writeFileSync(deltasPath, JSON.stringify(deltas, null, 2));
+      console.log(`✓ Saved deltas to ${deltasPath}`);
+      
+      // Apply changes
+      try {
+        const applyOutput = execSync(`bun ${auditPath} --apply`, {
+          cwd: __dirname,
+          encoding: 'utf-8',
+        });
+        console.log(`✓ Applied updates:\n${applyOutput}`);
+      } catch (err: any) {
+        console.error(`✗ Apply failed: ${err?.message || err}`);
+      }
+    } else {
+      console.log('✓ No mismatches detected; data is synchronized');
+    }
+  } catch (err: any) {
+    console.warn(`⚠️ Startup audit failed (non-fatal): ${err?.message || err}`);
+    console.log('Proceeding with normal polling anyway...');
+  }
+}
 
 async function runOnce() {
   const pbUrl = (() => {
     const explicit = process.env.PB_DB;
     if (explicit) return explicit;
-    // Poll interval (seconds) shared for logging remaining time per loop
-    const pollIntervalSec = Number(process.env.POLL_INTERVAL_SEC || 60);
     const redirect = process.env.REDIRECT_URI;
     if (!redirect) throw new Error('Missing PB_DB or REDIRECT_URI');
-      const startedAt = Date.now();
     const u = new URL(redirect);
     return `${u.protocol}//${u.host}`;
   })();
@@ -78,15 +124,8 @@ async function runOnce() {
         console.warn('⚠️ Missing key; skipping row');
         continue;
       }
-      const hashInput = JSON.stringify(Object.keys(obj).sort().reduce((acc: any, k) => { acc[k] = obj[k]; return acc; }, {}));
-      const prevHash = hashCache.get(String(keyVal));
-      if (prevHash === hashInput) {
-        skipped++;
-        continue;
-      }
       const silent = processed % 100 !== 0;
       await upsertPocketBaseByKey(pb, collection, keyName, obj, silent);
-      hashCache.set(String(keyVal), hashInput);
       processed++;
       // Progress indicator
       if (processed % 100 === 0) {
@@ -106,8 +145,11 @@ async function main() {
   validateEnv();
   const intervalSec = Number(process.env.POLL_INTERVAL_SEC || 60);
   console.log(`✓ Starting poller (interval ${intervalSec}s)`);
+  
+  // Run startup audit to detect and fix any mismatches
+  await runStartupAudit();
+  
   let running = false;
-      console.log(`✓ Starting poller (interval ${pollIntervalSec}s)`);
   const runSafely = async () => {
     if (running) return;
     running = true;

Excel_Listener/src/sync.ts

@@ -9,10 +9,33 @@ function requireEnv(name: string): string {
   return val;
 }
 
+// Excel-only fields (not synced to PocketBase)
+const EXCEL_ONLY_FIELDS = new Set([
+  'Contact_Notes', 'Clean_Phone_Number', 'Fax', 'Phone_Number_Extn',
+  'Secondary_HO', 'Primary_SL', 'POC_Name',
+]);
+
+// Excel-authoritative fields: always overwrite PB with Excel values
+const EXCEL_AUTHORITATIVE_FIELDS = new Set([
+  'Job_Folder_Link',
+  'Flag',
+  'Due_Date',
+  'Job_Full_Name',
+  'Job_QB_Link',
+  'Voxer_Link',
+  'Job_Codes',
+  'Estimate_Draft',
+  'Job_Status'
+]);
+
+const BOOLEAN_FIELDS = new Set(['Flag', 'Active', 'Estimate_Draft', 'Tax_Exempt']);
+const SENTINEL_ASAP_DATE = '1990-01-01T00:00:00.000Z';
+
 export function normalizeBoolean(val: any): boolean {
   if (val === true) return true;
+  if (val === false) return false;
   const s = String(val).toLowerCase();
-  return s === 'true' || s === '1' || s === 'yes' || s === 'y';
+  return s === 'true' || s === '1' || s === 'yes' || s === 'y' || s === 't';
 }
 
 function excelSerialToIso(num: number): string {
@@ -50,14 +73,71 @@ export function mapRowToObject(headers: string[], row: any[]): RowObject {
     const key = headers[i];
     const val = row[i];
     if (!key) continue;
+    
+    // Skip Excel-only fields
+    if (EXCEL_ONLY_FIELDS.has(key)) continue;
+    
     const lower = key.toLowerCase();
-    if (lower.includes('date')) obj[key] = toUtcIso(val);
-    else if (lower.startsWith('is_') || lower.endsWith('_flag') || lower === 'active' || lower.includes('tax_exempt')) obj[key] = normalizeBoolean(val);
-    else obj[key] = val;
+    
+    // Handle Due_Date with ASAP sentinel
+    if (lower === 'due_date') {
+      const valStr = String(val ?? '').trim();
+      if (valStr.toUpperCase() === 'ASAP') {
+        obj[key] = SENTINEL_ASAP_DATE;
+      } else {
+        obj[key] = toUtcIso(val);
+      }
+    }
+    // Date fields (except Due_Date already handled)
+    else if (lower.includes('date')) {
+      obj[key] = toUtcIso(val);
+    }
+    // Boolean fields
+    else if (BOOLEAN_FIELDS.has(key) || lower.startsWith('is_') || lower.endsWith('_flag') || lower === 'active' || lower.includes('tax_exempt')) {
+      obj[key] = normalizeBoolean(val);
+    }
+    // All other fields
+    else {
+      obj[key] = val;
+    }
   }
   return obj;
 }
 
+function mergeNotes(excelNotes: string | null | undefined, pbNotes: string | null | undefined): string {
+  // Normalize line endings and trim
+  const excel = (excelNotes ?? '').toString().replace(/\r\n/g, '\n').trim();
+  const pb = (pbNotes ?? '').toString().replace(/\r\n/g, '\n').trim();
+  
+  if (!excel && !pb) return '';
+  if (!excel) return pb;
+  if (!pb) return excel;
+  if (excel === pb) return excel;
+  
+  // Both have different content - preserve both
+  return `${excel}\n\n${pb}`;
+}
+
+function notesMergeNeeded(excelNotes: any, pbNotes: any): boolean {
+  // Check if a merge would change PocketBase content
+  const merged = mergeNotes(excelNotes, pbNotes);
+  // Normalize PB value the same way for comparison
+  const pbStr = (pbNotes ?? '').toString().replace(/\r\n/g, '\n').trim();
+  return merged !== pbStr;
+}
+
+function recordsEqual(existing: RowObject, candidate: RowObject): boolean {
+  // Compare only keys present in candidate; treat undefined and empty string as equal empties
+  for (const k of Object.keys(candidate)) {
+    const a = existing?.[k];
+    const b = candidate?.[k];
+    const aStr = a === undefined || a === null ? '' : String(a);
+    const bStr = b === undefined || b === null ? '' : String(b);
+    if (aStr !== bStr) return false;
+  }
+  return true;
+}
+
 export async function upsertPocketBaseByKey(pb: PocketBase, collection: string, keyName: string, record: RowObject, silent: boolean = false) {
   if (keyName.toLowerCase() !== 'job_number') {
     throw new Error('Primary key must be Job_Number (exact header name)');
@@ -66,9 +146,28 @@ export async function upsertPocketBaseByKey(pb: PocketBase, collection: string,
   if (!keyVal) throw new Error(`Missing key ${keyName} in record`);
   const filter = `${keyName}="${String(keyVal).replace(/"/g, '\\"')}"`;
   const found = await pb.collection(collection).getList(1, 1, { filter });
+  
   if (found?.items?.length) {
     const existing = found.items[0];
-    await pb.collection(collection).update(existing.id, record);
+    
+    // Handle Notes bidirectional merge - only if it would actually change
+    if ('Notes' in record && notesMergeNeeded(record.Notes, existing.Notes)) {
+      record.Notes = mergeNotes(record.Notes, existing.Notes);
+    }
+    
+    // Force Excel values for authoritative fields (overwrite even if unchanged in Excel)
+    const updatePayload = { ...record };
+    for (const field of EXCEL_AUTHORITATIVE_FIELDS) {
+      if (field in record) {
+        updatePayload[field] = record[field];
+      }
+    }
+    // Skip update if nothing would change
+    if (recordsEqual(existing, updatePayload)) {
+      if (!silent) console.log(`↺ No changes for PB ${collection} id=${existing.id}`);
+      return;
+    }
+    await pb.collection(collection).update(existing.id, updatePayload);
     if (!silent) console.log(`✓ Updated PB ${collection} id=${existing.id}`);
   } else {
     const created = await pb.collection(collection).create(record);

README.md

@@ -1,2 +1,13 @@
 # ExcelListenerUpdates
 
+## Top Rule: No Placeholders or Mock Data
+
+- Use only actual values. Do not include sample, placeholder, or mock data in code, docs, or commands.
+- If a placeholder seems necessary, ask for explicit approval and the real values before proceeding.
+
+See project-specific details in [Excel_Listener/README.md](Excel_Listener/README.md) and org standards in [global-copilot-instructions.md](global-copilot-instructions.md) and [project-copilot-instructions.md](project-copilot-instructions.md).
+
+## Version
+
+- v1.0.0-beta: stabilized Notes merge (normalized line endings, concat both sides), protected formula columns via single-cell updates, enforced fresh audit-before-apply, and reduced poller churn by skipping unchanged rows.
+

global-copilot-instructions.md

@@ -1,5 +1,18 @@
+# Top Rule: No Placeholders or Mock Data
+
+- Use only actual values. Do not introduce sample, placeholder, or mock data anywhere (code, docs, examples, commands).
+- If a placeholder seems unavoidable, stop and ask for explicit approval and the real values before proceeding.
+- Documentation should reference required variables without fake values or example tokens.
+
 # Global Copilot Instructions – No-Interaction Policy
 
+## Lessons Learned (Excel/PocketBase sync)
+- Normalize Notes before comparing/merging: convert `\r\n`→`\n`, trim, and compare full strings (no substring containment checks).
+- When Notes differ, keep both by concatenating with `\n\n`; never drop content or rely on substring containment.
+- Only write the Notes cell via table column `dataBodyRange/cell` to avoid overwriting formula columns.
+- Run a fresh audit before any apply/write; do not rely on stale deltas or cached data.
+- In poller, skip updates when `notesMergeNeeded` is false; do not rewrite every row.
+
 ## User Interaction Policy
 - Default to zero user interaction. Do not prompt users unless absolutely necessary.
 - Prefer agent/service account credentials for backend operations.

project-copilot-instructions.md

@@ -1,5 +1,18 @@
+# Top Rule: No Placeholders or Mock Data
+
+- Use only actual values. Do not include sample, placeholder, or mock data in code, docs, examples, or commands.
+- If a placeholder appears necessary, pause and request explicit approval and the real values before proceeding.
+- Examples must avoid fake tokens/IDs. Reference variables conceptually without inserting dummy values.
+
 # Project Copilot Instructions (Reusable, production-ready)
 
+## Lessons Learned (Notes sync)
+- Normalize Notes: convert `\r\n`→`\n`, trim both sides before compare/merge.
+- Merge policy: if different, concatenate Excel and PB with a blank line; do not use substring containment to decide equality.
+- Write Notes via table column `.../columns/Notes/dataBodyRange/cell(row=X,column=0)` to protect formula columns.
+- Run a fresh audit before apply; avoid stale `applied_updates_deltas.json` or cached fetches.
+- Poller must only update when `notesMergeNeeded` is true; otherwise skip to prevent touching every row.
+
 ## Core Policy
 - Zero user interaction. Backends must run unattended.
 - Tokens are delivered programmatically (env/API/CLI). No device-code or browser prompts.

tsconfig-global.md

@@ -1,3 +1,8 @@
+# Top Rule: No Placeholders or Mock Data
+
+- Use only actual values across code and documentation. Do not add sample, placeholder, or mock data.
+- If a placeholder seems required, stop and ask for approval and the real value before continuing.
+
 # TypeScript Configuration Standard
 
 ## Required Compiler Options