Task loading optimization through a 4 stage loading process
This commit is contained in:
+269
-149
@@ -4,6 +4,8 @@ import { pb } from "@/lib/pocketbase";
|
||||
import { toast } from "solid-sonner";
|
||||
import { TASGRID_COLLECTION, TAGS_COLLECTION, SHARE_RULES_COLLECTION, BUCKETS_COLLECTION, URGENCY_HOURS } from "@/lib/constants";
|
||||
|
||||
const LIST_FIELDS = 'id,collectionId,collectionName,created,updated,title,startDate,dueDate,priority,status,completed,tags,user,size,recurrence,sharedWith,deletedAt';
|
||||
|
||||
const getStorageKey = () => {
|
||||
const userId = pb.authStore.model?.id;
|
||||
return userId ? `tasgrid_data_${userId}` : null;
|
||||
@@ -729,67 +731,195 @@ export const initStore = async () => {
|
||||
console.warn("Failed to load tags:", tagErr);
|
||||
}
|
||||
|
||||
// 1.8 Sync System Tags & Share Rules
|
||||
await syncSystemTagsAndRules();
|
||||
// 1.8 Fetch Share Rules (CRITICAL for filter construction)
|
||||
// We need these loaded BEFORE we construct the focus filter
|
||||
await loadShareRules();
|
||||
|
||||
// 2. Fetch Tasks & Templates + Shared Tasks in PARALLEL
|
||||
const currentUserId = pb.authStore.model?.id;
|
||||
if (!currentUserId) return;
|
||||
|
||||
// Prepare parallel fetch promises (requestKey: null prevents auto-cancellation)
|
||||
const ownTasksPromise = pb.collection(TASGRID_COLLECTION).getFullList({
|
||||
filter: `user = "${currentUserId}"`,
|
||||
sort: '-created',
|
||||
requestKey: null
|
||||
});
|
||||
// --- PHASE 2: Multi-Stage Loading ---
|
||||
|
||||
const individuallySharedPromise = currentUserId
|
||||
? pb.collection(TASGRID_COLLECTION).getFullList({
|
||||
filter: `sharedWith ~ "${currentUserId}"`,
|
||||
sort: '-created',
|
||||
requestKey: null
|
||||
}).catch(err => { console.warn("Failed to load individually shared tasks:", err); return []; })
|
||||
: Promise.resolve([]);
|
||||
// --- WINDOWED SYNC STRATEGY ---
|
||||
|
||||
const shareRulesPromise = currentUserId
|
||||
? pb.collection(SHARE_RULES_COLLECTION).getFullList({
|
||||
filter: `targetUserId = "${currentUserId}"`,
|
||||
requestKey: null
|
||||
}).catch(err => { console.warn("Failed to load share rules:", err); return []; })
|
||||
: Promise.resolve([]);
|
||||
// STAGE 1: Instant Focus (Top 30 Incomplete - User Only)
|
||||
// Goal: < 100ms First Paint. Uses 'idx_user_focused'.
|
||||
try {
|
||||
const focusRecords = await pb.collection(TASGRID_COLLECTION).getList(1, 30, {
|
||||
filter: `user = "${currentUserId}" && completed = false && deletedAt = ""`,
|
||||
sort: '-priority,dueDate',
|
||||
requestKey: 'initial_focus'
|
||||
});
|
||||
|
||||
// 1.9 Fetch Tasks for Subscribed Buckets
|
||||
const subscribedBucketNames = store.subscribedBuckets
|
||||
.map(id => store.buckets.find(b => b.id === id)?.name)
|
||||
.filter(Boolean);
|
||||
// Filter out templates client-side to ensure tasks with empty/null tags aren't excluded by DB
|
||||
const focusTasks = focusRecords.items
|
||||
.map(mapRecordToTask)
|
||||
.filter(t => !t.tags?.includes("__template__"));
|
||||
|
||||
const bucketTasksPromise = (subscribedBucketNames.length > 0)
|
||||
? pb.collection(TASGRID_COLLECTION).getFullList({
|
||||
filter: subscribedBucketNames.map(name => `tags ~ "${name}"`).join(' || '),
|
||||
sort: '-created',
|
||||
requestKey: null
|
||||
}).catch(err => { console.warn("Failed to load bucket tasks:", err); return []; })
|
||||
: Promise.resolve([]);
|
||||
setStore("tasks", focusTasks);
|
||||
} catch (focusErr) {
|
||||
console.warn("Stage 1 load failed:", focusErr);
|
||||
}
|
||||
|
||||
// STAGE 2 & 3: Background Sync
|
||||
const backgroundSync = async () => {
|
||||
|
||||
// Wait for all initial fetches
|
||||
const [ownRecords, individuallySharedRecords, incomingRules, bucketRecords] = await Promise.all([
|
||||
ownTasksPromise,
|
||||
individuallySharedPromise,
|
||||
shareRulesPromise,
|
||||
bucketTasksPromise
|
||||
]);
|
||||
const bucketNames = store.subscribedBuckets
|
||||
.map(id => store.buckets.find(b => b.id === id)?.name)
|
||||
.filter(Boolean);
|
||||
|
||||
const allTasks: Task[] = [];
|
||||
const loadedTemplates: TaskTemplate[] = [];
|
||||
const seenIds = new Set<string>();
|
||||
// Add rules from shareRules store (which was just synced)
|
||||
const incomingRules = store.shareRules.filter(r => r.targetUserId === currentUserId);
|
||||
|
||||
// Process own tasks & templates
|
||||
ownRecords.forEach(r => {
|
||||
const tags = r.tags || [];
|
||||
if (tags.includes("__template__")) {
|
||||
// 2a. Fetch ALL Incomplete Tasks (Metadata Only) to ensure availability
|
||||
// We must replicate the complex sharing logic here
|
||||
const incompletePromises = [
|
||||
// Own Incomplete
|
||||
pb.collection(TASGRID_COLLECTION).getFullList({
|
||||
filter: `user = "${currentUserId}" && completed = false`,
|
||||
sort: '-created',
|
||||
fields: LIST_FIELDS,
|
||||
requestKey: null
|
||||
}),
|
||||
// Shared With Individual Incomplete
|
||||
pb.collection(TASGRID_COLLECTION).getFullList({
|
||||
filter: `sharedWith ~ "${currentUserId}" && completed = false`,
|
||||
sort: '-created',
|
||||
fields: LIST_FIELDS,
|
||||
requestKey: null
|
||||
}).catch(() => []),
|
||||
// Bucket Incomplete
|
||||
(bucketNames.length > 0) ? pb.collection(TASGRID_COLLECTION).getFullList({
|
||||
filter: `(${bucketNames.map(name => `tags ~ "${name}"`).join(' || ')}) && completed = false`,
|
||||
sort: '-created',
|
||||
fields: LIST_FIELDS,
|
||||
requestKey: null
|
||||
}).catch(() => []) : Promise.resolve([])
|
||||
];
|
||||
|
||||
// Share Rules Incomplete
|
||||
if (incomingRules.length > 0) {
|
||||
const ownerRules = new Map<string, { type: 'all' | 'tag', tagName?: string }[]>();
|
||||
incomingRules.forEach((rule: any) => {
|
||||
const existing = ownerRules.get(rule.ownerId) || [];
|
||||
existing.push({ type: rule.type, tagName: rule.tagName });
|
||||
ownerRules.set(rule.ownerId, existing);
|
||||
});
|
||||
|
||||
Array.from(ownerRules.entries()).forEach(([ownerId, rules]) => {
|
||||
const hasAllRule = rules.some(r => r.type === 'all');
|
||||
const tagRules = rules.filter(r => r.type === 'tag').map(r => r.tagName).filter(Boolean);
|
||||
let filter = `user = "${ownerId}" && completed = false`;
|
||||
if (!hasAllRule && tagRules.length > 0) filter += ` && (${tagRules.map(t => `tags ~ "${t}"`).join(' || ')})`;
|
||||
|
||||
incompletePromises.push(pb.collection(TASGRID_COLLECTION).getFullList({
|
||||
filter,
|
||||
sort: '-created',
|
||||
fields: LIST_FIELDS,
|
||||
requestKey: null
|
||||
}).catch(() => []));
|
||||
});
|
||||
}
|
||||
|
||||
const stage2Results = await Promise.all(incompletePromises);
|
||||
const allIncomplete = stage2Results.flat();
|
||||
|
||||
setStore("tasks", (currentTasks) => {
|
||||
const taskMap = new Map(currentTasks.map(t => [t.id, t]));
|
||||
allIncomplete.forEach((r: any) => {
|
||||
if (r.tags?.includes("__template__")) return;
|
||||
const existing = taskMap.get(r.id);
|
||||
// If existing has content, keep it. Otherwise use new record (which likely has no content yet)
|
||||
const newTask = mapRecordToTask(r);
|
||||
if (existing && existing.content) {
|
||||
newTask.content = existing.content;
|
||||
}
|
||||
taskMap.set(r.id, newTask);
|
||||
});
|
||||
return Array.from(taskMap.values());
|
||||
});
|
||||
|
||||
// 2b. Content Backfill for Incomplete Tasks
|
||||
// Fetch content for tasks that don't have it yet, in chunks
|
||||
const tasksNeedingContent = store.tasks.filter(t => !t.completed && t.content === undefined).map(t => t.id);
|
||||
if (tasksNeedingContent.length > 0) {
|
||||
// Fetch in chunks of 20 to avoid URL length limits
|
||||
const chunkSize = 20;
|
||||
for (let i = 0; i < tasksNeedingContent.length; i += chunkSize) {
|
||||
const ids = tasksNeedingContent.slice(i, i + chunkSize);
|
||||
// We fetch just ID and Content
|
||||
try {
|
||||
const contentRecords = await pb.collection(TASGRID_COLLECTION).getFullList({
|
||||
filter: ids.map(id => `id="${id}"`).join(' || '),
|
||||
fields: 'id,content',
|
||||
requestKey: null
|
||||
});
|
||||
|
||||
setStore("tasks", (tasks) => tasks.map(t => {
|
||||
const match = contentRecords.find((r: any) => r.id === t.id);
|
||||
if (match) return { ...t, content: match.content };
|
||||
return t;
|
||||
}));
|
||||
} catch (e) { console.warn("Backfill chunk failed", e); }
|
||||
}
|
||||
}
|
||||
|
||||
// STAGE 3: Recent History (Cap 150)
|
||||
const currentCount = store.tasks.length;
|
||||
const remainingSlots = 150 - currentCount;
|
||||
|
||||
if (remainingSlots > 0) {
|
||||
try {
|
||||
const recentCompleted = await pb.collection(TASGRID_COLLECTION).getList(1, remainingSlots, {
|
||||
filter: `user = "${currentUserId}" && completed = true`,
|
||||
sort: '-updated',
|
||||
fields: LIST_FIELDS,
|
||||
requestKey: null
|
||||
});
|
||||
|
||||
setStore("tasks", (prev) => {
|
||||
const newTasks = [...prev];
|
||||
recentCompleted.items.forEach((r: any) => {
|
||||
if (!newTasks.some(t => t.id === r.id)) {
|
||||
newTasks.push(mapRecordToTask(r));
|
||||
}
|
||||
});
|
||||
return newTasks;
|
||||
});
|
||||
} catch (e) {
|
||||
console.warn("Stage 3 load failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
checkRecurringTasks();
|
||||
|
||||
// Reconstructive migration
|
||||
const foundTagNames = new Set<string>();
|
||||
store.tasks.forEach(t => t.tags?.forEach(tag => foundTagNames.add(tag)));
|
||||
for (const tagName of foundTagNames) {
|
||||
if (tagName === "__template__") continue;
|
||||
if (!store.tagDefinitions.find(d => d.name === tagName)) {
|
||||
await upsertTagDefinition(tagName, 5, undefined, document.documentElement.classList.contains("dark") ? "dark" : "light");
|
||||
}
|
||||
}
|
||||
|
||||
// Sync System Tags & Share Rules (Heavy maintenance task)
|
||||
await syncSystemTagsAndRules();
|
||||
};
|
||||
|
||||
// Fire off background sync after a short delay
|
||||
setTimeout(backgroundSync, 100);
|
||||
|
||||
// Separate Templates load (less critical)
|
||||
try {
|
||||
const templates = await pb.collection(TASGRID_COLLECTION).getFullList({
|
||||
filter: `user = "${currentUserId}" && tags ~ "__template__"`,
|
||||
requestKey: 'templates_load'
|
||||
});
|
||||
const loadedTemplates = templates.map(r => {
|
||||
let meta: any = {};
|
||||
try { meta = JSON.parse(r.content || "{}"); } catch (e) { }
|
||||
loadedTemplates.push({
|
||||
return {
|
||||
id: r.id,
|
||||
name: r.title,
|
||||
title: meta.title || "",
|
||||
@@ -797,111 +927,16 @@ export const initStore = async () => {
|
||||
urgency: meta.urgency || 5,
|
||||
tags: meta.tags || [],
|
||||
content: meta.content || ""
|
||||
});
|
||||
} else {
|
||||
allTasks.push(mapRecordToTask(r));
|
||||
seenIds.add(r.id);
|
||||
}
|
||||
});
|
||||
|
||||
// Process individually shared tasks
|
||||
individuallySharedRecords.forEach((r: any) => {
|
||||
if (!seenIds.has(r.id)) {
|
||||
const tags = r.tags || [];
|
||||
if (!tags.includes("__template__")) {
|
||||
allTasks.push(mapRecordToTask(r));
|
||||
seenIds.add(r.id);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Process ShareRules - fetch tasks from each owner in parallel
|
||||
if (incomingRules.length > 0) {
|
||||
// Group rules by owner
|
||||
const ownerRules = new Map<string, { type: 'all' | 'tag', tagName?: string }[]>();
|
||||
incomingRules.forEach((rule: any) => {
|
||||
const existing = ownerRules.get(rule.ownerId) || [];
|
||||
existing.push({ type: rule.type, tagName: rule.tagName });
|
||||
ownerRules.set(rule.ownerId, existing);
|
||||
};
|
||||
});
|
||||
setStore("templates", loadedTemplates);
|
||||
} catch (err) { }
|
||||
|
||||
// Fetch from all owners in parallel
|
||||
const ownerFetchPromises = Array.from(ownerRules.entries()).map(async ([ownerId, rules]) => {
|
||||
const hasAllRule = rules.some(r => r.type === 'all');
|
||||
const tagRules = rules.filter(r => r.type === 'tag').map(r => r.tagName).filter(Boolean);
|
||||
|
||||
let filter = `user = "${ownerId}"`;
|
||||
if (!hasAllRule && tagRules.length > 0) {
|
||||
const tagFilters = tagRules.map(t => `tags ~ "${t}"`).join(' || ');
|
||||
filter += ` && (${tagFilters})`;
|
||||
}
|
||||
|
||||
try {
|
||||
return await pb.collection(TASGRID_COLLECTION).getFullList({ filter, sort: '-created', requestKey: null });
|
||||
} catch (err) {
|
||||
console.warn(`Failed to fetch tasks from owner ${ownerId}:`, err);
|
||||
return [];
|
||||
}
|
||||
});
|
||||
|
||||
const ownerResults = await Promise.all(ownerFetchPromises);
|
||||
ownerResults.flat().forEach((r: any) => {
|
||||
if (!seenIds.has(r.id)) {
|
||||
const tags = r.tags || [];
|
||||
if (!tags.includes("__template__")) {
|
||||
allTasks.push(mapRecordToTask(r));
|
||||
seenIds.add(r.id);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Process Bucket Tasks
|
||||
if (bucketRecords && bucketRecords.length > 0) {
|
||||
bucketRecords.forEach((r: any) => {
|
||||
if (!seenIds.has(r.id)) {
|
||||
const tags = r.tags || [];
|
||||
if (!tags.includes("__template__")) {
|
||||
allTasks.push(mapRecordToTask(r));
|
||||
seenIds.add(r.id);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Set store ONCE with all tasks and rules
|
||||
setStore("tasks", reconcile(allTasks));
|
||||
setStore("templates", loadedTemplates);
|
||||
// Make sure we include incomingRules in the store so ContextSwitcher can see them
|
||||
setStore("shareRules", incomingRules.map(mapRecordToShareRule));
|
||||
|
||||
// Check recurring tasks after load
|
||||
checkRecurringTasks();
|
||||
|
||||
// 2.7. Fetch Share Rules (owned by current user)
|
||||
await loadShareRules();
|
||||
// Start real-time subscription
|
||||
await subscribeToRealtime();
|
||||
|
||||
// 2.8 Fetch Filter Templates
|
||||
await loadFilterTemplates();
|
||||
|
||||
// 3. Subscribe to Realtime Updates
|
||||
await subscribeToRealtime();
|
||||
|
||||
// 4. Reconstructive Migration: Verify all tags in tasks have definitions
|
||||
const foundTagNames = new Set<string>();
|
||||
allTasks.forEach(t => t.tags?.forEach(tag => foundTagNames.add(tag)));
|
||||
loadedTemplates.forEach(t => t.tags?.forEach(tag => foundTagNames.add(tag)));
|
||||
|
||||
for (const tagName of foundTagNames) {
|
||||
if (tagName === "__template__") continue;
|
||||
const exists = store.tagDefinitions.find(d => d.name === tagName);
|
||||
if (!exists) {
|
||||
console.log(`Reconstructing missing tag: ${tagName}`);
|
||||
// Default to dark mode for reconstruction if we can't tell
|
||||
const currentTheme = document.documentElement.classList.contains("dark") ? "dark" : "light";
|
||||
await upsertTagDefinition(tagName, 5, undefined, currentTheme);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (store.tasks.length === 0) {
|
||||
console.error("Failed to load data:", err);
|
||||
@@ -939,7 +974,8 @@ export const toggleBucketSubscription = async (bucketId: string) => {
|
||||
if (bucketName) {
|
||||
const records = await pb.collection(TASGRID_COLLECTION).getFullList({
|
||||
filter: `tags ~ "${bucketName}"`,
|
||||
sort: '-created'
|
||||
sort: '-created',
|
||||
fields: LIST_FIELDS
|
||||
});
|
||||
|
||||
const newTasks: Task[] = [];
|
||||
@@ -1405,6 +1441,9 @@ export const syncSystemTagsAndRules = async () => {
|
||||
tagName: r.tagName
|
||||
}));
|
||||
|
||||
// Fetch task content on demand
|
||||
// We'll call this when a task is opened
|
||||
|
||||
// --- Sync Bucket Tags ---
|
||||
for (const bucket of store.buckets) {
|
||||
const tagExists = tagDefinitions.find(t => t.name === bucket.name);
|
||||
@@ -1551,6 +1590,87 @@ export const renameTagDefinition = async (oldName: string, newName: string) => {
|
||||
}
|
||||
};
|
||||
|
||||
// --- HISTORY ACTIONS ---
|
||||
|
||||
export const loadAllHistory = async () => {
|
||||
if (!pb.authStore.isValid) return;
|
||||
const userId = pb.authStore.model?.id;
|
||||
if (!userId) return;
|
||||
|
||||
try {
|
||||
toast.promise(
|
||||
(async () => {
|
||||
// Fetch ALL tasks for user (including completed)
|
||||
// We use a simplified query for the user's own history + shared items
|
||||
// Note: This matches the components of the background sync but without the 'completed=false' restriction
|
||||
const bucketNames = store.subscribedBuckets
|
||||
.map(id => store.buckets.find(b => b.id === id)?.name)
|
||||
.filter(Boolean);
|
||||
|
||||
const promises = [
|
||||
pb.collection(TASGRID_COLLECTION).getFullList({ filter: `user = "${userId}"`, sort: '-created', fields: LIST_FIELDS, requestKey: null }),
|
||||
pb.collection(TASGRID_COLLECTION).getFullList({ filter: `sharedWith ~ "${userId}"`, sort: '-created', fields: LIST_FIELDS, requestKey: null }).catch(() => [])
|
||||
];
|
||||
|
||||
if (bucketNames.length > 0) {
|
||||
promises.push(pb.collection(TASGRID_COLLECTION).getFullList({
|
||||
filter: bucketNames.map(name => `tags ~ "${name}"`).join(' || '),
|
||||
sort: '-created',
|
||||
fields: LIST_FIELDS,
|
||||
requestKey: null
|
||||
}).catch(() => []));
|
||||
}
|
||||
|
||||
// Add Share Rules
|
||||
const incomingRules = store.shareRules.filter(r => r.targetUserId === userId);
|
||||
if (incomingRules.length > 0) {
|
||||
incomingRules.forEach((rule: any) => {
|
||||
let filter = `user = "${rule.ownerId}"`;
|
||||
if (rule.type === 'tag' && rule.tagName) filter += ` && tags ~ "${rule.tagName}"`;
|
||||
promises.push(pb.collection(TASGRID_COLLECTION).getFullList({ filter, sort: '-created', fields: LIST_FIELDS, requestKey: null }).catch(() => []));
|
||||
});
|
||||
}
|
||||
|
||||
const results = await Promise.all(promises);
|
||||
const allRecords = results.flat();
|
||||
|
||||
setStore("tasks", (currentTasks) => {
|
||||
const taskMap = new Map(currentTasks.map(t => [t.id, t]));
|
||||
let addedCount = 0;
|
||||
allRecords.forEach((r: any) => {
|
||||
if (r.tags?.includes("__template__")) return;
|
||||
if (!taskMap.has(r.id)) {
|
||||
taskMap.set(r.id, mapRecordToTask(r));
|
||||
addedCount++;
|
||||
}
|
||||
});
|
||||
return Array.from(taskMap.values());
|
||||
});
|
||||
|
||||
return `Loaded history.`;
|
||||
})(),
|
||||
{
|
||||
loading: 'Loading full history...',
|
||||
success: (msg) => msg,
|
||||
error: 'Failed to load history'
|
||||
}
|
||||
);
|
||||
} catch (err) {
|
||||
console.error("Failed to load history:", err);
|
||||
}
|
||||
};
|
||||
|
||||
export const loadTaskContent = async (id: string) => {
|
||||
if (!pb.authStore.isValid) return;
|
||||
try {
|
||||
const record = await pb.collection(TASGRID_COLLECTION).getOne(id, { requestKey: null });
|
||||
// Only update if content is still undefined (i.e. not edited locally while loading)
|
||||
setStore("tasks", t => t.id === id && t.content === undefined, { content: record.content });
|
||||
} catch (err) {
|
||||
console.error("Failed to load task content:", err);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// -- Templates --
|
||||
|
||||
|
||||
Reference in New Issue
Block a user