feat(timeline,background): Add live remote post fetching and background sync

- Implement live post fetching from followed remote users with parallel requests and timeouts
- Add remote-sync background task to periodically cache posts from followed remote users
- Optimize swarm node detection and prioritize swarm API over ActivityPub for faster responses
- Add timeout wrapper (5s per node) to prevent slow remote nodes from blocking timeline requests
- Enhance scheduler to support background sync tasks with configurable intervals
- Transform remote posts to match local format with proper media and author metadata
- Improve error handling and logging for remote user post fetching
- Replace cached-only approach with hybrid live/cached strategy for fresher content

src/lib/background/remote-sync.ts

@@ -0,0 +1,110 @@
+/**
+ * Remote Follows Sync
+ * 
+ * Periodically syncs posts from remote users that local users follow.
+ * This ensures the home timeline shows fresh posts from followed remote users.
+ */
+
+import { db, remoteFollows } from '@/db';
+import { resolveRemoteUser } from '@/lib/activitypub/fetch';
+import { cacheRemoteUserPosts } from '@/lib/activitypub/cache';
+import { cacheSwarmUserPosts, isSwarmNode } from '@/lib/swarm/interactions';
+
+// Track last sync time per remote handle to avoid over-fetching
+const lastSyncTimes = new Map<string, number>();
+const MIN_SYNC_INTERVAL_MS = 60 * 1000; // Don't sync same user more than once per minute
+
+interface SyncResult {
+  synced: number;
+  skipped: number;
+  errors: number;
+  details: Array<{ handle: string; cached: number; error?: string }>;
+}
+
+/**
+ * Sync posts from all remote users that any local user follows
+ */
+export async function syncRemoteFollowsPosts(origin: string): Promise<SyncResult> {
+  const result: SyncResult = { synced: 0, skipped: 0, errors: 0, details: [] };
+
+  try {
+    // Get all unique remote handles that are being followed
+    const allRemoteFollows = await db.query.remoteFollows.findMany();
+    
+    // Deduplicate by target handle (multiple users might follow the same remote user)
+    const uniqueHandles = new Map<string, typeof allRemoteFollows[0]>();
+    for (const follow of allRemoteFollows) {
+      if (!uniqueHandles.has(follow.targetHandle)) {
+        uniqueHandles.set(follow.targetHandle, follow);
+      }
+    }
+
+    const now = Date.now();
+
+    for (const [targetHandle, follow] of uniqueHandles) {
+      try {
+        // Check if we've synced this user recently
+        const lastSync = lastSyncTimes.get(targetHandle);
+        if (lastSync && (now - lastSync) < MIN_SYNC_INTERVAL_MS) {
+          result.skipped++;
+          continue;
+        }
+
+        // Parse handle to get username and domain
+        const atIndex = targetHandle.lastIndexOf('@');
+        if (atIndex === -1) {
+          result.skipped++;
+          continue;
+        }
+
+        const handle = targetHandle.slice(0, atIndex);
+        const domain = targetHandle.slice(atIndex + 1);
+
+        // Check if this is a swarm node
+        const isSwarm = await isSwarmNode(domain);
+
+        let cached = 0;
+        if (isSwarm) {
+          // Use swarm sync for swarm nodes
+          const swarmResult = await cacheSwarmUserPosts(handle, domain, targetHandle, 20);
+          cached = swarmResult.cached;
+        } else {
+          // Use ActivityPub sync for federated nodes
+          const remoteProfile = await resolveRemoteUser(handle, domain);
+          if (remoteProfile?.outbox) {
+            const apResult = await cacheRemoteUserPosts(remoteProfile, targetHandle, origin, 20);
+            cached = apResult.cached;
+          }
+        }
+
+        lastSyncTimes.set(targetHandle, now);
+        
+        if (cached > 0) {
+          result.synced++;
+          result.details.push({ handle: targetHandle, cached });
+        } else {
+          result.skipped++;
+        }
+      } catch (error) {
+        result.errors++;
+        result.details.push({ 
+          handle: targetHandle, 
+          cached: 0, 
+          error: error instanceof Error ? error.message : 'Unknown error' 
+        });
+      }
+    }
+  } catch (error) {
+    console.error('[RemoteSync] Error syncing remote follows:', error);
+    result.errors++;
+  }
+
+  return result;
+}
+
+/**
+ * Clear the sync cache (useful for testing or forcing a full resync)
+ */
+export function clearSyncCache(): void {
+  lastSyncTimes.clear();
+}

src/lib/background/scheduler.ts

@@ -4,6 +4,7 @@
  * Runs periodic tasks within the Next.js process:
  * - Bot autonomous posting (every 1 minute)
  * - Swarm gossip (every 5 minutes)
+ * - Remote follows sync (every 10 minutes)
  * - Swarm announcement (on startup)
  */
 
@@ -11,9 +12,11 @@ import { processAllAutonomousBots } from '@/lib/bots/autonomous';
 import { runGossipRound } from '@/lib/swarm/gossip';
 import { announceToSeeds } from '@/lib/swarm/discovery';
 import { getSwarmStats } from '@/lib/swarm/registry';
+import { syncRemoteFollowsPosts } from '@/lib/background/remote-sync';
 
 const BOT_INTERVAL_MS = 60 * 1000; // 1 minute
 const GOSSIP_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
+const REMOTE_SYNC_INTERVAL_MS = 60 * 1000; // 1 minute - keep feeds fresh
 const STARTUP_DELAY_MS = 10 * 1000; // Wait 10s for server to be ready
 
 let isStarted = false;
@@ -82,13 +85,33 @@ async function announceToSwarm() {
   }
 }
 
-export function startBackgroundTasks() {
+async function runRemoteSync(origin: string) {
+  try {
+    const result = await syncRemoteFollowsPosts(origin);
+    if (result.synced > 0 || result.errors > 0) {
+      log('REMOTE_SYNC', `Synced ${result.synced} users, skipped ${result.skipped}, errors ${result.errors}`);
+      if (result.details.length > 0) {
+        const newPosts = result.details.filter(d => d.cached > 0);
+        if (newPosts.length > 0) {
+          log('REMOTE_SYNC', `New posts: ${newPosts.map(d => `${d.handle}: ${d.cached}`).join(', ')}`);
+        }
+      }
+    }
+  } catch (error) {
+    log('REMOTE_SYNC', `Error: ${error}`);
+  }
+}
+
+export function startBackgroundTasks(origin?: string) {
   // Prevent double-start (Next.js can call register() multiple times in dev)
   if (isStarted) return;
   isStarted = true;
 
+  // Default origin for remote sync (can be overridden)
+  const syncOrigin = origin || process.env.NEXT_PUBLIC_APP_URL || 'http://localhost:3000';
+
   log('STARTUP', 'Background task scheduler starting...');
-  log('STARTUP', `Bot interval: ${BOT_INTERVAL_MS / 1000}s, Gossip interval: ${GOSSIP_INTERVAL_MS / 1000}s`);
+  log('STARTUP', `Bot interval: ${BOT_INTERVAL_MS / 1000}s, Gossip interval: ${GOSSIP_INTERVAL_MS / 1000}s, Remote sync interval: ${REMOTE_SYNC_INTERVAL_MS / 1000}s`);
 
   // Wait for server to be fully ready before starting tasks
   setTimeout(async () => {
@@ -100,9 +123,13 @@ export function startBackgroundTasks() {
     // Run initial bot check
     await runBotTasks();
     
+    // Run initial remote sync (after 15s to let server stabilize)
+    setTimeout(() => runRemoteSync(syncOrigin), 15 * 1000);
+    
     // Schedule recurring tasks
     setInterval(runBotTasks, BOT_INTERVAL_MS);
     setInterval(runSwarmGossip, GOSSIP_INTERVAL_MS);
+    setInterval(() => runRemoteSync(syncOrigin), REMOTE_SYNC_INTERVAL_MS);
     
     // First gossip after 30s (let announcement propagate)
     setTimeout(runSwarmGossip, 30 * 1000);