From 4fe99a04c79c0ad56296411e56a9b2bb5b62a008 Mon Sep 17 00:00:00 2001
From: Christopher <admin@gaion.chat>
Date: Thu, 22 Jan 2026 16:41:26 -0800
Subject: [PATCH] Fixed post duplication

---
 src/app/api/posts/route.ts | 29 ++++++++++++++++++++++++-----
 1 file changed, 24 insertions(+), 5 deletions(-)

diff --git a/src/app/api/posts/route.ts b/src/app/api/posts/route.ts
index 8f73887..1c25c0b 100644
--- a/src/app/api/posts/route.ts
+++ b/src/app/api/posts/route.ts
@@ -113,17 +113,36 @@ export async function POST(request: Request) {
     }
 }
 
+// Normalize content for deduplication (strip HTML entities, URLs, whitespace)
+const normalizeForDedup = (content: string): string => {
+    return content
+        .replace(/&[a-z]+;/gi, '') // Remove HTML entities like &lsquo;
+        .replace(/&#\d+;/g, '') // Remove numeric entities
+        .replace(/https?:\/\/[^\s]+/gi, '') // Remove URLs
+        .replace(/[^\w\s]/g, '') // Remove punctuation
+        .replace(/\s+/g, ' ') // Normalize whitespace
+        .toLowerCase()
+        .trim()
+        .slice(0, 100); // Compare first 100 chars
+};
+
 // Helper to transform cached remote posts to match local post format
-// Also deduplicates by apId to prevent showing same post multiple times
+// Deduplicates by apId AND by similar content from same author
 const transformRemotePosts = (remotePostsData: typeof remotePosts.$inferSelect[]) => {
     const seenApIds = new Set<string>();
+    const seenContentKeys = new Set<string>(); // author+normalizedContent
     const uniquePosts: typeof remotePosts.$inferSelect[] = [];
 
     for (const rp of remotePostsData) {
-        if (!seenApIds.has(rp.apId)) {
-            seenApIds.add(rp.apId);
-            uniquePosts.push(rp);
-        }
+        if (seenApIds.has(rp.apId)) continue;
+
+        // Content-based dedup: same author + similar content = skip
+        const contentKey = `${rp.authorHandle}:${normalizeForDedup(rp.content)}`;
+        if (seenContentKeys.has(contentKey)) continue;
+
+        seenApIds.add(rp.apId);
+        seenContentKeys.add(contentKey);
+        uniquePosts.push(rp);
     }
 
     return uniquePosts.map(rp => {