From d2d8d9b74614d3d6559a93b88906e91f6860632b Mon Sep 17 00:00:00 2001 From: Christopher Date: Thu, 22 Jan 2026 16:25:25 -0800 Subject: [PATCH] Cachine recent posts --- src/app/[handle]/page.tsx | 15 +- src/app/api/posts/route.ts | 68 ++++++- src/app/api/users/[handle]/follow/route.ts | 8 + src/app/api/users/[handle]/posts/route.ts | 24 ++- src/db/schema.ts | 29 +++ src/lib/activitypub/cache.ts | 210 +++++++++++++++++++++ 6 files changed, 339 insertions(+), 15 deletions(-) create mode 100644 src/lib/activitypub/cache.ts diff --git a/src/app/[handle]/page.tsx b/src/app/[handle]/page.tsx index ffed044..47c3970 100644 --- a/src/app/[handle]/page.tsx +++ b/src/app/[handle]/page.tsx @@ -51,6 +51,7 @@ export default function ProfilePage() { const [activeTab, setActiveTab] = useState<'posts' | 'followers' | 'following'>('posts'); const [followers, setFollowers] = useState([]); const [following, setFollowing] = useState([]); + const [postsLoading, setPostsLoading] = useState(true); const [followersLoading, setFollowersLoading] = useState(false); const [followingLoading, setFollowingLoading] = useState(false); const [isEditing, setIsEditing] = useState(false); @@ -84,10 +85,12 @@ export default function ProfilePage() { }) .catch(() => setLoading(false)); + setPostsLoading(true); fetch(`/api/users/${handle}/posts`) .then(res => res.json()) .then(data => setPosts(data.posts || [])) - .catch(() => { }); + .catch(() => { }) + .finally(() => setPostsLoading(false)); }, [handle]); const handleLike = async (postId: string, currentLiked: boolean) => { @@ -562,14 +565,18 @@ export default function ProfilePage() { {/* Content */} {activeTab === 'posts' && ( - posts.length === 0 ? ( + postsLoading ? ( +
+

Loading...

+
+ ) : posts.length === 0 ? (

No posts yet

) : ( - posts.map(post => ( + posts.map((post, index) => ( f.targetHandle); + + // Get cached remote posts from followed users + let remotePostsData: typeof remotePosts.$inferSelect[] = []; + if (followedRemoteHandles.length > 0) { + remotePostsData = await db.query.remotePosts.findMany({ + where: inArray(remotePosts.authorHandle, followedRemoteHandles), + orderBy: [desc(remotePosts.publishedAt)], + limit: limit, + }); + } + + // Transform remote posts to match local post format + const transformedRemotePosts = remotePostsData.map(rp => { + const mediaData = rp.mediaJson ? JSON.parse(rp.mediaJson) : []; + return { + id: rp.id, + content: rp.content, + createdAt: rp.publishedAt, + likesCount: 0, + repostsCount: 0, + repliesCount: 0, + isRemote: true, + apId: rp.apId, + linkPreviewUrl: rp.linkPreviewUrl, + linkPreviewTitle: rp.linkPreviewTitle, + linkPreviewDescription: rp.linkPreviewDescription, + linkPreviewImage: rp.linkPreviewImage, + author: { + id: rp.authorActorUrl, + handle: rp.authorHandle, + displayName: rp.authorDisplayName, + avatarUrl: rp.authorAvatarUrl, + isRemote: true, + }, + media: mediaData.map((m: { url: string; altText?: string }, idx: number) => ({ + id: `${rp.id}-media-${idx}`, + url: m.url, + altText: m.altText || null, + })), + replyTo: null, + }; + }); + + // Merge and sort by date + const allPosts = [...localPosts, ...transformedRemotePosts] + .sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime()) + .slice(0, limit); + + feedPosts = allPosts as any; } catch { // Not authenticated, return public timeline feedPosts = await db.query.posts.findMany({ @@ -305,7 +359,7 @@ export async function GET(request: Request) { if (session?.user && feedPosts && feedPosts.length > 0) { const viewer = session.user; - const postIds = feedPosts.map(p => p.id).filter(Boolean); + const postIds = feedPosts.map((p: { id: string }) => p.id).filter(Boolean); if (postIds.length > 0) { const viewerLikes = await db.query.likes.findMany({ @@ -324,7 +378,7 @@ export async function GET(request: Request) { }); const repostedPostIds = new Set(viewerReposts.map(r => r.repostOfId)); - feedPosts = feedPosts.map(p => ({ + feedPosts = feedPosts.map((p: { id: string }) => ({ ...p, isLiked: likedPostIds.has(p.id), isReposted: repostedPostIds.has(p.id), diff --git a/src/app/api/users/[handle]/follow/route.ts b/src/app/api/users/[handle]/follow/route.ts index 77edfa5..d0adab8 100644 --- a/src/app/api/users/[handle]/follow/route.ts +++ b/src/app/api/users/[handle]/follow/route.ts @@ -6,6 +6,7 @@ import { requireAuth } from '@/lib/auth'; import { resolveRemoteUser } from '@/lib/activitypub/fetch'; import { createFollowActivity, createUndoActivity } from '@/lib/activitypub/activities'; import { deliverActivity } from '@/lib/activitypub/outbox'; +import { cacheRemoteUserPosts } from '@/lib/activitypub/cache'; type RouteContext = { params: Promise<{ handle: string }> }; @@ -132,6 +133,13 @@ export async function POST(request: Request, context: RouteContext) { inboxUrl: targetInbox, activityId, }); + + // Cache the remote user's recent posts in the background + const origin = new URL(request.url).origin; + cacheRemoteUserPosts(remoteProfile, targetHandle, origin, 20) + .then(result => console.log(`Cached ${result.cached} posts for ${targetHandle}`)) + .catch(err => console.error('Error caching remote posts:', err)); + return NextResponse.json({ success: true, following: true, remote: true }); } diff --git a/src/app/api/users/[handle]/posts/route.ts b/src/app/api/users/[handle]/posts/route.ts index 8a210ca..8e16aa4 100644 --- a/src/app/api/users/[handle]/posts/route.ts +++ b/src/app/api/users/[handle]/posts/route.ts @@ -133,6 +133,7 @@ export async function GET(request: Request, context: RouteContext) { bio: sanitizeText(remoteProfile.summary), }; const posts = []; + const seenIds = new Set(); const origin = new URL(request.url).origin; for (const item of outboxItems) { const activity = item?.type === 'Create' ? item : null; @@ -140,13 +141,20 @@ export async function GET(request: Request, context: RouteContext) { if (!object || typeof object === 'string' || object.type !== 'Note') { continue; } + // Deduplicate by object ID + const postId = object.id || activity.id; + if (seenIds.has(postId)) { + continue; + } + seenIds.add(postId); + const attachments = Array.isArray(object.attachment) ? object.attachment : []; const { text, urls } = extractTextAndUrls(object.content); const normalizedUrl = urls.length > 0 ? normalizeUrl(urls[0]) : null; const linkPreview = normalizedUrl ? await fetchLinkPreview(normalizedUrl, origin) : null; const contentText = linkPreview && normalizedUrl ? stripFirstUrl(text, normalizedUrl) : text; posts.push({ - id: object.id || activity.id, + id: postId, content: contentText || '', createdAt: object.published || activity.published || new Date().toISOString(), likesCount: 0, @@ -156,7 +164,7 @@ export async function GET(request: Request, context: RouteContext) { media: attachments .filter((attachment: any) => attachment?.url) .map((attachment: any, index: number) => ({ - id: `${object.id || activity.id || 'media'}-${index}`, + id: `${postId || 'media'}-${index}`, url: attachment.url, altText: sanitizeText(attachment.name) || null, })), @@ -193,6 +201,7 @@ export async function GET(request: Request, context: RouteContext) { bio: sanitizeText(remoteProfile.summary), }; const posts = []; + const seenIds = new Set(); const origin = new URL(request.url).origin; for (const item of outboxItems) { const activity = item?.type === 'Create' ? item : null; @@ -200,13 +209,20 @@ export async function GET(request: Request, context: RouteContext) { if (!object || typeof object === 'string' || object.type !== 'Note') { continue; } + // Deduplicate by object ID + const postId = object.id || activity.id; + if (seenIds.has(postId)) { + continue; + } + seenIds.add(postId); + const attachments = Array.isArray(object.attachment) ? object.attachment : []; const { text, urls } = extractTextAndUrls(object.content); const normalizedUrl = urls.length > 0 ? normalizeUrl(urls[0]) : null; const linkPreview = normalizedUrl ? await fetchLinkPreview(normalizedUrl, origin) : null; const contentText = linkPreview && normalizedUrl ? stripFirstUrl(text, normalizedUrl) : text; posts.push({ - id: object.id || activity.id, + id: postId, content: contentText || '', createdAt: object.published || activity.published || new Date().toISOString(), likesCount: 0, @@ -216,7 +232,7 @@ export async function GET(request: Request, context: RouteContext) { media: attachments .filter((attachment: any) => attachment?.url) .map((attachment: any, index: number) => ({ - id: `${object.id || activity.id || 'media'}-${index}`, + id: `${postId || 'media'}-${index}`, url: attachment.url, altText: sanitizeText(attachment.name) || null, })), diff --git a/src/db/schema.ts b/src/db/schema.ts index 61aebd7..a7e9669 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -223,6 +223,35 @@ export const remoteFollowers = pgTable('remote_followers', { index('remote_followers_actor_idx').on(table.actorUrl), ]); +// ============================================ +// REMOTE POSTS (cached posts from federated users) +// ============================================ + +export const remotePosts = pgTable('remote_posts', { + id: uuid('id').primaryKey().defaultRandom(), + apId: text('ap_id').notNull().unique(), // ActivityPub ID (URL) of the post + authorHandle: text('author_handle').notNull(), // e.g., user@mastodon.social + authorActorUrl: text('author_actor_url').notNull(), // Remote actor URL + authorDisplayName: text('author_display_name'), + authorAvatarUrl: text('author_avatar_url'), + content: text('content').notNull(), + publishedAt: timestamp('published_at').notNull(), // Original publish time + // Link preview + linkPreviewUrl: text('link_preview_url'), + linkPreviewTitle: text('link_preview_title'), + linkPreviewDescription: text('link_preview_description'), + linkPreviewImage: text('link_preview_image'), + // Media attachments stored as JSON + mediaJson: text('media_json'), // JSON array of {url, altText} + // Metadata + fetchedAt: timestamp('fetched_at').defaultNow().notNull(), + createdAt: timestamp('created_at').defaultNow().notNull(), +}, (table) => [ + index('remote_posts_author_idx').on(table.authorHandle), + index('remote_posts_published_idx').on(table.publishedAt), + index('remote_posts_ap_id_idx').on(table.apId), +]); + // ============================================ // LIKES // ============================================ diff --git a/src/lib/activitypub/cache.ts b/src/lib/activitypub/cache.ts new file mode 100644 index 0000000..37bd8fb --- /dev/null +++ b/src/lib/activitypub/cache.ts @@ -0,0 +1,210 @@ +import { db, remotePosts } from '@/db'; +import { eq } from 'drizzle-orm'; + +interface RemoteProfile { + id: string; + preferredUsername?: string; + name?: string; + icon?: string | { url?: string }; + summary?: string; + outbox?: string; +} + +interface OutboxItem { + type?: string; + object?: { + id?: string; + type?: string; + content?: string; + published?: string; + attachment?: Array<{ url?: string; name?: string }>; + }; + id?: string; + published?: string; +} + +const decodeEntities = (value: string) => + value + .replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCharCode(parseInt(hex, 16))) + .replace(/&#(\d+);/g, (_, num) => String.fromCharCode(Number(num))) + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'"); + +const sanitizeText = (value?: string | null) => { + if (!value) return null; + const withoutTags = value.replace(/<[^>]*>/g, ' '); + const decoded = decodeEntities(withoutTags); + return decoded.replace(/\s+/g, ' ').trim() || null; +}; + +const extractTextAndUrls = (value?: string | null) => { + if (!value) return { text: '', urls: [] as string[] }; + let html = value; + html = html.replace(//gi, ' '); + html = html.replace(/]*href=["']([^"']+)["'][^>]*>(.*?)<\/a>/gi, (_, href, text) => { + const cleanedHref = decodeEntities(String(href)); + const cleanedText = decodeEntities(String(text)).replace(/<[^>]*>/g, ' ').trim(); + return cleanedHref || cleanedText; + }); + const withoutTags = html.replace(/<[^>]*>/g, ' '); + const decoded = decodeEntities(withoutTags); + const text = decoded.replace(/\s+/g, ' ').trim(); + const urls = Array.from(text.matchAll(/https?:\/\/[^\s]+/gi)).map((match) => match[0]); + return { text, urls }; +}; + +const normalizeUrl = (value: string) => value.replace(/[)\].,!?]+$/, ''); + +const stripFirstUrl = (text: string, url: string) => { + const idx = text.indexOf(url); + if (idx === -1) return text; + const before = text.slice(0, idx).trimEnd(); + const after = text.slice(idx + url.length).trimStart(); + return `${before} ${after}`.trim(); +}; + +const fetchOutboxItems = async (outboxUrl: string, limit: number = 20): Promise => { + try { + const res = await fetch(outboxUrl, { + headers: { + 'Accept': 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"', + }, + signal: AbortSignal.timeout(10000), + }); + if (!res.ok) return []; + const data = await res.json(); + const first = data?.first; + if (first) { + if (typeof first === 'string') { + const pageRes = await fetch(first, { + headers: { + 'Accept': 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"', + }, + signal: AbortSignal.timeout(10000), + }); + if (!pageRes.ok) return []; + const page = await pageRes.json(); + return page?.orderedItems || page?.items || []; + } + return first?.orderedItems || first?.items || []; + } + const items = data?.orderedItems || data?.items || []; + return items.slice(0, limit); + } catch (error) { + console.error('Error fetching outbox:', error); + return []; + } +}; + +export async function cacheRemoteUserPosts( + remoteProfile: RemoteProfile, + authorHandle: string, // e.g., user@mastodon.social + origin: string, // Used for link previews + limit: number = 20 +): Promise<{ cached: number; errors: number }> { + if (!remoteProfile.outbox) { + return { cached: 0, errors: 0 }; + } + + const outboxItems = await fetchOutboxItems(remoteProfile.outbox, limit); + if (outboxItems.length === 0) { + return { cached: 0, errors: 0 }; + } + + const authorActorUrl = remoteProfile.id; + const authorDisplayName = remoteProfile.name || remoteProfile.preferredUsername || authorHandle; + const authorAvatarUrl = typeof remoteProfile.icon === 'string' + ? remoteProfile.icon + : remoteProfile.icon?.url; + + let cached = 0; + let errors = 0; + const seenIds = new Set(); + + for (const item of outboxItems) { + try { + const activity = item?.type === 'Create' ? item : null; + const object = activity?.object; + if (!object || typeof object === 'string' || object.type !== 'Note') { + continue; + } + + const apId = object.id || activity?.id; + if (!apId || seenIds.has(apId)) { + continue; + } + seenIds.add(apId); + + // Check if already cached + const existing = await db.query.remotePosts.findFirst({ + where: eq(remotePosts.apId, apId), + }); + if (existing) { + continue; + } + + const attachments = Array.isArray(object.attachment) ? object.attachment : []; + const { text, urls } = extractTextAndUrls(object.content); + const normalizedUrl = urls.length > 0 ? normalizeUrl(urls[0]) : null; + + // Fetch link preview if there's a URL + let linkPreview: { url?: string; title?: string | null; description?: string | null; image?: string | null } | null = null; + if (normalizedUrl) { + try { + const previewUrl = new URL('/api/media/preview', origin); + previewUrl.searchParams.set('url', normalizedUrl); + const res = await fetch(previewUrl.toString(), { + headers: { 'Accept': 'application/json' }, + signal: AbortSignal.timeout(4000), + }); + if (res.ok) { + const data = await res.json(); + linkPreview = { + url: data?.url || normalizedUrl, + title: data?.title || null, + description: data?.description || null, + image: data?.image || null, + }; + } + } catch { + // Link preview fetch failed, continue without it + } + } + + const contentText = linkPreview && normalizedUrl ? stripFirstUrl(text, normalizedUrl) : text; + const mediaJson = attachments + .filter((a: { url?: string }) => a?.url) + .map((a: { url?: string; name?: string }) => ({ + url: a.url, + altText: sanitizeText(a.name) || null, + })); + + const publishedAt = object.published ? new Date(object.published) : new Date(); + + await db.insert(remotePosts).values({ + apId, + authorHandle, + authorActorUrl, + authorDisplayName, + authorAvatarUrl: authorAvatarUrl || null, + content: contentText || '', + publishedAt, + linkPreviewUrl: linkPreview?.url || normalizedUrl, + linkPreviewTitle: linkPreview?.title || null, + linkPreviewDescription: linkPreview?.description || null, + linkPreviewImage: linkPreview?.image || null, + mediaJson: mediaJson.length > 0 ? JSON.stringify(mediaJson) : null, + }); + + cached++; + } catch (error) { + console.error('Error caching post:', error); + errors++; + } + } + + return { cached, errors }; +}