Cachine recent posts

2026-01-22 16:25:25 -08:00
parent c7af395755
commit d2d8d9b746
6 changed files with 339 additions and 15 deletions
@@ -51,6 +51,7 @@ export default function ProfilePage() {
    const [activeTab, setActiveTab] = useState<'posts' | 'followers' | 'following'>('posts');
    const [followers, setFollowers] = useState<UserSummary[]>([]);
    const [following, setFollowing] = useState<UserSummary[]>([]);
+    const [postsLoading, setPostsLoading] = useState(true);
    const [followersLoading, setFollowersLoading] = useState(false);
    const [followingLoading, setFollowingLoading] = useState(false);
    const [isEditing, setIsEditing] = useState(false);
@@ -84,10 +85,12 @@ export default function ProfilePage() {
            })
            .catch(() => setLoading(false));

+        setPostsLoading(true);
        fetch(`/api/users/${handle}/posts`)
            .then(res => res.json())
            .then(data => setPosts(data.posts || []))
-            .catch(() => { });
+            .catch(() => { })
+            .finally(() => setPostsLoading(false));
    }, [handle]);

    const handleLike = async (postId: string, currentLiked: boolean) => {
@@ -562,14 +565,18 @@ export default function ProfilePage() {

            {/* Content */}
            {activeTab === 'posts' && (
-                posts.length === 0 ? (
+                postsLoading ? (
+                    <div style={{ padding: '48px', textAlign: 'center', color: 'var(--foreground-tertiary)' }}>
+                        <p>Loading...</p>
+                    </div>
+                ) : posts.length === 0 ? (
                    <div style={{ padding: '48px', textAlign: 'center', color: 'var(--foreground-tertiary)' }}>
                        <p>No posts yet</p>
                    </div>
                ) : (
-                    posts.map(post => (
+                    posts.map((post, index) => (
                        <PostCard
-                            key={post.id}
+                            key={`${post.id}-${index}`}
                            post={post}
                            onLike={handleLike}
                            onRepost={handleRepost}
@@ -1,5 +1,5 @@
 import { NextResponse } from 'next/server';
-import { db, posts, users, media, follows, mutes, blocks, likes } from '@/db';
+import { db, posts, users, media, follows, mutes, blocks, likes, remoteFollows, remotePosts } from '@/db';
 import { requireAuth } from '@/lib/auth';
 import { eq, desc, and, inArray, isNull, notInArray, or } from 'drizzle-orm';
 import type { SQL } from 'drizzle-orm';
@@ -267,9 +267,8 @@ export async function GET(request: Request) {
            try {
                const user = await requireAuth();

-                // Get posts from people the user follows + their own posts
-                // For now, just return all posts (we'll add following filter later)
-                feedPosts = await db.query.posts.findMany({
+                // Get local posts from people the user follows + their own posts
+                const localPosts = await db.query.posts.findMany({
                    where: baseFilter,
                    with: {
                        author: true,
@@ -279,8 +278,63 @@ export async function GET(request: Request) {
                        },
                    },
                    orderBy: [desc(posts.createdAt)],
-                    limit,
+                    limit: limit * 2, // Get more to account for mixing with remote
                });
+
+                // Get handles of remote users we follow
+                const followedRemoteUsers = await db.query.remoteFollows.findMany({
+                    where: eq(remoteFollows.followerId, user.id),
+                });
+                const followedRemoteHandles = followedRemoteUsers.map(f => f.targetHandle);
+
+                // Get cached remote posts from followed users
+                let remotePostsData: typeof remotePosts.$inferSelect[] = [];
+                if (followedRemoteHandles.length > 0) {
+                    remotePostsData = await db.query.remotePosts.findMany({
+                        where: inArray(remotePosts.authorHandle, followedRemoteHandles),
+                        orderBy: [desc(remotePosts.publishedAt)],
+                        limit: limit,
+                    });
+                }
+
+                // Transform remote posts to match local post format
+                const transformedRemotePosts = remotePostsData.map(rp => {
+                    const mediaData = rp.mediaJson ? JSON.parse(rp.mediaJson) : [];
+                    return {
+                        id: rp.id,
+                        content: rp.content,
+                        createdAt: rp.publishedAt,
+                        likesCount: 0,
+                        repostsCount: 0,
+                        repliesCount: 0,
+                        isRemote: true,
+                        apId: rp.apId,
+                        linkPreviewUrl: rp.linkPreviewUrl,
+                        linkPreviewTitle: rp.linkPreviewTitle,
+                        linkPreviewDescription: rp.linkPreviewDescription,
+                        linkPreviewImage: rp.linkPreviewImage,
+                        author: {
+                            id: rp.authorActorUrl,
+                            handle: rp.authorHandle,
+                            displayName: rp.authorDisplayName,
+                            avatarUrl: rp.authorAvatarUrl,
+                            isRemote: true,
+                        },
+                        media: mediaData.map((m: { url: string; altText?: string }, idx: number) => ({
+                            id: `${rp.id}-media-${idx}`,
+                            url: m.url,
+                            altText: m.altText || null,
+                        })),
+                        replyTo: null,
+                    };
+                });
+
+                // Merge and sort by date
+                const allPosts = [...localPosts, ...transformedRemotePosts]
+                    .sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime())
+                    .slice(0, limit);
+
+                feedPosts = allPosts as any;
            } catch {
                // Not authenticated, return public timeline
                feedPosts = await db.query.posts.findMany({
@@ -305,7 +359,7 @@ export async function GET(request: Request) {

            if (session?.user && feedPosts && feedPosts.length > 0) {
                const viewer = session.user;
-                const postIds = feedPosts.map(p => p.id).filter(Boolean);
+                const postIds = feedPosts.map((p: { id: string }) => p.id).filter(Boolean);

                if (postIds.length > 0) {
                    const viewerLikes = await db.query.likes.findMany({
@@ -324,7 +378,7 @@ export async function GET(request: Request) {
                    });
                    const repostedPostIds = new Set(viewerReposts.map(r => r.repostOfId));

-                    feedPosts = feedPosts.map(p => ({
+                    feedPosts = feedPosts.map((p: { id: string }) => ({
                        ...p,
                        isLiked: likedPostIds.has(p.id),
                        isReposted: repostedPostIds.has(p.id),
@@ -6,6 +6,7 @@ import { requireAuth } from '@/lib/auth';
 import { resolveRemoteUser } from '@/lib/activitypub/fetch';
 import { createFollowActivity, createUndoActivity } from '@/lib/activitypub/activities';
 import { deliverActivity } from '@/lib/activitypub/outbox';
+import { cacheRemoteUserPosts } from '@/lib/activitypub/cache';

 type RouteContext = { params: Promise<{ handle: string }> };

@@ -132,6 +133,13 @@ export async function POST(request: Request, context: RouteContext) {
                inboxUrl: targetInbox,
                activityId,
            });
+
+            // Cache the remote user's recent posts in the background
+            const origin = new URL(request.url).origin;
+            cacheRemoteUserPosts(remoteProfile, targetHandle, origin, 20)
+                .then(result => console.log(`Cached ${result.cached} posts for ${targetHandle}`))
+                .catch(err => console.error('Error caching remote posts:', err));
+
            return NextResponse.json({ success: true, following: true, remote: true });
        }

@@ -133,6 +133,7 @@ export async function GET(request: Request, context: RouteContext) {
                bio: sanitizeText(remoteProfile.summary),
            };
            const posts = [];
+            const seenIds = new Set<string>();
            const origin = new URL(request.url).origin;
            for (const item of outboxItems) {
                const activity = item?.type === 'Create' ? item : null;
@@ -140,13 +141,20 @@ export async function GET(request: Request, context: RouteContext) {
                if (!object || typeof object === 'string' || object.type !== 'Note') {
                    continue;
                }
+                // Deduplicate by object ID
+                const postId = object.id || activity.id;
+                if (seenIds.has(postId)) {
+                    continue;
+                }
+                seenIds.add(postId);
+
                const attachments = Array.isArray(object.attachment) ? object.attachment : [];
                const { text, urls } = extractTextAndUrls(object.content);
                const normalizedUrl = urls.length > 0 ? normalizeUrl(urls[0]) : null;
                const linkPreview = normalizedUrl ? await fetchLinkPreview(normalizedUrl, origin) : null;
                const contentText = linkPreview && normalizedUrl ? stripFirstUrl(text, normalizedUrl) : text;
                posts.push({
-                    id: object.id || activity.id,
+                    id: postId,
                    content: contentText || '',
                    createdAt: object.published || activity.published || new Date().toISOString(),
                    likesCount: 0,
@@ -156,7 +164,7 @@ export async function GET(request: Request, context: RouteContext) {
                    media: attachments
                        .filter((attachment: any) => attachment?.url)
                        .map((attachment: any, index: number) => ({
-                            id: `${object.id || activity.id || 'media'}-${index}`,
+                            id: `${postId || 'media'}-${index}`,
                            url: attachment.url,
                            altText: sanitizeText(attachment.name) || null,
                        })),
@@ -193,6 +201,7 @@ export async function GET(request: Request, context: RouteContext) {
                bio: sanitizeText(remoteProfile.summary),
            };
            const posts = [];
+            const seenIds = new Set<string>();
            const origin = new URL(request.url).origin;
            for (const item of outboxItems) {
                const activity = item?.type === 'Create' ? item : null;
@@ -200,13 +209,20 @@ export async function GET(request: Request, context: RouteContext) {
                if (!object || typeof object === 'string' || object.type !== 'Note') {
                    continue;
                }
+                // Deduplicate by object ID
+                const postId = object.id || activity.id;
+                if (seenIds.has(postId)) {
+                    continue;
+                }
+                seenIds.add(postId);
+
                const attachments = Array.isArray(object.attachment) ? object.attachment : [];
                const { text, urls } = extractTextAndUrls(object.content);
                const normalizedUrl = urls.length > 0 ? normalizeUrl(urls[0]) : null;
                const linkPreview = normalizedUrl ? await fetchLinkPreview(normalizedUrl, origin) : null;
                const contentText = linkPreview && normalizedUrl ? stripFirstUrl(text, normalizedUrl) : text;
                posts.push({
-                    id: object.id || activity.id,
+                    id: postId,
                    content: contentText || '',
                    createdAt: object.published || activity.published || new Date().toISOString(),
                    likesCount: 0,
@@ -216,7 +232,7 @@ export async function GET(request: Request, context: RouteContext) {
                    media: attachments
                        .filter((attachment: any) => attachment?.url)
                        .map((attachment: any, index: number) => ({
-                            id: `${object.id || activity.id || 'media'}-${index}`,
+                            id: `${postId || 'media'}-${index}`,
                            url: attachment.url,
                            altText: sanitizeText(attachment.name) || null,
                        })),
@@ -223,6 +223,35 @@ export const remoteFollowers = pgTable('remote_followers', {
  index('remote_followers_actor_idx').on(table.actorUrl),
 ]);

+// ============================================
+// REMOTE POSTS (cached posts from federated users)
+// ============================================
+
+export const remotePosts = pgTable('remote_posts', {
+  id: uuid('id').primaryKey().defaultRandom(),
+  apId: text('ap_id').notNull().unique(), // ActivityPub ID (URL) of the post
+  authorHandle: text('author_handle').notNull(), // e.g., user@mastodon.social
+  authorActorUrl: text('author_actor_url').notNull(), // Remote actor URL
+  authorDisplayName: text('author_display_name'),
+  authorAvatarUrl: text('author_avatar_url'),
+  content: text('content').notNull(),
+  publishedAt: timestamp('published_at').notNull(), // Original publish time
+  // Link preview
+  linkPreviewUrl: text('link_preview_url'),
+  linkPreviewTitle: text('link_preview_title'),
+  linkPreviewDescription: text('link_preview_description'),
+  linkPreviewImage: text('link_preview_image'),
+  // Media attachments stored as JSON
+  mediaJson: text('media_json'), // JSON array of {url, altText}
+  // Metadata
+  fetchedAt: timestamp('fetched_at').defaultNow().notNull(),
+  createdAt: timestamp('created_at').defaultNow().notNull(),
+}, (table) => [
+  index('remote_posts_author_idx').on(table.authorHandle),
+  index('remote_posts_published_idx').on(table.publishedAt),
+  index('remote_posts_ap_id_idx').on(table.apId),
+]);
+
 // ============================================
 // LIKES
 // ============================================
@@ -0,0 +1,210 @@
+import { db, remotePosts } from '@/db';
+import { eq } from 'drizzle-orm';
+
+interface RemoteProfile {
+    id: string;
+    preferredUsername?: string;
+    name?: string;
+    icon?: string | { url?: string };
+    summary?: string;
+    outbox?: string;
+}
+
+interface OutboxItem {
+    type?: string;
+    object?: {
+        id?: string;
+        type?: string;
+        content?: string;
+        published?: string;
+        attachment?: Array<{ url?: string; name?: string }>;
+    };
+    id?: string;
+    published?: string;
+}
+
+const decodeEntities = (value: string) =>
+    value
+        .replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)))
+        .replace(/&#(\d+);/g, (_, num) => String.fromCharCode(Number(num)))
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'");
+
+const sanitizeText = (value?: string | null) => {
+    if (!value) return null;
+    const withoutTags = value.replace(/<[^>]*>/g, ' ');
+    const decoded = decodeEntities(withoutTags);
+    return decoded.replace(/\s+/g, ' ').trim() || null;
+};
+
+const extractTextAndUrls = (value?: string | null) => {
+    if (!value) return { text: '', urls: [] as string[] };
+    let html = value;
+    html = html.replace(/<br\s*\/?>/gi, ' ');
+    html = html.replace(/<a[^>]*href=["']([^"']+)["'][^>]*>(.*?)<\/a>/gi, (_, href, text) => {
+        const cleanedHref = decodeEntities(String(href));
+        const cleanedText = decodeEntities(String(text)).replace(/<[^>]*>/g, ' ').trim();
+        return cleanedHref || cleanedText;
+    });
+    const withoutTags = html.replace(/<[^>]*>/g, ' ');
+    const decoded = decodeEntities(withoutTags);
+    const text = decoded.replace(/\s+/g, ' ').trim();
+    const urls = Array.from(text.matchAll(/https?:\/\/[^\s]+/gi)).map((match) => match[0]);
+    return { text, urls };
+};
+
+const normalizeUrl = (value: string) => value.replace(/[)\].,!?]+$/, '');
+
+const stripFirstUrl = (text: string, url: string) => {
+    const idx = text.indexOf(url);
+    if (idx === -1) return text;
+    const before = text.slice(0, idx).trimEnd();
+    const after = text.slice(idx + url.length).trimStart();
+    return `${before} ${after}`.trim();
+};
+
+const fetchOutboxItems = async (outboxUrl: string, limit: number = 20): Promise<OutboxItem[]> => {
+    try {
+        const res = await fetch(outboxUrl, {
+            headers: {
+                'Accept': 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
+            },
+            signal: AbortSignal.timeout(10000),
+        });
+        if (!res.ok) return [];
+        const data = await res.json();
+        const first = data?.first;
+        if (first) {
+            if (typeof first === 'string') {
+                const pageRes = await fetch(first, {
+                    headers: {
+                        'Accept': 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
+                    },
+                    signal: AbortSignal.timeout(10000),
+                });
+                if (!pageRes.ok) return [];
+                const page = await pageRes.json();
+                return page?.orderedItems || page?.items || [];
+            }
+            return first?.orderedItems || first?.items || [];
+        }
+        const items = data?.orderedItems || data?.items || [];
+        return items.slice(0, limit);
+    } catch (error) {
+        console.error('Error fetching outbox:', error);
+        return [];
+    }
+};
+
+export async function cacheRemoteUserPosts(
+    remoteProfile: RemoteProfile,
+    authorHandle: string, // e.g., user@mastodon.social
+    origin: string, // Used for link previews
+    limit: number = 20
+): Promise<{ cached: number; errors: number }> {
+    if (!remoteProfile.outbox) {
+        return { cached: 0, errors: 0 };
+    }
+
+    const outboxItems = await fetchOutboxItems(remoteProfile.outbox, limit);
+    if (outboxItems.length === 0) {
+        return { cached: 0, errors: 0 };
+    }
+
+    const authorActorUrl = remoteProfile.id;
+    const authorDisplayName = remoteProfile.name || remoteProfile.preferredUsername || authorHandle;
+    const authorAvatarUrl = typeof remoteProfile.icon === 'string'
+        ? remoteProfile.icon
+        : remoteProfile.icon?.url;
+
+    let cached = 0;
+    let errors = 0;
+    const seenIds = new Set<string>();
+
+    for (const item of outboxItems) {
+        try {
+            const activity = item?.type === 'Create' ? item : null;
+            const object = activity?.object;
+            if (!object || typeof object === 'string' || object.type !== 'Note') {
+                continue;
+            }
+
+            const apId = object.id || activity?.id;
+            if (!apId || seenIds.has(apId)) {
+                continue;
+            }
+            seenIds.add(apId);
+
+            // Check if already cached
+            const existing = await db.query.remotePosts.findFirst({
+                where: eq(remotePosts.apId, apId),
+            });
+            if (existing) {
+                continue;
+            }
+
+            const attachments = Array.isArray(object.attachment) ? object.attachment : [];
+            const { text, urls } = extractTextAndUrls(object.content);
+            const normalizedUrl = urls.length > 0 ? normalizeUrl(urls[0]) : null;
+
+            // Fetch link preview if there's a URL
+            let linkPreview: { url?: string; title?: string | null; description?: string | null; image?: string | null } | null = null;
+            if (normalizedUrl) {
+                try {
+                    const previewUrl = new URL('/api/media/preview', origin);
+                    previewUrl.searchParams.set('url', normalizedUrl);
+                    const res = await fetch(previewUrl.toString(), {
+                        headers: { 'Accept': 'application/json' },
+                        signal: AbortSignal.timeout(4000),
+                    });
+                    if (res.ok) {
+                        const data = await res.json();
+                        linkPreview = {
+                            url: data?.url || normalizedUrl,
+                            title: data?.title || null,
+                            description: data?.description || null,
+                            image: data?.image || null,
+                        };
+                    }
+                } catch {
+                    // Link preview fetch failed, continue without it
+                }
+            }
+
+            const contentText = linkPreview && normalizedUrl ? stripFirstUrl(text, normalizedUrl) : text;
+            const mediaJson = attachments
+                .filter((a: { url?: string }) => a?.url)
+                .map((a: { url?: string; name?: string }) => ({
+                    url: a.url,
+                    altText: sanitizeText(a.name) || null,
+                }));
+
+            const publishedAt = object.published ? new Date(object.published) : new Date();
+
+            await db.insert(remotePosts).values({
+                apId,
+                authorHandle,
+                authorActorUrl,
+                authorDisplayName,
+                authorAvatarUrl: authorAvatarUrl || null,
+                content: contentText || '',
+                publishedAt,
+                linkPreviewUrl: linkPreview?.url || normalizedUrl,
+                linkPreviewTitle: linkPreview?.title || null,
+                linkPreviewDescription: linkPreview?.description || null,
+                linkPreviewImage: linkPreview?.image || null,
+                mediaJson: mediaJson.length > 0 ? JSON.stringify(mediaJson) : null,
+            });
+
+            cached++;
+        } catch (error) {
+            console.error('Error caching post:', error);
+            errors++;
+        }
+    }
+
+    return { cached, errors };
+}