Add docker publish flow, node blocklist & API updates
Replace docker/metadata GH Action with a docker-publish.sh driven workflow (supports auto-versioning, extra tags, multi-arch builds and optional builder). Update docs and READMEs to use the updater and new publish flow. Add server-side swarm/node blocklist support and admin nodes API. Improve auth endpoints (me, logout, switch, session accounts) and support account switching. Extend bots API to support custom LLM provider and optional endpoint. Enforce node blocklist on chat send/receive, refactor link preview handling into dedicated preview modules, and enhance notifications and posts like/repost handlers to accept both signed actions and regular auth flows. Misc: remove admin update UI details, add helper libs (media previews, node-blocklist, reposts, notifications, etc.), and minor housekeeping (pyc, workflow tweaks).
This commit is contained in:
cyph3rasi
@@ -12,7 +12,7 @@ import { db, botContentItems, bots } from '@/db';
|
||||
import { eq, and } from 'drizzle-orm';
|
||||
import { ContentGenerator, type Bot as ContentGeneratorBot, type ContentItem } from './contentGenerator';
|
||||
import { canPost } from './rateLimiter';
|
||||
import { decryptApiKey, deserializeEncryptedData } from './encryption';
|
||||
import { decryptApiKey, deserializeEncryptedData, type LLMProvider } from './encryption';
|
||||
import { parseScheduleConfig, isDue } from './scheduler';
|
||||
import { triggerPost } from './posting';
|
||||
|
||||
@@ -213,8 +213,9 @@ function toContentGeneratorBot(bot: typeof bots.$inferSelect, handle: string): C
|
||||
name: bot.name,
|
||||
handle: handle,
|
||||
personalityConfig: JSON.parse(bot.personalityConfig),
|
||||
llmProvider: bot.llmProvider as 'openrouter' | 'openai' | 'anthropic',
|
||||
llmProvider: bot.llmProvider as LLMProvider,
|
||||
llmModel: bot.llmModel,
|
||||
llmEndpoint: bot.llmEndpoint,
|
||||
llmApiKeyEncrypted: bot.llmApiKeyEncrypted,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import { generateAndUploadAvatarToUserStorage } from '@/lib/storage/s3';
|
||||
import { decryptPrivateKey, deserializeEncryptedKey } from '@/lib/crypto/private-key';
|
||||
import { eq, and, count } from 'drizzle-orm';
|
||||
import { generateKeyPair } from '@/lib/crypto/keys';
|
||||
import { isIP } from 'net';
|
||||
import {
|
||||
encryptApiKey,
|
||||
decryptApiKey,
|
||||
@@ -52,6 +53,7 @@ export interface BotCreateInput {
|
||||
personality: PersonalityConfig;
|
||||
llmProvider: LLMProvider;
|
||||
llmModel: string;
|
||||
llmEndpoint?: string;
|
||||
llmApiKey: string;
|
||||
schedule?: ScheduleConfig;
|
||||
autonomousMode?: boolean;
|
||||
@@ -65,6 +67,7 @@ export interface BotUpdateInput {
|
||||
personality?: PersonalityConfig;
|
||||
llmProvider?: LLMProvider;
|
||||
llmModel?: string;
|
||||
llmEndpoint?: string | null;
|
||||
llmApiKey?: string;
|
||||
schedule?: ScheduleConfig | null;
|
||||
autonomousMode?: boolean;
|
||||
@@ -83,6 +86,7 @@ export interface Bot {
|
||||
personalityConfig: PersonalityConfig;
|
||||
llmProvider: LLMProvider;
|
||||
llmModel: string;
|
||||
llmEndpoint?: string | null;
|
||||
scheduleConfig: ScheduleConfig | null;
|
||||
autonomousMode: boolean;
|
||||
isActive: boolean;
|
||||
@@ -248,6 +252,65 @@ export function validateScheduleConfig(config: ScheduleConfig): void {
|
||||
}
|
||||
}
|
||||
|
||||
function isPrivateHostname(hostname: string): boolean {
|
||||
const normalized = hostname.toLowerCase();
|
||||
|
||||
if (
|
||||
normalized === 'localhost' ||
|
||||
normalized.endsWith('.localhost') ||
|
||||
normalized.endsWith('.local')
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const ipVersion = isIP(normalized);
|
||||
if (ipVersion === 4) {
|
||||
const octets = normalized.split('.').map(Number);
|
||||
const [first, second] = octets;
|
||||
|
||||
return (
|
||||
first === 10 ||
|
||||
first === 127 ||
|
||||
(first === 169 && second === 254) ||
|
||||
(first === 172 && second >= 16 && second <= 31) ||
|
||||
(first === 192 && second === 168)
|
||||
);
|
||||
}
|
||||
|
||||
if (ipVersion === 6) {
|
||||
return normalized === '::1' || normalized.startsWith('fc') || normalized.startsWith('fd') || normalized.startsWith('fe80:');
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export function normalizeAndValidateLlmEndpoint(endpoint: string): string {
|
||||
if (!endpoint || typeof endpoint !== 'string') {
|
||||
throw new BotValidationError('Custom endpoint is required');
|
||||
}
|
||||
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(endpoint.trim());
|
||||
} catch {
|
||||
throw new BotValidationError('Custom endpoint must be a valid URL');
|
||||
}
|
||||
|
||||
if (parsed.protocol !== 'https:') {
|
||||
throw new BotValidationError('Custom endpoint must use HTTPS');
|
||||
}
|
||||
|
||||
if (parsed.username || parsed.password) {
|
||||
throw new BotValidationError('Custom endpoint cannot include embedded credentials');
|
||||
}
|
||||
|
||||
if (isPrivateHostname(parsed.hostname)) {
|
||||
throw new BotValidationError('Custom endpoint must be publicly reachable and cannot target localhost or a private network');
|
||||
}
|
||||
|
||||
return parsed.toString();
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// HELPER FUNCTIONS
|
||||
// ============================================
|
||||
@@ -269,6 +332,7 @@ function dbBotToBot(dbBot: typeof bots.$inferSelect, botUser: typeof users.$infe
|
||||
personalityConfig: JSON.parse(dbBot.personalityConfig) as PersonalityConfig,
|
||||
llmProvider: dbBot.llmProvider as LLMProvider,
|
||||
llmModel: dbBot.llmModel,
|
||||
llmEndpoint: dbBot.llmEndpoint,
|
||||
scheduleConfig: dbBot.scheduleConfig ? JSON.parse(dbBot.scheduleConfig) as ScheduleConfig : null,
|
||||
autonomousMode: dbBot.autonomousMode,
|
||||
isActive: dbBot.isActive,
|
||||
@@ -308,6 +372,10 @@ export async function createBot(ownerId: string, config: BotCreateInput): Promis
|
||||
if (config.schedule) {
|
||||
validateScheduleConfig(config.schedule);
|
||||
}
|
||||
|
||||
const normalizedEndpoint = config.llmProvider === 'custom'
|
||||
? normalizeAndValidateLlmEndpoint(config.llmEndpoint || '')
|
||||
: null;
|
||||
|
||||
// Validate API key format
|
||||
const apiKeyValidation = validateApiKeyFormat(config.llmApiKey, config.llmProvider);
|
||||
@@ -391,6 +459,7 @@ export async function createBot(ownerId: string, config: BotCreateInput): Promis
|
||||
personalityConfig: JSON.stringify(config.personality),
|
||||
llmProvider: config.llmProvider,
|
||||
llmModel: config.llmModel,
|
||||
llmEndpoint: normalizedEndpoint,
|
||||
llmApiKeyEncrypted: serializeEncryptedData(encryptedApiKey),
|
||||
scheduleConfig: config.schedule ? JSON.stringify(config.schedule) : null,
|
||||
autonomousMode: config.autonomousMode ?? false,
|
||||
@@ -443,15 +512,7 @@ export async function updateBot(botId: string, config: BotUpdateInput): Promise<
|
||||
if (config.schedule !== undefined && config.schedule !== null) {
|
||||
validateScheduleConfig(config.schedule);
|
||||
}
|
||||
|
||||
// Validate API key if provided
|
||||
if (config.llmApiKey !== undefined && config.llmProvider !== undefined) {
|
||||
const apiKeyValidation = validateApiKeyFormat(config.llmApiKey, config.llmProvider);
|
||||
if (!apiKeyValidation.valid) {
|
||||
throw new BotValidationError(apiKeyValidation.error || 'Invalid API key');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Check if bot exists and get its user account
|
||||
const existingBot = await db.query.bots.findFirst({
|
||||
where: eq(bots.id, botId),
|
||||
@@ -469,6 +530,25 @@ export async function updateBot(botId: string, config: BotUpdateInput): Promise<
|
||||
if (!botUser) {
|
||||
throw new BotNotFoundError(botId);
|
||||
}
|
||||
|
||||
const targetProvider = (config.llmProvider ?? existingBot.llmProvider) as LLMProvider;
|
||||
|
||||
if (targetProvider === 'custom') {
|
||||
const endpointToValidate = config.llmEndpoint !== undefined ? config.llmEndpoint : existingBot.llmEndpoint;
|
||||
if (!endpointToValidate) {
|
||||
throw new BotValidationError('Custom endpoint is required');
|
||||
}
|
||||
config.llmEndpoint = normalizeAndValidateLlmEndpoint(endpointToValidate);
|
||||
} else if (config.llmEndpoint !== undefined || config.llmProvider !== undefined) {
|
||||
config.llmEndpoint = null;
|
||||
}
|
||||
|
||||
if (config.llmApiKey !== undefined) {
|
||||
const apiKeyValidation = validateApiKeyFormat(config.llmApiKey, targetProvider);
|
||||
if (!apiKeyValidation.valid) {
|
||||
throw new BotValidationError(apiKeyValidation.error || 'Invalid API key');
|
||||
}
|
||||
}
|
||||
|
||||
// Build update object for bot config
|
||||
const botUpdateData: Partial<typeof bots.$inferInsert> = {
|
||||
@@ -508,6 +588,12 @@ export async function updateBot(botId: string, config: BotUpdateInput): Promise<
|
||||
if (config.llmModel !== undefined) {
|
||||
botUpdateData.llmModel = config.llmModel;
|
||||
}
|
||||
|
||||
if (config.llmEndpoint !== undefined) {
|
||||
botUpdateData.llmEndpoint = config.llmEndpoint;
|
||||
} else if (config.llmProvider !== undefined && config.llmProvider !== 'custom') {
|
||||
botUpdateData.llmEndpoint = null;
|
||||
}
|
||||
|
||||
if (config.llmApiKey !== undefined) {
|
||||
const encryptedApiKey = encryptApiKey(config.llmApiKey);
|
||||
|
||||
@@ -12,6 +12,8 @@ import {
|
||||
calculateBackoffDelay,
|
||||
shouldRetrySource,
|
||||
isSourceDueForFetch,
|
||||
shouldExcludeRedditPost,
|
||||
fetchRedditPosts,
|
||||
BASE_BACKOFF_DELAY_MS,
|
||||
MAX_BACKOFF_DELAY_MS,
|
||||
MAX_CONSECUTIVE_ERRORS,
|
||||
@@ -195,6 +197,88 @@ describe('isSourceDueForFetch', () => {
|
||||
|
||||
});
|
||||
|
||||
describe('Reddit filtering', () => {
|
||||
const originalFetch = global.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('excludes stickied and pinned reddit posts', () => {
|
||||
expect(shouldExcludeRedditPost({
|
||||
id: 't3_stickied',
|
||||
title: 'Community thread',
|
||||
permalink: '/r/test/comments/stickied/community_thread/',
|
||||
url: 'https://reddit.com/r/test/comments/stickied/community_thread/',
|
||||
created_utc: 1710000000,
|
||||
stickied: true,
|
||||
})).toBe(true);
|
||||
|
||||
expect(shouldExcludeRedditPost({
|
||||
id: 't3_pinned',
|
||||
title: 'Pinned thread',
|
||||
permalink: '/r/test/comments/pinned/pinned_thread/',
|
||||
url: 'https://reddit.com/r/test/comments/pinned/pinned_thread/',
|
||||
created_utc: 1710000000,
|
||||
pinned: true,
|
||||
})).toBe(true);
|
||||
|
||||
expect(shouldExcludeRedditPost({
|
||||
id: 't3_normal',
|
||||
title: 'Normal post',
|
||||
permalink: '/r/test/comments/normal/normal_post/',
|
||||
url: 'https://reddit.com/r/test/comments/normal/normal_post/',
|
||||
created_utc: 1710000000,
|
||||
})).toBe(false);
|
||||
});
|
||||
|
||||
it('fetchRedditPosts removes community highlights before returning items', async () => {
|
||||
const payload = {
|
||||
data: {
|
||||
children: [
|
||||
{
|
||||
data: {
|
||||
id: 'abc123',
|
||||
name: 't3_abc123',
|
||||
title: 'Community Highlights',
|
||||
selftext: 'Pinned moderator thread',
|
||||
permalink: '/r/test/comments/abc123/community_highlights/',
|
||||
url: 'https://www.reddit.com/r/test/comments/abc123/community_highlights/',
|
||||
created_utc: 1710000000,
|
||||
stickied: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
data: {
|
||||
id: 'def456',
|
||||
name: 't3_def456',
|
||||
title: 'Actual content post',
|
||||
selftext: 'Useful content',
|
||||
permalink: '/r/test/comments/def456/actual_content_post/',
|
||||
url: 'https://example.com/article',
|
||||
created_utc: 1710000100,
|
||||
stickied: false,
|
||||
pinned: false,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
global.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
text: async () => JSON.stringify(payload),
|
||||
} as Response);
|
||||
|
||||
const items = await fetchRedditPosts('test', { maxItems: 10 });
|
||||
|
||||
expect(items).toHaveLength(1);
|
||||
expect(items[0]?.title).toBe('Actual content post');
|
||||
expect(items[0]?.id).toBe('t3_def456');
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================
|
||||
// BACKOFF DELAY PROPERTY TESTS
|
||||
// ============================================
|
||||
|
||||
@@ -41,6 +41,7 @@ export interface ContentItemInput {
|
||||
title: string;
|
||||
content: string | null;
|
||||
url: string;
|
||||
url_overridden_by_dest?: string;
|
||||
publishedAt: Date;
|
||||
}
|
||||
|
||||
@@ -325,16 +326,39 @@ export async function fetchRedditPosts(
|
||||
subreddit: string,
|
||||
options: FetchOptions = {}
|
||||
): Promise<FeedItem[]> {
|
||||
// Use RSS feed instead of JSON API - more reliable and doesn't require auth
|
||||
const rssUrl = `https://www.reddit.com/r/${subreddit}/hot.rss`;
|
||||
|
||||
const requestedMaxItems = options.maxItems ?? DEFAULT_MAX_ITEMS;
|
||||
const fetchLimit = Math.min(Math.max(requestedMaxItems + 10, requestedMaxItems), 100);
|
||||
const url = new URL(`${REDDIT_API_BASE}/r/${subreddit}/hot.json`);
|
||||
url.searchParams.set('raw_json', '1');
|
||||
url.searchParams.set('limit', String(fetchLimit));
|
||||
|
||||
const responseText = await fetchUrl(url.toString(), {
|
||||
timeout: options.timeout,
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
},
|
||||
});
|
||||
|
||||
let response: RedditListingResponse;
|
||||
try {
|
||||
return await fetchRSSFeed(rssUrl, options);
|
||||
} catch (error) {
|
||||
// If RSS fails, try the old.reddit.com RSS which sometimes works better
|
||||
const oldRedditUrl = `https://old.reddit.com/r/${subreddit}/hot.rss`;
|
||||
return await fetchRSSFeed(oldRedditUrl, options);
|
||||
response = JSON.parse(responseText) as RedditListingResponse;
|
||||
} catch {
|
||||
throw new ParseError('Failed to parse Reddit JSON response');
|
||||
}
|
||||
|
||||
const posts = response.data?.children
|
||||
?.map((child) => child.data)
|
||||
.filter((post): post is RedditListingChildData => Boolean(post)) || [];
|
||||
|
||||
const filteredPosts = posts.filter((post) => !shouldExcludeRedditPost(post));
|
||||
|
||||
return filteredPosts.slice(0, requestedMaxItems).map((post): FeedItem => ({
|
||||
id: post.name || post.id,
|
||||
title: post.title,
|
||||
content: post.selftext || '',
|
||||
url: `${REDDIT_API_BASE}${post.permalink}`,
|
||||
publishedAt: new Date(post.created_utc * 1000),
|
||||
}));
|
||||
}
|
||||
|
||||
// ============================================
|
||||
@@ -384,6 +408,31 @@ interface BraveNewsResponse {
|
||||
results?: BraveNewsResult[];
|
||||
}
|
||||
|
||||
interface RedditListingChildData {
|
||||
id: string;
|
||||
name?: string;
|
||||
title: string;
|
||||
selftext?: string;
|
||||
permalink: string;
|
||||
url: string;
|
||||
url_overridden_by_dest?: string;
|
||||
created_utc: number;
|
||||
stickied?: boolean;
|
||||
pinned?: boolean;
|
||||
}
|
||||
|
||||
interface RedditListingResponse {
|
||||
data?: {
|
||||
children?: Array<{
|
||||
data?: RedditListingChildData;
|
||||
}>;
|
||||
};
|
||||
}
|
||||
|
||||
export function shouldExcludeRedditPost(post: RedditListingChildData): boolean {
|
||||
return Boolean(post.stickied || post.pinned);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch articles from a news API.
|
||||
*
|
||||
|
||||
@@ -26,6 +26,7 @@ export interface Bot {
|
||||
personalityConfig: PersonalityConfig;
|
||||
llmProvider: LLMProvider;
|
||||
llmModel: string;
|
||||
llmEndpoint?: string | null;
|
||||
llmApiKeyEncrypted: string;
|
||||
}
|
||||
|
||||
@@ -35,6 +36,7 @@ export interface Bot {
|
||||
export interface ContentItem {
|
||||
id: string;
|
||||
sourceId: string;
|
||||
externalId?: string;
|
||||
title: string;
|
||||
content: string | null;
|
||||
url: string;
|
||||
@@ -467,6 +469,7 @@ export class ContentGenerator {
|
||||
provider: bot.llmProvider,
|
||||
apiKey: bot.llmApiKeyEncrypted,
|
||||
model: bot.llmModel,
|
||||
endpoint: bot.llmEndpoint,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ import * as crypto from 'crypto';
|
||||
// TYPES
|
||||
// ============================================
|
||||
|
||||
export type LLMProvider = 'openrouter' | 'openai' | 'anthropic';
|
||||
export type LLMProvider = 'openrouter' | 'openai' | 'anthropic' | 'custom';
|
||||
|
||||
export interface EncryptedData {
|
||||
encrypted: string; // Base64 encoded ciphertext + auth tag
|
||||
@@ -40,6 +40,7 @@ const API_KEY_PATTERNS: Record<LLMProvider, RegExp> = {
|
||||
openrouter: /^sk-or-[a-zA-Z0-9_-]+$/,
|
||||
anthropic: /^sk-ant-[a-zA-Z0-9_-]+$/,
|
||||
openai: /^sk-[a-zA-Z0-9_-]+$/,
|
||||
custom: /^[\s\S]+$/,
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -330,7 +331,7 @@ export function validateAndDetectApiKey(apiKey: string): ApiKeyValidationResult
|
||||
* @returns True if the provider is supported
|
||||
*/
|
||||
export function isSupportedProvider(provider: string): provider is LLMProvider {
|
||||
return provider === 'openrouter' || provider === 'openai' || provider === 'anthropic';
|
||||
return provider === 'openrouter' || provider === 'openai' || provider === 'anthropic' || provider === 'custom';
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -339,5 +340,5 @@ export function isSupportedProvider(provider: string): provider is LLMProvider {
|
||||
* @returns Array of supported provider names
|
||||
*/
|
||||
export function getSupportedProviders(): LLMProvider[] {
|
||||
return ['openrouter', 'openai', 'anthropic'];
|
||||
return ['openrouter', 'openai', 'anthropic', 'custom'];
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ export interface LLMConfig {
|
||||
provider: LLMProvider;
|
||||
apiKey: string; // Encrypted before storage
|
||||
model: string;
|
||||
endpoint?: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -123,6 +124,7 @@ export const PROVIDER_ENDPOINTS: Record<LLMProvider, string> = {
|
||||
openrouter: 'https://openrouter.ai/api/v1/chat/completions',
|
||||
openai: 'https://api.openai.com/v1/chat/completions',
|
||||
anthropic: 'https://api.anthropic.com/v1/messages',
|
||||
custom: '',
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -132,6 +134,7 @@ export const DEFAULT_MODELS: Record<LLMProvider, string> = {
|
||||
openrouter: 'openai/gpt-3.5-turbo',
|
||||
openai: 'gpt-3.5-turbo',
|
||||
anthropic: 'claude-3-haiku-20240307',
|
||||
custom: 'gpt-4o-mini',
|
||||
};
|
||||
|
||||
// ============================================
|
||||
@@ -299,7 +302,8 @@ export function parseOpenRouterResponse(
|
||||
*/
|
||||
export function parseOpenAIResponse(
|
||||
data: Record<string, unknown>,
|
||||
model: string
|
||||
model: string,
|
||||
provider: LLMProvider = 'openai'
|
||||
): LLMCompletionResponse {
|
||||
const choices = data.choices as Array<{ message: { content: string } }>;
|
||||
const usage = data.usage as { prompt_tokens: number; completion_tokens: number; total_tokens: number };
|
||||
@@ -312,7 +316,7 @@ export function parseOpenAIResponse(
|
||||
total: usage?.total_tokens ?? 0,
|
||||
},
|
||||
model: (data.model as string) ?? model,
|
||||
provider: 'openai',
|
||||
provider,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -359,6 +363,7 @@ export function buildHeaders(provider: LLMProvider, apiKey: string): Record<stri
|
||||
headers['X-Title'] = 'Synapsis Bot';
|
||||
break;
|
||||
case 'openai':
|
||||
case 'custom':
|
||||
headers['Authorization'] = `Bearer ${apiKey}`;
|
||||
break;
|
||||
case 'anthropic':
|
||||
@@ -386,6 +391,7 @@ export class LLMClient {
|
||||
private provider: LLMProvider;
|
||||
private apiKey: string;
|
||||
private model: string;
|
||||
private endpoint: string | null;
|
||||
private retryConfig: RetryConfig;
|
||||
private timeoutMs: number;
|
||||
|
||||
@@ -403,6 +409,7 @@ export class LLMClient {
|
||||
) {
|
||||
this.provider = config.provider;
|
||||
this.model = config.model || DEFAULT_MODELS[config.provider];
|
||||
this.endpoint = config.endpoint || null;
|
||||
this.retryConfig = retryConfig;
|
||||
this.timeoutMs = timeoutMs;
|
||||
|
||||
@@ -480,7 +487,16 @@ export class LLMClient {
|
||||
* @throws LLMClientError if the request fails
|
||||
*/
|
||||
private async makeRequest(request: LLMCompletionRequest): Promise<LLMCompletionResponse> {
|
||||
const endpoint = PROVIDER_ENDPOINTS[this.provider];
|
||||
const endpoint = this.provider === 'custom' ? this.endpoint : PROVIDER_ENDPOINTS[this.provider];
|
||||
if (!endpoint) {
|
||||
throw new LLMClientError(
|
||||
'Custom provider requires an endpoint',
|
||||
'INVALID_REQUEST',
|
||||
this.provider,
|
||||
undefined,
|
||||
false
|
||||
);
|
||||
}
|
||||
const headers = buildHeaders(this.provider, this.apiKey);
|
||||
const body = this.buildRequestBody(request);
|
||||
|
||||
@@ -566,6 +582,7 @@ export class LLMClient {
|
||||
case 'openrouter':
|
||||
return buildOpenRouterRequest(request, this.model);
|
||||
case 'openai':
|
||||
case 'custom':
|
||||
return buildOpenAIRequest(request, this.model);
|
||||
case 'anthropic':
|
||||
return buildAnthropicRequest(request, this.model);
|
||||
@@ -580,7 +597,9 @@ export class LLMClient {
|
||||
case 'openrouter':
|
||||
return parseOpenRouterResponse(data, this.model);
|
||||
case 'openai':
|
||||
return parseOpenAIResponse(data, this.model);
|
||||
return parseOpenAIResponse(data, this.model, 'openai');
|
||||
case 'custom':
|
||||
return parseOpenAIResponse(data, this.model, 'custom');
|
||||
case 'anthropic':
|
||||
return parseAnthropicResponse(data, this.model);
|
||||
}
|
||||
@@ -618,12 +637,14 @@ export function createLLMClient(
|
||||
export function createLLMClientFromBot(
|
||||
provider: LLMProvider,
|
||||
encryptedApiKey: string,
|
||||
model: string
|
||||
model: string,
|
||||
endpoint?: string | null
|
||||
): LLMClient {
|
||||
return new LLMClient({
|
||||
provider,
|
||||
apiKey: encryptedApiKey,
|
||||
model,
|
||||
endpoint,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -643,7 +664,7 @@ export function validateLLMConfig(config: unknown): { valid: boolean; errors: st
|
||||
const configObj = config as Record<string, unknown>;
|
||||
|
||||
// Validate provider
|
||||
const validProviders: LLMProvider[] = ['openrouter', 'openai', 'anthropic'];
|
||||
const validProviders: LLMProvider[] = ['openrouter', 'openai', 'anthropic', 'custom'];
|
||||
if (!configObj.provider || !validProviders.includes(configObj.provider as LLMProvider)) {
|
||||
errors.push(`Provider must be one of: ${validProviders.join(', ')}`);
|
||||
}
|
||||
@@ -657,6 +678,10 @@ export function validateLLMConfig(config: unknown): { valid: boolean; errors: st
|
||||
if (configObj.model !== undefined && typeof configObj.model !== 'string') {
|
||||
errors.push('Model must be a string');
|
||||
}
|
||||
|
||||
if (configObj.provider === 'custom' && (!configObj.endpoint || typeof configObj.endpoint !== 'string')) {
|
||||
errors.push('Endpoint is required for custom provider');
|
||||
}
|
||||
|
||||
return { valid: errors.length === 0, errors };
|
||||
}
|
||||
|
||||
@@ -446,6 +446,7 @@ export async function processMention(mentionId: string): Promise<MentionResponse
|
||||
personalityConfig: JSON.parse(bot.personalityConfig),
|
||||
llmProvider: bot.llmProvider as any,
|
||||
llmModel: bot.llmModel,
|
||||
llmEndpoint: bot.llmEndpoint,
|
||||
llmApiKeyEncrypted: bot.llmApiKeyEncrypted,
|
||||
};
|
||||
|
||||
|
||||
+67
-103
@@ -7,12 +7,15 @@
|
||||
* Requirements: 5.4, 11.5, 11.6
|
||||
*/
|
||||
|
||||
import { db, posts, users, bots, botContentItems, botActivityLogs } from '@/db';
|
||||
import { db, posts, users, bots, botContentItems, botActivityLogs, botContentSources } from '@/db';
|
||||
import { eq, and, inArray } from 'drizzle-orm';
|
||||
import { ContentGenerator, type Bot as ContentGeneratorBot, type ContentItem } from './contentGenerator';
|
||||
import { canPost, recordPost } from './rateLimiter';
|
||||
import { getBotById } from './botManager';
|
||||
import { decryptApiKey, deserializeEncryptedData } from './encryption';
|
||||
import { decryptApiKey, deserializeEncryptedData, type LLMProvider } from './encryption';
|
||||
import { fetchRedditRichPreview } from '@/lib/media/redditPreview';
|
||||
import type { LinkPreviewData } from '@/lib/media/linkPreview';
|
||||
import { fetchGenericLinkPreview } from '@/lib/media/genericPreview';
|
||||
|
||||
// ============================================
|
||||
// TYPES
|
||||
@@ -139,8 +142,9 @@ function toContentGeneratorBot(bot: typeof bots.$inferSelect & { user: { handle:
|
||||
name: bot.name,
|
||||
handle: bot.user.handle,
|
||||
personalityConfig: JSON.parse(bot.personalityConfig),
|
||||
llmProvider: bot.llmProvider as 'openrouter' | 'openai' | 'anthropic',
|
||||
llmProvider: bot.llmProvider as LLMProvider,
|
||||
llmModel: bot.llmModel,
|
||||
llmEndpoint: bot.llmEndpoint,
|
||||
llmApiKeyEncrypted: bot.llmApiKeyEncrypted,
|
||||
};
|
||||
}
|
||||
@@ -175,6 +179,7 @@ async function getContentItemById(contentItemId: string): Promise<ContentItem |
|
||||
return {
|
||||
id: item.id,
|
||||
sourceId: item.sourceId,
|
||||
externalId: item.externalId,
|
||||
title: item.title,
|
||||
content: item.content,
|
||||
url: item.url,
|
||||
@@ -250,6 +255,7 @@ async function getNextUnprocessedContentItem(botId: string): Promise<ContentItem
|
||||
return {
|
||||
id: item.id,
|
||||
sourceId: item.sourceId,
|
||||
externalId: item.externalId,
|
||||
title: item.title,
|
||||
content: item.content,
|
||||
url: item.url,
|
||||
@@ -630,62 +636,8 @@ function isRedditUrl(url: string): boolean {
|
||||
* Fetch link preview for Reddit URLs using their oEmbed API.
|
||||
* Reddit blocks regular scraping but provides oEmbed for embedding.
|
||||
*/
|
||||
async function fetchRedditPreview(url: string): Promise<{
|
||||
url: string;
|
||||
title: string | null;
|
||||
description: string | null;
|
||||
image: string | null;
|
||||
} | null> {
|
||||
try {
|
||||
// Reddit's oEmbed endpoint
|
||||
const oembedUrl = `https://www.reddit.com/oembed?url=${encodeURIComponent(url)}`;
|
||||
|
||||
const response = await fetch(oembedUrl, {
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
},
|
||||
signal: AbortSignal.timeout(5000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
console.log(`Reddit oEmbed returned ${response.status} for ${url}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
// Extract title - try title field first, then parse from HTML
|
||||
let title = data.title || null;
|
||||
if (!title && data.html) {
|
||||
// Try to extract title from the embed HTML
|
||||
const titleMatch = data.html.match(/href="[^"]+">([^<]+)<\/a>/);
|
||||
if (titleMatch && titleMatch[1] && titleMatch[1] !== 'Comment') {
|
||||
title = titleMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
// Build description from subreddit info if available
|
||||
let description = null;
|
||||
if (data.author_name) {
|
||||
description = `Posted by ${data.author_name}`;
|
||||
} else if (data.html) {
|
||||
// Try to extract subreddit from HTML
|
||||
const subredditMatch = data.html.match(/r\/([a-zA-Z0-9_]+)/);
|
||||
if (subredditMatch) {
|
||||
description = `r/${subredditMatch[1]}`;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
url,
|
||||
title,
|
||||
description,
|
||||
image: data.thumbnail_url || null,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch Reddit oEmbed preview:', error);
|
||||
return null;
|
||||
}
|
||||
async function fetchRedditPreview(url: string): Promise<LinkPreviewData | null> {
|
||||
return fetchRedditRichPreview(url);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -695,12 +647,7 @@ async function fetchRedditPreview(url: string): Promise<{
|
||||
* @param url - The URL to fetch preview for
|
||||
* @returns Link preview data or null
|
||||
*/
|
||||
async function fetchLinkPreview(url: string): Promise<{
|
||||
url: string;
|
||||
title: string | null;
|
||||
description: string | null;
|
||||
image: string | null;
|
||||
} | null> {
|
||||
async function fetchLinkPreview(url: string): Promise<LinkPreviewData | null> {
|
||||
// Use Reddit-specific handler
|
||||
if (isRedditUrl(url)) {
|
||||
return fetchRedditPreview(url);
|
||||
@@ -708,48 +655,61 @@ async function fetchLinkPreview(url: string): Promise<{
|
||||
|
||||
// Generic OG tag scraping for other sites
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; SynapsisBot/1.0; +https://synapsis.social)',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
},
|
||||
signal: AbortSignal.timeout(5000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
|
||||
// Simple regex extraction for OG tags
|
||||
const getMeta = (property: string): string | null => {
|
||||
const regex = new RegExp(`<meta[^>]+(?:property|name)=["'](?:og:)?${property}["'][^>]+content=["']([^"']+)["']`, 'i');
|
||||
const match = html.match(regex);
|
||||
if (match) return match[1];
|
||||
|
||||
const regexRev = new RegExp(`<meta[^>]+content=["']([^"']+)["'][^>]+(?:property|name)=["'](?:og:)?${property}["']`, 'i');
|
||||
const matchRev = html.match(regexRev);
|
||||
return matchRev ? matchRev[1] : null;
|
||||
};
|
||||
|
||||
const title = getMeta('title') || html.match(/<title>([^<]+)<\/title>/i)?.[1];
|
||||
const description = getMeta('description');
|
||||
const image = getMeta('image');
|
||||
|
||||
return {
|
||||
url,
|
||||
title: title?.trim() || null,
|
||||
description: description?.trim() || null,
|
||||
image: image?.trim() || null,
|
||||
};
|
||||
return await fetchGenericLinkPreview(url);
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch link preview:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function extractRedditPostId(contentItem: ContentItem): string | null {
|
||||
if (contentItem.externalId) {
|
||||
const normalizedExternalId = contentItem.externalId.replace(/^t3_/, '');
|
||||
if (/^[a-z0-9]+$/i.test(normalizedExternalId)) {
|
||||
return normalizedExternalId;
|
||||
}
|
||||
}
|
||||
|
||||
if (contentItem.url) {
|
||||
try {
|
||||
const pathnameParts = new URL(contentItem.url).pathname.split('/').filter(Boolean);
|
||||
const commentsIndex = pathnameParts.indexOf('comments');
|
||||
if (commentsIndex >= 0 && pathnameParts[commentsIndex + 1]) {
|
||||
return pathnameParts[commentsIndex + 1];
|
||||
}
|
||||
} catch {
|
||||
// Fall back to returning null below.
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function resolveContentItemSourceUrl(contentItem?: ContentItem | null): Promise<string | undefined> {
|
||||
if (!contentItem?.url) {
|
||||
return contentItem?.url;
|
||||
}
|
||||
|
||||
const source = await db.query.botContentSources.findFirst({
|
||||
where: eq(botContentSources.id, contentItem.sourceId),
|
||||
columns: {
|
||||
type: true,
|
||||
subreddit: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (source?.type !== 'reddit' || !source.subreddit) {
|
||||
return contentItem.url;
|
||||
}
|
||||
|
||||
const postId = extractRedditPostId(contentItem);
|
||||
if (!postId) {
|
||||
return contentItem.url;
|
||||
}
|
||||
|
||||
return `https://www.reddit.com/r/${source.subreddit}/comments/${postId}/`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a post in the database.
|
||||
* Posts are created under the bot's own user account.
|
||||
@@ -762,7 +722,7 @@ async function fetchLinkPreview(url: string): Promise<{
|
||||
async function createPostInDatabase(
|
||||
botId: string,
|
||||
content: string,
|
||||
sourceUrl?: string
|
||||
contentItem?: ContentItem | null
|
||||
): Promise<typeof posts.$inferSelect> {
|
||||
// Get bot config
|
||||
const bot = await db.query.bots.findFirst({
|
||||
@@ -789,6 +749,7 @@ async function createPostInDatabase(
|
||||
}
|
||||
|
||||
// Fetch link preview if source URL provided
|
||||
const sourceUrl = await resolveContentItemSourceUrl(contentItem);
|
||||
let linkPreview: Awaited<ReturnType<typeof fetchLinkPreview>> = null;
|
||||
if (sourceUrl) {
|
||||
linkPreview = await fetchLinkPreview(sourceUrl);
|
||||
@@ -809,6 +770,9 @@ async function createPostInDatabase(
|
||||
linkPreviewTitle: linkPreview?.title || null,
|
||||
linkPreviewDescription: linkPreview?.description || null,
|
||||
linkPreviewImage: linkPreview?.image || null,
|
||||
linkPreviewType: linkPreview?.type || null,
|
||||
linkPreviewVideoUrl: linkPreview?.videoUrl || null,
|
||||
linkPreviewMediaJson: linkPreview?.media ? JSON.stringify(linkPreview.media) : null,
|
||||
}).returning();
|
||||
|
||||
// Update bot user's post count
|
||||
@@ -1079,7 +1043,7 @@ export async function triggerPost(
|
||||
}
|
||||
|
||||
// Create post in database with source URL for link preview (if content item exists)
|
||||
const post = await createPostInDatabase(botId, postContent, contentItem?.url);
|
||||
const post = await createPostInDatabase(botId, postContent, contentItem);
|
||||
|
||||
// Record post for rate limiting
|
||||
if (!skipRateLimitCheck) {
|
||||
|
||||
Reference in New Issue
Block a user