From e165ed8a2411af93104b0f9c96047eb5f438e2fe Mon Sep 17 00:00:00 2001 From: Nathan Rashleigh Date: Sat, 11 Jan 2025 14:01:20 +1100 Subject: [PATCH 1/2] fix mobile. trim --- client/src/App.css | 2 +- client/src/App.js | 9 ++++++--- src/util.reset.meili.ts | 6 ++++++ 3 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 src/util.reset.meili.ts diff --git a/client/src/App.css b/client/src/App.css index 9b741da..eb63852 100644 --- a/client/src/App.css +++ b/client/src/App.css @@ -165,7 +165,7 @@ h1 { padding: 0; } - .ais-Hits-list { + .ais-Hits-list, .ais-InfiniteHits-list { grid-template-columns: 1fr; } } diff --git a/client/src/App.js b/client/src/App.js index c66bca8..d9ef738 100644 --- a/client/src/App.js +++ b/client/src/App.js @@ -25,6 +25,7 @@ const { searchClient } = instantMeiliSearch( { finitePagination: false, placeholderSearch: true, + attributesToSnippet: ['text:40'], } ); @@ -40,7 +41,9 @@ const placeholders = [ 'we are not the same', 'anarchy', 'goblin', - 'wise' + 'wise', + 'look inside', + 'gorilla warfare', ] @@ -69,8 +72,8 @@ const App = () => ( ); const Hit = ({ hit }) => ( -
- +
+ {hit.text}
diff --git a/src/util.reset.meili.ts b/src/util.reset.meili.ts new file mode 100644 index 0000000..ea24c84 --- /dev/null +++ b/src/util.reset.meili.ts @@ -0,0 +1,6 @@ +import "dotenv/config" + +import createServices from "./services" +const { memes } = await createServices() +await memes.resetMeili(); + From 3cfe6b3d540d894920362a631d674628a7aa6594 Mon Sep 17 00:00:00 2001 From: Nathan Rashleigh Date: Wed, 9 Apr 2025 23:44:13 +1000 Subject: [PATCH 2/2] wut --- archiver/index.ts | 6 +++--- archiver/src/scraper.ts | 43 +++++++++++++++++++++++++---------------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/archiver/index.ts b/archiver/index.ts index f312d57..f8475e1 100644 --- a/archiver/index.ts +++ b/archiver/index.ts @@ -46,11 +46,11 @@ async function main() { const scraper = await discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments) const CHANNEL_IDS = [ - // 367936426962190336n, // TPM shitposts - // 367936467592413194n, // tpm screenshots + 367936426962190336n, // TPM shitposts + 367936467592413194n, // tpm screenshots // 1028511952277864501n, // tpm sludge // 725835129326665768n, // machinegirl memes - 1095060717117374687n, // aella memes + // 1095060717117374687n, // aella memes ] async function migrateDb() { diff --git a/archiver/src/scraper.ts b/archiver/src/scraper.ts index 4047f24..7e0d10e 100644 --- a/archiver/src/scraper.ts +++ b/archiver/src/scraper.ts @@ -8,8 +8,6 @@ import type { ImageAnnotatorClient } from '@google-cloud/vision'; import type { Logger } from 'pino'; import type Minio from 'minio' -import sharp from 'sharp'; - import type { AttachmentService } from './attachments' import { IMAGE_TYPES, VIDEO_TYPES } from './attachments' @@ -93,6 +91,8 @@ export default async function discordScraper( const guild = await fetchGuildDetails(channel.guild_id) const log = baseLog.child({ guild: guild.name, channel: channel.name, channelId }) + log.info("processing...") + async function filterAttachments(messages): Promise { const acceptedContentTypes = [ ...IMAGE_TYPES, @@ -105,7 +105,7 @@ export default async function discordScraper( .flatMap(m => m.attachments.map(a => [a, m])) .filter(([a, m]) => acceptedContentTypes.includes(a.content_type)); - log.debug({ validAttachments: validAttachments.length }, 'attachments found in messages') + log.info({ validAttachments: validAttachments.length }, 'attachments found in messages') return Promise.all(validAttachments .map(async ([a, m]) => { @@ -134,32 +134,30 @@ export default async function discordScraper( })) } - async function* fetchNew(channelId, log, latestMessageId): AsyncIterable { + async function* fetchNew(channelId, log, newMessages): AsyncIterable { // reverse the messages because we want to yield them in chronological order // the api call returns them most recent first - const messages = (await fetchChannelMessagesAfter(channelId, latestMessageId)).reverse() + const messages = newMessages.reverse() if (messages.length === 0) { log.info('reached latest message') } - log.debug({ messages: messages.length, latestMessageId }, 'fetched new messages') yield* await filterAttachments(messages) } - async function* fetchOld(channelId, log, earliestMessageId): AsyncIterable { - const messages = await fetchChannelMessagesBefore(channelId, earliestMessageId) - if (messages.length === 0) { + async function* fetchOld(channelId, log, oldMessages): AsyncIterable { + if (oldMessages.length === 0) { log.info('reached beginning of history') } - log.debug({ messages: messages.length, earliestMessageId }, 'fetched older messages') - yield* await filterAttachments(messages) + yield* await filterAttachments(oldMessages) } const ensureHistory = async () => { const stored = await prisma.channelHistory.findFirst({ where: { id: channelId }}) if (stored) return stored + const channel = await fetchChannel(channelId) const lastMessageId = channel.last_message_id return await prisma.channelHistory.create({ data: { latestMessageId: lastMessageId, earliestMessageId: lastMessageId, id: channelId } @@ -169,15 +167,26 @@ export default async function discordScraper( const history = await ensureHistory() async function processLoop() { + // updated by the attachment process const history = await ensureHistory() - const newAttachments = fetchNew(channelId, log, history.latestMessageId) - const oldAttachments = fetchOld(channelId, log, history.earliestMessageId) + + const newMessages = await fetchChannelMessagesAfter(channelId, history.latestMessageId) + const newAttachments = fetchNew(channelId, log, newMessages) + const latestMessageId = _.first(newMessages) ? BigInt(_.first(newMessages).id) : 0n + if (latestMessageId) await prisma.channelHistory.update({ data: { latestMessageId }, where: { id: history.id }}) + log.info({ messages: newMessages.length, latestMessageId }, 'fetched new messages') + + const oldMessages = await fetchChannelMessagesBefore(channelId, history.earliestMessageId) + const oldAttachments = fetchOld(channelId, log, oldMessages) + const earliestMessageId = _.first(oldMessages) ? BigInt(_.first(oldMessages).id) : 0n + if (earliestMessageId) await prisma.channelHistory.update({ data: { earliestMessageId }, where: { id: history.id }}) + log.info({ messages: oldMessages.length, earliestMessageId }, 'fetched older messages') let oldCount = 0; let newCount = 0; for await (const attachment of newAttachments) { - log.debug({ msg: 'scraped new attachment', attachment: _.omit(attachment, ['guild', 'message', 'channel', 'vision']) }) + log.info({ msg: 'scraped new attachment', attachment: _.omit(attachment, ['guild', 'message', 'channel', 'vision']) }) const newHistory = { id: history.id, latestMessageId: attachment.msgId } await attachments.process(attachment, newHistory) // pretty oldschool, the async iterable from the fetch fn does not have a count @@ -186,18 +195,18 @@ export default async function discordScraper( } for await (const attachment of oldAttachments) { - log.debug({ msg: 'scraped old attachment', attachment: _.omit(attachment, ['guild', 'message', 'channel', 'vision']) }) + log.info({ msg: 'scraped old attachment', attachment: _.omit(attachment, ['guild', 'message', 'channel', 'vision']) }) const newHistory = { id: history.id, earliestMessageId: attachment.msgId } await attachments.process(attachment, newHistory) newCount++ } - log.debug({ oldCount, newCount }, 'processed attachments') + log.info({ oldCount, newCount }, 'processed attachments') const shouldWait = newCount === 0 && oldCount === 0 const PROCESS_DELAY_MS = 1000 * 60 // 1 minute if (shouldWait) { - log.debug({ PROCESS_DELAY_MS }, 'waiting for new messages') + log.info({ PROCESS_DELAY_MS }, 'waiting for new messages') await setTimeout(PROCESS_DELAY_MS) }