Merge branch 'master' of ssh://strix:/home/ntr/repos/gimmeme

archiver/src/scraper.ts

@@ -90,6 +90,8 @@ export default async function discordScraper(
         const guild = await fetchGuildDetails(channel.guild_id)
         const log = baseLog.child({ guild: guild.name, channel: channel.name, channelId })
 
+        log.info("processing...")
+
         async function filterAttachments(messages): Promise<Attachment[]> {
             const acceptedContentTypes = [
                 ...IMAGE_TYPES,
@@ -102,7 +104,7 @@ export default async function discordScraper(
                 .flatMap(m => m.attachments.map(a => [a, m]))
                 .filter(([a, m]) => acceptedContentTypes.includes(a.content_type));
 
-            log.debug({ validAttachments: validAttachments.length }, 'attachments found in messages')
+            log.info({ validAttachments: validAttachments.length }, 'attachments found in messages')
 
             return Promise.all(validAttachments
                 .map(async ([a, m]) => {
@@ -132,32 +134,30 @@ export default async function discordScraper(
                 }))
         }
 
-        async function* fetchNew(channelId, log, latestMessageId): AsyncIterable<Attachment> {
+        async function* fetchNew(channelId, log, newMessages): AsyncIterable<Attachment> {
             // reverse the messages because we want to yield them in chronological order
             // the api call returns them most recent first
-            const messages = (await fetchChannelMessagesAfter(channelId, latestMessageId)).reverse()
+            const messages = newMessages.reverse()
             if (messages.length === 0) {
                 log.info('reached latest message')
             }
 
-            log.debug({ messages: messages.length, latestMessageId }, 'fetched new messages')
             yield* await filterAttachments(messages)
         }
 
-        async function* fetchOld(channelId, log, earliestMessageId): AsyncIterable<Attachment> {
+        async function* fetchOld(channelId, log, oldMessages): AsyncIterable<Attachment> {
-            const messages = await fetchChannelMessagesBefore(channelId, earliestMessageId)
+            if (oldMessages.length === 0) {
-            if (messages.length === 0) {
                 log.info('reached beginning of history')
             }
 
-            log.debug({ messages: messages.length, earliestMessageId }, 'fetched older messages')
+            yield* await filterAttachments(oldMessages)
-            yield* await filterAttachments(messages)
         }
 
         const ensureHistory = async () => {
             const stored = await prisma.channelHistory.findFirst({ where: { id: channelId }})
             if (stored) return stored
 
+            const channel = await fetchChannel(channelId)
             const lastMessageId = channel.last_message_id
             return await prisma.channelHistory.create({ data:
                 { latestMessageId: lastMessageId, earliestMessageId: lastMessageId, id: channelId }
@@ -165,15 +165,26 @@ export default async function discordScraper(
         }
 
         async function processLoop() {
+            // updated by the attachment process
             const history = await ensureHistory()
-            const newAttachments = fetchNew(channelId, log, history.latestMessageId)
+
-            const oldAttachments = fetchOld(channelId, log, history.earliestMessageId)
+            const newMessages = await fetchChannelMessagesAfter(channelId, history.latestMessageId)
+            const newAttachments = fetchNew(channelId, log, newMessages)
+            const latestMessageId = _.first(newMessages) ? BigInt(_.first(newMessages).id) : 0n
+            if (latestMessageId) await prisma.channelHistory.update({ data: { latestMessageId }, where: { id: history.id }})
+            log.info({ messages: newMessages.length, latestMessageId }, 'fetched new messages')
+
+            const oldMessages = await fetchChannelMessagesBefore(channelId, history.earliestMessageId)
+            const oldAttachments = fetchOld(channelId, log, oldMessages)
+            const earliestMessageId = _.first(oldMessages) ? BigInt(_.first(oldMessages).id) : 0n
+            if (earliestMessageId) await prisma.channelHistory.update({ data: { earliestMessageId }, where: { id: history.id }})
+            log.info({ messages: oldMessages.length, earliestMessageId }, 'fetched older messages')
 
             let oldCount = 0;
             let newCount = 0;
 
             for await (const attachment of newAttachments) {
-                log.debug({ msg: 'scraped new attachment', attachment: _.omit(attachment, ['guild', 'message', 'channel', 'vision']) })
+                log.info({ msg: 'scraped new attachment', attachment: _.omit(attachment, ['guild', 'message', 'channel', 'vision']) })
                 const newHistory = { id: history.id, latestMessageId: attachment.msgId }
                 await attachments.process(attachment, newHistory)
                 // pretty oldschool, the async iterable from the fetch fn does not have a count
@@ -182,18 +193,18 @@ export default async function discordScraper(
             }
 
             for await (const attachment of oldAttachments) {
-                log.debug({ msg: 'scraped old attachment', attachment: _.omit(attachment, ['guild', 'message', 'channel', 'vision']) })
+                log.info({ msg: 'scraped old attachment', attachment: _.omit(attachment, ['guild', 'message', 'channel', 'vision']) })
                 const newHistory = { id: history.id, earliestMessageId: attachment.msgId }
                 await attachments.process(attachment, newHistory)
                 newCount++
             }
 
-            log.debug({ oldCount, newCount }, 'processed attachments')
+            log.info({ oldCount, newCount }, 'processed attachments')
 
             const shouldWait = newCount === 0 && oldCount === 0
             const PROCESS_DELAY_MS = 1000 * 60 // 1 minute
             if (shouldWait) {
-                log.debug({ PROCESS_DELAY_MS }, 'waiting for new messages')
+                log.info({ PROCESS_DELAY_MS }, 'waiting for new messages')
                 await setTimeout(PROCESS_DELAY_MS)
             }
 

client/src/App.css

@@ -165,7 +165,7 @@ h1 {
     padding: 0;
   }
 
-  .ais-Hits-list {
+  .ais-Hits-list, .ais-InfiniteHits-list {
     grid-template-columns: 1fr;
   }
 }

client/src/App.js

@@ -25,6 +25,7 @@ const { searchClient } = instantMeiliSearch(
   {
     finitePagination: false,
     placeholderSearch: true,
+    attributesToSnippet: ['text:40'],
   }
 );
 
@@ -40,7 +41,9 @@ const placeholders = [
   'we are not the same',
   'anarchy',
   'goblin',
-  'wise'
+  'wise',
+  'look inside',
+  'gorilla warfare',
 ]
 
 
@@ -69,8 +72,8 @@ const App = () => (
 );
 
 const Hit = ({ hit }) => (
-  <div class="hit" key={hit.id}>
+  <div className="hit" key={hit.id}>
-    <a href={hit.url} class="img" target="_blank" rel="noreferrer">
+    <a href={hit.url} className="img" target="_blank" rel="noreferrer">
       <img src={hit.url} alt={hit.text} />
     </a>
     <div className="info">

src/util.reset.meili.ts

@@ -0,0 +1,6 @@
+import "dotenv/config"
+
+import createServices from "./services"
+const { memes } = await createServices()
+await memes.resetMeili();
+