posted at for sorting

index.ts

@@ -37,16 +37,18 @@ const meili = new MeiliSearch({
 
 const memes = await memeService(log, prisma, meili)
 const attachments = await attachmentService(log, minio, prisma, memes)
-const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments)
+const scraper = await discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments)
 
 const CHANNEL_IDS = [
     367936426962190336n,  // TPM shitposts
+    367936467592413194n,  // tpm screenshots
+    1028511952277864501n,  // tpm sludge
+    725835129326665768n, // machinegirl memes
     1095060717117374687n, // aella memes
 ]
 
 async function main() {
     await Promise.all(CHANNEL_IDS.map(id => scraper.processChannel(id)))
-    await memes.updateMeili();
 }
 
 await main()

prisma/migrations/20240414022802_add_timestamp_and_postedat/migration.sql

@@ -0,0 +1,25 @@
+/*
+  Warnings:
+
+  - Added the required column `postedAt` to the `attachments` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `timestamp` to the `memes` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "attachments"
+  ADD COLUMN     "postedAt"
+  TIMESTAMP(3);
+
+UPDATE "attachments"
+  set "postedAt" = TIMESTAMP 'epoch' + ((id >> 22) + 1420070400000) * INTERVAL '1 millisecond';
+
+-- AlterTable
+ALTER TABLE "memes"
+  ADD COLUMN     "postedAt"
+  TIMESTAMP(3);
+
+UPDATE "memes"
+  set "postedAt" = TIMESTAMP 'epoch' + (("attachmentId" >> 22) + 1420070400000) * INTERVAL '1 millisecond';
+
+ALTER TABLE "attachments" ALTER COLUMN "postedAt" SET NOT NULL;
+ALTER TABLE "memes" ALTER COLUMN "postedAt" SET NOT NULL;

prisma/schema.prisma

@@ -27,6 +27,7 @@ model Attachment {
 
     createdAt  DateTime @default(now())
     updatedAt  DateTime @default(now())
+    postedAt   DateTime
 
     @@index([channelId])
     @@map("attachments")
@@ -38,6 +39,12 @@ model Meme {
     text       String
     tags       Json
 
+    // need an timestamp for giving to meili to sort by
+    // i think a usize would actually work but can't
+    // convince the meili-js lib to serialize it properly to json
+    // this one actually has to be .valueOf()'d before we send it
+    // cause js numbers are bigger than postgres int4
+    postedAt   DateTime
     createdAt  DateTime @default(now())
     updatedAt  DateTime @default(now())
 

search.html

@@ -1,40 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8" />
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@meilisearch/instant-meilisearch/templates/basic_search.css" />
-  </head>
-  <body>
-      <div id="app" class="wrapper"></div>
-  </body>
-  <script src="https://unpkg.com/react@18/umd/react.development.js" crossorigin></script>
-  <script src="https://unpkg.com/react-dom@18/umd/react-dom.development.js" crossorigin></script>
-  <script src="https://cdn.jsdelivr.net/npm/react-instantsearch-dom@6.7.0/dist/umd/ReactInstantSearchDOM.js"></script>
-  <script src="https://cdn.jsdelivr.net/npm/@meilisearch/instant-meilisearch/dist/instant-meilisearch.umd.min.js"></script>
-  <script>
-    const { InstantSearch, SearchBox, Hits, Highlight, Configure } = ReactInstantSearchDOM;
-    const searchClient = instantMeiliSearch(
-      "http://localhost:7700",
-      '33e714590199031d076954182c55a47b09ae7546e3dbf6f35862eb99cc950e1e'
-    );
-    const App = () => (
-      React.createElement(InstantSearch, {
-        indexName: "memes",
-        searchClient: searchClient
-      }, [
-        React.createElement(SearchBox, { key: 1 }),
-        React.createElement(Hits, { hitComponent: Hit, key: 2 }),
-        React.createElement(Configure, { hitsPerPage: 10 })]
-      )
-    );
-
-    const Hit = (props) => (
-      React.createElement(Highlight, {
-        attribute: "id",
-        hit: props.hit
-      })
-    );
-    ReactDOM.createRoot(document.querySelector('#app')).render(App());
-    console.log(searchClient)
-  </script>
-</html>

src/attachments.ts

@@ -73,13 +73,14 @@ export default async function attachmentService(log: Logger, minio: Minio.Client
 
                 await memes.persist(dbMeme)
 
+                log.info(dbMeme, 'new meme')
                 return { meme: dbMeme, upload, attachment: dbA }
             }, { timeout: 60000 })
         } catch (e) {
 
             // this happens pretty regularly it seems
             // when ppl reply to an attachment it is included in their message
-            if (e.meta.modelName !== 'Meme' && e.meta.target[0] !== 'attachmentId') {
+            if (e.meta?.modelName !== 'Meme' && e.meta?.target[0] !== 'attachmentId') {
                 log.error(e, 'failed to process attachment')
                 log.error(a, 'failed to process attachment')
             }

src/memes.ts

@@ -2,6 +2,8 @@ import type { PrismaClient, Attachment, Meme } from '@prisma/client';
 import type { Meilisearch } from 'meilisearch';
 import type { Logger } from 'pino';
 
+import _ from 'lodash'
+
 import * as attachments from './attachments'
 
 const STORAGE_BUCKET = process.env.STORAGE_BUCKET
@@ -29,6 +31,7 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
             text,
             tags: tags,
             attachmentId: a.id,
+            postedAt: a.postedAt,
             createdAt: undefined,
             updatedAt: undefined
         }
@@ -37,21 +40,27 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
     }
 
     async function ensureMeiliIndex() {
-        const delTask = await meili.deleteIndex('memes')
-        const delResult = await meili.waitForTask(delTask.taskUid)
         const task = await meili.createIndex('memes', { primaryKey: 'id' })
         const result = await meili.waitForTask(task.taskUid)
         log.debug(result);
-        return meili.index('memes')
+        const index = await meili.index('memes')
+        await index.updateSortableAttributes(['postedAt'])
+        return index;
     }
 
-    async function updateMeili() {
+    // BLOWS IT UP
-        await ensureMeiliIndex();
+    async function resetMeili() {
-        const index = meili.index('memes')
+        const delTask = await meili.deleteIndex('memes')
-        const memes = await prisma.meme.findMany();
+        const delResult = await meili.waitForTask(delTask.taskUid)
-        const task = await index.updateDocuments(memes)
+        const index = await ensureMeiliIndex()
+        const memes = (await prisma.meme.findMany())
+            .map(m => ({ ...m, postedAt: m.postedAt.valueOf() }));
+
+        for await (const batch of _.chunk(memes, 1000)) {
+            const task = await index.updateDocuments(batch)
             const res = await meili.waitForTask(task.taskUid)
-        log.debug(res)
+            log.info({ res }, 'added meme batch')
+        }
     }
 
     async function persist(m: Meme) {
@@ -60,11 +69,13 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
     }
 
     await ensureMeiliIndex();
-    await updateMeili();
+
+    // BAM
+    // await resetMeili();
 
     return {
         ensureMeiliIndex,
-        updateMeili,
+        resetMeili,
         fromAttachment,
         persist,
     }

src/scraper.test.ts

@@ -1,44 +0,0 @@
-import "dotenv/config"
-
-import fs from 'fs/promises'
-
-import pino from 'pino'
-
-import * as Minio from 'minio'
-import Vision from '@google-cloud/vision'
-import { PrismaClient } from '@prisma/client'
-import { MeiliSearch } from 'meilisearch'
-
-import discordScraper from './scraper'
-import attachmentService from "./attachments"
-
-// @ts-ignore - ts don't know wtf it talking about
-BigInt.prototype.toJSON = function bigIntStringify() {
-    return this.toString();
-};
-
-const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { assert: { type: "json" } })).default
-const STORAGE_BUCKET = process.env.STORAGE_BUCKET
-
-const log = pino({ base: {} });
-const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS })
-const prisma = new PrismaClient()
-const minio = new Minio.Client({
-    endPoint: process.env.STORAGE_URL,
-    accessKey: process.env.STORAGE_ACCESS_KEY,
-    secretKey: process.env.STORAGE_SECRET_KEY,
-})
-const meili = new MeiliSearch({
-  host: process.env.MEILI_URL,
-  apiKey: process.env.MEILI_MASTER_KEY,
-})
-
-const attachments = attachmentService(log, minio, prisma)
-const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log)
-
-const CHANNEL_IDS = [
-    367936426962190336n,  // TPM shitposts
-    1095060717117374687n, // aella memes
-]
-
-await Promise.all(CHANNEL_IDS.map(id => scraper.processChannel(id)))

src/scraper.ts

@@ -15,7 +15,13 @@ import { IMAGE_TYPES, VIDEO_TYPES } from './attachments'
 // TODO
 // also fetch embeds and their descriptions
 
-export default function scraper(
+function convertSnowflakeToDate(snowflake) {
+    const DISCORD_EPOCH = 1420070400000
+    const milliseconds = global.BigInt(snowflake) >> 22n
+    return new Date(Number(milliseconds) + DISCORD_EPOCH)
+}
+
+export default async function scraper(
     discordAuth,
     vision: ImageAnnotatorClient,
     minio: Minio.Client,
@@ -77,6 +83,12 @@ export default function scraper(
         return res[0] || {}
     }
 
+    async function fetchMissingVision() {
+        const attachments = await prisma.attachment
+            .findMany({ where: { vision: {}, contentType: { in: IMAGE_TYPES } }})
+        baseLog.info({attachments: attachments.length }, 'missing vision attachments')
+    }
+
     async function processChannel(channelId) {
         const channel = await fetchChannel(channelId)
         const guild = await fetchGuildDetails(channel.guild_id)
@@ -85,7 +97,7 @@ export default function scraper(
         async function filterAttachments(messages): Promise<Attachment[]> {
             const acceptedContentTypes = [
                 ...IMAGE_TYPES,
-                ...VIDEO_TYPES,
+                // ...VIDEO_TYPES,
             ]
 
             const validAttachments = messages
@@ -110,19 +122,19 @@ export default function scraper(
                         memeId: crypto.randomUUID(),
 
                         // @ts-ignore it's fine
-                        vision: {}, // await detectText(attachment),
+                        vision: {},
                         guild,
                         channel,
                         message: m,
-
+                        postedAt: convertSnowflakeToDate(a.id),
                         createdAt: undefined,
                         updatedAt: undefined
                     }
-                    return attachment
+                    return { ...attachment, vision: await detectText(attachment) }
                 }))
         }
 
-        async function* fetchNew(channelId, log, latestMessageId): AsyncGenerator<Attachment> {
+        async function* fetchNew(channelId, log, latestMessageId): AsyncIterable<Attachment> {
             // reverse the messages because we want to yield them in chronological order
             // the api call returns them most recent first
             const messages = (await fetchChannelMessagesAfter(channelId, latestMessageId)).reverse()
@@ -137,7 +149,7 @@ export default function scraper(
             yield* fetchNew(channelId, log, cursor)
         }
 
-        async function* fetchOld(channelId, log, earliestMessageId): AsyncGenerator<Attachment, void, Attachment> {
+        async function* fetchOld(channelId, log, earliestMessageId): AsyncIterable<Attachment> {
             const messages = await fetchChannelMessagesBefore(channelId, earliestMessageId)
             if (messages.length === 0) {
                 log.info('no new messages')