posted at for sorting

prisma/migrations/20240414022802_add_timestamp_and_postedat/migration.sql

@@ -0,0 +1,25 @@
+/*
+  Warnings:
+
+  - Added the required column `postedAt` to the `attachments` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `timestamp` to the `memes` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "attachments"
+  ADD COLUMN     "postedAt"
+  TIMESTAMP(3);
+
+UPDATE "attachments"
+  set "postedAt" = TIMESTAMP 'epoch' + ((id >> 22) + 1420070400000) * INTERVAL '1 millisecond';
+
+-- AlterTable
+ALTER TABLE "memes"
+  ADD COLUMN     "postedAt"
+  TIMESTAMP(3);
+
+UPDATE "memes"
+  set "postedAt" = TIMESTAMP 'epoch' + (("attachmentId" >> 22) + 1420070400000) * INTERVAL '1 millisecond';
+
+ALTER TABLE "attachments" ALTER COLUMN "postedAt" SET NOT NULL;
+ALTER TABLE "memes" ALTER COLUMN "postedAt" SET NOT NULL;

prisma/schema.prisma

@@ -27,6 +27,7 @@ model Attachment {
 
     createdAt  DateTime @default(now())
     updatedAt  DateTime @default(now())
+    postedAt   DateTime
 
     @@index([channelId])
     @@map("attachments")
@@ -38,6 +39,12 @@ model Meme {
     text       String
     tags       Json
 
+    // need an timestamp for giving to meili to sort by
+    // i think a usize would actually work but can't
+    // convince the meili-js lib to serialize it properly to json
+    // this one actually has to be .valueOf()'d before we send it
+    // cause js numbers are bigger than postgres int4
+    postedAt   DateTime
     createdAt  DateTime @default(now())
     updatedAt  DateTime @default(now())
 

src/memes.ts

@@ -31,6 +31,7 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
             text,
             tags: tags,
             attachmentId: a.id,
+            postedAt: a.postedAt,
             createdAt: undefined,
             updatedAt: undefined
         }
@@ -42,7 +43,9 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
         const task = await meili.createIndex('memes', { primaryKey: 'id' })
         const result = await meili.waitForTask(task.taskUid)
         log.debug(result);
-        return meili.index('memes')
+        const index = await meili.index('memes')
+        await index.updateSortableAttributes(['postedAt'])
+        return index;
     }
 
     // BLOWS IT UP
@@ -50,7 +53,8 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
         const delTask = await meili.deleteIndex('memes')
         const delResult = await meili.waitForTask(delTask.taskUid)
         const index = await ensureMeiliIndex()
-        const memes = await prisma.meme.findMany();
+        const memes = (await prisma.meme.findMany())
+            .map(m => ({ ...m, postedAt: m.postedAt.valueOf() }));
 
         for await (const batch of _.chunk(memes, 1000)) {
             const task = await index.updateDocuments(batch)
@@ -66,6 +70,9 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
 
     await ensureMeiliIndex();
 
+    // BAM
+    // await resetMeili();
+
     return {
         ensureMeiliIndex,
         resetMeili,

src/scraper.ts

@@ -15,6 +15,12 @@ import { IMAGE_TYPES, VIDEO_TYPES } from './attachments'
 // TODO
 // also fetch embeds and their descriptions
 
+function convertSnowflakeToDate(snowflake) {
+    const DISCORD_EPOCH = 1420070400000
+    const milliseconds = global.BigInt(snowflake) >> 22n
+    return new Date(Number(milliseconds) + DISCORD_EPOCH)
+}
+
 export default async function scraper(
     discordAuth,
     vision: ImageAnnotatorClient,
@@ -91,7 +97,7 @@ export default async function scraper(
         async function filterAttachments(messages): Promise<Attachment[]> {
             const acceptedContentTypes = [
                 ...IMAGE_TYPES,
-                ...VIDEO_TYPES,
+                // ...VIDEO_TYPES,
             ]
 
             const validAttachments = messages
@@ -120,6 +126,7 @@ export default async function scraper(
                         guild,
                         channel,
                         message: m,
+                        postedAt: convertSnowflakeToDate(a.id),
                         createdAt: undefined,
                         updatedAt: undefined
                     }