diff --git a/prisma/migrations/20240414022802_add_timestamp_and_postedat/migration.sql b/prisma/migrations/20240414022802_add_timestamp_and_postedat/migration.sql new file mode 100644 index 0000000..26b3755 --- /dev/null +++ b/prisma/migrations/20240414022802_add_timestamp_and_postedat/migration.sql @@ -0,0 +1,25 @@ +/* + Warnings: + + - Added the required column `postedAt` to the `attachments` table without a default value. This is not possible if the table is not empty. + - Added the required column `timestamp` to the `memes` table without a default value. This is not possible if the table is not empty. + +*/ +-- AlterTable +ALTER TABLE "attachments" + ADD COLUMN "postedAt" + TIMESTAMP(3); + +UPDATE "attachments" + set "postedAt" = TIMESTAMP 'epoch' + ((id >> 22) + 1420070400000) * INTERVAL '1 millisecond'; + +-- AlterTable +ALTER TABLE "memes" + ADD COLUMN "postedAt" + TIMESTAMP(3); + +UPDATE "memes" + set "postedAt" = TIMESTAMP 'epoch' + (("attachmentId" >> 22) + 1420070400000) * INTERVAL '1 millisecond'; + +ALTER TABLE "attachments" ALTER COLUMN "postedAt" SET NOT NULL; +ALTER TABLE "memes" ALTER COLUMN "postedAt" SET NOT NULL; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index fec3e12..20d2b30 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -27,6 +27,7 @@ model Attachment { createdAt DateTime @default(now()) updatedAt DateTime @default(now()) + postedAt DateTime @@index([channelId]) @@map("attachments") @@ -38,6 +39,12 @@ model Meme { text String tags Json + // need an timestamp for giving to meili to sort by + // i think a usize would actually work but can't + // convince the meili-js lib to serialize it properly to json + // this one actually has to be .valueOf()'d before we send it + // cause js numbers are bigger than postgres int4 + postedAt DateTime createdAt DateTime @default(now()) updatedAt DateTime @default(now()) diff --git a/src/memes.ts b/src/memes.ts index 2d7a6f9..8e1aada 100644 --- a/src/memes.ts +++ b/src/memes.ts @@ -31,6 +31,7 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient, text, tags: tags, attachmentId: a.id, + postedAt: a.postedAt, createdAt: undefined, updatedAt: undefined } @@ -42,7 +43,9 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient, const task = await meili.createIndex('memes', { primaryKey: 'id' }) const result = await meili.waitForTask(task.taskUid) log.debug(result); - return meili.index('memes') + const index = await meili.index('memes') + await index.updateSortableAttributes(['postedAt']) + return index; } // BLOWS IT UP @@ -50,7 +53,8 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient, const delTask = await meili.deleteIndex('memes') const delResult = await meili.waitForTask(delTask.taskUid) const index = await ensureMeiliIndex() - const memes = await prisma.meme.findMany(); + const memes = (await prisma.meme.findMany()) + .map(m => ({ ...m, postedAt: m.postedAt.valueOf() })); for await (const batch of _.chunk(memes, 1000)) { const task = await index.updateDocuments(batch) @@ -66,6 +70,9 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient, await ensureMeiliIndex(); + // BAM + // await resetMeili(); + return { ensureMeiliIndex, resetMeili, diff --git a/src/scraper.ts b/src/scraper.ts index 4116104..1b0b37c 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -15,6 +15,12 @@ import { IMAGE_TYPES, VIDEO_TYPES } from './attachments' // TODO // also fetch embeds and their descriptions +function convertSnowflakeToDate(snowflake) { + const DISCORD_EPOCH = 1420070400000 + const milliseconds = global.BigInt(snowflake) >> 22n + return new Date(Number(milliseconds) + DISCORD_EPOCH) +} + export default async function scraper( discordAuth, vision: ImageAnnotatorClient, @@ -91,7 +97,7 @@ export default async function scraper( async function filterAttachments(messages): Promise { const acceptedContentTypes = [ ...IMAGE_TYPES, - ...VIDEO_TYPES, + // ...VIDEO_TYPES, ] const validAttachments = messages @@ -120,6 +126,7 @@ export default async function scraper( guild, channel, message: m, + postedAt: convertSnowflakeToDate(a.id), createdAt: undefined, updatedAt: undefined }