ntr/memestream
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

vision fix

Browse Source
This commit is contained in:
ntr committed 2024-04-13 21:19:37 +10:00
1 parent 29a3bcce39
commit ed0a42eeb3
6 files changed
+30 -102
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

index.ts
+4 -2
View File
@@ -37,16 +37,18 @@ const meili = new MeiliSearch({
const memes = await memeService(log, prisma, meili)
const attachments = await attachmentService(log, minio, prisma, memes)
const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments)
const scraper = await discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments)
const CHANNEL_IDS = [
367936426962190336n, // TPM shitposts
367936467592413194n, // tpm screenshots
1028511952277864501n, // tpm sludge
725835129326665768n, // machinegirl memes
1095060717117374687n, // aella memes
]
async function main() {
await Promise.all(CHANNEL_IDS.map(id => scraper.processChannel(id)))
await memes.updateMeili();
}
await main()
search.html
-40
View File
@@ -1,40 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@meilisearch/instant-meilisearch/templates/basic_search.css" />
</head>
<body>
<div id="app" class="wrapper"></div>
</body>
<script src="https://unpkg.com/react@18/umd/react.development.js" crossorigin></script>
<script src="https://unpkg.com/react-dom@18/umd/react-dom.development.js" crossorigin></script>
<script src="https://cdn.jsdelivr.net/npm/react-instantsearch-dom@6.7.0/dist/umd/ReactInstantSearchDOM.js"></script>
<script src="https://cdn.jsdelivr.net/npm/@meilisearch/instant-meilisearch/dist/instant-meilisearch.umd.min.js"></script>
<script>
const { InstantSearch, SearchBox, Hits, Highlight, Configure } = ReactInstantSearchDOM;
const searchClient = instantMeiliSearch(
"http://localhost:7700",
'33e714590199031d076954182c55a47b09ae7546e3dbf6f35862eb99cc950e1e'
);
const App = () => (
React.createElement(InstantSearch, {
indexName: "memes",
searchClient: searchClient
}, [
React.createElement(SearchBox, { key: 1 }),
React.createElement(Hits, { hitComponent: Hit, key: 2 }),
React.createElement(Configure, { hitsPerPage: 10 })]
)
);
const Hit = (props) => (
React.createElement(Highlight, {
attribute: "id",
hit: props.hit
})
);
ReactDOM.createRoot(document.querySelector('#app')).render(App());
console.log(searchClient)
</script>
</html>
src/attachments.ts
+2 -1
View File
@@ -73,13 +73,14 @@ export default async function attachmentService(log: Logger, minio: Minio.Client
await memes.persist(dbMeme)
log.info({ msg: 'new meme', dbMeme })
return { meme: dbMeme, upload, attachment: dbA }
}, { timeout: 60000 })
} catch (e) {
// this happens pretty regularly it seems
// when ppl reply to an attachment it is included in their message
if (e.meta.modelName !== 'Meme' && e.meta.target[0] !== 'attachmentId') {
if (e.meta?.modelName !== 'Meme' && e.meta?.target[0] !== 'attachmentId') {
log.error(e, 'failed to process attachment')
log.error(a, 'failed to process attachment')
}
src/memes.ts
+13 -9
View File
@@ -2,6 +2,8 @@ import type { PrismaClient, Attachment, Meme } from '@prisma/client';
import type { Meilisearch } from 'meilisearch';
import type { Logger } from 'pino';
import _ from 'lodash'
import * as attachments from './attachments'
const STORAGE_BUCKET = process.env.STORAGE_BUCKET
@@ -37,21 +39,24 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
}
async function ensureMeiliIndex() {
const delTask = await meili.deleteIndex('memes')
const delResult = await meili.waitForTask(delTask.taskUid)
const task = await meili.createIndex('memes', { primaryKey: 'id' })
const result = await meili.waitForTask(task.taskUid)
log.debug(result);
return meili.index('memes')
}
async function updateMeili() {
await ensureMeiliIndex();
const index = meili.index('memes')
// BLOWS IT UP
async function resetMeili() {
const delTask = await meili.deleteIndex('memes')
const delResult = await meili.waitForTask(delTask.taskUid)
const index = await ensureMeiliIndex()
const memes = await prisma.meme.findMany();
const task = await index.updateDocuments(memes)
for await (const batch of _.chunk(memes, 1000)) {
const task = await index.updateDocuments(batch)
const res = await meili.waitForTask(task.taskUid)
log.debug(res)
log.info({ res }, 'added meme batch')
}
}
async function persist(m: Meme) {
@@ -60,11 +65,10 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
}
await ensureMeiliIndex();
await updateMeili();
return {
ensureMeiliIndex,
updateMeili,
resetMeili,
fromAttachment,
persist,
}
src/scraper.test.ts
-44
View File
@@ -1,44 +0,0 @@
import "dotenv/config"
import fs from 'fs/promises'
import pino from 'pino'
import * as Minio from 'minio'
import Vision from '@google-cloud/vision'
import { PrismaClient } from '@prisma/client'
import { MeiliSearch } from 'meilisearch'
import discordScraper from './scraper'
import attachmentService from "./attachments"
// @ts-ignore - ts don't know wtf it talking about
BigInt.prototype.toJSON = function bigIntStringify() {
return this.toString();
};
const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { assert: { type: "json" } })).default
const STORAGE_BUCKET = process.env.STORAGE_BUCKET
const log = pino({ base: {} });
const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS })
const prisma = new PrismaClient()
const minio = new Minio.Client({
endPoint: process.env.STORAGE_URL,
accessKey: process.env.STORAGE_ACCESS_KEY,
secretKey: process.env.STORAGE_SECRET_KEY,
})
const meili = new MeiliSearch({
host: process.env.MEILI_URL,
apiKey: process.env.MEILI_MASTER_KEY,
})
const attachments = attachmentService(log, minio, prisma)
const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log)
const CHANNEL_IDS = [
367936426962190336n, // TPM shitposts
1095060717117374687n, // aella memes
]
await Promise.all(CHANNEL_IDS.map(id => scraper.processChannel(id)))
src/scraper.ts
+11 -6
View File
@@ -15,7 +15,7 @@ import { IMAGE_TYPES, VIDEO_TYPES } from './attachments'
// TODO
// also fetch embeds and their descriptions
export default function scraper(
export default async function scraper(
discordAuth,
vision: ImageAnnotatorClient,
minio: Minio.Client,
@@ -77,6 +77,12 @@ export default function scraper(
return res[0] || {}
}
async function fetchMissingVision() {
const attachments = await prisma.attachment
.findMany({ where: { vision: {}, contentType: { in: IMAGE_TYPES } }})
baseLog.info({attachments: attachments.length }, 'missing vision attachments')
}
async function processChannel(channelId) {
const channel = await fetchChannel(channelId)
const guild = await fetchGuildDetails(channel.guild_id)
@@ -110,19 +116,18 @@ export default function scraper(
memeId: crypto.randomUUID(),
// @ts-ignore it's fine
vision: {}, // await detectText(attachment),
vision: {},
guild,
channel,
message: m,
createdAt: undefined,
updatedAt: undefined
}
return attachment
return { ...attachment, vision: await detectText(attachment) }
}))
}
async function* fetchNew(channelId, log, latestMessageId): AsyncGenerator<Attachment> {
async function* fetchNew(channelId, log, latestMessageId): AsyncIterable<Attachment> {
// reverse the messages because we want to yield them in chronological order
// the api call returns them most recent first
const messages = (await fetchChannelMessagesAfter(channelId, latestMessageId)).reverse()
@@ -137,7 +142,7 @@ export default function scraper(
yield* fetchNew(channelId, log, cursor)
}
async function* fetchOld(channelId, log, earliestMessageId): AsyncGenerator<Attachment, void, Attachment> {
async function* fetchOld(channelId, log, earliestMessageId): AsyncIterable<Attachment> {
const messages = await fetchChannelMessagesBefore(channelId, earliestMessageId)
if (messages.length === 0) {
log.info('no new messages')