ntr/memestream
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sublime fkt but everything working

Browse Source
This commit is contained in:
Nathan Rashleigh committed 2024-04-10 12:33:47 +10:00
1 parent b655adc305
commit 66228047f4
12 files changed
+363 -123
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/attachments.ts
+71
View File
@@ -0,0 +1,71 @@
import sharp from 'sharp';
import axios from 'axios'
import mime from 'mime-types'
import type Minio from 'minio'
import type { PrismaClient, Attachment } from '@prisma/client';
import * as memes from './memes'
const STORAGE_BUCKET = process.env.STORAGE_BUCKET
export const IMAGE_TYPES = [
'image/png',
'image/jpeg',
'image/webp',
'image/gif',
]
export const VIDEO_TYPES = [
'video/mp4',
'video/quicktime'
]
export function isImage(a: Attachment) {
return IMAGE_TYPES.includes(a.contentType)
}
export function objectName(a: Attachment) {
if (isImage(a))
return `${a.memeId}.webp`
return `${a.memeId}.${mime.extension(a.contentType)}`
}
export default function attachmentService(minio: Minio.Client, prisma: PrismaClient) {
async function storeAttachment(a: Attachment) {
const download = await axios.get(a.url, { responseType: 'stream' })
const stream = () => {
if (isImage(a)) {
const converter = sharp().webp()
download.data.pipe(converter)
return converter
}
return download.data
}
return minio.putObject(STORAGE_BUCKET, objectName(a), stream())
}
async function deleteAttachment(a: Attachment) {
return minio.removeObject(STORAGE_BUCKET, objectName(a))
}
async function process(a: Attachment) {
const meme = memes.fromAttachment(a)
const upload = await storeAttachment(a)
return prisma.$transaction(async prisma => {
const dbMeme = await prisma.meme.upsert({ create: meme, update: meme, where: { id: meme.id } })
const dbA = prisma.attachment.upsert({ create: a, update: a, where: { id: a.id } })
console.log('pending addition to db', dbMeme)
const upload = await storeAttachment(a)
console.log('successfully uploaded', upload)
return { meme: dbMeme, upload, attachment: dbA }
})
}
return {
process,
}
}
src/memes.ts
+31
View File
@@ -0,0 +1,31 @@
import type { PrismaClient, Attachment, Meme } from '@prisma/client';
import * as attachments from './attachments'
function memeUrl(a: Attachment) {
const STORAGE_URL = process.env.STORAGE_URL
const path = attachments.objectName(a)
const u = new URL(path, `https://${STORAGE_URL}`)
return u.href
}
export function fromAttachment(a: Attachment) {
const text = a.vision.fullTextAnnotation?.text || ''
const tags = [
a.guild.name,
a.channel.name,
a.message.author.username,
]
const m: Meme = {
id: a.memeId,
url: memeUrl(a),
text,
tags: tags,
attachmentId: a.id,
createdAt: undefined,
updatedAt: undefined
}
return m
}
src/scraper.ts
+17 -90
View File
@@ -1,7 +1,4 @@
import path from 'path'
import axios from 'axios'
import mime from 'mime-types'
import type { PrismaClient, Attachment } from '@prisma/client';
import type { ImageAnnotatorClient } from '@google-cloud/vision';
@@ -9,21 +6,14 @@ import type Minio from 'minio'
import sharp from 'sharp';
const IMAGE_TYPES = [
'image/png',
'image/jpeg',
'image/webp',
'image/gif',
]
import * as attachments from './attachments'
const VIDEO_TYPES = [
'video/mp4',
'video/quicktime'
]
const STORAGE_BUCKET = process.env.STORAGE_BUCKET
export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio: Minio.Client) {
export default function scraper(
discordAuth,
vision: ImageAnnotatorClient,
minio: Minio.Client,
prisma: PrismaClient,
) {
const headers = {
'Authorization': discordAuth,
}
@@ -50,7 +40,7 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio
async function detectText(attachment: Attachment) {
// bail out of anything not an image
if (!IMAGE_TYPES.includes(attachment.contentType)) return {}
if (!attachments.IMAGE_TYPES.includes(attachment.contentType)) return {}
const annotateRequest = {
image: {
@@ -65,41 +55,24 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio
return res[0] || {}
}
async function downloadAttachment(a: Attachment) {
const download = await axios.get(a.url, { responseType: 'stream' })
const isImage = IMAGE_TYPES.includes(a.contentType)
const objectName = () => {
if (isImage)
return `${a.memeId}.webp`
return `${a.memeId}.${mime.extension(a.contentType)}`
}
const stream = async () => {
if (isImage) {
const converter = sharp().webp()
download.data.pipe(converter)
return converter
}
return download.data
}
return minio.putObject(STORAGE_BUCKET, objectName(), await stream())
}
function filterAttachments(messages, guild, channel) {
const acceptedContentTypes = [
...IMAGE_TYPES,
...VIDEO_TYPES,
...attachments.IMAGE_TYPES,
...attachments.VIDEO_TYPES,
]
return Promise.all(messages
const validAttachments = messages
.filter(m => m.attachments)
// bundle message and attachment together
.map(m => m.attachments.map(a => [a, m]))
.flat()
.filter(([a, m]) => acceptedContentTypes.includes(a.content_type))
const attachmentIds = validAttachments.map(([a, m]) => a.id)
const alreadyProcessedIds = prisma.attachment.find
return Promise.all(validAttachments
.map(async ([a, m]) => {
const attachment: Attachment = {
id: BigInt(a.id),
@@ -145,51 +118,5 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio
return {
getChannelAttachments,
downloadAttachment,
}
}
// async def download_attachment(attachment: Attachment):
// async with http_session.get(attachment.url) as response:
// if response.status == 200:
// (filename, ext) = attachment.file_details()
// object_name = attachment.object_name()
// print(f"uploading object_name={object_name} filename={filename} ext={ext}")
// res_bytes = await response.read()
// # make a webp conversion for obvious image types
// if attachment.convert_to_webp():
// # have to read the whole img before converting to webp
// image = Image.open(BytesIO(res_bytes))
// webp_output = BytesIO()
// image.save(webp_output, format='WebP')
// webp_output.seek(0)
// return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=webp_output)
// return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=res_bytes)
// def create_attachments(message, channel, guild):
// urls = [(parse_url(attachment['url']), attachment['id']) for attachment in message['attachments']]
// def generate_attachment(url: Url, attachmentId: str):
// if not url.path:
// return None
// (filename, ext) = os.path.splitext(os.path.basename(url.path))
// extensions = ['jpg', 'png', 'jpeg', 'webp', 'mov', 'gif', 'mp4']
// if ext.lower()[1:] not in extensions:
// return None
// return Attachment(
// id=uuid4(),
// attachmentId=int(attachmentId),
// msgId=int(message['id']),
// message=message,
// guild=guild,
// url=url.url,
// channel=channel
// )
// return [generate_attachment(*url) for url in urls]
src/test.attachment.ts
+121
View File
@@ -0,0 +1,121 @@
export default {
id: 1227338263132635217n,
msgId: 1227338263010873456n,
channelId: 367936426962190336n,
url: "https://cdn.discordapp.com/attachments/367936426962190336/1227338263132635217/WOW.jpg?ex=66280ad4&is=661595d4&hm=023a7a708935cf1c0cab01085486b939e1c08f92789f7a9b5c0c8ac5914c401c&",
contentType: "image/jpeg",
message: {
id: "1227338263010873456",
type: 0,
content: "",
channel_id: "367936426962190336",
author: {
id: "505353587174408193",
username: "tgcharlet",
avatar: "e800ffab640eb8684020054301668f79",
discriminator: "0",
public_flags: 0,
flags: 0,
banner: null,
accent_color: null,
global_name: "Chehrlet",
avatar_decoration_data: null,
banner_color: null,
clan: null
},
attachments: [
{
id: "1227338263132635217",
filename: "WOW.jpg",
size: 55508,
url: "https://cdn.discordapp.com/attachments/367936426962190336/1227338263132635217/WOW.jpg?ex=66280ad4&is=661595d4&hm=023a7a708935cf1c0cab01085486b939e1c08f92789f7a9b5c0c8ac5914c401c&",
proxy_url: "https://media.discordapp.net/attachments/367936426962190336/1227338263132635217/WOW.jpg?ex=66280ad4&is=661595d4&hm=023a7a708935cf1c0cab01085486b939e1c08f92789f7a9b5c0c8ac5914c401c&",
width: 1080,
height: 1022,
content_type: "image/jpeg",
content_scan_version: 1,
placeholder: "YPcFD4LU/YlNhpxWh4iLiJhn+iRpT8AH",
placeholder_version: 1
}
]
},
memeId: "7456180d-fa94-4465-bcb6-ea80524c67e8",
vision: {
textAnnotations: [],
localizedObjectAnnotations: [],
safeSearchAnnotation: null,
imagePropertiesAnnotation: null,
error: null,
cropHintsAnnotation: null,
fullTextAnnotation: {
text: "damn thats crazy"
},
webDetection: null,
productSearchResults: null,
context: null
},
guild: {
id: "239638846688460802",
name: "The Philosopher's Meme",
icon: "bda0d70d3bb2ec4df2dee343029d990d",
description: "An applied philosophical community of practice for creatives, intellectuals, researchers, and developers. https://tpm.ac",
home_header: null,
splash: "0d80f531b01ab5abc8d153386f2af3e6",
discovery_splash: "b35170de857b5eb58b1065131afa0826",
banner: null,
owner_id: "103361901899038720",
application_id: null,
region: "us-south",
afk_channel_id: null,
afk_timeout: 300,
system_channel_id: "934578834534182953",
system_channel_flags: 0,
widget_enabled: true,
widget_channel_id: "301188489490202626",
verification_level: 3,
default_message_notifications: 1,
mfa_level: 1,
explicit_content_filter: 2,
max_presences: null,
max_members: 500000,
max_stage_video_channel_users: 50,
max_video_channel_users: 25,
vanity_url_code: null,
premium_tier: 1,
premium_subscription_count: 3,
preferred_locale: "en-US",
rules_channel_id: "301188489490202626",
safety_alerts_channel_id: "837815188970537031",
public_updates_channel_id: "837815188970537031",
hub_type: null,
premium_progress_bar_enabled: true,
latest_onboarding_question_id: "1136794072313507860",
nsfw: false,
nsfw_level: 0,
incidents_data: null,
inventory_settings: null,
embed_enabled: true,
embed_channel_id: "301188489490202626"
},
channel: {
id: "367936426962190336",
type: 0,
last_message_id: "1227404638450483340",
flags: 0,
last_pin_timestamp: "2022-07-12T12:05:47+00:00",
guild_id: "239638846688460802",
name: "shitposts_memes",
parent_id: "1023698856191471667",
rate_limit_per_user: 0,
topic: "memes, shitposts; memeing, shitposting",
position: 38,
nsfw: false,
icon_emoji: {
id: null,
name: "🤪"
},
theme_color: null
},
createdAt: new Date(),
updatedAt: new Date(),
}