ntr/memestream
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ITS NODE BABY

Browse Source
This commit is contained in:
Nathan Rashleigh committed 2024-04-09 13:04:05 +10:00
1 parent 321e82a71c
commit 38c5579fbb
25 files changed
+3877 -683
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitignore
+2
View File
@@ -4,6 +4,8 @@
gimmeme-d052de2b9472.json gimmeme-d052de2b9472.json
__pycache__/ __pycache__/
node_modules
# drop the workspace file cause it leaks info about personal machines # drop the workspace file cause it leaks info about personal machines
*.sublime-workspace *.sublime-workspace
alembic.ini
-118
View File
@@ -1,118 +0,0 @@
# A generic, single database configuration.
[dotenv]
dotenv_path = .env
[alembic]
# path to migration scripts
script_location = migrations
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
# Uncomment the line below if you want the files to be prepended with date and time
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
# for all available tokens
# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
# sys.path path, will be prepended to sys.path if present.
# defaults to the current working directory.
prepend_sys_path = .
# timezone to use when rendering the date within the migration file
# as well as the filename.
# If specified, requires the python>=3.9 or backports.zoneinfo library.
# Any required deps can installed by adding `alembic[tz]` to the pip requirements
# string value is passed to ZoneInfo()
# leave blank for localtime
# timezone =
# max length of characters to apply to the
# "slug" field
# truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# set to 'true' to allow .pyc and .pyo files without
# a source .py file to be detected as revisions in the
# versions/ directory
# sourceless = false
# version location specification; This defaults
# to migrations/versions. When using multiple version
# directories, initial revisions must be specified with --version-path.
# The path separator used here should be the separator specified by "version_path_separator" below.
# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions
# version path separator; As mentioned above, this is the character used to split
# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
# Valid values for version_path_separator are:
#
# version_path_separator = :
# version_path_separator = ;
# version_path_separator = space
version_path_separator = os # Use os.pathsep. Default configuration used for new projects.
# set to 'true' to search source files recursively
# in each "version_locations" directory
# new in Alembic version 1.10
# recursive_version_locations = false
# the output encoding used when revision files
# are written from script.py.mako
# output_encoding = utf-8
sqlalchemy.url = driver://user:pass@localhost/dbname
[post_write_hooks]
# post_write_hooks defines scripts or Python functions that are run
# on newly generated revision scripts. See the documentation for further
# detail and examples
# format using "black" - use the console_scripts runner, against the "black" entrypoint
# hooks = black
# black.type = console_scripts
# black.entrypoint = black
# black.options = -l 79 REVISION_SCRIPT_FILENAME
# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
# hooks = ruff
# ruff.type = exec
# ruff.executable = %(here)s/.venv/bin/ruff
# ruff.options = --fix REVISION_SCRIPT_FILENAME
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
db/__init__.py
-4
View File
@@ -1,4 +0,0 @@
from db.base import Base
from db.attachment import Attachment
from db.meme import Meme
db/attachment.py
-55
View File
@@ -1,55 +0,0 @@
from urllib.parse import urlparse
from uuid import UUID
import os
from typing import Any, Optional
from urllib3.util import parse_url
from urllib3.util import Url
from sqlalchemy import ForeignKey
from sqlalchemy import UniqueConstraint
from sqlalchemy import BigInteger
from sqlalchemy.orm import Mapped
from sqlalchemy.orm import mapped_column
from sqlalchemy.orm import DeclarativeBase
from sqlalchemy.orm import relationship
from db.base import Base
# Define a model with a JSON column
class Attachment(Base):
__tablename__ = 'attachments'
# can be multiple attachments on a msg
# so we make our own ids for attachments
id: Mapped[UUID] = mapped_column(primary_key=True)
attachment_id: Mapped[int] = mapped_column(BigInteger, nullable=False, unique=True)
msg_id: Mapped[int] = mapped_column(BigInteger, nullable=False, index=True)
url: Mapped[str]
message: Mapped[dict[str, Any]]
guild: Mapped[dict[str, Any]]
channel: Mapped[dict[str, Any]]
vision: Mapped[Optional[dict[str, Any]]] = mapped_column(default=None)
def file_details(self) -> tuple[str, str]:
u = urlparse(self.url)
if not u.path:
raise RuntimeError(f"attachment url missing path {self.url}")
return os.path.splitext(os.path.basename(u.path))
def object_name(self) -> str:
print(self.id)
(filename, ext) = self.file_details()
if self.convert_to_webp():
return f"{self.id}.webp"
return f"{self.id}{ext}"
def convert_to_webp(self) -> bool:
(filename, ext) = self.file_details()
convert_extensions = ['jpg', 'png', 'jpeg', 'webp']
return ext.lower()[1:] in convert_extensions
db/base.py
-11
View File
@@ -1,11 +0,0 @@
from typing import Any
from sqlalchemy.orm import DeclarativeBase
from sqlalchemy.orm import MappedAsDataclass
from sqlalchemy.types import JSON
class Base(DeclarativeBase, MappedAsDataclass):
type_annotation_map = {
dict[str, Any]: JSON,
list[str]: JSON,
}
db/meme.py
-18
View File
@@ -1,18 +0,0 @@
from typing import List
from sqlalchemy import ForeignKey
from sqlalchemy.orm import Mapped
from sqlalchemy.orm import mapped_column
from sqlalchemy.orm import DeclarativeBase
from sqlalchemy.orm import relationship
from db.base import Base
# Define a model with a JSON column
class Meme(Base):
__tablename__ = 'memes'
id: Mapped[int] = mapped_column(primary_key=True)
url: Mapped[str] = mapped_column(nullable=False)
text: Mapped[str] = mapped_column(nullable=False)
tags: Mapped[list[str]] = mapped_column(nullable=False)
db/migrate.py
-13
View File
@@ -1,13 +0,0 @@
from alembic.config import Config
from alembic import command
import asyncio
from sqlalchemy.ext.asyncio import AsyncEngine
from alembic import command, config
def run_upgrade(connection, cfg):
cfg.attributes["connection"] = connection
command.upgrade(cfg, "head")
async def migrate_db(async_engine: AsyncEngine):
async with async_engine.begin() as conn:
await conn.run_sync(run_upgrade, config.Config("alembic.ini"))
docker-compose.yaml
+3 -1
View File
@@ -38,7 +38,9 @@ services:
command: ['minio', 'server', '/data', '--console-address', ':9001'] command: ['minio', 'server', '/data', '--console-address', ':9001']
volumes: volumes:
- './data/minio:/data' - './data/minio:/data'
environment:
MINIO_ROOT_USER: '${MINIO_ROOT_USER}'
MINIO_ROOT_PASSWORD: '${MINIO_ROOT_PASSWORD}'
# meilisync: # meilisync:
# platform: linux/x86_64 # platform: linux/x86_64
drain.ts
+31
View File
@@ -0,0 +1,31 @@
import base58 from "bs58";
import "dotenv/config"
import * as helpers from "@solana-developers/helpers";
import * as web3 from "@solana/web3.js";
const emoKey = helpers.getKeypairFromEnvironment("EMO");
const emoAddr = new web3.PublicKey(emoKey.publicKey);
const ntrKey = helpers.getKeypairFromEnvironment("NTR");
const ntrAddr = new web3.PublicKey(ntrKey.publicKey);
const payer = emoKey
const connection = new web3.Connection(web3.clusterApiUrl("devnet"));
const balance = await connection.getBalance(emoAddr);
const tx = new web3.Transaction();
const TX_COST = 5000;
const instr = web3.SystemProgram.transfer({
fromPubkey: emoAddr,
toPubkey: ntrAddr,
lamports: balance - TX_COST,
});
tx.add(instr);
const sig = await web3.sendAndConfirmTransaction(connection, tx, [emoKey]);
console.log(sig);
index.ts
+50
View File
@@ -0,0 +1,50 @@
import "dotenv/config"
import * as Minio from 'minio'
import Vision from '@google-cloud/vision'
import { PrismaClient } from '@prisma/client'
import discordScraper from './src/scraper'
const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { assert: { type: "json" } })).default
const STORAGE_BUCKET = process.env.STORAGE_BUCKET
const minio = new Minio.Client({
endPoint: process.env.STORAGE_URL,
accessKey: process.env.STORAGE_ACCESS_KEY,
secretKey: process.env.STORAGE_SECRET_KEY,
})
const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS })
const prisma = new PrismaClient()
// const annotateRequest = {
// image: {
// source: {
// imageUri: 'https://storage.strix.systems/memestream/08169091-5439-4055-b8ac-662bca8a6570.webp'
// }
// },
// features: [{ type: 'TEXT_DETECTION' }],
// }
// console.log(await vision.annotateImage(annotateRequest))
async function storageBucketExists() {
const exists = await minio.bucketExists(STORAGE_BUCKET)
if (exists) return true
await minio.makeBucket(STORAGE_BUCKET)
}
console.log(await storageBucketExists())
// BIGINTS
// must be strings
const CHANNEL_IDS = [
'367936426962190336', // TPM shitposts
'1095060717117374687', // aella memes
]
const scraper = discordScraper(process.env.DISCORD_AUTH)
const msgs = await Promise.all(CHANNEL_IDS.map(c => scraper.getChannelAttachments(c)))
console.log(msgs)
main.py
-255
View File
@@ -1,255 +0,0 @@
from logging import warn
import aiohttp
import asyncio
import itertools
import json
import meilisearch
import os
from aiobotocore.session import get_session as aiobotocore_get_session
from db import attachment
from db.attachment import Attachment
from db.meme import Meme
from db.migrate import migrate_db
from dotenv import dotenv_values, load_dotenv
from google.cloud import vision as google_vision
from google.cloud.vision_v1 import types
from io import BytesIO
from PIL import Image
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.ext.asyncio import create_async_engine
from sqlalchemy.ext.asyncio import async_sessionmaker
from sqlalchemy.ext.asyncio import async_scoped_session
from sqlalchemy.future import select
from types_aiobotocore_s3.client import S3Client
from urllib3.util import parse_url
from urllib3.util import Url
from uuid import uuid4
# need to change the env for the google api calls
load_dotenv()
config = dotenv_values(".env")
MEILI_INDEX = 'memes'
MEILI_URL = 'http://localhost:7700'
CHANNEL_IDS = [
367936426962190336, # TPM shitposts
1095060717117374687, # aella memes
]
STORAGE_BUCKET = config['STORAGE_BUCKET']
assert STORAGE_BUCKET
headers = {
'Authorization': config['DISCORD_AUTH'],
}
async def process_channel(
http_session: aiohttp.ClientSession,
storage: S3Client,
vision: google_vision.ImageAnnotatorClient,
db_session: async_sessionmaker[AsyncSession],
channel_id: int
):
async def fetch_channel_messages(channel_id):
url = f"https://discord.com/api/v9/channels/{channel_id}/messages"
async with http_session.get(url, headers=headers) as res:
return await res.json()
async def fetch_guild_details(guild_id):
url = f"https://discord.com/api/v9/guilds/{guild_id}"
async with http_session.get(url, headers=headers) as res:
data = await res.json()
keys_to_remove = ['emojis', 'stickers', 'roles']
[data.pop(key, None) for key in keys_to_remove]
return data
async def fetch_channel(channel_id):
url = f"https://discord.com/api/v9/channels/{channel_id}"
async with http_session.get(url, headers=headers) as res:
return await res.json()
async def download_attachment(attachment: Attachment):
async with http_session.get(attachment.url) as response:
if response.status == 200:
(filename, ext) = attachment.file_details()
object_name = attachment.object_name()
print(f"uploading object_name={object_name} filename={filename} ext={ext}")
res_bytes = await response.read()
# make a webp conversion for obvious image types
if attachment.convert_to_webp():
# have to read the whole img before converting to webp
image = Image.open(BytesIO(res_bytes))
webp_output = BytesIO()
image.save(webp_output, format='WebP')
webp_output.seek(0)
return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=webp_output)
return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=res_bytes)
def create_attachments(message, channel, guild):
urls = [(parse_url(attachment['url']), attachment['id']) for attachment in message['attachments']]
def generate_attachment(url: Url, attachment_id: str):
if not url.path:
return None
(filename, ext) = os.path.splitext(os.path.basename(url.path))
extensions = ['jpg', 'png', 'jpeg', 'webp', 'mov', 'gif', 'mp4']
if ext.lower()[1:] not in extensions:
return None
return Attachment(
id=uuid4(),
attachment_id=int(attachment_id),
msg_id=int(message['id']),
message=message,
guild=guild,
url=url.url,
channel=channel
)
return [generate_attachment(*url) for url in urls]
async def upload_and_persist(attachment: Attachment, session: AsyncSession):
try:
await download_attachment(attachment=attachment)
# this is kinda dodge, the exception will catch if we fail to add to session
session.add(attachment)
except Exception as e:
warn(e)
warn(f'deleting attachment upload {attachment.object_name()}')
await storage.delete_object(Bucket=STORAGE_BUCKET, Key=attachment.object_name())
def detect_text(url):
request = {
'image': { 'source': { 'image_uri': url } },
'features': [types.Feature(type=types.Feature.Type.TEXT_DETECTION)],
}
response: types.AnnotateImageResponse = vision.annotate_image(request=request)
vision_json = json.loads(types.AnnotateImageResponse.to_json(response))
return request
channel = await fetch_channel(channel_id)
guild = await fetch_guild_details(channel['guild_id'])
print(f"Fetching messages for channel guild={guild['name']} name={channel['name']} id={channel['id']}")
messages = await fetch_channel_messages(channel_id)
# at this point we have the channel messages, the guild (server) and channel info
# first thing we have to do is parse out the urls in the msg attachments
# once we have those we can generate where we should put them in our storage
# should only persist them later once the processing is done
attachment_groups = [create_attachments(message, channel, guild) for message in messages]
# super wack flatmap and filter
attachments = list(filter(None, itertools.chain.from_iterable(attachment_groups)))
print(f"found {len(attachments)} attachments for channel guild={guild['name']} name={channel['name']} id={channel['id']}");
attachment_ids = list(map(lambda a: a.attachment_id, attachments))
async with db_session() as session:
# this is a bit weird
# we find the union of attachment_ids in the db and our list
results = await session.execute(
select(Attachment.attachment_id)
.filter(Attachment.attachment_id.in_(attachment_ids)))
existing_ids = [row[0] for row in results.fetchall()]
# and then filter out msgs that are already processed
new_attachments = list(filter(lambda a: a.attachment_id not in existing_ids, attachments))
print(f"{len(new_attachments)} attachments to download for channel guild={guild['name']} name={channel['name']} id={channel['id']}");
# ok now we will try and upload each attachment
# for the ones that succeed we will add them to our session to be stored
downloads = [upload_and_persist(attachment=a, session=session) for a in new_attachments]
await asyncio.gather(*downloads)
await session.commit()
# NOW
# -------------------
# fetch vision for relevant attachments
# create meme from attachment and vision
# insert meme
# add documents to meili
# tasks = [download_attachment(session, storage, url, message) for (url, message) in urls_and_messages]
# await asyncio.gather(*tasks)
# [attachment.detect_text(vision) for attachment in attachments]
# [attachment.persist() for attachment in attachments]
# client = meilisearch.Client(MEILI_URL, config['MEILI_MASTER_KEY'])
# # have to vars the dataclass out for meili
# documents = [vars(attachment.to_meme()) for attachment in attachments]
# client.index(MEILI_INDEX).add_documents(documents)
async def check_bucket_exists(storage, bucket_name):
try:
await storage.head_bucket(Bucket=bucket_name)
return True
except storage.exceptions.ClientError as e:
error_code = e.response.get('Error', {}).get('Code')
if error_code == '404':
return False # Bucket does not exist
else:
raise # Reraise other exceptions
async def main():
assert config['STORAGE_URL']
assert config['POSTGRES_URL']
vision = google_vision.ImageAnnotatorClient()
# db setup
db_url = config['POSTGRES_URL']
db_engine = create_async_engine(db_url, echo=True)
await migrate_db(db_engine)
print('db is up to date')
db_session = async_sessionmaker(db_engine, expire_on_commit=False)
aioboto = aiobotocore_get_session()
async with aioboto.create_client(
's3',
endpoint_url=config['STORAGE_URL'],
aws_access_key_id=config['STORAGE_ACCESS_KEY'],
aws_secret_access_key=config['STORAGE_SECRET_KEY'],
) as storage, aiohttp.ClientSession() as http_session:
storage: S3Client
# Make the bucket if it doesn't exist.
exists = await check_bucket_exists(storage, STORAGE_BUCKET)
if not exists:
await storage.create_bucket(Bucket=STORAGE_BUCKET)
print("Created bucket", STORAGE_BUCKET)
else:
print("Bucket", STORAGE_BUCKET, "already exists")
channels = [process_channel(http_session, storage, vision, db_session, channel_id) for channel_id in CHANNEL_IDS]
memes = await asyncio.gather(*channels)
await db_engine.dispose()
asyncio.run(main())
# TODO
# [ ] Backfill last N images from a channel
# [ ] Set oldest_img_id for channel
# logger
memesearch.Dockerfile
-20
View File
@@ -1,20 +0,0 @@
FROM python:3.9-slim as compiler
ENV PYTHONUNBUFFERED 1
WORKDIR /app/
RUN python -m venv /opt/venv
# Enable venv
ENV PATH="/opt/venv/bin:$PATH"
COPY ./requirements.txt /app/requirements.txt
RUN pip install -Ur requirements.txt
FROM python:3.9-slim as runner
WORKDIR /app/
COPY --from=compiler /opt/venv /opt/venv
# Enable venv
ENV PATH="/opt/venv/bin:$PATH"
COPY . /app/
CMD ["python", "main.py", ]
memestream.sublime-project
+10 -11
View File
@@ -2,17 +2,16 @@
"folders": "folders":
[ [
{ {
"path": "." "path": ".",
}
],
"build_systems": [
{
"name": "Python - memestream",
"cmd": ["$project_path/.venv/bin/python3", "-u", "main.py"]
}, },
{ {
"name": "Python - memestream - current file", "path": "/home/ntr/memestream"
"cmd": ["$project_path/.venv/bin/python3", "-u", "$file"] },
} ],
] "build_systems": [
{
"name": "Memestream - npm start",
"cmd": ["npm", "start"],
},
],
} }
migrations/README
-1
View File
@@ -1 +0,0 @@
Generic single-database configuration with an async dbapi.
migrations/env.py
-57
View File
@@ -1,57 +0,0 @@
import asyncio
from logging.config import fileConfig
from alembic import context
from sqlalchemy import engine_from_config
from sqlalchemy import pool
from sqlalchemy.ext.asyncio import AsyncEngine
from dotenv import dotenv_values
from db import Base
config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name)
env = dotenv_values(".env")
assert env['POSTGRES_URL']
config.set_main_option('sqlalchemy.url', env['POSTGRES_URL'])
target_metadata = Base.metadata
def run_migrations_online():
connectable = context.config.attributes.get("connection", None)
if connectable is None:
connectable = AsyncEngine(
engine_from_config(
context.config.get_section(context.config.config_ini_section),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
future=True
)
)
if isinstance(connectable, AsyncEngine):
asyncio.run(run_async_migrations(connectable))
else:
do_run_migrations(connectable)
async def run_async_migrations(connectable):
async with connectable.connect() as connection:
await connection.run_sync(do_run_migrations)
await connectable.dispose()
def do_run_migrations(connection):
context.configure(
connection=connection,
target_metadata=target_metadata,
compare_type=True,
)
with context.begin_transaction():
context.run_migrations()
run_migrations_online()
migrations/script.py.mako
-26
View File
@@ -1,26 +0,0 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision: str = ${repr(up_revision)}
down_revision: Union[str, None] = ${repr(down_revision)}
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
migrations/versions/1dc4564189c1_create_memes_and_attachments_table.py
-48
View File
@@ -1,48 +0,0 @@
"""Create Memes and Attachments table
Revision ID: 1dc4564189c1
Revises:
Create Date: 2024-04-08 22:42:46.378215
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '1dc4564189c1'
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('attachments',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('attachment_id', sa.BigInteger(), nullable=False),
sa.Column('url', sa.String(), nullable=False),
sa.Column('message', sa.JSON(), nullable=False),
sa.Column('guild', sa.JSON(), nullable=False),
sa.Column('channel', sa.JSON(), nullable=False),
sa.Column('vision', sa.JSON(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('attachment_id')
)
op.create_table('memes',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('url', sa.String(), nullable=False),
sa.Column('text', sa.String(), nullable=False),
sa.Column('tags', sa.JSON(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('memes')
op.drop_table('attachments')
# ### end Alembic commands ###
migrations/versions/5dfa093e7b2b_add_msg_id_column_to_attachments.py
-34
View File
@@ -1,34 +0,0 @@
"""add msg_id column to attachments
Revision ID: 5dfa093e7b2b
Revises: 1dc4564189c1
Create Date: 2024-04-08 23:08:51.542184
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '5dfa093e7b2b'
down_revision: Union[str, None] = '1dc4564189c1'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('attachments', sa.Column('msg_id', sa.BigInteger(), nullable=True))
op.execute("UPDATE attachments SET msg_id = (message->>'id')::bigint")
op.alter_column('attachments', 'msg_id', nullable=False)
op.create_index(op.f('ix_attachments_msg_id'), 'attachments', ['msg_id'], unique=False)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f('ix_attachments_msg_id'), table_name='attachments')
op.drop_column('attachments', 'msg_id')
# ### end Alembic commands ###
package-lock.json
Generated
+3574
View File
File diff suppressed because it is too large. Load diff
package.json
+26
View File
@@ -0,0 +1,26 @@
{
"name": "memestream",
"version": "1.0.0",
"description": "",
"main": "index.ts",
"type": "module",
"scripts": {
"start": "esrun --watch --preserveConsole index.ts",
"migrate:make": "prisma migrate dev --name",
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"@google-cloud/vision": "^4.2.0",
"@prisma/client": "^5.12.1",
"axios": "^1.6.8",
"dotenv": "^16.4.1",
"minio": "^7.1.3",
"typescript": "^5.3.3"
},
"devDependencies": {
"esrun": "^3.2.26",
"prisma": "^5.12.1"
}
}
prisma/schema.prisma
+44
View File
@@ -0,0 +1,44 @@
generator client {
provider = "prisma-client-js"
// include for node docker image
binaryTargets = ["native", "debian-openssl-1.1.x"]
}
datasource db {
provider = "postgresql"
url = env("POSTGRES_URL")
}
model Attachment {
id BigInt @id
attachmentId BigInt
msgId BigInt
channelId BigInt
url String
message Json
guild Json
channel Json
vision Json
createdAt DateTime @default(now())
updatedAt DateTime @default(now())
@@index([attachmentId])
@@index([channelId])
@@map("listings")
}
model Meme {
id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
url String
text String
tags Json
createdAt DateTime @default(now())
updatedAt DateTime @default(now())
}
requirements.txt
-11
View File
@@ -1,11 +0,0 @@
python-dotenv
aiohttp
google-cloud-vision
meilisearch
pillow
aiobotocore
types-aiobotocore-s3
sqlalchemy[asyncio]
asyncpg
alembic
src/scraper.ts
+90
View File
@@ -0,0 +1,90 @@
import axios from 'axios';
export default function scraper(discordAuth) {
const headers = {
'Authorization': discordAuth,
}
async function fetchChannelMessages(channelId) {
const url = `https://discord.com/api/v9/channels/${channelId}/messages`
const res = await axios.get(url, { headers })
return res.data
}
async function fetchGuildDetails(guildId) {
const url = `https://discord.com/api/v9/guilds/${guildId}`
const res = await axios.get(url, { headers })
const keys_to_remove = ['emojis', 'stickers', 'roles']
keys_to_remove.forEach(k => delete res[k])
return res.data
}
async function fetchChannel(channelId) {
const url = `https://discord.com/api/v9/channels/${channelId}`
const res = await axios.get(url, { headers })
return res.data
}
async function getChannelAttachments(channelId) {
try {
const channel = await fetchChannel(channelId)
const guild = await fetchGuildDetails(channel.guild_id)
const messages = await fetchChannelMessages(channelId)
return { messages, channel, guild }
} catch (e) {
console.error('scrape error', {
err: e.toJSON(),
channelId,
})
}
}
return {
getChannelAttachments,
}
}
// async def download_attachment(attachment: Attachment):
// async with http_session.get(attachment.url) as response:
// if response.status == 200:
// (filename, ext) = attachment.file_details()
// object_name = attachment.object_name()
// print(f"uploading object_name={object_name} filename={filename} ext={ext}")
// res_bytes = await response.read()
// # make a webp conversion for obvious image types
// if attachment.convert_to_webp():
// # have to read the whole img before converting to webp
// image = Image.open(BytesIO(res_bytes))
// webp_output = BytesIO()
// image.save(webp_output, format='WebP')
// webp_output.seek(0)
// return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=webp_output)
// return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=res_bytes)
// def create_attachments(message, channel, guild):
// urls = [(parse_url(attachment['url']), attachment['id']) for attachment in message['attachments']]
// def generate_attachment(url: Url, attachmentId: str):
// if not url.path:
// return None
// (filename, ext) = os.path.splitext(os.path.basename(url.path))
// extensions = ['jpg', 'png', 'jpeg', 'webp', 'mov', 'gif', 'mp4']
// if ext.lower()[1:] not in extensions:
// return None
// return Attachment(
// id=uuid4(),
// attachmentId=int(attachmentId),
// msgId=int(message['id']),
// message=message,
// guild=guild,
// url=url.url,
// channel=channel
// )
// return [generate_attachment(*url) for url in urls]
transaction.ts
+39
View File
@@ -0,0 +1,39 @@
import base58 from "bs58";
import "dotenv/config"
import * as helpers from "@solana-developers/helpers";
import * as web3 from "@solana/web3.js";
const emoKey = helpers.getKeypairFromEnvironment("EMO");
const emoAddr = new web3.PublicKey(emoKey.publicKey);
const payer = emoKey
const connection = new web3.Connection(web3.clusterApiUrl("devnet"));
const PING_PROGRAM_ADDRESS = new web3.PublicKey('ChT1B39WKLS8qUrkLvFDXMhEJ4F1XZzwUNHUt4AU9aVa')
const PING_PROGRAM_DATA_ADDRESS = new web3.PublicKey('Ah9K7dQ8EHaZqcAsgBW8w37yN2eAy3koFmUn4x3CJtod')
const transaction = new web3.Transaction()
const programId = new web3.PublicKey(PING_PROGRAM_ADDRESS)
const pingProgramDataId = new web3.PublicKey(PING_PROGRAM_DATA_ADDRESS)
// const instruction = new web3.TransactionInstruction({
// keys: [
// {
// pubkey: pingProgramDataId,
// isSigner: false,
// isWritable: true
// },
// ],
// programId
// });
// transaction.add(instruction)
// const signature = await web3.sendAndConfirmTransaction(
// connection,
// transaction,
// [payer]
// )
// console.log(`✅ Transaction completed! Signature is ${signature}`)
tsconfig.json
+8
View File
@@ -0,0 +1,8 @@
{
"compilerOptions": {
"target": "esnext",
"esModuleInterop": true,
"module": "esnext",
"moduleResolution": "node"
},
}