ITS NODE BABY
This commit is contained in:
Nathan Rashleigh committed
1 parent
321e82a71c
commit
38c5579fbb
25 files changed
+3877
-683
No files matched your search
@@ -4,6 +4,8 @@
|
||||
gimmeme-d052de2b9472.json
|
||||
__pycache__/
|
||||
|
||||
node_modules
|
||||
|
||||
# drop the workspace file cause it leaks info about personal machines
|
||||
*.sublime-workspace
|
||||
|
||||
|
||||
-118
@@ -1,118 +0,0 @@
|
||||
# A generic, single database configuration.
|
||||
|
||||
[dotenv]
|
||||
dotenv_path = .env
|
||||
|
||||
[alembic]
|
||||
# path to migration scripts
|
||||
script_location = migrations
|
||||
|
||||
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
|
||||
# Uncomment the line below if you want the files to be prepended with date and time
|
||||
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
|
||||
# for all available tokens
|
||||
# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
|
||||
|
||||
# sys.path path, will be prepended to sys.path if present.
|
||||
# defaults to the current working directory.
|
||||
prepend_sys_path = .
|
||||
|
||||
# timezone to use when rendering the date within the migration file
|
||||
# as well as the filename.
|
||||
# If specified, requires the python>=3.9 or backports.zoneinfo library.
|
||||
# Any required deps can installed by adding `alembic[tz]` to the pip requirements
|
||||
# string value is passed to ZoneInfo()
|
||||
# leave blank for localtime
|
||||
# timezone =
|
||||
|
||||
# max length of characters to apply to the
|
||||
# "slug" field
|
||||
# truncate_slug_length = 40
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
# set to 'true' to allow .pyc and .pyo files without
|
||||
# a source .py file to be detected as revisions in the
|
||||
# versions/ directory
|
||||
# sourceless = false
|
||||
|
||||
# version location specification; This defaults
|
||||
# to migrations/versions. When using multiple version
|
||||
# directories, initial revisions must be specified with --version-path.
|
||||
# The path separator used here should be the separator specified by "version_path_separator" below.
|
||||
# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions
|
||||
|
||||
# version path separator; As mentioned above, this is the character used to split
|
||||
# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
|
||||
# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
|
||||
# Valid values for version_path_separator are:
|
||||
#
|
||||
# version_path_separator = :
|
||||
# version_path_separator = ;
|
||||
# version_path_separator = space
|
||||
version_path_separator = os # Use os.pathsep. Default configuration used for new projects.
|
||||
|
||||
# set to 'true' to search source files recursively
|
||||
# in each "version_locations" directory
|
||||
# new in Alembic version 1.10
|
||||
# recursive_version_locations = false
|
||||
|
||||
# the output encoding used when revision files
|
||||
# are written from script.py.mako
|
||||
# output_encoding = utf-8
|
||||
|
||||
sqlalchemy.url = driver://user:pass@localhost/dbname
|
||||
|
||||
[post_write_hooks]
|
||||
# post_write_hooks defines scripts or Python functions that are run
|
||||
# on newly generated revision scripts. See the documentation for further
|
||||
# detail and examples
|
||||
|
||||
# format using "black" - use the console_scripts runner, against the "black" entrypoint
|
||||
# hooks = black
|
||||
# black.type = console_scripts
|
||||
# black.entrypoint = black
|
||||
# black.options = -l 79 REVISION_SCRIPT_FILENAME
|
||||
|
||||
# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
|
||||
# hooks = ruff
|
||||
# ruff.type = exec
|
||||
# ruff.executable = %(here)s/.venv/bin/ruff
|
||||
# ruff.options = --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Logging configuration
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
@@ -1,4 +0,0 @@
|
||||
from db.base import Base
|
||||
|
||||
from db.attachment import Attachment
|
||||
from db.meme import Meme
|
||||
@@ -1,55 +0,0 @@
|
||||
from urllib.parse import urlparse
|
||||
from uuid import UUID
|
||||
import os
|
||||
|
||||
from typing import Any, Optional
|
||||
|
||||
from urllib3.util import parse_url
|
||||
from urllib3.util import Url
|
||||
|
||||
from sqlalchemy import ForeignKey
|
||||
from sqlalchemy import UniqueConstraint
|
||||
from sqlalchemy import BigInteger
|
||||
from sqlalchemy.orm import Mapped
|
||||
from sqlalchemy.orm import mapped_column
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from db.base import Base
|
||||
|
||||
# Define a model with a JSON column
|
||||
class Attachment(Base):
|
||||
__tablename__ = 'attachments'
|
||||
|
||||
# can be multiple attachments on a msg
|
||||
# so we make our own ids for attachments
|
||||
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||
attachment_id: Mapped[int] = mapped_column(BigInteger, nullable=False, unique=True)
|
||||
msg_id: Mapped[int] = mapped_column(BigInteger, nullable=False, index=True)
|
||||
|
||||
url: Mapped[str]
|
||||
|
||||
message: Mapped[dict[str, Any]]
|
||||
guild: Mapped[dict[str, Any]]
|
||||
channel: Mapped[dict[str, Any]]
|
||||
vision: Mapped[Optional[dict[str, Any]]] = mapped_column(default=None)
|
||||
|
||||
def file_details(self) -> tuple[str, str]:
|
||||
u = urlparse(self.url)
|
||||
if not u.path:
|
||||
raise RuntimeError(f"attachment url missing path {self.url}")
|
||||
return os.path.splitext(os.path.basename(u.path))
|
||||
|
||||
def object_name(self) -> str:
|
||||
print(self.id)
|
||||
(filename, ext) = self.file_details()
|
||||
if self.convert_to_webp():
|
||||
return f"{self.id}.webp"
|
||||
|
||||
return f"{self.id}{ext}"
|
||||
|
||||
def convert_to_webp(self) -> bool:
|
||||
(filename, ext) = self.file_details()
|
||||
convert_extensions = ['jpg', 'png', 'jpeg', 'webp']
|
||||
return ext.lower()[1:] in convert_extensions
|
||||
|
||||
-11
@@ -1,11 +0,0 @@
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
from sqlalchemy.orm import MappedAsDataclass
|
||||
from sqlalchemy.types import JSON
|
||||
|
||||
class Base(DeclarativeBase, MappedAsDataclass):
|
||||
type_annotation_map = {
|
||||
dict[str, Any]: JSON,
|
||||
list[str]: JSON,
|
||||
}
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
from typing import List
|
||||
|
||||
from sqlalchemy import ForeignKey
|
||||
from sqlalchemy.orm import Mapped
|
||||
from sqlalchemy.orm import mapped_column
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from db.base import Base
|
||||
|
||||
# Define a model with a JSON column
|
||||
class Meme(Base):
|
||||
__tablename__ = 'memes'
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
url: Mapped[str] = mapped_column(nullable=False)
|
||||
text: Mapped[str] = mapped_column(nullable=False)
|
||||
tags: Mapped[list[str]] = mapped_column(nullable=False)
|
||||
@@ -1,13 +0,0 @@
|
||||
from alembic.config import Config
|
||||
from alembic import command
|
||||
import asyncio
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
from alembic import command, config
|
||||
|
||||
def run_upgrade(connection, cfg):
|
||||
cfg.attributes["connection"] = connection
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
async def migrate_db(async_engine: AsyncEngine):
|
||||
async with async_engine.begin() as conn:
|
||||
await conn.run_sync(run_upgrade, config.Config("alembic.ini"))
|
||||
+3
-1
@@ -38,7 +38,9 @@ services:
|
||||
command: ['minio', 'server', '/data', '--console-address', ':9001']
|
||||
volumes:
|
||||
- './data/minio:/data'
|
||||
|
||||
environment:
|
||||
MINIO_ROOT_USER: '${MINIO_ROOT_USER}'
|
||||
MINIO_ROOT_PASSWORD: '${MINIO_ROOT_PASSWORD}'
|
||||
|
||||
# meilisync:
|
||||
# platform: linux/x86_64
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import base58 from "bs58";
|
||||
import "dotenv/config"
|
||||
import * as helpers from "@solana-developers/helpers";
|
||||
import * as web3 from "@solana/web3.js";
|
||||
|
||||
const emoKey = helpers.getKeypairFromEnvironment("EMO");
|
||||
const emoAddr = new web3.PublicKey(emoKey.publicKey);
|
||||
|
||||
const ntrKey = helpers.getKeypairFromEnvironment("NTR");
|
||||
const ntrAddr = new web3.PublicKey(ntrKey.publicKey);
|
||||
|
||||
const payer = emoKey
|
||||
const connection = new web3.Connection(web3.clusterApiUrl("devnet"));
|
||||
|
||||
const balance = await connection.getBalance(emoAddr);
|
||||
|
||||
const tx = new web3.Transaction();
|
||||
const TX_COST = 5000;
|
||||
|
||||
const instr = web3.SystemProgram.transfer({
|
||||
fromPubkey: emoAddr,
|
||||
toPubkey: ntrAddr,
|
||||
lamports: balance - TX_COST,
|
||||
});
|
||||
|
||||
tx.add(instr);
|
||||
|
||||
const sig = await web3.sendAndConfirmTransaction(connection, tx, [emoKey]);
|
||||
|
||||
console.log(sig);
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import "dotenv/config"
|
||||
|
||||
import * as Minio from 'minio'
|
||||
import Vision from '@google-cloud/vision'
|
||||
import { PrismaClient } from '@prisma/client'
|
||||
|
||||
import discordScraper from './src/scraper'
|
||||
|
||||
const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { assert: { type: "json" } })).default
|
||||
const STORAGE_BUCKET = process.env.STORAGE_BUCKET
|
||||
|
||||
const minio = new Minio.Client({
|
||||
endPoint: process.env.STORAGE_URL,
|
||||
accessKey: process.env.STORAGE_ACCESS_KEY,
|
||||
secretKey: process.env.STORAGE_SECRET_KEY,
|
||||
})
|
||||
const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS })
|
||||
const prisma = new PrismaClient()
|
||||
|
||||
// const annotateRequest = {
|
||||
// image: {
|
||||
// source: {
|
||||
// imageUri: 'https://storage.strix.systems/memestream/08169091-5439-4055-b8ac-662bca8a6570.webp'
|
||||
// }
|
||||
// },
|
||||
// features: [{ type: 'TEXT_DETECTION' }],
|
||||
// }
|
||||
|
||||
// console.log(await vision.annotateImage(annotateRequest))
|
||||
|
||||
async function storageBucketExists() {
|
||||
const exists = await minio.bucketExists(STORAGE_BUCKET)
|
||||
if (exists) return true
|
||||
await minio.makeBucket(STORAGE_BUCKET)
|
||||
}
|
||||
|
||||
console.log(await storageBucketExists())
|
||||
|
||||
// BIGINTS
|
||||
// must be strings
|
||||
const CHANNEL_IDS = [
|
||||
'367936426962190336', // TPM shitposts
|
||||
'1095060717117374687', // aella memes
|
||||
]
|
||||
|
||||
const scraper = discordScraper(process.env.DISCORD_AUTH)
|
||||
|
||||
const msgs = await Promise.all(CHANNEL_IDS.map(c => scraper.getChannelAttachments(c)))
|
||||
|
||||
console.log(msgs)
|
||||
@@ -1,255 +0,0 @@
|
||||
from logging import warn
|
||||
import aiohttp
|
||||
import asyncio
|
||||
import itertools
|
||||
import json
|
||||
import meilisearch
|
||||
import os
|
||||
|
||||
from aiobotocore.session import get_session as aiobotocore_get_session
|
||||
from db import attachment
|
||||
from db.attachment import Attachment
|
||||
from db.meme import Meme
|
||||
from db.migrate import migrate_db
|
||||
from dotenv import dotenv_values, load_dotenv
|
||||
from google.cloud import vision as google_vision
|
||||
from google.cloud.vision_v1 import types
|
||||
from io import BytesIO
|
||||
from PIL import Image
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
from sqlalchemy.ext.asyncio import async_scoped_session
|
||||
from sqlalchemy.future import select
|
||||
|
||||
from types_aiobotocore_s3.client import S3Client
|
||||
from urllib3.util import parse_url
|
||||
from urllib3.util import Url
|
||||
from uuid import uuid4
|
||||
|
||||
# need to change the env for the google api calls
|
||||
load_dotenv()
|
||||
config = dotenv_values(".env")
|
||||
|
||||
MEILI_INDEX = 'memes'
|
||||
MEILI_URL = 'http://localhost:7700'
|
||||
|
||||
CHANNEL_IDS = [
|
||||
367936426962190336, # TPM shitposts
|
||||
1095060717117374687, # aella memes
|
||||
]
|
||||
|
||||
STORAGE_BUCKET = config['STORAGE_BUCKET']
|
||||
assert STORAGE_BUCKET
|
||||
|
||||
headers = {
|
||||
'Authorization': config['DISCORD_AUTH'],
|
||||
}
|
||||
|
||||
async def process_channel(
|
||||
http_session: aiohttp.ClientSession,
|
||||
storage: S3Client,
|
||||
vision: google_vision.ImageAnnotatorClient,
|
||||
db_session: async_sessionmaker[AsyncSession],
|
||||
channel_id: int
|
||||
):
|
||||
async def fetch_channel_messages(channel_id):
|
||||
url = f"https://discord.com/api/v9/channels/{channel_id}/messages"
|
||||
async with http_session.get(url, headers=headers) as res:
|
||||
return await res.json()
|
||||
|
||||
async def fetch_guild_details(guild_id):
|
||||
url = f"https://discord.com/api/v9/guilds/{guild_id}"
|
||||
async with http_session.get(url, headers=headers) as res:
|
||||
data = await res.json()
|
||||
|
||||
keys_to_remove = ['emojis', 'stickers', 'roles']
|
||||
[data.pop(key, None) for key in keys_to_remove]
|
||||
|
||||
return data
|
||||
|
||||
async def fetch_channel(channel_id):
|
||||
url = f"https://discord.com/api/v9/channels/{channel_id}"
|
||||
async with http_session.get(url, headers=headers) as res:
|
||||
return await res.json()
|
||||
|
||||
async def download_attachment(attachment: Attachment):
|
||||
async with http_session.get(attachment.url) as response:
|
||||
if response.status == 200:
|
||||
(filename, ext) = attachment.file_details()
|
||||
object_name = attachment.object_name()
|
||||
|
||||
print(f"uploading object_name={object_name} filename={filename} ext={ext}")
|
||||
res_bytes = await response.read()
|
||||
|
||||
# make a webp conversion for obvious image types
|
||||
if attachment.convert_to_webp():
|
||||
# have to read the whole img before converting to webp
|
||||
image = Image.open(BytesIO(res_bytes))
|
||||
webp_output = BytesIO()
|
||||
image.save(webp_output, format='WebP')
|
||||
webp_output.seek(0)
|
||||
return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=webp_output)
|
||||
|
||||
return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=res_bytes)
|
||||
|
||||
def create_attachments(message, channel, guild):
|
||||
urls = [(parse_url(attachment['url']), attachment['id']) for attachment in message['attachments']]
|
||||
|
||||
def generate_attachment(url: Url, attachment_id: str):
|
||||
if not url.path:
|
||||
return None
|
||||
|
||||
(filename, ext) = os.path.splitext(os.path.basename(url.path))
|
||||
extensions = ['jpg', 'png', 'jpeg', 'webp', 'mov', 'gif', 'mp4']
|
||||
if ext.lower()[1:] not in extensions:
|
||||
return None
|
||||
|
||||
return Attachment(
|
||||
id=uuid4(),
|
||||
attachment_id=int(attachment_id),
|
||||
msg_id=int(message['id']),
|
||||
message=message,
|
||||
guild=guild,
|
||||
url=url.url,
|
||||
channel=channel
|
||||
)
|
||||
|
||||
return [generate_attachment(*url) for url in urls]
|
||||
|
||||
async def upload_and_persist(attachment: Attachment, session: AsyncSession):
|
||||
try:
|
||||
await download_attachment(attachment=attachment)
|
||||
# this is kinda dodge, the exception will catch if we fail to add to session
|
||||
session.add(attachment)
|
||||
except Exception as e:
|
||||
warn(e)
|
||||
warn(f'deleting attachment upload {attachment.object_name()}')
|
||||
await storage.delete_object(Bucket=STORAGE_BUCKET, Key=attachment.object_name())
|
||||
|
||||
def detect_text(url):
|
||||
request = {
|
||||
'image': { 'source': { 'image_uri': url } },
|
||||
'features': [types.Feature(type=types.Feature.Type.TEXT_DETECTION)],
|
||||
}
|
||||
|
||||
response: types.AnnotateImageResponse = vision.annotate_image(request=request)
|
||||
vision_json = json.loads(types.AnnotateImageResponse.to_json(response))
|
||||
|
||||
return request
|
||||
|
||||
|
||||
channel = await fetch_channel(channel_id)
|
||||
guild = await fetch_guild_details(channel['guild_id'])
|
||||
print(f"Fetching messages for channel guild={guild['name']} name={channel['name']} id={channel['id']}")
|
||||
messages = await fetch_channel_messages(channel_id)
|
||||
|
||||
# at this point we have the channel messages, the guild (server) and channel info
|
||||
|
||||
# first thing we have to do is parse out the urls in the msg attachments
|
||||
# once we have those we can generate where we should put them in our storage
|
||||
# should only persist them later once the processing is done
|
||||
attachment_groups = [create_attachments(message, channel, guild) for message in messages]
|
||||
|
||||
# super wack flatmap and filter
|
||||
attachments = list(filter(None, itertools.chain.from_iterable(attachment_groups)))
|
||||
|
||||
print(f"found {len(attachments)} attachments for channel guild={guild['name']} name={channel['name']} id={channel['id']}");
|
||||
|
||||
attachment_ids = list(map(lambda a: a.attachment_id, attachments))
|
||||
|
||||
async with db_session() as session:
|
||||
# this is a bit weird
|
||||
# we find the union of attachment_ids in the db and our list
|
||||
results = await session.execute(
|
||||
select(Attachment.attachment_id)
|
||||
.filter(Attachment.attachment_id.in_(attachment_ids)))
|
||||
|
||||
existing_ids = [row[0] for row in results.fetchall()]
|
||||
|
||||
# and then filter out msgs that are already processed
|
||||
new_attachments = list(filter(lambda a: a.attachment_id not in existing_ids, attachments))
|
||||
print(f"{len(new_attachments)} attachments to download for channel guild={guild['name']} name={channel['name']} id={channel['id']}");
|
||||
|
||||
# ok now we will try and upload each attachment
|
||||
# for the ones that succeed we will add them to our session to be stored
|
||||
downloads = [upload_and_persist(attachment=a, session=session) for a in new_attachments]
|
||||
await asyncio.gather(*downloads)
|
||||
await session.commit()
|
||||
|
||||
# NOW
|
||||
# -------------------
|
||||
# fetch vision for relevant attachments
|
||||
# create meme from attachment and vision
|
||||
# insert meme
|
||||
# add documents to meili
|
||||
|
||||
# tasks = [download_attachment(session, storage, url, message) for (url, message) in urls_and_messages]
|
||||
# await asyncio.gather(*tasks)
|
||||
|
||||
# [attachment.detect_text(vision) for attachment in attachments]
|
||||
# [attachment.persist() for attachment in attachments]
|
||||
|
||||
# client = meilisearch.Client(MEILI_URL, config['MEILI_MASTER_KEY'])
|
||||
|
||||
# # have to vars the dataclass out for meili
|
||||
# documents = [vars(attachment.to_meme()) for attachment in attachments]
|
||||
# client.index(MEILI_INDEX).add_documents(documents)
|
||||
|
||||
async def check_bucket_exists(storage, bucket_name):
|
||||
try:
|
||||
await storage.head_bucket(Bucket=bucket_name)
|
||||
return True
|
||||
except storage.exceptions.ClientError as e:
|
||||
error_code = e.response.get('Error', {}).get('Code')
|
||||
if error_code == '404':
|
||||
return False # Bucket does not exist
|
||||
else:
|
||||
raise # Reraise other exceptions
|
||||
|
||||
async def main():
|
||||
assert config['STORAGE_URL']
|
||||
assert config['POSTGRES_URL']
|
||||
|
||||
vision = google_vision.ImageAnnotatorClient()
|
||||
|
||||
# db setup
|
||||
db_url = config['POSTGRES_URL']
|
||||
|
||||
db_engine = create_async_engine(db_url, echo=True)
|
||||
await migrate_db(db_engine)
|
||||
print('db is up to date')
|
||||
|
||||
db_session = async_sessionmaker(db_engine, expire_on_commit=False)
|
||||
|
||||
aioboto = aiobotocore_get_session()
|
||||
async with aioboto.create_client(
|
||||
's3',
|
||||
endpoint_url=config['STORAGE_URL'],
|
||||
aws_access_key_id=config['STORAGE_ACCESS_KEY'],
|
||||
aws_secret_access_key=config['STORAGE_SECRET_KEY'],
|
||||
) as storage, aiohttp.ClientSession() as http_session:
|
||||
|
||||
storage: S3Client
|
||||
|
||||
# Make the bucket if it doesn't exist.
|
||||
exists = await check_bucket_exists(storage, STORAGE_BUCKET)
|
||||
if not exists:
|
||||
await storage.create_bucket(Bucket=STORAGE_BUCKET)
|
||||
print("Created bucket", STORAGE_BUCKET)
|
||||
else:
|
||||
print("Bucket", STORAGE_BUCKET, "already exists")
|
||||
|
||||
channels = [process_channel(http_session, storage, vision, db_session, channel_id) for channel_id in CHANNEL_IDS]
|
||||
memes = await asyncio.gather(*channels)
|
||||
|
||||
await db_engine.dispose()
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
|
||||
# TODO
|
||||
# [ ] Backfill last N images from a channel
|
||||
# [ ] Set oldest_img_id for channel
|
||||
# logger
|
||||
@@ -1,20 +0,0 @@
|
||||
FROM python:3.9-slim as compiler
|
||||
ENV PYTHONUNBUFFERED 1
|
||||
|
||||
WORKDIR /app/
|
||||
|
||||
RUN python -m venv /opt/venv
|
||||
# Enable venv
|
||||
ENV PATH="/opt/venv/bin:$PATH"
|
||||
|
||||
COPY ./requirements.txt /app/requirements.txt
|
||||
RUN pip install -Ur requirements.txt
|
||||
|
||||
FROM python:3.9-slim as runner
|
||||
WORKDIR /app/
|
||||
COPY --from=compiler /opt/venv /opt/venv
|
||||
|
||||
# Enable venv
|
||||
ENV PATH="/opt/venv/bin:$PATH"
|
||||
COPY . /app/
|
||||
CMD ["python", "main.py", ]
|
||||
+10
-11
@@ -2,17 +2,16 @@
|
||||
"folders":
|
||||
[
|
||||
{
|
||||
"path": "."
|
||||
}
|
||||
],
|
||||
"build_systems": [
|
||||
{
|
||||
"name": "Python - memestream",
|
||||
"cmd": ["$project_path/.venv/bin/python3", "-u", "main.py"]
|
||||
"path": ".",
|
||||
},
|
||||
{
|
||||
"name": "Python - memestream - current file",
|
||||
"cmd": ["$project_path/.venv/bin/python3", "-u", "$file"]
|
||||
}
|
||||
]
|
||||
"path": "/home/ntr/memestream"
|
||||
},
|
||||
],
|
||||
"build_systems": [
|
||||
{
|
||||
"name": "Memestream - npm start",
|
||||
"cmd": ["npm", "start"],
|
||||
},
|
||||
],
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
Generic single-database configuration with an async dbapi.
|
||||
@@ -1,57 +0,0 @@
|
||||
import asyncio
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import engine_from_config
|
||||
from sqlalchemy import pool
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
|
||||
from dotenv import dotenv_values
|
||||
from db import Base
|
||||
|
||||
config = context.config
|
||||
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
env = dotenv_values(".env")
|
||||
assert env['POSTGRES_URL']
|
||||
config.set_main_option('sqlalchemy.url', env['POSTGRES_URL'])
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
def run_migrations_online():
|
||||
connectable = context.config.attributes.get("connection", None)
|
||||
if connectable is None:
|
||||
connectable = AsyncEngine(
|
||||
engine_from_config(
|
||||
context.config.get_section(context.config.config_ini_section),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
future=True
|
||||
)
|
||||
)
|
||||
|
||||
if isinstance(connectable, AsyncEngine):
|
||||
asyncio.run(run_async_migrations(connectable))
|
||||
else:
|
||||
do_run_migrations(connectable)
|
||||
|
||||
|
||||
async def run_async_migrations(connectable):
|
||||
async with connectable.connect() as connection:
|
||||
await connection.run_sync(do_run_migrations)
|
||||
await connectable.dispose()
|
||||
|
||||
|
||||
def do_run_migrations(connection):
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
compare_type=True,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
run_migrations_online()
|
||||
@@ -1,26 +0,0 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -1,48 +0,0 @@
|
||||
"""Create Memes and Attachments table
|
||||
|
||||
Revision ID: 1dc4564189c1
|
||||
Revises:
|
||||
Create Date: 2024-04-08 22:42:46.378215
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '1dc4564189c1'
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('attachments',
|
||||
sa.Column('id', sa.Uuid(), nullable=False),
|
||||
sa.Column('attachment_id', sa.BigInteger(), nullable=False),
|
||||
sa.Column('url', sa.String(), nullable=False),
|
||||
sa.Column('message', sa.JSON(), nullable=False),
|
||||
sa.Column('guild', sa.JSON(), nullable=False),
|
||||
sa.Column('channel', sa.JSON(), nullable=False),
|
||||
sa.Column('vision', sa.JSON(), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('attachment_id')
|
||||
)
|
||||
op.create_table('memes',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('url', sa.String(), nullable=False),
|
||||
sa.Column('text', sa.String(), nullable=False),
|
||||
sa.Column('tags', sa.JSON(), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table('memes')
|
||||
op.drop_table('attachments')
|
||||
# ### end Alembic commands ###
|
||||
@@ -1,34 +0,0 @@
|
||||
"""add msg_id column to attachments
|
||||
|
||||
Revision ID: 5dfa093e7b2b
|
||||
Revises: 1dc4564189c1
|
||||
Create Date: 2024-04-08 23:08:51.542184
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '5dfa093e7b2b'
|
||||
down_revision: Union[str, None] = '1dc4564189c1'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.add_column('attachments', sa.Column('msg_id', sa.BigInteger(), nullable=True))
|
||||
op.execute("UPDATE attachments SET msg_id = (message->>'id')::bigint")
|
||||
op.alter_column('attachments', 'msg_id', nullable=False)
|
||||
op.create_index(op.f('ix_attachments_msg_id'), 'attachments', ['msg_id'], unique=False)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index(op.f('ix_attachments_msg_id'), table_name='attachments')
|
||||
op.drop_column('attachments', 'msg_id')
|
||||
# ### end Alembic commands ###
|
||||
Generated
+3574
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"name": "memestream",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.ts",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "esrun --watch --preserveConsole index.ts",
|
||||
"migrate:make": "prisma migrate dev --name",
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@google-cloud/vision": "^4.2.0",
|
||||
"@prisma/client": "^5.12.1",
|
||||
"axios": "^1.6.8",
|
||||
"dotenv": "^16.4.1",
|
||||
"minio": "^7.1.3",
|
||||
"typescript": "^5.3.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"esrun": "^3.2.26",
|
||||
"prisma": "^5.12.1"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
generator client {
|
||||
provider = "prisma-client-js"
|
||||
// include for node docker image
|
||||
binaryTargets = ["native", "debian-openssl-1.1.x"]
|
||||
}
|
||||
|
||||
datasource db {
|
||||
provider = "postgresql"
|
||||
url = env("POSTGRES_URL")
|
||||
}
|
||||
|
||||
model Attachment {
|
||||
id BigInt @id
|
||||
attachmentId BigInt
|
||||
msgId BigInt
|
||||
channelId BigInt
|
||||
|
||||
url String
|
||||
|
||||
message Json
|
||||
guild Json
|
||||
channel Json
|
||||
vision Json
|
||||
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @default(now())
|
||||
|
||||
@@index([attachmentId])
|
||||
@@index([channelId])
|
||||
|
||||
@@map("listings")
|
||||
}
|
||||
|
||||
model Meme {
|
||||
id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
|
||||
url String
|
||||
text String
|
||||
tags Json
|
||||
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @default(now())
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
python-dotenv
|
||||
aiohttp
|
||||
google-cloud-vision
|
||||
meilisearch
|
||||
pillow
|
||||
aiobotocore
|
||||
types-aiobotocore-s3
|
||||
|
||||
sqlalchemy[asyncio]
|
||||
asyncpg
|
||||
alembic
|
||||
@@ -0,0 +1,90 @@
|
||||
import axios from 'axios';
|
||||
|
||||
export default function scraper(discordAuth) {
|
||||
const headers = {
|
||||
'Authorization': discordAuth,
|
||||
}
|
||||
|
||||
async function fetchChannelMessages(channelId) {
|
||||
const url = `https://discord.com/api/v9/channels/${channelId}/messages`
|
||||
const res = await axios.get(url, { headers })
|
||||
return res.data
|
||||
}
|
||||
|
||||
async function fetchGuildDetails(guildId) {
|
||||
const url = `https://discord.com/api/v9/guilds/${guildId}`
|
||||
const res = await axios.get(url, { headers })
|
||||
const keys_to_remove = ['emojis', 'stickers', 'roles']
|
||||
keys_to_remove.forEach(k => delete res[k])
|
||||
return res.data
|
||||
}
|
||||
|
||||
async function fetchChannel(channelId) {
|
||||
const url = `https://discord.com/api/v9/channels/${channelId}`
|
||||
const res = await axios.get(url, { headers })
|
||||
return res.data
|
||||
}
|
||||
|
||||
async function getChannelAttachments(channelId) {
|
||||
try {
|
||||
const channel = await fetchChannel(channelId)
|
||||
const guild = await fetchGuildDetails(channel.guild_id)
|
||||
const messages = await fetchChannelMessages(channelId)
|
||||
return { messages, channel, guild }
|
||||
} catch (e) {
|
||||
console.error('scrape error', {
|
||||
err: e.toJSON(),
|
||||
channelId,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
getChannelAttachments,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// async def download_attachment(attachment: Attachment):
|
||||
// async with http_session.get(attachment.url) as response:
|
||||
// if response.status == 200:
|
||||
// (filename, ext) = attachment.file_details()
|
||||
// object_name = attachment.object_name()
|
||||
|
||||
// print(f"uploading object_name={object_name} filename={filename} ext={ext}")
|
||||
// res_bytes = await response.read()
|
||||
|
||||
// # make a webp conversion for obvious image types
|
||||
// if attachment.convert_to_webp():
|
||||
// # have to read the whole img before converting to webp
|
||||
// image = Image.open(BytesIO(res_bytes))
|
||||
// webp_output = BytesIO()
|
||||
// image.save(webp_output, format='WebP')
|
||||
// webp_output.seek(0)
|
||||
// return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=webp_output)
|
||||
|
||||
// return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=res_bytes)
|
||||
|
||||
// def create_attachments(message, channel, guild):
|
||||
// urls = [(parse_url(attachment['url']), attachment['id']) for attachment in message['attachments']]
|
||||
|
||||
// def generate_attachment(url: Url, attachmentId: str):
|
||||
// if not url.path:
|
||||
// return None
|
||||
|
||||
// (filename, ext) = os.path.splitext(os.path.basename(url.path))
|
||||
// extensions = ['jpg', 'png', 'jpeg', 'webp', 'mov', 'gif', 'mp4']
|
||||
// if ext.lower()[1:] not in extensions:
|
||||
// return None
|
||||
|
||||
// return Attachment(
|
||||
// id=uuid4(),
|
||||
// attachmentId=int(attachmentId),
|
||||
// msgId=int(message['id']),
|
||||
// message=message,
|
||||
// guild=guild,
|
||||
// url=url.url,
|
||||
// channel=channel
|
||||
// )
|
||||
|
||||
// return [generate_attachment(*url) for url in urls]
|
||||
@@ -0,0 +1,39 @@
|
||||
import base58 from "bs58";
|
||||
import "dotenv/config"
|
||||
import * as helpers from "@solana-developers/helpers";
|
||||
import * as web3 from "@solana/web3.js";
|
||||
|
||||
const emoKey = helpers.getKeypairFromEnvironment("EMO");
|
||||
const emoAddr = new web3.PublicKey(emoKey.publicKey);
|
||||
|
||||
const payer = emoKey
|
||||
const connection = new web3.Connection(web3.clusterApiUrl("devnet"));
|
||||
|
||||
const PING_PROGRAM_ADDRESS = new web3.PublicKey('ChT1B39WKLS8qUrkLvFDXMhEJ4F1XZzwUNHUt4AU9aVa')
|
||||
const PING_PROGRAM_DATA_ADDRESS = new web3.PublicKey('Ah9K7dQ8EHaZqcAsgBW8w37yN2eAy3koFmUn4x3CJtod')
|
||||
|
||||
const transaction = new web3.Transaction()
|
||||
const programId = new web3.PublicKey(PING_PROGRAM_ADDRESS)
|
||||
const pingProgramDataId = new web3.PublicKey(PING_PROGRAM_DATA_ADDRESS)
|
||||
|
||||
|
||||
// const instruction = new web3.TransactionInstruction({
|
||||
// keys: [
|
||||
// {
|
||||
// pubkey: pingProgramDataId,
|
||||
// isSigner: false,
|
||||
// isWritable: true
|
||||
// },
|
||||
// ],
|
||||
// programId
|
||||
// });
|
||||
|
||||
// transaction.add(instruction)
|
||||
|
||||
// const signature = await web3.sendAndConfirmTransaction(
|
||||
// connection,
|
||||
// transaction,
|
||||
// [payer]
|
||||
// )
|
||||
|
||||
// console.log(`✅ Transaction completed! Signature is ${signature}`)
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "esnext",
|
||||
"esModuleInterop": true,
|
||||
"module": "esnext",
|
||||
"moduleResolution": "node"
|
||||
},
|
||||
}
|
||||
Reference in new issue
Block a user