docker

docker-compose.yaml

@@ -0,0 +1,56 @@
+---
+
+services:
+  pg:
+    image: postgres:latest
+    environment:
+      POSTGRES_USER: '${POSTGRES_USER}'
+      POSTGRES_DB: '${POSTGRES_DB}'
+      POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
+    ports:
+      - '5432:5432'
+    volumes:
+      - "./data/pg:/var/lib/postgresql/data"
+
+  redis:
+    image: redis:latest
+    ports:
+      - '6379:6379'
+    command: ['redis-server', '/usr/local/etc/redis/redis.conf']
+    volumes:
+      - "./data/redis:/data"
+      - "./redis.conf:/usr/local/etc/redis/redis.conf"
+
+  meili:
+    image: getmeili/meilisearch:v1.7
+    ports:
+      - '7700:7700'
+    volumes:
+      - './data/meili:/meili_data'
+    environment:
+      MEILI_MASTER_KEY: '${MEILI_MASTER_KEY}'
+
+  minio:
+    image: quay.io/minio/minio
+    ports:
+      - 9000:9000
+      - 9001:9001
+    command: ['minio', 'server', '/data', '--console-address', ':9001']
+    volumes:
+      - './data/minio:/data'
+
+
+  # meilisync:
+  #   platform: linux/x86_64
+  #   image: long2ice/meilisync
+  #   volumes:
+  #     - ./config.yml:/meilisync/config.yml
+
+  # nginx:
+  #   image: nginx:latest
+  #   depends_on:
+  #     - 'api'
+  #   volumes:
+  #     - './nginx/localhost.nginx.conf:/etc/nginx/conf.d/default.conf'
+  #   ports:
+  #     - '80:80'

main.py

@@ -4,18 +4,24 @@ import aiohttp
 import json
 import meilisearch
 
+from io import BytesIO
 from dataclasses import dataclass
-from typing import Dict, Any
+from typing import Dict, Any, List
 
 from dotenv import dotenv_values, load_dotenv
 from google.cloud import vision as google_vision
 from google.protobuf.json_format import MessageToDict, MessageToJson
 from google.cloud.vision_v1 import types
+from PIL import Image
 
 # need to change the env for the google api calls
 load_dotenv()
 config = dotenv_values(".env")
 
+DATA_DIR = './data/memesearch'
+MEILI_INDEX = 'memes'
+MEILI_URL = 'http://localhost:7700'
+
 CHANNEL_IDS = [
     367936426962190336, # TPM shitposts
     1095060717117374687, # aella memes
@@ -31,6 +37,13 @@ headers = {
 # [ ] save img as json + imagefile
 # [ ] check extensions (ignore mp4 maybe)
 
+@dataclass
+class MemeDocument:
+    id: int
+    url: str
+    text: str | None
+    tags: List[str]
+
 @dataclass
 class MemeMessage:
     id: int
@@ -38,8 +51,8 @@ class MemeMessage:
     guild_name: str
     channel_name: str
 
-    text: str | None
     vision_json: str | None
+    text: str | None
 
     # maybe don't need these
     # kind of extra
@@ -47,22 +60,34 @@ class MemeMessage:
     guild_json: Dict[str, Any]
     channel_json: Dict[str, Any]
 
+    def output_dir(self):
+        return f'{DATA_DIR}/guilds/{self.guild_name}/{self.channel_name}'
+
+    def img_dir(self):
+        return f"{DATA_DIR}/imgs/"
+
     async def save_img(self, session: aiohttp.ClientSession):
-        output_dir = self.output_dir()
+        webp_output = f'{DATA_DIR}/imgs/'
 
         # there's a big param of some kind after the image url that we remove
         url_filename = os.path.basename(self.url).split('?')[0]
         (filename, ext) = os.path.splitext(url_filename)
         print(f"filename={filename} ext={ext}")
-        os.makedirs(output_dir, exist_ok=True);
+
+        os.makedirs(self.output_dir(), exist_ok=True);
+        os.makedirs(self.img_dir(), exist_ok=True);
 
         async with session.get(self.url) as response:
             if response.status == 200:
-                with open(f'{output_dir}/{self.id}{ext}', 'wb') as f:
+                res_bytes = await response.read()
-                    f.write(await response.read())
 
-    def output_dir(self):
+                with open(f'{self.output_dir()}/{self.id}{ext}', 'wb') as f:
-        return f'./data/{self.guild_name}/{self.channel_name}'
+                    f.write(res_bytes)
+
+                convert_extensions = ['jpg', 'png', 'jpeg', 'webp']
+                if ext.lower() in convert_extensions:
+                    image = Image.open(BytesIO(res_bytes))
+                    image.save(f"{self.img_dir()}/{self.id}.webp", format='WebP')
 
     def annotate_image_request(self, vision):
         request = {
@@ -82,12 +107,13 @@ class MemeMessage:
         with open(f'{self.output_dir()}/{self.id}.json', 'w') as f:
             json.dump(vars(self), f, indent=4)
 
-    def to_meili_document(self):
+    def to_document(self):
-        return {
+        return MemeDocument(
-            'id': self.id,
+            id=self.id,
-            'text': self.text,
+            url=f'{DATA_DIR}/{self.id}.webp',
-            'url': self.url,
+            text=self.text,
-        }
+            tags=[self.guild_name, self.channel_name],
+        )
 
 def filter_message_img_urls(messages):
     url_lists = map(
@@ -142,9 +168,11 @@ async def download_channel_imgs(session, vision, channel_id):
     [meme.annotate_image_request(vision) for meme in memes]
     [meme.persist() for meme in memes]
 
-    client = meilisearch.Client('http://localhost:7700', 'aSampleMasterKey')
+    client = meilisearch.Client(MEILI_URL, config['MEILI_MASTER_KEY'])
-    documents = [meme.to_meili_document() for meme in memes]
+
-    client.index('memes').add_documents(documents)
+    # have to vars the dataclass out for meili
+    documents = [vars(meme.to_document()) for meme in memes]
+    client.index(MEILI_INDEX).add_documents(documents)
 
 async def main():
     vision = google_vision.ImageAnnotatorClient()

memesearch.Dockerfile

@@ -0,0 +1,20 @@
+FROM python:3.9-slim as compiler
+ENV PYTHONUNBUFFERED 1
+
+WORKDIR /app/
+
+RUN python -m venv /opt/venv
+# Enable venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+COPY ./requirements.txt /app/requirements.txt
+RUN pip install -Ur requirements.txt
+
+FROM python:3.9-slim as runner
+WORKDIR /app/
+COPY --from=compiler /opt/venv /opt/venv
+
+# Enable venv
+ENV PATH="/opt/venv/bin:$PATH"
+COPY . /app/
+CMD ["python", "main.py", ]

redis.conf

@@ -0,0 +1 @@
+appendfsync everysec

requirements.txt

@@ -2,3 +2,4 @@ python-dotenv
 aiohttp
 google-cloud-vision
 meilisearch
+pillow

search.html

@@ -0,0 +1,40 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@meilisearch/instant-meilisearch/templates/basic_search.css" />
+  </head>
+  <body>
+      <div id="app" class="wrapper"></div>
+  </body>
+  <script src="https://unpkg.com/react@18/umd/react.development.js" crossorigin></script>
+  <script src="https://unpkg.com/react-dom@18/umd/react-dom.development.js" crossorigin></script>
+  <script src="https://cdn.jsdelivr.net/npm/react-instantsearch-dom@6.7.0/dist/umd/ReactInstantSearchDOM.js"></script>
+  <script src="https://cdn.jsdelivr.net/npm/@meilisearch/instant-meilisearch/dist/instant-meilisearch.umd.min.js"></script>
+  <script>
+    const { InstantSearch, SearchBox, Hits, Highlight, Configure } = ReactInstantSearchDOM;
+    const searchClient = instantMeiliSearch(
+      "http://localhost:7700",
+      '73xjT1ueTlcXPEKJtzvJxtrlLNOR5q85fJ_TOzoRgcA'
+    );
+    const App = () => (
+      React.createElement(InstantSearch, {
+        indexName: "memes",
+        searchClient: searchClient
+      }, [
+        React.createElement(SearchBox, { key: 1 }),
+        React.createElement(Hits, { hitComponent: Hit, key: 2 }),
+        React.createElement(Configure, { hitsPerPage: 10 })]
+      )
+    );
+
+    const Hit = (props) => (
+      React.createElement(Highlight, {
+        attribute: "id",
+        hit: props.hit
+      })
+    );
+    ReactDOM.createRoot(document.querySelector('#app')).render(App());
+    console.log(searchClient)
+  </script>
+</html>