big bang

.dockerignore

@@ -0,0 +1,2 @@
+data/
+docker-compose.yml

.gitignore

@@ -0,0 +1,2 @@
+data/
+docker-compose.yml

Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.9-bullseye
+WORKDIR /app/
+
+# Install dependencies
+RUN apt-get update && apt-get install -y \
+    ffmpeg libolm-dev \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+# Install Whisper
+RUN pip install git+https://github.com/openai/whisper.git
+
+# Install model files
+RUN whisper --model tiny dummy.wav; exit 0
+#RUN whisper --model base dummy.wav; exit 0
+#RUN whisper --model small dummy.wav; exit 0
+#RUN whisper --model medium dummy.wav; exit 0
+#RUN whisper --model large dummy.wav; exit 0
+#RUN whisper --model tiny.en dummy.wav; exit 0
+#RUN whisper --model base.en dummy.wav; exit 0
+#RUN whisper --model small.en dummy.wav; exit 0
+#RUN whisper --model medium.en dummy.wav; exit 0
+
+ADD requirements.txt /app/
+
+RUN pip install -r requirements.txt
+
+VOLUME /data/
+
+ADD . /app/
+
+CMD ["python", "-u", "main.py"]

README.md

@@ -0,0 +1,3 @@
+# Matrix Speech-To-Text Bot
+
+Transcribes audio messages using [OpenAI Whisper](https://github.com/openai/whisper)

docker-compose.yml.sample

@@ -0,0 +1,12 @@
+version: "3.7"
+
+services:
+  matrix-stt-bot:
+    image: ftcaplan/matrix-stt-bot
+    volumes:
+      - ./data/:/data/
+    environment:
+      - "HOMESERVER=https://matrix.example.com"
+      - "USERNAME=@stt-bot:example.com"
+      - "PASSWORD=<password>"
+      

main.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+from urllib.parse import urlparse
+import tempfile
+import os
+
+import whisper
+import simplematrixbotlib as botlib
+import nio
+
+model = whisper.load_model("tiny")
+
+creds = botlib.Creds(
+  homeserver=os.environ['HOMESERVER'],
+  username=os.environ['USERNAME'],
+  password=os.getenv('PASSWORD', None),
+  login_token=os.getenv('LOGIN_TOKEN', None),
+  access_token=os.getenv('ACCESS_TOKEN', None),
+  session_stored_file="/data/session.txt"
+)
+
+config = botlib.Config()
+config.encryption_enabled = True
+config.emoji_verify = True
+config.ignore_unverified_devices = True
+config.store_path = '/data/crypto_store/'
+bot = botlib.Bot(creds, config)
+
+@bot.listener.on_custom_event(nio.RoomMessageAudio)
+async def on_audio_message(room, event):
+  print(event.sender, event.body, event.url)
+  match = botlib.MessageMatch(room, event, bot)
+  if match.is_not_from_this_bot():
+    bot.async_client.room_typing(room, )
+    url = urlparse(event.url)
+    response = await bot.async_client.download(server_name=url.netloc, media_id=url.path[1:])
+    print(response)
+    with tempfile.NamedTemporaryFile("w+b") as file:
+      file.write(response.body)
+      file.flush()
+      print(file.name)
+      result = model.transcribe(file.name)
+
+    await bot.api.send_text_message(
+      room_id=room.room_id,
+      message=f"Transcription of {response.filename}: {result['text']}",
+      msgtype="m.notice")
+
+bot.run()

requirements.txt

@@ -0,0 +1,2 @@
+simplematrixbotlib==2.7.0
+matrix-nio[e2e]==0.19