Alek Ratzloff
8c4bb5ac60
Markov now uses a sqlite3 database instead of flat JSON files. This should significantly speed up saving time, plus reduce the amount of RAM that it uses. Saving and loading large JSON files was very slow and caused issues with other plugins, especially when messages were received. Additionally, in order to save RAM, a cache was used and periodically flushed when not used, adding some complications to the implementation. This has all been removed since things get committed on the fly with the database implementation. The main trade-off we have to make is the disk space used by the database. This is OK though, because disk space is cheap while RAM is not. Signed-off-by: Alek Ratzloff <alekratz@gmail.com>
49 lines
1.3 KiB
Python
49 lines
1.3 KiB
Python
import logging
|
|
import asyncio
|
|
import sys
|
|
import re
|
|
|
|
from omnibot.config import ServerConfig
|
|
from omnibot.bot import Bot
|
|
from plugins.markov import Markov
|
|
|
|
|
|
async def main():
|
|
"""
|
|
Hacky "load my IRC logs" script
|
|
"""
|
|
|
|
# TODO - add config path selection
|
|
logging.basicConfig(
|
|
level=logging.DEBUG,
|
|
format="%(asctime)s - %(name)-12s - %(levelname)-8s - %(message)s",
|
|
)
|
|
log = logging.getLogger(__name__)
|
|
|
|
channel = sys.argv[1]
|
|
files = sys.argv[2:]
|
|
server_config = ServerConfig()
|
|
with open("config.toml") as fp:
|
|
server_config.load(fp)
|
|
# This only works on one plugin per config
|
|
bot = Bot(server_config)
|
|
plugin = [plugin for plugin in bot.plugins if isinstance(plugin, Markov)][0]
|
|
|
|
LINE_RE = re.compile(r"^\[[^\]]+\] <(?P<name>[^>]+)> (?P<message>.+)$")
|
|
|
|
for fname in files:
|
|
log.info("Loading %s", fname)
|
|
with open(fname) as fp:
|
|
lines = list(fp)
|
|
for line in lines:
|
|
if mat := LINE_RE.search(line):
|
|
name = mat["name"]
|
|
message = mat["message"].strip()
|
|
if name != server_config.nick and message and message[0] != "!":
|
|
plugin.add(channel, name, message, commit=False)
|
|
await plugin.save()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|