From 4776b1857bf8613fe2dee983c5d9ab47024fad32 Mon Sep 17 00:00:00 2001 From: imcompiling Date: Tue, 14 May 2024 11:16:10 -0400 Subject: [PATCH 01/21] Start using fastapi. --- classes/clan_message.py | 6 ++++++ dakiny_bot.py | 3 +++ requirements.txt | 3 ++- 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 classes/clan_message.py diff --git a/classes/clan_message.py b/classes/clan_message.py new file mode 100644 index 0000000..da37135 --- /dev/null +++ b/classes/clan_message.py @@ -0,0 +1,6 @@ +from dakiny_bot import app + + +@app.post("/message/") +async def hello(): + print("we did a webhook") diff --git a/dakiny_bot.py b/dakiny_bot.py index b1679fb..a51eed8 100644 --- a/dakiny_bot.py +++ b/dakiny_bot.py @@ -4,6 +4,7 @@ import sys import hydra import nextcord +from fastapi import FastAPI from hydra.core.config_store import ConfigStore from nextcord.ext import commands @@ -27,6 +28,8 @@ handler.setFormatter( ) logger.addHandler(handler) +app = FastAPI() + @hydra.main(config_path="conf", config_name="config") def main(cfg: DakinyConfig): diff --git a/requirements.txt b/requirements.txt index 2ffe3d1..205c64d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ python-dotenv==1.0.0 pytz==2022.7.1 hydra-core==1.3.2 sqlalchemy<2.0 -pandas==1.5.2 \ No newline at end of file +pandas==1.5.2 +fastapi==0.111.0 \ No newline at end of file From 3fec5fab41c5830f171df0f747f8b0a2e68c282c Mon Sep 17 00:00:00 2001 From: imcompiling Date: Tue, 14 May 2024 12:19:11 -0400 Subject: [PATCH 02/21] Webhook is using external file. --- classes/clan_message.py | 7 +++++-- main.py | 7 +++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/classes/clan_message.py b/classes/clan_message.py index da37135..ef7c376 100644 --- a/classes/clan_message.py +++ b/classes/clan_message.py @@ -1,6 +1,9 @@ -from dakiny_bot import app +from fastapi import APIRouter + +router = APIRouter() -@app.post("/message/") +@router.post("/message/") async def hello(): print("we did a webhook") + return {"message": "webhook get"} diff --git a/main.py b/main.py index a51eed8..0e9d034 100644 --- a/main.py +++ b/main.py @@ -4,10 +4,12 @@ import sys import hydra import nextcord +import uvicorn from fastapi import FastAPI from hydra.core.config_store import ConfigStore from nextcord.ext import commands +import classes.clan_message as clan_message import database_setup.create_database as create_database from classes.nominate_highcouncil_views import NominateHighCouncilView from classes.nominate_views import NominateView @@ -29,6 +31,7 @@ handler.setFormatter( logger.addHandler(handler) app = FastAPI() +app.include_router(clan_message.router) @hydra.main(config_path="conf", config_name="config") @@ -64,6 +67,10 @@ def main(cfg: DakinyConfig): bot.add_view(VoteHighcouncilView(cfg, bot.get_guild(cfg.params.guild_id))) print(f"{bot.user.name} has connected to Discord!") + config = uvicorn.Config("main:app", port=5000, log_level="info") + server = uvicorn.Server(config) + await server.serve() + # Run Discord bot bot.run(bot.config.params.discord_token) From 4ae31159dce4fbdb8ab3a1ff57184ae534d83afb Mon Sep 17 00:00:00 2001 From: imcompiling Date: Tue, 14 May 2024 16:04:52 -0400 Subject: [PATCH 03/21] Move the webhook listener into a cog so that I have access to the bot for the action method. --- classes/clan_message.py | 9 -------- cogs/clan_messages/__init__.py | 38 ++++++++++++++++++++++++++++++++++ main.py | 10 --------- 3 files changed, 38 insertions(+), 19 deletions(-) delete mode 100644 classes/clan_message.py create mode 100644 cogs/clan_messages/__init__.py diff --git a/classes/clan_message.py b/classes/clan_message.py deleted file mode 100644 index ef7c376..0000000 --- a/classes/clan_message.py +++ /dev/null @@ -1,9 +0,0 @@ -from fastapi import APIRouter - -router = APIRouter() - - -@router.post("/message/") -async def hello(): - print("we did a webhook") - return {"message": "webhook get"} diff --git a/cogs/clan_messages/__init__.py b/cogs/clan_messages/__init__.py new file mode 100644 index 0000000..9d479a9 --- /dev/null +++ b/cogs/clan_messages/__init__.py @@ -0,0 +1,38 @@ +import uvicorn +from fastapi import FastAPI +from nextcord.ext import commands + +app = FastAPI() + + +class ClanMessages(commands.Cog, name="Clan Messages"): + def __init__(self, bot: commands.Bot): + self.bot = bot + + @commands.Cog.listener() + async def on_ready(self): + app.add_api_route("/message/", self.hello, methods=["POST"]) + config = uvicorn.Config(app=app, port=5000, log_level="info") + server = uvicorn.Server(config) + await server.serve() + + async def hello(self): + print("we did a webhook") + return {"message": "webhook get"} + + async def update_messages(self): + # TODO everything + + # git pull + # if no change, return + # for ever directory, sort the files by number then post + # posting involves checking if there is a message for that particular file then editing it or creating a + # new one. the message could be deleted, which would cause a new message to be sent and database updated. + pass + + +# This function will be called when this extension is loaded. +# It is necessary to add these functions to the bot. +def setup(bot: commands.Bot): + """Adds function to bot""" + bot.add_cog(ClanMessages(bot)) diff --git a/main.py b/main.py index 0e9d034..b1679fb 100644 --- a/main.py +++ b/main.py @@ -4,12 +4,9 @@ import sys import hydra import nextcord -import uvicorn -from fastapi import FastAPI from hydra.core.config_store import ConfigStore from nextcord.ext import commands -import classes.clan_message as clan_message import database_setup.create_database as create_database from classes.nominate_highcouncil_views import NominateHighCouncilView from classes.nominate_views import NominateView @@ -30,9 +27,6 @@ handler.setFormatter( ) logger.addHandler(handler) -app = FastAPI() -app.include_router(clan_message.router) - @hydra.main(config_path="conf", config_name="config") def main(cfg: DakinyConfig): @@ -67,10 +61,6 @@ def main(cfg: DakinyConfig): bot.add_view(VoteHighcouncilView(cfg, bot.get_guild(cfg.params.guild_id))) print(f"{bot.user.name} has connected to Discord!") - config = uvicorn.Config("main:app", port=5000, log_level="info") - server = uvicorn.Server(config) - await server.serve() - # Run Discord bot bot.run(bot.config.params.discord_token) From 9d78070c459da47d10a6abc883371e2a874c4975 Mon Sep 17 00:00:00 2001 From: imcompiling Date: Tue, 14 May 2024 16:34:31 -0400 Subject: [PATCH 04/21] Add gitpython requirement. Clone the repo. --- .gitignore | 5 +++-- cogs/clan_messages/__init__.py | 20 ++++++++++++++++++++ requirements.txt | 3 ++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 2b4fd5b..351e035 100644 --- a/.gitignore +++ b/.gitignore @@ -130,7 +130,7 @@ dmypy.json # logging nextcord.log -# data file for bot +# bot files highscores_data.json highscores_config.json deleted_backups/ @@ -139,4 +139,5 @@ TODO.txt sql_tester.py conf outputs -database \ No newline at end of file +database +dakiny-messages/ \ No newline at end of file diff --git a/cogs/clan_messages/__init__.py b/cogs/clan_messages/__init__.py index 9d479a9..1e85952 100644 --- a/cogs/clan_messages/__init__.py +++ b/cogs/clan_messages/__init__.py @@ -1,16 +1,25 @@ +import os + import uvicorn from fastapi import FastAPI +from git import Repo from nextcord.ext import commands app = FastAPI() +REPO_URL = "https://gitea.cheezers.org/imcompiling/dakiny-messages.git" +REPO_DIR = "./dakiny-messages" class ClanMessages(commands.Cog, name="Clan Messages"): def __init__(self, bot: commands.Bot): self.bot = bot + self.clone_repo() @commands.Cog.listener() async def on_ready(self): + """ + Triggers when the bot is in ready state. Adds the webhook listener to FastAPI and serves. + """ app.add_api_route("/message/", self.hello, methods=["POST"]) config = uvicorn.Config(app=app, port=5000, log_level="info") server = uvicorn.Server(config) @@ -18,6 +27,7 @@ class ClanMessages(commands.Cog, name="Clan Messages"): async def hello(self): print("we did a webhook") + await self.update_messages() return {"message": "webhook get"} async def update_messages(self): @@ -30,6 +40,16 @@ class ClanMessages(commands.Cog, name="Clan Messages"): # new one. the message could be deleted, which would cause a new message to be sent and database updated. pass + def clone_repo(self): + """ + Checks if the repo exists and creates repo object, clones if it does not. + """ + # check if repo has been cloned. open repo object + if os.path.isdir(REPO_DIR): + self.repo = Repo(REPO_DIR) + else: + self.repo = Repo.clone_from(REPO_URL, REPO_DIR) + # This function will be called when this extension is loaded. # It is necessary to add these functions to the bot. diff --git a/requirements.txt b/requirements.txt index 205c64d..c397c41 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ pytz==2022.7.1 hydra-core==1.3.2 sqlalchemy<2.0 pandas==1.5.2 -fastapi==0.111.0 \ No newline at end of file +fastapi==0.111.0 +GitPython==3.1.43 \ No newline at end of file From a3e195925ab628597d02bbc49bffbb76232692d2 Mon Sep 17 00:00:00 2001 From: imcompiling Date: Tue, 14 May 2024 17:14:54 -0400 Subject: [PATCH 05/21] Get on the right branch --- cogs/clan_messages/__init__.py | 21 ++++++++++++++++++--- config.py | 1 + 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/cogs/clan_messages/__init__.py b/cogs/clan_messages/__init__.py index 1e85952..a9c2712 100644 --- a/cogs/clan_messages/__init__.py +++ b/cogs/clan_messages/__init__.py @@ -5,6 +5,8 @@ from fastapi import FastAPI from git import Repo from nextcord.ext import commands +from config import DakinyConfig + app = FastAPI() REPO_URL = "https://gitea.cheezers.org/imcompiling/dakiny-messages.git" REPO_DIR = "./dakiny-messages" @@ -13,7 +15,8 @@ REPO_DIR = "./dakiny-messages" class ClanMessages(commands.Cog, name="Clan Messages"): def __init__(self, bot: commands.Bot): self.bot = bot - self.clone_repo() + self.config: DakinyConfig = self.bot.config + self.repo_setup() @commands.Cog.listener() async def on_ready(self): @@ -38,9 +41,8 @@ class ClanMessages(commands.Cog, name="Clan Messages"): # for ever directory, sort the files by number then post # posting involves checking if there is a message for that particular file then editing it or creating a # new one. the message could be deleted, which would cause a new message to be sent and database updated. - pass - def clone_repo(self): + def repo_setup(self): """ Checks if the repo exists and creates repo object, clones if it does not. """ @@ -50,6 +52,19 @@ class ClanMessages(commands.Cog, name="Clan Messages"): else: self.repo = Repo.clone_from(REPO_URL, REPO_DIR) + self.origin = self.repo.remote("origin") + + if self.config.params.git_branch == "test": + self.repo.create_head("test", self.origin.refs.test).set_tracking_branch( + self.origin.refs.test + ).checkout() + else: + self.repo.create_head("main", self.origin.refs.main).set_tracking_branch( + self.origin.refs.main + ).checkout() + + self.origin.pull() + # This function will be called when this extension is loaded. # It is necessary to add these functions to the bot. diff --git a/config.py b/config.py index b779fbc..6510907 100644 --- a/config.py +++ b/config.py @@ -23,6 +23,7 @@ class Params: event_participate_points: int event_post_channel_id: int general_channel_id: int + git_branch: str guild_id: int high_council_role_id: int voting_parent_id: int From f51f66697ffe5a3ffe981135628b81ef3a095694 Mon Sep 17 00:00:00 2001 From: imcompiling Date: Tue, 14 May 2024 17:54:58 -0400 Subject: [PATCH 06/21] Start working on update_messages function. --- cogs/clan_messages/__init__.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/cogs/clan_messages/__init__.py b/cogs/clan_messages/__init__.py index a9c2712..7f27c14 100644 --- a/cogs/clan_messages/__init__.py +++ b/cogs/clan_messages/__init__.py @@ -10,6 +10,7 @@ from config import DakinyConfig app = FastAPI() REPO_URL = "https://gitea.cheezers.org/imcompiling/dakiny-messages.git" REPO_DIR = "./dakiny-messages" +MESSAGES_DIR = f"{REPO_DIR}/messages" class ClanMessages(commands.Cog, name="Clan Messages"): @@ -23,6 +24,7 @@ class ClanMessages(commands.Cog, name="Clan Messages"): """ Triggers when the bot is in ready state. Adds the webhook listener to FastAPI and serves. """ + await self.update_messages() app.add_api_route("/message/", self.hello, methods=["POST"]) config = uvicorn.Config(app=app, port=5000, log_level="info") server = uvicorn.Server(config) @@ -34,13 +36,25 @@ class ClanMessages(commands.Cog, name="Clan Messages"): return {"message": "webhook get"} async def update_messages(self): - # TODO everything + """ + _summary_ + """ + + self.origin.pull() - # git pull - # if no change, return # for ever directory, sort the files by number then post # posting involves checking if there is a message for that particular file then editing it or creating a # new one. the message could be deleted, which would cause a new message to be sent and database updated. + dirs = [ + f"{MESSAGES_DIR}/{dir_name}" + for dir_name in os.listdir(MESSAGES_DIR) + if os.path.isdir(f"{MESSAGES_DIR}/{dir_name}") + ] + for dir_name in dirs: + sorted_files = sorted( + os.listdir(dir_name), key=lambda filename: int(filename.split(".")[0]) + ) + print(sorted_files) def repo_setup(self): """ From 9ab4d9e33986e170a7fdec6851a5aaef3844b3c2 Mon Sep 17 00:00:00 2001 From: imcompiling Date: Tue, 14 May 2024 18:01:16 -0400 Subject: [PATCH 07/21] Add the messages table to database. --- database_setup/database.sql | 5 +++++ database_setup/sql_data_classes.py | 9 +++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/database_setup/database.sql b/database_setup/database.sql index 10b7061..7f31fcf 100644 --- a/database_setup/database.sql +++ b/database_setup/database.sql @@ -46,4 +46,9 @@ CREATE TABLE ParticipationPoints ( DiscordMember boolean not null, Points int unsigned DEFAULT 0, PRIMARY KEY (Cycle, UserId) +); + +CREATE TABLE Messages ( + FileName varchar(100) PRIMARY KEY, + MessageId bigint unsigned not null ); \ No newline at end of file diff --git a/database_setup/sql_data_classes.py b/database_setup/sql_data_classes.py index 9fa382a..007d432 100644 --- a/database_setup/sql_data_classes.py +++ b/database_setup/sql_data_classes.py @@ -12,9 +12,10 @@ Base = automap_base() Base.prepare(autoload_with=engine) # define the classes -ElectionInformation = Base.classes.ElectionInformation CouncilMembers = Base.classes.CouncilMembers -Votes = Base.classes.Votes -UserBirthday = Base.classes.UserBirthday -Presences = Base.classes.Presences +ElectionInformation = Base.classes.ElectionInformation +Messages = Base.classes.Messages ParticipationPoints = Base.classes.ParticipationPoints +Presences = Base.classes.Presences +UserBirthday = Base.classes.UserBirthday +Votes = Base.classes.Votes From a1f955abbe3dca4347da9163de15b68929cee909 Mon Sep 17 00:00:00 2001 From: imcompiling Date: Tue, 14 May 2024 18:15:58 -0400 Subject: [PATCH 08/21] Fill out more of the update message method. --- cogs/clan_messages/__init__.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/cogs/clan_messages/__init__.py b/cogs/clan_messages/__init__.py index 7f27c14..04a0006 100644 --- a/cogs/clan_messages/__init__.py +++ b/cogs/clan_messages/__init__.py @@ -31,9 +31,8 @@ class ClanMessages(commands.Cog, name="Clan Messages"): await server.serve() async def hello(self): - print("we did a webhook") await self.update_messages() - return {"message": "webhook get"} + return async def update_messages(self): """ @@ -46,15 +45,24 @@ class ClanMessages(commands.Cog, name="Clan Messages"): # posting involves checking if there is a message for that particular file then editing it or creating a # new one. the message could be deleted, which would cause a new message to be sent and database updated. dirs = [ - f"{MESSAGES_DIR}/{dir_name}" + dir_name for dir_name in os.listdir(MESSAGES_DIR) if os.path.isdir(f"{MESSAGES_DIR}/{dir_name}") ] for dir_name in dirs: + dir_path = f"{MESSAGES_DIR}/{dir_name}" sorted_files = sorted( - os.listdir(dir_name), key=lambda filename: int(filename.split(".")[0]) + os.listdir(dir_path), key=lambda filename: int(filename.split(".")[0]) ) - print(sorted_files) + for file_name in sorted_files: + file_path = f"{dir_path}/{file_name}" + with open(file_path, "r", encoding="utf8") as reader: + print(reader.read(2000)) + + # get the channel from dir_name + # if doesn't exist, make one + # write or edit message + # save message id if written def repo_setup(self): """ From 36c738bfaa37fa62c065af81e365ff4d6f8bf0b4 Mon Sep 17 00:00:00 2001 From: imcompiling Date: Wed, 15 May 2024 13:27:18 -0400 Subject: [PATCH 09/21] Finish the update messages function. --- cogs/clan_messages/__init__.py | 99 +++++++++++++++++++++++++++++----- database_setup/database.sql | 4 +- 2 files changed, 88 insertions(+), 15 deletions(-) diff --git a/cogs/clan_messages/__init__.py b/cogs/clan_messages/__init__.py index 04a0006..b94f405 100644 --- a/cogs/clan_messages/__init__.py +++ b/cogs/clan_messages/__init__.py @@ -1,11 +1,15 @@ import os +import nextcord import uvicorn from fastapi import FastAPI from git import Repo from nextcord.ext import commands +from sqlalchemy import select, update +from sqlalchemy.orm import Session from config import DakinyConfig +from database_setup.sql_data_classes import Messages, engine app = FastAPI() REPO_URL = "https://gitea.cheezers.org/imcompiling/dakiny-messages.git" @@ -40,6 +44,7 @@ class ClanMessages(commands.Cog, name="Clan Messages"): """ self.origin.pull() + guild: nextcord.Guild = self.bot.get_guild(self.config.params.guild_id) # for ever directory, sort the files by number then post # posting involves checking if there is a message for that particular file then editing it or creating a @@ -49,20 +54,88 @@ class ClanMessages(commands.Cog, name="Clan Messages"): for dir_name in os.listdir(MESSAGES_DIR) if os.path.isdir(f"{MESSAGES_DIR}/{dir_name}") ] - for dir_name in dirs: - dir_path = f"{MESSAGES_DIR}/{dir_name}" - sorted_files = sorted( - os.listdir(dir_path), key=lambda filename: int(filename.split(".")[0]) - ) - for file_name in sorted_files: - file_path = f"{dir_path}/{file_name}" - with open(file_path, "r", encoding="utf8") as reader: - print(reader.read(2000)) - # get the channel from dir_name - # if doesn't exist, make one - # write or edit message - # save message id if written + with Session(engine) as session: + for dir_name in dirs: + dir_path = f"{MESSAGES_DIR}/{dir_name}" + channel = await self.get_clan_message_channel(session, dir_name, guild) + sorted_files = sorted( + os.listdir(dir_path), + key=lambda filename: int(filename.split(".")[0]), + ) + + for file_name in sorted_files: + message_name = f"{dir_name}/{file_name}" + file_path = f"{dir_path}/{file_name}" + with open(file_path, "r", encoding="utf8") as reader: + stmt = select(Messages.Id).where(Messages.Path == message_name) + msg_id = session.scalar(stmt) + if msg_id is None: + # don't have a message for this file + msg = await channel.send(reader.read(2000)) + new_message = Messages(Path=message_name, Id=msg.id) + session.add(new_message) + session.commit() + else: + try: + msg: nextcord.Message = await channel.fetch_message( + msg_id + ) + await msg.edit(reader.read(2000)) + except nextcord.NotFound: + msg = await channel.send(reader.read(2000)) + stmt = ( + update(Messages) + .where(Messages.Path == message_name) + .values(Id=msg.id) + ) + session.execute(stmt) + session.commit() + + async def get_clan_message_channel( + self, session: Session, channel_name: str, guild: nextcord.Guild + ) -> nextcord.TextChannel: + """ + Used to obtain a channel from the guild object. Searches database for an id to get the channel. If that is not + found, then searches through all channels in the guild to find one with a matching name. If that also doesn't + turn up a channel, then creates it. Adds an entry into database for last two paths. + Assumes that there is an open session to the database. + + Args: + session (Session): Open connection to database. + channel_name (str): Name of channel to obtain. + guild (nextcord.Guild): Guild object to get channel from. + + Returns: + nextcord.TextChannel: Channel object with name channel_name. + """ + stmt = select(Messages.Id).where(Messages.Path == channel_name) + channel_id = session.scalar(stmt) + if channel_id is None: + ret_channel = None + # channel is not tracked. see if it exists + channels = await guild.fetch_channels() + for channel in channels: + if ( + channel.name == channel_name + and channel.__class__ is nextcord.TextChannel + ): + ret_channel = channel + + if ret_channel is None: + # channel doesn't exist, we must create it + ret_channel = await guild.create_text_channel( + channel_name, + reason="Creating channel as required for by design of Dakiny-Messages repo.", + ) + + # record channel information + new_item = Messages(Path=channel_name, Id=ret_channel.id) + session.add(new_item) + session.commit() + return ret_channel + + return await guild.fetch_channel(channel_id) def repo_setup(self): """ diff --git a/database_setup/database.sql b/database_setup/database.sql index 7f31fcf..bab1494 100644 --- a/database_setup/database.sql +++ b/database_setup/database.sql @@ -49,6 +49,6 @@ CREATE TABLE ParticipationPoints ( ); CREATE TABLE Messages ( - FileName varchar(100) PRIMARY KEY, - MessageId bigint unsigned not null + Path varchar(100) PRIMARY KEY, + Id bigint unsigned not null ); \ No newline at end of file From b9362f7eca182bd5cf3e8a793dea98162ffe061d Mon Sep 17 00:00:00 2001 From: imcompiling Date: Wed, 15 May 2024 13:33:07 -0400 Subject: [PATCH 10/21] Handle a channel being deleted after entered into database. --- cogs/clan_messages/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cogs/clan_messages/__init__.py b/cogs/clan_messages/__init__.py index b94f405..e340251 100644 --- a/cogs/clan_messages/__init__.py +++ b/cogs/clan_messages/__init__.py @@ -135,7 +135,14 @@ class ClanMessages(commands.Cog, name="Clan Messages"): session.commit() return ret_channel - return await guild.fetch_channel(channel_id) + try: + return await guild.fetch_channel(channel_id) + except nextcord.NotFound: + # the channel was deleted for some reason + return await guild.create_text_channel( + channel_name, + reason="Creating channel as required for by design of Dakiny-Messages repo.", + ) def repo_setup(self): """ From ee6b30179108f20f41350abbd0f38319be0fddf9 Mon Sep 17 00:00:00 2001 From: imcompiling Date: Wed, 15 May 2024 13:46:13 -0400 Subject: [PATCH 11/21] Fill out some docustrings --- cogs/clan_messages/__init__.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/cogs/clan_messages/__init__.py b/cogs/clan_messages/__init__.py index e340251..f538ebb 100644 --- a/cogs/clan_messages/__init__.py +++ b/cogs/clan_messages/__init__.py @@ -1,3 +1,7 @@ +""" +File contains logic for creating a webhook listener, pulling from git, and updating all messages. +""" + import os import nextcord @@ -18,6 +22,11 @@ MESSAGES_DIR = f"{REPO_DIR}/messages" class ClanMessages(commands.Cog, name="Clan Messages"): + """ + Cog to be loaded into the bot. Used to be able to have a fastapi post listener and access to the bot at + the same time. + """ + def __init__(self, bot: commands.Bot): self.bot = bot self.config: DakinyConfig = self.bot.config @@ -29,18 +38,14 @@ class ClanMessages(commands.Cog, name="Clan Messages"): Triggers when the bot is in ready state. Adds the webhook listener to FastAPI and serves. """ await self.update_messages() - app.add_api_route("/message/", self.hello, methods=["POST"]) + app.add_api_route("/messages/", self.update_messages, methods=["POST"]) config = uvicorn.Config(app=app, port=5000, log_level="info") server = uvicorn.Server(config) await server.serve() - async def hello(self): - await self.update_messages() - return - async def update_messages(self): """ - _summary_ + Pulls from git and then updates all the discord messages as described by the files in the git repo. """ self.origin.pull() @@ -92,6 +97,8 @@ class ClanMessages(commands.Cog, name="Clan Messages"): session.execute(stmt) session.commit() + return {"message": "Successfully updated all discord messages."} + async def get_clan_message_channel( self, session: Session, channel_name: str, guild: nextcord.Guild ) -> nextcord.TextChannel: From cf5b5f80681fa545a2dce2f0ed1e9bf477b989c9 Mon Sep 17 00:00:00 2001 From: imcompiling Date: Wed, 15 May 2024 13:57:51 -0400 Subject: [PATCH 12/21] Move update clan messages into utils, a slash command will need the functionality. --- cogs/clan_messages/__init__.py | 107 ++---------------------------- utils.py | 116 ++++++++++++++++++++++++++++++++- 2 files changed, 119 insertions(+), 104 deletions(-) diff --git a/cogs/clan_messages/__init__.py b/cogs/clan_messages/__init__.py index f538ebb..fb8522f 100644 --- a/cogs/clan_messages/__init__.py +++ b/cogs/clan_messages/__init__.py @@ -9,16 +9,12 @@ import uvicorn from fastapi import FastAPI from git import Repo from nextcord.ext import commands -from sqlalchemy import select, update -from sqlalchemy.orm import Session from config import DakinyConfig -from database_setup.sql_data_classes import Messages, engine +from utils import REPO_DIR, update_clan_messages app = FastAPI() REPO_URL = "https://gitea.cheezers.org/imcompiling/dakiny-messages.git" -REPO_DIR = "./dakiny-messages" -MESSAGES_DIR = f"{REPO_DIR}/messages" class ClanMessages(commands.Cog, name="Clan Messages"): @@ -45,112 +41,17 @@ class ClanMessages(commands.Cog, name="Clan Messages"): async def update_messages(self): """ - Pulls from git and then updates all the discord messages as described by the files in the git repo. + Pulls from git and then calls method to update all the discord messages as described by the + files in the git repo. """ self.origin.pull() guild: nextcord.Guild = self.bot.get_guild(self.config.params.guild_id) - # for ever directory, sort the files by number then post - # posting involves checking if there is a message for that particular file then editing it or creating a - # new one. the message could be deleted, which would cause a new message to be sent and database updated. - dirs = [ - dir_name - for dir_name in os.listdir(MESSAGES_DIR) - if os.path.isdir(f"{MESSAGES_DIR}/{dir_name}") - ] - - with Session(engine) as session: - for dir_name in dirs: - dir_path = f"{MESSAGES_DIR}/{dir_name}" - channel = await self.get_clan_message_channel(session, dir_name, guild) - sorted_files = sorted( - os.listdir(dir_path), - key=lambda filename: int(filename.split(".")[0]), - ) - - for file_name in sorted_files: - message_name = f"{dir_name}/{file_name}" - file_path = f"{dir_path}/{file_name}" - with open(file_path, "r", encoding="utf8") as reader: - stmt = select(Messages.Id).where(Messages.Path == message_name) - msg_id = session.scalar(stmt) - if msg_id is None: - # don't have a message for this file - msg = await channel.send(reader.read(2000)) - new_message = Messages(Path=message_name, Id=msg.id) - session.add(new_message) - session.commit() - else: - try: - msg: nextcord.Message = await channel.fetch_message( - msg_id - ) - await msg.edit(reader.read(2000)) - except nextcord.NotFound: - msg = await channel.send(reader.read(2000)) - stmt = ( - update(Messages) - .where(Messages.Path == message_name) - .values(Id=msg.id) - ) - session.execute(stmt) - session.commit() + await update_clan_messages(guild) return {"message": "Successfully updated all discord messages."} - async def get_clan_message_channel( - self, session: Session, channel_name: str, guild: nextcord.Guild - ) -> nextcord.TextChannel: - """ - Used to obtain a channel from the guild object. Searches database for an id to get the channel. If that is not - found, then searches through all channels in the guild to find one with a matching name. If that also doesn't - turn up a channel, then creates it. Adds an entry into database for last two paths. - Assumes that there is an open session to the database. - - Args: - session (Session): Open connection to database. - channel_name (str): Name of channel to obtain. - guild (nextcord.Guild): Guild object to get channel from. - - Returns: - nextcord.TextChannel: Channel object with name channel_name. - """ - stmt = select(Messages.Id).where(Messages.Path == channel_name) - channel_id = session.scalar(stmt) - if channel_id is None: - ret_channel = None - # channel is not tracked. see if it exists - channels = await guild.fetch_channels() - for channel in channels: - if ( - channel.name == channel_name - and channel.__class__ is nextcord.TextChannel - ): - ret_channel = channel - - if ret_channel is None: - # channel doesn't exist, we must create it - ret_channel = await guild.create_text_channel( - channel_name, - reason="Creating channel as required for by design of Dakiny-Messages repo.", - ) - - # record channel information - new_item = Messages(Path=channel_name, Id=ret_channel.id) - session.add(new_item) - session.commit() - return ret_channel - - try: - return await guild.fetch_channel(channel_id) - except nextcord.NotFound: - # the channel was deleted for some reason - return await guild.create_text_channel( - channel_name, - reason="Creating channel as required for by design of Dakiny-Messages repo.", - ) - def repo_setup(self): """ Checks if the repo exists and creates repo object, clones if it does not. diff --git a/utils.py b/utils.py index ed43245..6ee956e 100644 --- a/utils.py +++ b/utils.py @@ -2,21 +2,25 @@ File contains util functions. """ +import os from datetime import datetime import nextcord -from sqlalchemy import select +from sqlalchemy import select, update from sqlalchemy.orm import Session from config import DakinyConfig from database_setup.sql_data_classes import ( CouncilMembers, + Messages, ParticipationPoints, Votes, engine, ) NOMINEE_SPLITTER = "========================================\n\n" +REPO_DIR = "./dakiny-messages" +MESSAGES_DIR = f"{REPO_DIR}/messages" def get_datetime(day: str) -> datetime: @@ -386,3 +390,113 @@ def record_event_host(config: DakinyConfig, user: int): points.Points += host_point_value session.commit() + + +async def update_clan_messages(guild: nextcord.Guild): + """ + Updates all clan messages to contents in MESSAGES_DIR. + Does not to any git work. Is expected to be called after git is pulled. + + for ever directory, sort the files by number then post + posting involves checking if there is a message for that particular file then editing it or creating a + new one. the message could be deleted, which would cause a new message to be sent and database updated. + + Channel fetching/management is handled by another function. + + Args: + guild (nextcord.Guild): Guild object that all channels and messages belong to. + """ + + dirs = [ + dir_name + for dir_name in os.listdir(MESSAGES_DIR) + if os.path.isdir(f"{MESSAGES_DIR}/{dir_name}") + ] + + with Session(engine) as session: + for dir_name in dirs: + dir_path = f"{MESSAGES_DIR}/{dir_name}" + channel = await get_clan_message_channel(session, dir_name, guild) + sorted_files = sorted( + os.listdir(dir_path), + key=lambda filename: int(filename.split(".")[0]), + ) + + for file_name in sorted_files: + message_name = f"{dir_name}/{file_name}" + file_path = f"{dir_path}/{file_name}" + with open(file_path, "r", encoding="utf8") as reader: + stmt = select(Messages.Id).where(Messages.Path == message_name) + msg_id = session.scalar(stmt) + if msg_id is None: + # don't have a message for this file + msg = await channel.send(reader.read(2000)) + new_message = Messages(Path=message_name, Id=msg.id) + session.add(new_message) + session.commit() + else: + try: + msg: nextcord.Message = await channel.fetch_message(msg_id) + await msg.edit(reader.read(2000)) + except nextcord.NotFound: + msg = await channel.send(reader.read(2000)) + stmt = ( + update(Messages) + .where(Messages.Path == message_name) + .values(Id=msg.id) + ) + session.execute(stmt) + session.commit() + + +async def get_clan_message_channel( + session: Session, channel_name: str, guild: nextcord.Guild +) -> nextcord.TextChannel: + """ + Used to obtain a channel from the guild object. Searches database for an id to get the channel. If that is not + found, then searches through all channels in the guild to find one with a matching name. If that also doesn't + turn up a channel, then creates it. Adds an entry into database for last two paths. + Assumes that there is an open session to the database. + + Args: + session (Session): Open connection to database. + channel_name (str): Name of channel to obtain. + guild (nextcord.Guild): Guild object to get channel from. + + Returns: + nextcord.TextChannel: Channel object with name channel_name. + """ + stmt = select(Messages.Id).where(Messages.Path == channel_name) + channel_id = session.scalar(stmt) + if channel_id is None: + ret_channel = None + # channel is not tracked. see if it exists + channels = await guild.fetch_channels() + for channel in channels: + if ( + channel.name == channel_name + and channel.__class__ is nextcord.TextChannel + ): + ret_channel = channel + + if ret_channel is None: + # channel doesn't exist, we must create it + ret_channel = await guild.create_text_channel( + channel_name, + reason="Creating channel as required for by design of Dakiny-Messages repo.", + ) + + # record channel information + new_item = Messages(Path=channel_name, Id=ret_channel.id) + session.add(new_item) + session.commit() + return ret_channel + + try: + return await guild.fetch_channel(channel_id) + except nextcord.NotFound: + # the channel was deleted for some reason + return await guild.create_text_channel( + channel_name, + reason="Creating channel as required for by design of Dakiny-Messages repo.", + ) From 52ee85f17c4c7bf710f254a67a9f749c8636eed5 Mon Sep 17 00:00:00 2001 From: imcompiling Date: Wed, 15 May 2024 14:07:27 -0400 Subject: [PATCH 13/21] Add a refresh command. --- cogs/commands/__init__.py | 2 ++ cogs/commands/refresh.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 cogs/commands/refresh.py diff --git a/cogs/commands/__init__.py b/cogs/commands/__init__.py index 013fef5..725f143 100644 --- a/cogs/commands/__init__.py +++ b/cogs/commands/__init__.py @@ -5,6 +5,7 @@ Cog loader. Adds all command cogs to the bot. from nextcord.ext import commands from cogs.commands.record import RecordCog +from cogs.commands.refresh import RefreshCog from cogs.commands.remove import RemoveCog from cogs.commands.request import RequestCog from cogs.commands.set import setCog @@ -20,6 +21,7 @@ def setup(bot: commands.Bot): """ bot.add_cog(RecordCog(bot)) + bot.add_cog(RefreshCog()) bot.add_cog(RemoveCog()) bot.add_cog(RequestCog(bot)) bot.add_cog(TestCog(bot)) diff --git a/cogs/commands/refresh.py b/cogs/commands/refresh.py new file mode 100644 index 0000000..dddfb7e --- /dev/null +++ b/cogs/commands/refresh.py @@ -0,0 +1,32 @@ +""" +Contained commands and views for refresh commands. +""" + +import nextcord +from nextcord.ext import commands + +from utils import update_clan_messages + + +class RefreshCog(commands.Cog, name="Refresh Commands"): + """ + Class holds refresh commands. + """ + + @nextcord.slash_command( + default_member_permissions=nextcord.Permissions(administrator=True) + ) + async def refresh(self, interaction: nextcord.Interaction): + """ + Parent slash command for refresh. Never called. + """ + + @refresh.subcommand(description="Force refresh all of the bot managed messages.") + async def event(self, interaction: nextcord.Interaction): + """ + Calls update_clan_messages() to update all messages. + + Args: + interaction (nextcord.Interaction): Interaction with the command. + """ + await update_clan_messages(interaction.guild) From 5726875c594506bfeec855e6e3957c6c3e85a7dc Mon Sep 17 00:00:00 2001 From: imcompiling Date: Wed, 15 May 2024 14:15:11 -0400 Subject: [PATCH 14/21] Store updated channel id. --- utils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 6ee956e..b462915 100644 --- a/utils.py +++ b/utils.py @@ -496,7 +496,15 @@ async def get_clan_message_channel( return await guild.fetch_channel(channel_id) except nextcord.NotFound: # the channel was deleted for some reason - return await guild.create_text_channel( + ret_channel = await guild.create_text_channel( channel_name, reason="Creating channel as required for by design of Dakiny-Messages repo.", ) + stmt = ( + update(Messages) + .where(Messages.Path == channel_name) + .values(Id=ret_channel.id) + ) + session.execute(stmt) + session.commit() + return ret_channel From f822ed2e2d285f3f176544d9128537c63b59ec30 Mon Sep 17 00:00:00 2001 From: imcompiling Date: Wed, 15 May 2024 14:17:27 -0400 Subject: [PATCH 15/21] Correct sub command name. Add response. --- cogs/commands/refresh.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cogs/commands/refresh.py b/cogs/commands/refresh.py index dddfb7e..72ac037 100644 --- a/cogs/commands/refresh.py +++ b/cogs/commands/refresh.py @@ -22,7 +22,7 @@ class RefreshCog(commands.Cog, name="Refresh Commands"): """ @refresh.subcommand(description="Force refresh all of the bot managed messages.") - async def event(self, interaction: nextcord.Interaction): + async def messages(self, interaction: nextcord.Interaction): """ Calls update_clan_messages() to update all messages. @@ -30,3 +30,6 @@ class RefreshCog(commands.Cog, name="Refresh Commands"): interaction (nextcord.Interaction): Interaction with the command. """ await update_clan_messages(interaction.guild) + await interaction.response.send_message( + "Messages have been refreshed.", ephemeral=True + ) From 7333100b1d00cfe68f2eebf5fa5d3880ecb230e4 Mon Sep 17 00:00:00 2001 From: imcompiling Date: Wed, 15 May 2024 14:52:02 -0400 Subject: [PATCH 16/21] Need git on the docker image now. --- Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index aff3029..6a485b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.10.6-slim -# install dependencies for mysqlclient -# RUN apt-get update && apt-get -y upgrade && apt-get -y install python3-dev default-libmysqlclient-dev build-essential +# install dependencies +RUN apt-get update && apt-get -y upgrade && apt-get -y install git RUN pip install --upgrade pip @@ -20,7 +20,4 @@ COPY database_setup database_setup/ RUN pip install -r requirements.txt -# remove build deps that aren't needed to run -# RUN apt-get -y remove python3-dev build-essential && apt-get -y autoremove - CMD ["python3", "-u", "./main.py"] \ No newline at end of file From dfd4ef9347b396267fe5b953d9df226b1c16b683 Mon Sep 17 00:00:00 2001 From: imcompiling Date: Wed, 15 May 2024 17:27:35 -0400 Subject: [PATCH 17/21] Need to defer followup on refresh because it takes a while. --- cogs/commands/refresh.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cogs/commands/refresh.py b/cogs/commands/refresh.py index 72ac037..429fa85 100644 --- a/cogs/commands/refresh.py +++ b/cogs/commands/refresh.py @@ -29,7 +29,6 @@ class RefreshCog(commands.Cog, name="Refresh Commands"): Args: interaction (nextcord.Interaction): Interaction with the command. """ + await interaction.response.defer() await update_clan_messages(interaction.guild) - await interaction.response.send_message( - "Messages have been refreshed.", ephemeral=True - ) + await interaction.followup.send("Messages have been refreshed.", ephemeral=True) From e1be3218b3b6f58694a82a183cbbf0590f3524b3 Mon Sep 17 00:00:00 2001 From: imcompiling Date: Wed, 15 May 2024 20:40:31 -0400 Subject: [PATCH 18/21] Add expose to see if that fixes issue. --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 6a485b7..4fe2b95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,4 +20,7 @@ COPY database_setup database_setup/ RUN pip install -r requirements.txt +# Expose to outside of container +EXPOSE 5000 + CMD ["python3", "-u", "./main.py"] \ No newline at end of file From 443b92dfd344a3bdd62769ce94d2c656e0115db0 Mon Sep 17 00:00:00 2001 From: imcompiling Date: Wed, 15 May 2024 20:43:19 -0400 Subject: [PATCH 19/21] Revert "Add expose to see if that fixes issue." This reverts commit e1be3218b3b6f58694a82a183cbbf0590f3524b3. --- Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4fe2b95..6a485b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,4 @@ COPY database_setup database_setup/ RUN pip install -r requirements.txt -# Expose to outside of container -EXPOSE 5000 - CMD ["python3", "-u", "./main.py"] \ No newline at end of file From 70e4a84db38f78d485edc3cae98b178d077f2550 Mon Sep 17 00:00:00 2001 From: imcompiling Date: Wed, 15 May 2024 23:16:12 -0400 Subject: [PATCH 20/21] Set uvicorn to listen on all addresses. --- cogs/clan_messages/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cogs/clan_messages/__init__.py b/cogs/clan_messages/__init__.py index fb8522f..fb1e90a 100644 --- a/cogs/clan_messages/__init__.py +++ b/cogs/clan_messages/__init__.py @@ -35,7 +35,7 @@ class ClanMessages(commands.Cog, name="Clan Messages"): """ await self.update_messages() app.add_api_route("/messages/", self.update_messages, methods=["POST"]) - config = uvicorn.Config(app=app, port=5000, log_level="info") + config = uvicorn.Config(app=app, host="0.0.0.0", port=5000, log_level="info") server = uvicorn.Server(config) await server.serve() From eedaf1c71f41b4b50d306fccbe427ddee51cf45f Mon Sep 17 00:00:00 2001 From: imcompiling Date: Thu, 16 May 2024 09:13:34 -0400 Subject: [PATCH 21/21] Don't need to refresh messages on bot start. --- cogs/clan_messages/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cogs/clan_messages/__init__.py b/cogs/clan_messages/__init__.py index fb1e90a..209e27b 100644 --- a/cogs/clan_messages/__init__.py +++ b/cogs/clan_messages/__init__.py @@ -33,7 +33,6 @@ class ClanMessages(commands.Cog, name="Clan Messages"): """ Triggers when the bot is in ready state. Adds the webhook listener to FastAPI and serves. """ - await self.update_messages() app.add_api_route("/messages/", self.update_messages, methods=["POST"]) config = uvicorn.Config(app=app, host="0.0.0.0", port=5000, log_level="info") server = uvicorn.Server(config)