From 78834f53196a6d8e6992df41a1a124098fadc5bc Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 12:16:23 +0100 Subject: [PATCH 01/22] Rename history to state as it's now storing more than just the history --- .gitignore | 2 +- matching.py | 10 ++-- matching_test.py | 26 +++++----- matchy.py | 26 ++-------- state.py | 125 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 149 insertions(+), 40 deletions(-) create mode 100644 state.py diff --git a/.gitignore b/.gitignore index 1c6e046..2296932 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ __pycache__ config.json -history.json +state.json .venv \ No newline at end of file diff --git a/matching.py b/matching.py index 49dc5c7..3b51a1d 100644 --- a/matching.py +++ b/matching.py @@ -3,7 +3,7 @@ import logging import random from datetime import datetime, timedelta from typing import Protocol, runtime_checkable -import history +import state # Number of days to step forward from the start of history for each match attempt @@ -88,7 +88,7 @@ def get_member_group_eligibility_score(member: Member, def attempt_create_groups(matchees: list[Member], - hist: history.History, + hist: state.State, oldest_relevant_ts: datetime, per_group: int) -> tuple[bool, list[list[Member]]]: """History aware group matching""" @@ -106,7 +106,7 @@ def attempt_create_groups(matchees: list[Member], matchee_matches = hist.matchees.get( str(matchee.id), {}).get("matches", {}) relevant_matches = list(int(id) for id, ts in matchee_matches.items() - if history.ts_to_datetime(ts) >= oldest_relevant_ts) + if state.ts_to_datetime(ts) >= oldest_relevant_ts) # Try every single group from the current group onwards # Progressing through the groups like this ensures we slowly fill them up with compatible people @@ -144,7 +144,7 @@ def datetime_range(start_time: datetime, increment: timedelta, end: datetime): def members_to_groups(matchees: list[Member], - hist: history.History = history.History(), + hist: state.State = state.State(), per_group: int = 3, allow_fallback: bool = False) -> list[list[Member]]: """Generate the groups from the set of matchees""" @@ -152,7 +152,7 @@ def members_to_groups(matchees: list[Member], rand = random.Random(117) # Some stable randomness # Grab the oldest timestamp - history_start = hist.oldest() or datetime.now() + history_start = hist.oldest_history() or datetime.now() # Walk from the start of time until now using the timestep increment for oldest_relevant_datetime in datetime_range(history_start, _ATTEMPT_TIMESTEP_INCREMENT, datetime.now()): diff --git a/matching_test.py b/matching_test.py index 4331fe9..8409202 100644 --- a/matching_test.py +++ b/matching_test.py @@ -5,7 +5,7 @@ import discord import pytest import random import matching -import history +import state from datetime import datetime, timedelta @@ -44,16 +44,16 @@ class Member(): return self._id -def inner_validate_members_to_groups(matchees: list[Member], hist: history.History, per_group: int): +def inner_validate_members_to_groups(matchees: list[Member], tmp_state: state.State, per_group: int): """Inner function to validate the main output of the groups function""" - groups = matching.members_to_groups(matchees, hist, per_group) + groups = matching.members_to_groups(matchees, tmp_state, per_group) # We should always have one group assert len(groups) # Log the groups to history # This will validate the internals - hist.log_groups_to_history(groups) + tmp_state.log_groups(groups) # Ensure each group contains within the bounds of expected members for group in groups: @@ -81,8 +81,8 @@ def inner_validate_members_to_groups(matchees: list[Member], hist: history.Histo ], ids=['single', "larger_groups", "100_members", "5_group", "pairs", "356_big_groups"]) def test_members_to_groups_no_history(matchees, per_group): """Test simple group matching works""" - hist = history.History() - inner_validate_members_to_groups(matchees, hist, per_group) + tmp_state = state.State() + inner_validate_members_to_groups(matchees, tmp_state, per_group) def items_found_in_lists(list_of_lists, items): @@ -95,8 +95,8 @@ def items_found_in_lists(list_of_lists, items): @pytest.mark.parametrize("history_data, matchees, per_group, checks", [ # Slightly more difficult test - # Describe a history where we previously matched up some people and ensure they don't get rematched ( + # Describe a history where we previously matched up some people and ensure they don't get rematched [ { "ts": datetime.now() - timedelta(days=1), @@ -156,13 +156,13 @@ def items_found_in_lists(list_of_lists, items): ], ids=['simple_history', 'fallback']) def test_members_to_groups_with_history(history_data, matchees, per_group, checks): """Test more advanced group matching works""" - hist = history.History() + tmp_state = state.State() # Replay the history for d in history_data: - hist.log_groups_to_history(d["groups"], d["ts"]) + tmp_state.log_groups(d["groups"], d["ts"]) - groups = inner_validate_members_to_groups(matchees, hist, per_group) + groups = inner_validate_members_to_groups(matchees, tmp_state, per_group) # Run the custom validate functions for check in checks: @@ -208,8 +208,8 @@ def test_members_to_groups_stress_test(): rand.shuffle(history_data) # Replay the history - hist = history.History() + tmp_state = state.State() for d in history_data: - hist.log_groups_to_history(d["groups"], d["ts"]) + tmp_state.log_groups(d["groups"], d["ts"]) - inner_validate_members_to_groups(matchees, hist, per_group) + inner_validate_members_to_groups(matchees, tmp_state, per_group) diff --git a/matchy.py b/matchy.py index 7c7c891..a76d78c 100755 --- a/matchy.py +++ b/matchy.py @@ -6,13 +6,13 @@ import discord from discord import app_commands from discord.ext import commands import matching -import history +import state import config import re Config = config.load() -History = history.load() +State = state.load() logger = logging.getLogger("matchy") logger.setLevel(logging.INFO) @@ -68,22 +68,6 @@ async def close(ctx: commands.Context): await bot.close() -# @bot.tree.command(description="Sign up as a matchee in this server") -# @commands.guild_only() -# async def join(interaction: discord.Interaction): -# # TODO: Sign up -# await interaction.response.send_message( -# f"Awesome, great to have you on board {interaction.user.mention}!", ephemeral=True) - - -# @bot.tree.command(description="Leave the matchee list in this server") -# @commands.guild_only() -# async def leave(interaction: discord.Interaction): -# # TODO: Remove the user -# await interaction.response.send_message( -# f"No worries, see you soon {interaction.user.mention}!", ephemeral=True) - - @bot.tree.command(description="Match up matchees") @commands.guild_only() @app_commands.describe(members_min="Minimum matchees per match (defaults to 3)", @@ -114,7 +98,7 @@ async def match(interaction: discord.Interaction, members_min: int = None, match matchees = list( m for m in interaction.channel.members if matchee in m.roles) groups = matching.members_to_groups( - matchees, History, members_min, allow_fallback=True) + matchees, State, members_min, allow_fallback=True) # Post about all the groups with a button to send to the channel groups_list = '\n'.join(matching.group_to_message(g) for g in groups) @@ -173,7 +157,7 @@ class DynamicGroupButton(discord.ui.DynamicItem[discord.ui.Button], matchees = list( m for m in interaction.channel.members if matchee in m.roles) groups = matching.members_to_groups( - matchees, History, self.min, allow_fallback=True) + matchees, State, self.min, allow_fallback=True) # Send the groups for msg in (matching.group_to_message(g) for g in groups): @@ -183,7 +167,7 @@ class DynamicGroupButton(discord.ui.DynamicItem[discord.ui.Button], await interaction.channel.send("That's all folks, happy matching and remember - DFTBA!") # Save the groups to the history - History.save_groups_to_history(groups) + State.save_groups(groups) logger.info("Done. Matched %s matchees into %s groups.", len(matchees), len(groups)) diff --git a/state.py b/state.py new file mode 100644 index 0000000..524943a --- /dev/null +++ b/state.py @@ -0,0 +1,125 @@ +"""Store bot state""" +import os +from datetime import datetime +from schema import Schema, And, Use, Optional +from typing import Protocol +import files +import copy + +_FILE = "state.json" + +# Warning: Changing any of the below needs proper thought to ensure backwards compatibility +_DEFAULT_DICT = { + "history": {}, + "matchees": {} +} +_TIME_FORMAT = "%a %b %d %H:%M:%S %Y" +_SCHEMA = Schema( + { + Optional("history"): { + Optional(str): { # a datetime + "groups": [ + { + "members": [ + # The ID of each matchee in the match + And(Use(int)) + ] + } + ] + } + }, + Optional("matchees"): { + Optional(str): { + Optional("matches"): { + # Matchee ID and Datetime pair + Optional(str): And(Use(str)) + } + } + } + } +) + + +class Member(Protocol): + @property + def id(self) -> int: + pass + + +def ts_to_datetime(ts: str) -> datetime: + """Convert a ts to datetime using the internal format""" + return datetime.strptime(ts, _TIME_FORMAT) + + +def validate(dict: dict): + """Initialise and validate the state""" + _SCHEMA.validate(dict) + + +class State(): + def __init__(self, data: dict = _DEFAULT_DICT): + """Initialise and validate the state""" + validate(data) + self.__dict__ = copy.deepcopy(data) + + @property + def history(self) -> list[dict]: + return self.__dict__["history"] + + @property + def matchees(self) -> dict[str, dict]: + return self.__dict__["matchees"] + + def save(self) -> None: + """Save out the state""" + files.save(_FILE, self.__dict__) + + def oldest_history(self) -> datetime: + """Grab the oldest timestamp in history""" + if not self.history: + return None + times = (ts_to_datetime(dt) for dt in self.history.keys()) + return sorted(times)[0] + + def log_groups(self, groups: list[list[Member]], ts: datetime = datetime.now()) -> None: + """Log the groups""" + tmp_state = State(self.__dict__) + ts = datetime.strftime(ts, _TIME_FORMAT) + + # Grab or create the hitory item for this set of groups + history_item = {} + tmp_state.history[ts] = history_item + history_item_groups = [] + history_item["groups"] = history_item_groups + + for group in groups: + + # Add the group data + history_item_groups.append({ + "members": list(m.id for m in group) + }) + + # Update the matchee data with the matches + for m in group: + matchee = tmp_state.matchees.get(str(m.id), {}) + matchee_matches = matchee.get("matches", {}) + + for o in (o for o in group if o.id != m.id): + matchee_matches[str(o.id)] = ts + + matchee["matches"] = matchee_matches + tmp_state.matchees[str(m.id)] = matchee + + # Validate before storing the result + validate(self.__dict__) + self.__dict__ = tmp_state.__dict__ + + def save_groups(self, groups: list[list[Member]]) -> None: + """Save out the groups to the state file""" + self.log_groups(groups) + self.save() + + +def load() -> State: + """Load the state""" + return State(files.load(_FILE) if os.path.isfile(_FILE) else _DEFAULT_DICT) From d3a22ff090fa04e3d0cf93af651ce17d35f2b501 Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 17:53:37 +0100 Subject: [PATCH 02/22] Significant set of changes * Use /join and /leave instead of roles * Use scopes to check for user rights rather than using the config file * Add /list to show the set of current people signed up * Add a bunch more testing for various things * Version both the config and the state --- README.md | 10 +-- config.py | 73 +++++++++++---- history.py | 125 -------------------------- matching.py | 100 ++++++++++++--------- matching_test.py | 99 ++++++++++++++++++-- matchy.py | 130 +++++++++++++++++---------- state.py | 230 +++++++++++++++++++++++++++++++++++++---------- state_test.py | 65 ++++++++++++++ 8 files changed, 537 insertions(+), 295 deletions(-) delete mode 100644 history.py create mode 100644 state_test.py diff --git a/README.md b/README.md index 056bacd..d0ec770 100644 --- a/README.md +++ b/README.md @@ -20,18 +20,16 @@ Only usable by `OWNER` users, reloads the config and syncs commands, or closes d Matchy is configured by a `config.json` file that takes this format: ``` { + "version": 1, "token": "<>", - "owners": [ - <> - ] } ``` User IDs can be grabbed by turning on Discord's developer mode and right clicking on a user. ## TODO * Write bot tests with [dpytest](https://dpytest.readthedocs.io/en/latest/tutorials/getting_started.html) +* Implement /pause to pause a user for a little while +* Move more constants to the config * Add scheduling functionality -* Version the config and history files -* Implement /signup rather than using roles -* Implement authorisation scopes instead of just OWNER values +* Fix logging in some sub files * Improve the weirdo \ No newline at end of file diff --git a/config.py b/config.py index 9882d99..2719d73 100644 --- a/config.py +++ b/config.py @@ -1,38 +1,77 @@ """Very simple config loading library""" from schema import Schema, And, Use import files +import os +import logging + +logger = logging.getLogger("config") +logger.setLevel(logging.INFO) _FILE = "config.json" + +# Warning: Changing any of the below needs proper thought to ensure backwards compatibility +_VERSION = 1 + + +class _Keys(): + TOKEN = "token" + VERSION = "version" + + # Removed + OWNERS = "owners" + + _SCHEMA = Schema( { - # Discord bot token - "token": And(Use(str)), + # The current version + _Keys.VERSION: And(Use(int)), - # ids of owners authorised to use owner-only commands - "owners": And(Use(list[int])), + # Discord bot token + _Keys.TOKEN: And(Use(str)), } ) +def _migrate_to_v1(d: dict): + # Owners moved to History in v1 + # Note: owners will be required to be re-added to the state.json + owners = d.pop(_Keys.OWNERS) + logger.warn( + "Migration removed owners from config, these must be re-added to the state.json") + logger.warn("Owners: %s", owners) + + +# Set of migration functions to apply +_MIGRATIONS = [ + _migrate_to_v1 +] + + class Config(): def __init__(self, data: dict): """Initialise and validate the config""" _SCHEMA.validate(data) - self.__dict__ = data + self._dict = data @property def token(self) -> str: - return self.__dict__["token"] - - @property - def owners(self) -> list[int]: - return self.__dict__["owners"] - - def reload(self) -> None: - """Reload the config back into the dict""" - self.__dict__ = load().__dict__ + return self._dict["token"] -def load() -> Config: - """Load the config""" - return Config(files.load(_FILE)) +def _migrate(dict: dict): + """Migrate a dict through versions""" + version = dict.get("version", 0) + for i in range(version, _VERSION): + _MIGRATIONS[i](dict) + dict["version"] = _VERSION + + +def load_from_file(file: str = _FILE) -> Config: + """ + Load the state from a file + Apply any required migrations + """ + assert os.path.isfile(file) + loaded = files.load(file) + _migrate(loaded) + return Config(loaded) diff --git a/history.py b/history.py deleted file mode 100644 index 06b77e4..0000000 --- a/history.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Store matching history""" -import os -from datetime import datetime -from schema import Schema, And, Use, Optional -from typing import Protocol -import files -import copy - -_FILE = "history.json" - -# Warning: Changing any of the below needs proper thought to ensure backwards compatibility -_DEFAULT_DICT = { - "history": {}, - "matchees": {} -} -_TIME_FORMAT = "%a %b %d %H:%M:%S %Y" -_SCHEMA = Schema( - { - Optional("history"): { - Optional(str): { # a datetime - "groups": [ - { - "members": [ - # The ID of each matchee in the match - And(Use(int)) - ] - } - ] - } - }, - Optional("matchees"): { - Optional(str): { - Optional("matches"): { - # Matchee ID and Datetime pair - Optional(str): And(Use(str)) - } - } - } - } -) - - -class Member(Protocol): - @property - def id(self) -> int: - pass - - -def ts_to_datetime(ts: str) -> datetime: - """Convert a ts to datetime using the history format""" - return datetime.strptime(ts, _TIME_FORMAT) - - -def validate(dict: dict): - """Initialise and validate the history""" - _SCHEMA.validate(dict) - - -class History(): - def __init__(self, data: dict = _DEFAULT_DICT): - """Initialise and validate the history""" - validate(data) - self.__dict__ = copy.deepcopy(data) - - @property - def history(self) -> list[dict]: - return self.__dict__["history"] - - @property - def matchees(self) -> dict[str, dict]: - return self.__dict__["matchees"] - - def save(self) -> None: - """Save out the history""" - files.save(_FILE, self.__dict__) - - def oldest(self) -> datetime: - """Grab the oldest timestamp in history""" - if not self.history: - return None - times = (ts_to_datetime(dt) for dt in self.history.keys()) - return sorted(times)[0] - - def log_groups_to_history(self, groups: list[list[Member]], ts: datetime = datetime.now()) -> None: - """Log the groups""" - tmp_history = History(self.__dict__) - ts = datetime.strftime(ts, _TIME_FORMAT) - - # Grab or create the hitory item for this set of groups - history_item = {} - tmp_history.history[ts] = history_item - history_item_groups = [] - history_item["groups"] = history_item_groups - - for group in groups: - - # Add the group data - history_item_groups.append({ - "members": list(m.id for m in group) - }) - - # Update the matchee data with the matches - for m in group: - matchee = tmp_history.matchees.get(str(m.id), {}) - matchee_matches = matchee.get("matches", {}) - - for o in (o for o in group if o.id != m.id): - matchee_matches[str(o.id)] = ts - - matchee["matches"] = matchee_matches - tmp_history.matchees[str(m.id)] = matchee - - # Validate before storing the result - validate(self.__dict__) - self.__dict__ = tmp_history.__dict__ - - def save_groups_to_history(self, groups: list[list[Member]]) -> None: - """Save out the groups to the history file""" - self.log_groups_to_history(groups) - self.save() - - -def load() -> History: - """Load the history""" - return History(files.load(_FILE) if os.path.isfile(_FILE) else _DEFAULT_DICT) diff --git a/matching.py b/matching.py index 3b51a1d..4f9b31d 100644 --- a/matching.py +++ b/matching.py @@ -1,6 +1,5 @@ """Utility functions for matchy""" import logging -import random from datetime import datetime, timedelta from typing import Protocol, runtime_checkable import state @@ -9,17 +8,16 @@ import state # Number of days to step forward from the start of history for each match attempt _ATTEMPT_TIMESTEP_INCREMENT = timedelta(days=7) -# Attempts for each of those time periods -_ATTEMPTS_PER_TIMESTEP = 3 -# Various eligability scoring factors for group meetups -_SCORE_CURRENT_MEMBERS = 2**1 -_SCORE_REPEAT_ROLE = 2**2 -_SCORE_REPEAT_MATCH = 2**3 -_SCORE_EXTRA_MEMBERS = 2**4 +class _ScoreFactors(int): + """Various eligability scoring factors for group meetups""" + REPEAT_ROLE = 2**2 + REPEAT_MATCH = 2**3 + EXTRA_MEMBER = 2**5 + + # Scores higher than this are fully rejected + UPPER_THRESHOLD = 2**6 -# Scores higher than this are fully rejected -_SCORE_UPPER_THRESHOLD = 2**6 logger = logging.getLogger("matching") logger.setLevel(logging.INFO) @@ -69,33 +67,42 @@ def members_to_groups_simple(matchees: list[Member], per_group: int) -> tuple[bo def get_member_group_eligibility_score(member: Member, group: list[Member], - relevant_matches: list[int], - per_group: int) -> int: + prior_matches: list[int], + per_group: int) -> float: """Rates a member against a group""" - rating = len(group) * _SCORE_CURRENT_MEMBERS + # An empty group is a "perfect" score atomatically + rating = 0 + if not group: + return rating - repeat_meetings = sum(m.id in relevant_matches for m in group) - rating += repeat_meetings * _SCORE_REPEAT_MATCH + # Add score based on prior matchups of this user + rating += sum(m.id in prior_matches for m in group) * \ + _ScoreFactors.REPEAT_MATCH - repeat_roles = sum(r in member.roles for r in (m.roles for m in group)) - rating += (repeat_roles * _SCORE_REPEAT_ROLE) + # Calculate the number of roles that match + all_role_ids = set(r.id for mr in [r.roles for r in group] for r in mr) + member_role_ids = [r.id for r in member.roles] + repeat_roles = sum(id in member_role_ids for id in all_role_ids) + rating += repeat_roles * _ScoreFactors.REPEAT_ROLE - extra_members = len(group) - per_group - if extra_members > 0: - rating += extra_members * _SCORE_EXTRA_MEMBERS + # Add score based on the number of extra members + # Calculate the member offset (+1 for this user) + extra_members = (len(group) - per_group) + 1 + if extra_members >= 0: + rating += extra_members * _ScoreFactors.EXTRA_MEMBER return rating def attempt_create_groups(matchees: list[Member], - hist: state.State, + current_state: state.State, oldest_relevant_ts: datetime, per_group: int) -> tuple[bool, list[list[Member]]]: """History aware group matching""" num_groups = max(len(matchees)//per_group, 1) # Set up the groups in place - groups = list([] for _ in range(num_groups)) + groups = [[] for _ in range(num_groups)] matchees_left = matchees.copy() @@ -103,21 +110,21 @@ def attempt_create_groups(matchees: list[Member], while matchees_left: # Get the next matchee to place matchee = matchees_left.pop() - matchee_matches = hist.matchees.get( - str(matchee.id), {}).get("matches", {}) - relevant_matches = list(int(id) for id, ts in matchee_matches.items() - if state.ts_to_datetime(ts) >= oldest_relevant_ts) + matchee_matches = current_state.get_user_matches(matchee.id) + relevant_matches = [int(id) for id, ts + in matchee_matches.items() + if state.ts_to_datetime(ts) >= oldest_relevant_ts] # Try every single group from the current group onwards # Progressing through the groups like this ensures we slowly fill them up with compatible people - scores: list[tuple[int, int]] = [] + scores: list[tuple[int, float]] = [] for group in groups: score = get_member_group_eligibility_score( - matchee, group, relevant_matches, num_groups) + matchee, group, relevant_matches, per_group) # If the score isn't too high, consider this group - if score <= _SCORE_UPPER_THRESHOLD: + if score <= _ScoreFactors.UPPER_THRESHOLD: scores.append((group, score)) # Optimisation: @@ -143,31 +150,41 @@ def datetime_range(start_time: datetime, increment: timedelta, end: datetime): current += increment +def iterate_all_shifts(list: list): + """Yields each shifted variation of the input list""" + yield list + for _ in range(len(list)-1): + list = list[1:] + [list[0]] + yield list + + def members_to_groups(matchees: list[Member], hist: state.State = state.State(), per_group: int = 3, allow_fallback: bool = False) -> list[list[Member]]: """Generate the groups from the set of matchees""" attempts = 0 # Tracking for logging purposes - rand = random.Random(117) # Some stable randomness + num_groups = len(matchees)//per_group + + # Bail early if there's no-one to match + if not matchees: + return [] # Grab the oldest timestamp - history_start = hist.oldest_history() or datetime.now() + history_start = hist.get_oldest_timestamp() or datetime.now() # Walk from the start of time until now using the timestep increment for oldest_relevant_datetime in datetime_range(history_start, _ATTEMPT_TIMESTEP_INCREMENT, datetime.now()): - # Have a few attempts before stepping forward in time - for _ in range(_ATTEMPTS_PER_TIMESTEP): - - rand.shuffle(matchees) # Shuffle the matchees each attempt + # Attempt with each starting matchee + for shifted_matchees in iterate_all_shifts(matchees): attempts += 1 groups = attempt_create_groups( - matchees, hist, oldest_relevant_datetime, per_group) + shifted_matchees, hist, oldest_relevant_datetime, per_group) # Fail the match if our groups aren't big enough - if (len(matchees)//per_group) <= 1 or (groups and all(len(g) >= per_group for g in groups)): + if num_groups <= 1 or (groups and all(len(g) >= per_group for g in groups)): logger.info("Matched groups after %s attempt(s)", attempts) return groups @@ -176,6 +193,10 @@ def members_to_groups(matchees: list[Member], logger.info("Fell back to simple groups after %s attempt(s)", attempts) return members_to_groups_simple(matchees, per_group) + # Simply assert false, this should never happen + # And should be caught by tests + assert False + def group_to_message(group: list[Member]) -> str: """Get the message to send for each group""" @@ -185,8 +206,3 @@ def group_to_message(group: list[Member]) -> str: else: mentions = mentions[0] return f"Matched up {mentions}!" - - -def get_role_from_guild(guild: Guild, role: str) -> Role: - """Find a role in a guild""" - return next((r for r in guild.roles if r.name == role), None) diff --git a/matching_test.py b/matching_test.py index 8409202..5a40897 100644 --- a/matching_test.py +++ b/matching_test.py @@ -1,5 +1,5 @@ """ - Test functions for Matchy + Test functions for the matching module """ import discord import pytest @@ -30,6 +30,7 @@ class Role(): class Member(): def __init__(self, id: int, roles: list[Role] = []): self._id = id + self._roles = roles @property def mention(self) -> str: @@ -37,7 +38,7 @@ class Member(): @property def roles(self) -> list[Role]: - return [] + return self._roles @property def id(self) -> int: @@ -153,7 +154,59 @@ def items_found_in_lists(list_of_lists, items): # Nothing specific to validate ] ), -], ids=['simple_history', 'fallback']) + # Specific test pulled out of the stress test + ( + [ + { + "ts": datetime.now() - timedelta(days=4), + "groups": [ + [Member(i) for i in [1, 2, 3, 4, 5, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15]] + ] + }, + { + "ts": datetime.now() - timedelta(days=5), + "groups": [ + [Member(i) for i in [1, 2, 3, 4, 5, 6, 7, 8]] + ] + } + ], + [Member(i) for i in [1, 2, 11, 4, 12, 3, 7, 5, 8, 10, 9, 6]], + 3, + [ + # Nothing specific to validate + ] + ), + # Silly example that failued due to bad role logic + ( + [ + # No history + ], + [ + # print([(m.id, [r.id for r in m.roles]) for m in matchees]) to get the below + Member(i, [Role(r) for r in roles]) for (i, roles) in + [ + (4, [1, 2, 3, 4, 5, 6, 7, 8]), + (8, [1]), + (9, [1, 2, 3, 4, 5]), + (6, [1, 2, 3]), + (11, [1, 2, 3]), + (7, [1, 2, 3, 4, 5, 6, 7]), + (1, [1, 2, 3, 4]), + (5, [1, 2, 3, 4, 5]), + (12, [1, 2, 3, 4]), + (10, [1]), + (13, [1, 2, 3, 4, 5, 6]), + (2, [1, 2, 3, 4, 5, 6]), + (3, [1, 2, 3, 4, 5, 6, 7]) + ] + ], + 2, + [ + # Nothing else + ] + ) +], ids=['simple_history', 'fallback', 'example_1', 'example_2']) def test_members_to_groups_with_history(history_data, matchees, per_group, checks): """Test more advanced group matching works""" tmp_state = state.State() @@ -180,8 +233,8 @@ def test_members_to_groups_stress_test(): # Slowly ramp a randomized shuffled list of members with randomised roles for num_members in range(1, 5): - matchees = list(Member(i, list(Role(i) for i in range(1, rand.randint(2, num_members*2 + 1)))) - for i in range(1, rand.randint(2, num_members*10 + 1))) + matchees = [Member(i, [Role(i) for i in range(1, rand.randint(2, num_members*2 + 1))]) + for i in range(1, rand.randint(2, num_members*10 + 1))] rand.shuffle(matchees) for num_history in range(8): @@ -190,14 +243,14 @@ def test_members_to_groups_stress_test(): # Start some time from now to the past time = datetime.now() - timedelta(days=rand.randint(0, num_history*5)) history_data = [] - for x in range(0, num_history): + for _ in range(0, num_history): run = { "ts": time } groups = [] for y in range(1, num_history): - groups.append(list(Member(i) - for i in range(1, max(num_members, rand.randint(2, num_members*10 + 1))))) + groups.append([Member(i) + for i in range(1, max(num_members, rand.randint(2, num_members*10 + 1)))]) run["groups"] = groups history_data.append(run) @@ -212,4 +265,32 @@ def test_members_to_groups_stress_test(): for d in history_data: tmp_state.log_groups(d["groups"], d["ts"]) - inner_validate_members_to_groups(matchees, tmp_state, per_group) + inner_validate_members_to_groups( + matchees, tmp_state, per_group) + + +def test_auth_scopes(): + tmp_state = state.State() + + id = "1" + tmp_state.set_user_scope(id, state.AuthScope.OWNER) + assert tmp_state.get_user_has_scope(id, state.AuthScope.OWNER) + assert tmp_state.get_user_has_scope(id, state.AuthScope.MATCHER) + + id = "2" + tmp_state.set_user_scope(id, state.AuthScope.MATCHER) + assert not tmp_state.get_user_has_scope(id, state.AuthScope.OWNER) + assert tmp_state.get_user_has_scope(id, state.AuthScope.MATCHER) + + tmp_state.validate() + + +def test_iterate_all_shifts(): + original = [1, 2, 3, 4] + lists = [val for val in matching.iterate_all_shifts(original)] + assert lists == [ + [1, 2, 3, 4], + [2, 3, 4, 1], + [3, 4, 1, 2], + [4, 1, 2, 3], + ] diff --git a/matchy.py b/matchy.py index a76d78c..7f5fc8e 100755 --- a/matchy.py +++ b/matchy.py @@ -11,8 +11,11 @@ import config import re -Config = config.load() -State = state.load() +STATE_FILE = "state.json" +CONFIG_FILE = "config.json" + +Config = config.load_from_file(CONFIG_FILE) +State = state.load_from_file(STATE_FILE) logger = logging.getLogger("matchy") logger.setLevel(logging.INFO) @@ -39,7 +42,7 @@ async def on_ready(): def owner_only(ctx: commands.Context) -> bool: """Checks the author is an owner""" - return ctx.message.author.id in Config.owners + return State.get_user_has_scope(ctx.message.author.id, state.AuthScope.OWNER) @bot.command() @@ -47,9 +50,10 @@ def owner_only(ctx: commands.Context) -> bool: @commands.check(owner_only) async def sync(ctx: commands.Context): """Handle sync command""" - msg = await ctx.reply("Reloading config...", ephemeral=True) - Config.reload() - logger.info("Reloaded config") + msg = await ctx.reply("Reloading state...", ephemeral=True) + global State + State = state.load_from_file(STATE_FILE) + logger.info("Reloaded state") await msg.edit(content="Syncing commands...") synced = await bot.tree.sync() @@ -68,96 +72,112 @@ async def close(ctx: commands.Context): await bot.close() +@bot.tree.command(description="Join the matchees for this channel") +@commands.guild_only() +async def join(interaction: discord.Interaction): + State.set_use_active_in_channel( + interaction.user.id, interaction.channel.id) + state.save_to_file(State, STATE_FILE) + await interaction.response.send_message( + f"Roger roger {interaction.user.mention}!\n" + + f"Added you to {interaction.channel.mention}!", + ephemeral=True, silent=True) + + +@bot.tree.command(description="Leave the matchees for this channel") +@commands.guild_only() +async def leave(interaction: discord.Interaction): + State.set_use_active_in_channel( + interaction.user.id, interaction.channel.id, False) + state.save_to_file(State, STATE_FILE) + await interaction.response.send_message( + f"No worries {interaction.user.mention}. Come back soon :)", ephemeral=True, silent=True) + + +@bot.tree.command(description="List the matchees for this channel") +@commands.guild_only() +async def list(interaction: discord.Interaction): + matchees = get_matchees_in_channel(interaction.channel) + mentions = [m.mention for m in matchees] + msg = "Current matchees in this channel:\n" + \ + f"{', '.join(mentions[:-1])} and {mentions[-1]}" + await interaction.response.send_message(msg, ephemeral=True, silent=True) + + @bot.tree.command(description="Match up matchees") @commands.guild_only() -@app_commands.describe(members_min="Minimum matchees per match (defaults to 3)", - matchee_role="Role for matchees (defaults to @Matchee)") -async def match(interaction: discord.Interaction, members_min: int = None, matchee_role: str = None): +@app_commands.describe(members_min="Minimum matchees per match (defaults to 3)") +async def match(interaction: discord.Interaction, members_min: int = None): """Match groups of channel members""" - logger.info("Handling request '/match group_min=%s matchee_role=%s'", - members_min, matchee_role) + logger.info("Handling request '/match group_min=%s", members_min) logger.info("User %s from %s in #%s", interaction.user, interaction.guild.name, interaction.channel.name) # Sort out the defaults, if not specified they'll come in as None if not members_min: members_min = 3 - if not matchee_role: - matchee_role = "Matchee" - # Grab the roles and verify the given role - matcher = matching.get_role_from_guild(interaction.guild, "Matcher") - matcher = matcher and matcher in interaction.user.roles - matchee = matching.get_role_from_guild(interaction.guild, matchee_role) - if not matchee: - await interaction.response.send_message(f"Server is missing '{matchee_role}' role :(", ephemeral=True) + # Grab the groups + groups = active_members_to_groups(interaction.channel, members_min) + + # Let the user know when there's nobody to match + if not groups: + await interaction.response.send_message("Nobody to match up :(", ephemeral=True, silent=True) return - # Create some example groups to show the user - matchees = list( - m for m in interaction.channel.members if matchee in m.roles) - groups = matching.members_to_groups( - matchees, State, members_min, allow_fallback=True) - # Post about all the groups with a button to send to the channel groups_list = '\n'.join(matching.group_to_message(g) for g in groups) msg = f"Roger! I've generated example groups for ya:\n\n{groups_list}" view = discord.utils.MISSING - if not matcher: + if State.get_user_has_scope(interaction.user.id, state.AuthScope.MATCHER): # Let a non-matcher know why they don't have the button - msg += "\n\nYou'll need the 'Matcher' role to post this to the channel, sorry!" + msg += f"\n\nYou'll need the {state.AuthScope.MATCHER} scope to post this to the channel, sorry!" else: # Otherwise set up the button msg += "\n\nClick the button to match up groups and send them to the channel.\n" view = discord.ui.View(timeout=None) - view.add_item(DynamicGroupButton(members_min, matchee_role)) + view.add_item(DynamicGroupButton(members_min)) await interaction.response.send_message(msg, ephemeral=True, silent=True, view=view) logger.info("Done.") +# Increment when adjusting the custom_id so we don't confuse old users +_BUTTON_CUSTOM_ID_VERSION = 1 + + class DynamicGroupButton(discord.ui.DynamicItem[discord.ui.Button], - template=r'match:min:(?P[0-9]+):role:(?P[@\w\s]+)'): - def __init__(self, min: int, role: str) -> None: + template=f'match:v{_BUTTON_CUSTOM_ID_VERSION}:' + r'min:(?P[0-9]+)'): + def __init__(self, min: int) -> None: super().__init__( discord.ui.Button( label='Match Groups!', style=discord.ButtonStyle.blurple, - custom_id=f'match:min:{min}:role:{role}', + custom_id=f'match:min:{min}', ) ) self.min: int = min - self.role: int = role # This is called when the button is clicked and the custom_id matches the template. @classmethod async def from_custom_id(cls, interaction: discord.Interaction, item: discord.ui.Button, match: re.Match[str], /): min = int(match['min']) - role = str(match['role']) - return cls(min, role) + return cls(min) async def callback(self, interaction: discord.Interaction) -> None: """Match up people when the button is pressed""" - logger.info("Handling button press min=%s role=%s'", - self.min, self.role) + logger.info("Handling button press min=%s", self.min) logger.info("User %s from %s in #%s", interaction.user, interaction.guild.name, interaction.channel.name) # Let the user know we've recieved the message await interaction.response.send_message(content="Matchy is matching matchees...", ephemeral=True) - # Grab the role - matchee = matching.get_role_from_guild(interaction.guild, self.role) - - # Create our groups! - matchees = list( - m for m in interaction.channel.members if matchee in m.roles) - groups = matching.members_to_groups( - matchees, State, self.min, allow_fallback=True) + groups = active_members_to_groups(interaction.channel, self.min) # Send the groups for msg in (matching.group_to_message(g) for g in groups): @@ -167,10 +187,26 @@ class DynamicGroupButton(discord.ui.DynamicItem[discord.ui.Button], await interaction.channel.send("That's all folks, happy matching and remember - DFTBA!") # Save the groups to the history - State.save_groups(groups) + State.log_groups(groups) + state.save_to_file(State, STATE_FILE) - logger.info("Done. Matched %s matchees into %s groups.", - len(matchees), len(groups)) + logger.info("Done! Matched into %s groups.", len(groups)) + + +def get_matchees_in_channel(channel: discord.channel): + """Fetches the matchees in a channel""" + # Gather up the prospective matchees + return [m for m in channel.members if State.get_user_active_in_channel(m.id, channel.id)] + + +def active_members_to_groups(channel: discord.channel, min_members: int): + """Helper to create groups from channel members""" + + # Gather up the prospective matchees + matchees = get_matchees_in_channel(channel) + + # Create our groups! + return matching.members_to_groups(matchees, State, min_members, allow_fallback=True) if __name__ == "__main__": diff --git a/state.py b/state.py index 524943a..344a00f 100644 --- a/state.py +++ b/state.py @@ -5,22 +5,65 @@ from schema import Schema, And, Use, Optional from typing import Protocol import files import copy +import logging + +logger = logging.getLogger("state") +logger.setLevel(logging.INFO) -_FILE = "state.json" # Warning: Changing any of the below needs proper thought to ensure backwards compatibility -_DEFAULT_DICT = { - "history": {}, - "matchees": {} -} +_VERSION = 1 + + +def _migrate_to_v1(d: dict): + logger.info("Renaming %s to %s", _Key.MATCHEES, _Key.USERS) + d[_Key.USERS] = d[_Key.MATCHEES] + del d[_Key.MATCHEES] + + +# Set of migration functions to apply +_MIGRATIONS = [ + _migrate_to_v1 +] + + +class AuthScope(str): + """Various auth scopes""" + OWNER = "owner" + MATCHER = "matcher" + + +class _Key(str): + """Various keys used in the schema""" + HISTORY = "history" + GROUPS = "groups" + MEMBERS = "members" + USERS = "users" + SCOPES = "scopes" + MATCHES = "matches" + ACTIVE = "active" + CHANNELS = "channels" + REACTIVATE = "reactivate" + VERSION = "version" + + # Unused + MATCHEES = "matchees" + + _TIME_FORMAT = "%a %b %d %H:%M:%S %Y" + + _SCHEMA = Schema( { - Optional("history"): { - Optional(str): { # a datetime - "groups": [ + # The current version + _Key.VERSION: And(Use(int)), + + Optional(_Key.HISTORY): { + # A datetime + Optional(str): { + _Key.GROUPS: [ { - "members": [ + _Key.MEMBERS: [ # The ID of each matchee in the match And(Use(int)) ] @@ -28,17 +71,33 @@ _SCHEMA = Schema( ] } }, - Optional("matchees"): { + Optional(_Key.USERS): { Optional(str): { - Optional("matches"): { + Optional(_Key.SCOPES): And(Use(list[str])), + Optional(_Key.MATCHES): { # Matchee ID and Datetime pair Optional(str): And(Use(str)) + }, + Optional(_Key.CHANNELS): { + # The channel ID + Optional(str): { + # Whether the user is signed up in this channel + _Key.ACTIVE: And(Use(bool)), + } } } - } + }, } ) +# Empty but schema-valid internal dict +_EMPTY_DICT = { + _Key.HISTORY: {}, + _Key.USERS: {}, + _Key.VERSION: _VERSION +} +assert _SCHEMA.validate(_EMPTY_DICT) + class Member(Protocol): @property @@ -51,75 +110,148 @@ def ts_to_datetime(ts: str) -> datetime: return datetime.strptime(ts, _TIME_FORMAT) -def validate(dict: dict): - """Initialise and validate the state""" - _SCHEMA.validate(dict) - - class State(): - def __init__(self, data: dict = _DEFAULT_DICT): + def __init__(self, data: dict = _EMPTY_DICT): """Initialise and validate the state""" - validate(data) - self.__dict__ = copy.deepcopy(data) + self.validate(data) + self._dict = copy.deepcopy(data) @property - def history(self) -> list[dict]: - return self.__dict__["history"] + def _history(self) -> dict[str]: + return self._dict[_Key.HISTORY] @property - def matchees(self) -> dict[str, dict]: - return self.__dict__["matchees"] + def _users(self) -> dict[str]: + return self._dict[_Key.USERS] - def save(self) -> None: - """Save out the state""" - files.save(_FILE, self.__dict__) + def validate(self, dict: dict = None): + """Initialise and validate a state dict""" + if not dict: + dict = self._dict + _SCHEMA.validate(dict) - def oldest_history(self) -> datetime: + def get_oldest_timestamp(self) -> datetime: """Grab the oldest timestamp in history""" - if not self.history: - return None - times = (ts_to_datetime(dt) for dt in self.history.keys()) - return sorted(times)[0] + times = (ts_to_datetime(dt) for dt in self._history.keys()) + return next(times, None) + + def get_user_matches(self, id: int) -> list[int]: + return self._users.get(str(id), {}).get(_Key.MATCHES, {}) def log_groups(self, groups: list[list[Member]], ts: datetime = datetime.now()) -> None: """Log the groups""" - tmp_state = State(self.__dict__) + tmp_state = State(self._dict) ts = datetime.strftime(ts, _TIME_FORMAT) # Grab or create the hitory item for this set of groups history_item = {} - tmp_state.history[ts] = history_item + tmp_state._history[ts] = history_item history_item_groups = [] - history_item["groups"] = history_item_groups + history_item[_Key.GROUPS] = history_item_groups for group in groups: # Add the group data history_item_groups.append({ - "members": list(m.id for m in group) + _Key.MEMBERS: [m.id for m in group] }) # Update the matchee data with the matches for m in group: - matchee = tmp_state.matchees.get(str(m.id), {}) - matchee_matches = matchee.get("matches", {}) + matchee = tmp_state._users.get(str(m.id), {}) + matchee_matches = matchee.get(_Key.MATCHES, {}) for o in (o for o in group if o.id != m.id): matchee_matches[str(o.id)] = ts - matchee["matches"] = matchee_matches - tmp_state.matchees[str(m.id)] = matchee + matchee[_Key.MATCHES] = matchee_matches + tmp_state._users[str(m.id)] = matchee # Validate before storing the result - validate(self.__dict__) - self.__dict__ = tmp_state.__dict__ + tmp_state.validate() + self._dict = tmp_state._dict - def save_groups(self, groups: list[list[Member]]) -> None: - """Save out the groups to the state file""" - self.log_groups(groups) - self.save() + def set_user_scope(self, id: str, scope: str, value: bool = True): + """Add an auth scope to a user""" + # Dive in + user = self._users.get(str(id), {}) + scopes = user.get(_Key.SCOPES, []) + + # Set the value + if value and scope not in scopes: + scopes.append(scope) + elif not value and scope in scopes: + scopes.remove(scope) + + # Roll out + user[_Key.SCOPES] = scopes + self._users[id] = user + + def get_user_has_scope(self, id: str, scope: str) -> bool: + """ + Check if a user has an auth scope + "owner" users have all scopes + """ + user = self._users.get(str(id), {}) + scopes = user.get(_Key.SCOPES, []) + return AuthScope.OWNER in scopes or scope in scopes + + def set_use_active_in_channel(self, id: str, channel_id: str, active: bool = True): + """Set a user as active (or not) on a given channel""" + # Dive in + user = self._users.get(str(id), {}) + channels = user.get(_Key.CHANNELS, {}) + channel = channels.get(str(channel_id), {}) + + # Set the value + channel[_Key.ACTIVE] = active + + # Unroll + channels[str(channel_id)] = channel + user[_Key.CHANNELS] = channels + self._users[str(id)] = user + + def get_user_active_in_channel(self, id: str, channel_id: str) -> bool: + """Get a users active channels""" + user = self._users.get(str(id), {}) + channels = user.get(_Key.CHANNELS, {}) + return str(channel_id) in [channel for (channel, props) in channels.items() if props.get(_Key.ACTIVE, False)] + + @property + def dict_internal(self) -> dict: + """Only to be used to get the internal dict as a copy""" + return copy.deepcopy(self._dict) -def load() -> State: - """Load the state""" - return State(files.load(_FILE) if os.path.isfile(_FILE) else _DEFAULT_DICT) +def _migrate(dict: dict): + """Migrate a dict through versions""" + version = dict.get("version", 0) + for i in range(version, _VERSION): + logger.info("Migrating from v%s to v%s", version, version+1) + _MIGRATIONS[i](dict) + dict[_Key.VERSION] = _VERSION + + +def load_from_file(file: str) -> State: + """ + Load the state from a file + Apply any required migrations + """ + loaded = _EMPTY_DICT + + # If there's a file load it and try to migrate + if os.path.isfile(file): + loaded = files.load(file) + _migrate(loaded) + + st = State(loaded) + + # Save out the migrated (or new) file + files.save(file, st._dict) + + return st + + +def save_to_file(state: State, file: str): + """Saves the state out to a file""" + files.save(file, state.dict_internal) diff --git a/state_test.py b/state_test.py new file mode 100644 index 0000000..bd97648 --- /dev/null +++ b/state_test.py @@ -0,0 +1,65 @@ +""" + Test functions for the state module +""" +import state +import tempfile +import os + + +def test_basic_state(): + """Simple validate basic state load""" + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, 'tmp.json') + state.load_from_file(path) + + +def test_simple_load_reload(): + """Test a basic load, save, reload""" + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, 'tmp.json') + st = state.load_from_file(path) + state.save_to_file(st, path) + + st = state.load_from_file(path) + state.save_to_file(st, path) + st = state.load_from_file(path) + + +def test_authscope(): + """Test setting and getting an auth scope""" + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, 'tmp.json') + st = state.load_from_file(path) + state.save_to_file(st, path) + + assert not st.get_user_has_scope(1, state.AuthScope.MATCHER) + + st = state.load_from_file(path) + st.set_user_scope(1, state.AuthScope.MATCHER) + state.save_to_file(st, path) + + st = state.load_from_file(path) + assert st.get_user_has_scope(1, state.AuthScope.MATCHER) + + st.set_user_scope(1, state.AuthScope.MATCHER, False) + assert not st.get_user_has_scope(1, state.AuthScope.MATCHER) + + +def test_channeljoin(): + """Test setting and getting an active channel""" + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, 'tmp.json') + st = state.load_from_file(path) + state.save_to_file(st, path) + + assert not st.get_user_active_in_channel(1, "2") + + st = state.load_from_file(path) + st.set_use_active_in_channel(1, "2", True) + state.save_to_file(st, path) + + st = state.load_from_file(path) + assert st.get_user_active_in_channel(1, "2") + + st.set_use_active_in_channel(1, "2", False) + assert not st.get_user_active_in_channel(1, "2") From f99f67789d84c1ec5719ce59ed01be04ea108947 Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 17:58:21 +0100 Subject: [PATCH 03/22] Update the readme for the new commands and formats --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d0ec770..c729f5e 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,14 @@ Matchy matches matchees. Matchy is a Discord bot that groups up users for fun and vibes. Matchy can be installed by clicking [here](https://discord.com/oauth2/authorize?client_id=1270849346987884696). ## Commands -### /match [group_min: int(3)] [matchee_role: str(@Matchee)] -Matches groups of users with a given role and posts those groups to the channel. Tracks historical matches and attempts to match users to make new connections with people with divergent roles, in an attempt to maximise diversity. +### /match [group_min: int(3)] +Matches groups of users in a channel and offers a button to pose those groups to the channel to users with `matcher` auth scope. Tracks historical matches and attempts to match users to make new connections with people with divergent roles, in an attempt to maximise diversity. + +### /join and /leave +Allows users to sign up and leave the group matching in the channel the command is used ### $sync and $close -Only usable by `OWNER` users, reloads the config and syncs commands, or closes down the bot. Only usable in DMs with the bot user. +Only usable by `OWNER` users, reloads the config and syncs commands, or closes down the bot. Only usable in DMs with the bot user. ## Dependencies * `python3` - Obviously @@ -24,12 +27,11 @@ Matchy is configured by a `config.json` file that takes this format: "token": "<>", } ``` -User IDs can be grabbed by turning on Discord's developer mode and right clicking on a user. ## TODO * Write bot tests with [dpytest](https://dpytest.readthedocs.io/en/latest/tutorials/getting_started.html) * Implement /pause to pause a user for a little while * Move more constants to the config * Add scheduling functionality -* Fix logging in some sub files +* Fix logging in some sub files (doesn't seem to actually be output?) * Improve the weirdo \ No newline at end of file From 22ad36fb09f7c1398c6dea3ce14d2ebfc58529d0 Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 18:04:21 +0100 Subject: [PATCH 04/22] Reorganise a little to put scripts in bin --- .github/workflows/test.yml | 7 ++----- coverage.sh => bin/coverage.sh | 0 run.sh => bin/run.sh | 0 bin/test.sh | 7 +++++++ 4 files changed, 9 insertions(+), 5 deletions(-) rename coverage.sh => bin/coverage.sh (100%) rename run.sh => bin/run.sh (100%) create mode 100755 bin/test.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b041b56..3ccdaff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,12 +18,9 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -r requirements.txt - - name: Analysing the code with flake8 + - name: Run tests run: | - flake8 --max-line-length 120 $(git ls-files '*.py') - - name: Run tests with pytest - run: | - pytest + bash bin/test.sh - name: Update release branch if: github.ref == 'refs/heads/main' run: | diff --git a/coverage.sh b/bin/coverage.sh similarity index 100% rename from coverage.sh rename to bin/coverage.sh diff --git a/run.sh b/bin/run.sh similarity index 100% rename from run.sh rename to bin/run.sh diff --git a/bin/test.sh b/bin/test.sh new file mode 100755 index 0000000..e23c9b8 --- /dev/null +++ b/bin/test.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +# Check formatting and linting +flake8 --max-line-length 120 $(git ls-files '*.py') + +# Run pytest +pytest \ No newline at end of file From 129721eb509ddeabf58af83c16d6c7e739798ce4 Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 18:05:28 +0100 Subject: [PATCH 05/22] Move python files into py dir --- .vscode/launch.json | 2 +- bin/run.sh | 2 +- config.py => py/config.py | 0 files.py => py/files.py | 0 matching.py => py/matching.py | 0 matching_test.py => py/matching_test.py | 0 matchy.py => py/matchy.py | 0 state.py => py/state.py | 0 state_test.py => py/state_test.py | 0 9 files changed, 2 insertions(+), 2 deletions(-) rename config.py => py/config.py (100%) rename files.py => py/files.py (100%) rename matching.py => py/matching.py (100%) rename matching_test.py => py/matching_test.py (100%) rename matchy.py => py/matchy.py (100%) rename state.py => py/state.py (100%) rename state_test.py => py/state_test.py (100%) diff --git a/.vscode/launch.json b/.vscode/launch.json index 3669159..26fafc0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,7 +8,7 @@ "name": "Python Debugger: Matchy", "type": "debugpy", "request": "launch", - "program": "matchy.py", + "program": "py/matchy.py", "console": "integratedTerminal" } ] diff --git a/bin/run.sh b/bin/run.sh index 4fff0ad..7b09ccb 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -8,4 +8,4 @@ if [ ! -d .venv ]; then fi source .venv/bin/activate python -m pip install -r requirements.txt -python matchy.py +python py/matchy.py \ No newline at end of file diff --git a/config.py b/py/config.py similarity index 100% rename from config.py rename to py/config.py diff --git a/files.py b/py/files.py similarity index 100% rename from files.py rename to py/files.py diff --git a/matching.py b/py/matching.py similarity index 100% rename from matching.py rename to py/matching.py diff --git a/matching_test.py b/py/matching_test.py similarity index 100% rename from matching_test.py rename to py/matching_test.py diff --git a/matchy.py b/py/matchy.py similarity index 100% rename from matchy.py rename to py/matchy.py diff --git a/state.py b/py/state.py similarity index 100% rename from state.py rename to py/state.py diff --git a/state_test.py b/py/state_test.py similarity index 100% rename from state_test.py rename to py/state_test.py From a480549ad30e374f8dd40a069f905290a4a76d80 Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 18:07:20 +0100 Subject: [PATCH 06/22] Ensure bash scripts exit on fail and explain what they're doing --- bin/coverage.sh | 3 +++ bin/test.sh | 2 ++ 2 files changed, 5 insertions(+) diff --git a/bin/coverage.sh b/bin/coverage.sh index f26ec2b..b1ddb63 100755 --- a/bin/coverage.sh +++ b/bin/coverage.sh @@ -1,2 +1,5 @@ #!/usr/bin/env bash +set -x +set -e + pytest --cov=. --cov-report=html \ No newline at end of file diff --git a/bin/test.sh b/bin/test.sh index e23c9b8..d019117 100755 --- a/bin/test.sh +++ b/bin/test.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +set -x +set -e # Check formatting and linting flake8 --max-line-length 120 $(git ls-files '*.py') From 7efe781e66d9250e931ceefa3d66bb03fe8f3c5e Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 19:02:47 +0100 Subject: [PATCH 07/22] Implement user pausing with /pause --- README.md | 1 - py/matchy.py | 20 +++++- py/state.py | 167 +++++++++++++++++++++++++++++------------------ py/state_test.py | 4 +- 4 files changed, 125 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index c729f5e..142b3c8 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,6 @@ Matchy is configured by a `config.json` file that takes this format: ## TODO * Write bot tests with [dpytest](https://dpytest.readthedocs.io/en/latest/tutorials/getting_started.html) -* Implement /pause to pause a user for a little while * Move more constants to the config * Add scheduling functionality * Fix logging in some sub files (doesn't seem to actually be output?) diff --git a/py/matchy.py b/py/matchy.py index 7f5fc8e..62bdaaa 100755 --- a/py/matchy.py +++ b/py/matchy.py @@ -75,7 +75,7 @@ async def close(ctx: commands.Context): @bot.tree.command(description="Join the matchees for this channel") @commands.guild_only() async def join(interaction: discord.Interaction): - State.set_use_active_in_channel( + State.set_user_active_in_channel( interaction.user.id, interaction.channel.id) state.save_to_file(State, STATE_FILE) await interaction.response.send_message( @@ -87,13 +87,26 @@ async def join(interaction: discord.Interaction): @bot.tree.command(description="Leave the matchees for this channel") @commands.guild_only() async def leave(interaction: discord.Interaction): - State.set_use_active_in_channel( + State.set_user_active_in_channel( interaction.user.id, interaction.channel.id, False) state.save_to_file(State, STATE_FILE) await interaction.response.send_message( f"No worries {interaction.user.mention}. Come back soon :)", ephemeral=True, silent=True) +@bot.tree.command(description="Pause your matching in this channel for a number of days") +@commands.guild_only() +@app_commands.describe(days="Days to pause for (defaults to 7)") +async def pause(interaction: discord.Interaction, days: int = None): + if not days: # Default to a week + days = 7 + State.set_user_paused_in_channel( + interaction.user.id, interaction.channel.id, days) + state.save_to_file(State, STATE_FILE) + await interaction.response.send_message( + f"Sure thing {interaction.user.mention}. Paused you for {days} days!", ephemeral=True, silent=True) + + @bot.tree.command(description="List the matchees for this channel") @commands.guild_only() async def list(interaction: discord.Interaction): @@ -195,6 +208,9 @@ class DynamicGroupButton(discord.ui.DynamicItem[discord.ui.Button], def get_matchees_in_channel(channel: discord.channel): """Fetches the matchees in a channel""" + # Reactivate any unpaused users + State.reactivate_users(channel.id) + # Gather up the prospective matchees return [m for m in channel.members if State.get_user_active_in_channel(m.id, channel.id)] diff --git a/py/state.py b/py/state.py index 344a00f..859b999 100644 --- a/py/state.py +++ b/py/state.py @@ -1,11 +1,12 @@ """Store bot state""" import os -from datetime import datetime +from datetime import datetime, timedelta from schema import Schema, And, Use, Optional from typing import Protocol import files import copy import logging +from contextlib import contextmanager logger = logging.getLogger("state") logger.setLevel(logging.INFO) @@ -83,6 +84,8 @@ _SCHEMA = Schema( Optional(str): { # Whether the user is signed up in this channel _Key.ACTIVE: And(Use(bool)), + # A timestamp for when to re-activate the user + Optional(_Key.REACTIVATE): And(Use(str)), } } } @@ -106,24 +109,21 @@ class Member(Protocol): def ts_to_datetime(ts: str) -> datetime: - """Convert a ts to datetime using the internal format""" + """Convert a string ts to datetime using the internal format""" return datetime.strptime(ts, _TIME_FORMAT) +def datetime_to_ts(ts: datetime) -> str: + """Convert a datetime to a string ts using the internal format""" + return datetime.strftime(ts, _TIME_FORMAT) + + class State(): def __init__(self, data: dict = _EMPTY_DICT): """Initialise and validate the state""" self.validate(data) self._dict = copy.deepcopy(data) - @property - def _history(self) -> dict[str]: - return self._dict[_Key.HISTORY] - - @property - def _users(self) -> dict[str]: - return self._dict[_Key.USERS] - def validate(self, dict: dict = None): """Initialise and validate a state dict""" if not dict: @@ -138,54 +138,50 @@ class State(): def get_user_matches(self, id: int) -> list[int]: return self._users.get(str(id), {}).get(_Key.MATCHES, {}) - def log_groups(self, groups: list[list[Member]], ts: datetime = datetime.now()) -> None: + def log_groups(self, groups: list[list[Member]], ts: datetime = None) -> None: """Log the groups""" - tmp_state = State(self._dict) - ts = datetime.strftime(ts, _TIME_FORMAT) + ts = datetime_to_ts(ts or datetime.now()) + with self._safe_wrap() as safe_state: + # Grab or create the hitory item for this set of groups + history_item = {} + safe_state._history[ts] = history_item + history_item_groups = [] + history_item[_Key.GROUPS] = history_item_groups - # Grab or create the hitory item for this set of groups - history_item = {} - tmp_state._history[ts] = history_item - history_item_groups = [] - history_item[_Key.GROUPS] = history_item_groups + for group in groups: - for group in groups: + # Add the group data + history_item_groups.append({ + _Key.MEMBERS: [m.id for m in group] + }) - # Add the group data - history_item_groups.append({ - _Key.MEMBERS: [m.id for m in group] - }) + # Update the matchee data with the matches + for m in group: + matchee = safe_state._users.get(str(m.id), {}) + matchee_matches = matchee.get(_Key.MATCHES, {}) - # Update the matchee data with the matches - for m in group: - matchee = tmp_state._users.get(str(m.id), {}) - matchee_matches = matchee.get(_Key.MATCHES, {}) + for o in (o for o in group if o.id != m.id): + matchee_matches[str(o.id)] = ts - for o in (o for o in group if o.id != m.id): - matchee_matches[str(o.id)] = ts - - matchee[_Key.MATCHES] = matchee_matches - tmp_state._users[str(m.id)] = matchee - - # Validate before storing the result - tmp_state.validate() - self._dict = tmp_state._dict + matchee[_Key.MATCHES] = matchee_matches + safe_state._users[str(m.id)] = matchee def set_user_scope(self, id: str, scope: str, value: bool = True): """Add an auth scope to a user""" - # Dive in - user = self._users.get(str(id), {}) - scopes = user.get(_Key.SCOPES, []) + with self._safe_wrap() as safe_state: + # Dive in + user = safe_state._users.get(str(id), {}) + scopes = user.get(_Key.SCOPES, []) - # Set the value - if value and scope not in scopes: - scopes.append(scope) - elif not value and scope in scopes: - scopes.remove(scope) + # Set the value + if value and scope not in scopes: + scopes.append(scope) + elif not value and scope in scopes: + scopes.remove(scope) - # Roll out - user[_Key.SCOPES] = scopes - self._users[id] = user + # Roll out + user[_Key.SCOPES] = scopes + safe_state._users[str(id)] = user def get_user_has_scope(self, id: str, scope: str) -> bool: """ @@ -196,20 +192,9 @@ class State(): scopes = user.get(_Key.SCOPES, []) return AuthScope.OWNER in scopes or scope in scopes - def set_use_active_in_channel(self, id: str, channel_id: str, active: bool = True): + def set_user_active_in_channel(self, id: str, channel_id: str, active: bool = True): """Set a user as active (or not) on a given channel""" - # Dive in - user = self._users.get(str(id), {}) - channels = user.get(_Key.CHANNELS, {}) - channel = channels.get(str(channel_id), {}) - - # Set the value - channel[_Key.ACTIVE] = active - - # Unroll - channels[str(channel_id)] = channel - user[_Key.CHANNELS] = channels - self._users[str(id)] = user + self._set_user_channel_prop(id, channel_id, _Key.ACTIVE, active) def get_user_active_in_channel(self, id: str, channel_id: str) -> bool: """Get a users active channels""" @@ -217,11 +202,69 @@ class State(): channels = user.get(_Key.CHANNELS, {}) return str(channel_id) in [channel for (channel, props) in channels.items() if props.get(_Key.ACTIVE, False)] + def set_user_paused_in_channel(self, id: str, channel_id: str, days: int): + """Sets a user as paused in a channel""" + # Deactivate the user in the channel first + self.set_user_active_in_channel(id, channel_id, False) + + # Set the reactivate time the number of days in the future + ts = datetime.now() + timedelta(days=days) + self._set_user_channel_prop( + id, channel_id, _Key.REACTIVATE, datetime_to_ts(ts)) + + def reactivate_users(self, channel_id: str): + """Reactivate any users who've passed their reactivation time on this channel""" + with self._safe_wrap() as safe_state: + for user in safe_state._users.values(): + channels = user.get(_Key.CHANNELS, {}) + channel = channels.get(str(channel_id), {}) + if channel and not channel[_Key.ACTIVE]: + reactivate = channel.get(_Key.REACTIVATE, None) + # Check if we've gone past the reactivation time and re-activate + if reactivate and datetime.now() > ts_to_datetime(reactivate): + channel[_Key.ACTIVE] = True + @property - def dict_internal(self) -> dict: + def dict_internal_copy(self) -> dict: """Only to be used to get the internal dict as a copy""" return copy.deepcopy(self._dict) + @property + def _history(self) -> dict[str]: + return self._dict[_Key.HISTORY] + + @property + def _users(self) -> dict[str]: + return self._dict[_Key.USERS] + + def _set_user_channel_prop(self, id: str, channel_id: str, key: str, value): + """Set a user channel property helper""" + with self._safe_wrap() as safe_state: + # Dive in + user = safe_state._users.get(str(id), {}) + channels = user.get(_Key.CHANNELS, {}) + channel = channels.get(str(channel_id), {}) + + # Set the value + channel[key] = value + + # Unroll + channels[str(channel_id)] = channel + user[_Key.CHANNELS] = channels + safe_state._users[str(id)] = user + + @contextmanager + def _safe_wrap(self): + """Safely run any function wrapped in a validate""" + # Wrap in a temporary state to validate first to prevent corruption + tmp_state = State(self._dict) + try: + yield tmp_state + finally: + # Validate and then overwrite our dict with the new one + tmp_state.validate() + self._dict = tmp_state._dict + def _migrate(dict: dict): """Migrate a dict through versions""" @@ -254,4 +297,4 @@ def load_from_file(file: str) -> State: def save_to_file(state: State, file: str): """Saves the state out to a file""" - files.save(file, state.dict_internal) + files.save(file, state.dict_internal_copy) diff --git a/py/state_test.py b/py/state_test.py index bd97648..b79a426 100644 --- a/py/state_test.py +++ b/py/state_test.py @@ -55,11 +55,11 @@ def test_channeljoin(): assert not st.get_user_active_in_channel(1, "2") st = state.load_from_file(path) - st.set_use_active_in_channel(1, "2", True) + st.set_user_active_in_channel(1, "2", True) state.save_to_file(st, path) st = state.load_from_file(path) assert st.get_user_active_in_channel(1, "2") - st.set_use_active_in_channel(1, "2", False) + st.set_user_active_in_channel(1, "2", False) assert not st.get_user_active_in_channel(1, "2") From 5f66d3454cc7777b1b6cc83b7fa3f94cf66cb907 Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 19:03:44 +0100 Subject: [PATCH 08/22] Add a readme entry for pausing --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 142b3c8..f3f6dde 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,9 @@ Matches groups of users in a channel and offers a button to pose those groups to ### /join and /leave Allows users to sign up and leave the group matching in the channel the command is used +### /pause [days: int(7)] +Allows users to pause their matching in a channel for a given number of days + ### $sync and $close Only usable by `OWNER` users, reloads the config and syncs commands, or closes down the bot. Only usable in DMs with the bot user. From 78b3d002d8c1de4f446a9a73e2331f5e6e866e18 Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 19:04:11 +0100 Subject: [PATCH 09/22] Update the bot activity to the join message --- py/matchy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/matchy.py b/py/matchy.py index 62bdaaa..f3e970d 100755 --- a/py/matchy.py +++ b/py/matchy.py @@ -36,7 +36,7 @@ async def setup_hook(): async def on_ready(): """Bot is ready and connected""" logger.info("Bot is up and ready!") - activity = discord.Game("/match") + activity = discord.Game("/join") await bot.change_presence(status=discord.Status.online, activity=activity) From 29d40cd80f38284ef3a66eeea390f1eddd020915 Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 19:04:22 +0100 Subject: [PATCH 10/22] Fix up checking the auth scope for sending the button --- py/matchy.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/py/matchy.py b/py/matchy.py index f3e970d..c352f53 100755 --- a/py/matchy.py +++ b/py/matchy.py @@ -145,13 +145,14 @@ async def match(interaction: discord.Interaction, members_min: int = None): view = discord.utils.MISSING if State.get_user_has_scope(interaction.user.id, state.AuthScope.MATCHER): - # Let a non-matcher know why they don't have the button - msg += f"\n\nYou'll need the {state.AuthScope.MATCHER} scope to post this to the channel, sorry!" - else: # Otherwise set up the button msg += "\n\nClick the button to match up groups and send them to the channel.\n" view = discord.ui.View(timeout=None) view.add_item(DynamicGroupButton(members_min)) + else: + # Let a non-matcher know why they don't have the button + msg += f"\n\nYou'll need the { + state.AuthScope.MATCHER} scope to post this to the channel, sorry!" await interaction.response.send_message(msg, ephemeral=True, silent=True, view=view) From 10b08264e4ba517a2479f80f9f5a3cf53e6b462c Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 19:04:43 +0100 Subject: [PATCH 11/22] Ensure we properly version the button custom ID --- py/matchy.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/py/matchy.py b/py/matchy.py index c352f53..ffd8c6e 100755 --- a/py/matchy.py +++ b/py/matchy.py @@ -160,17 +160,18 @@ async def match(interaction: discord.Interaction, members_min: int = None): # Increment when adjusting the custom_id so we don't confuse old users -_BUTTON_CUSTOM_ID_VERSION = 1 +_MATCH_BUTTON_CUSTOM_ID_VERSION = 1 +_MATCH_BUTTON_CUSTOM_ID_PREFIX = f'match:v{_MATCH_BUTTON_CUSTOM_ID_VERSION}:' class DynamicGroupButton(discord.ui.DynamicItem[discord.ui.Button], - template=f'match:v{_BUTTON_CUSTOM_ID_VERSION}:' + r'min:(?P[0-9]+)'): + template=_MATCH_BUTTON_CUSTOM_ID_PREFIX + r'min:(?P[0-9]+)'): def __init__(self, min: int) -> None: super().__init__( discord.ui.Button( label='Match Groups!', style=discord.ButtonStyle.blurple, - custom_id=f'match:min:{min}', + custom_id=_MATCH_BUTTON_CUSTOM_ID_PREFIX + f'min:{min}', ) ) self.min: int = min From c77c83c929570ffe817839dcdbc6c8ccdf55f78d Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 19:09:38 +0100 Subject: [PATCH 12/22] Pull the stress test parametes out so they're more obvious and can be tweaked --- py/matching_test.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/py/matching_test.py b/py/matching_test.py index 5a40897..742d3a6 100644 --- a/py/matching_test.py +++ b/py/matching_test.py @@ -222,6 +222,14 @@ def test_members_to_groups_with_history(history_data, matchees, per_group, check assert check(groups) +# Allows controling of the scale of the stress test +# Try and keep it under 10s when committed, but otherwise these numbers can be fudged +# Especially to test a wide range of weird situations +_STRESS_TEST_RANGE_PER_GROUP = range(2, 6) +_STRESS_TEST_RANGE_NUM_MEMBERS = range(1, 5) +_STRESS_TEST_RANGE_NUM_HISTORIES = range(8) + + def test_members_to_groups_stress_test(): """stress test firing significant random data at the code""" @@ -229,15 +237,15 @@ def test_members_to_groups_stress_test(): rand = random.Random(123) # Slowly ramp up the group size - for per_group in range(2, 6): + for per_group in _STRESS_TEST_RANGE_PER_GROUP: # Slowly ramp a randomized shuffled list of members with randomised roles - for num_members in range(1, 5): + for num_members in _STRESS_TEST_RANGE_NUM_MEMBERS: matchees = [Member(i, [Role(i) for i in range(1, rand.randint(2, num_members*2 + 1))]) for i in range(1, rand.randint(2, num_members*10 + 1))] rand.shuffle(matchees) - for num_history in range(8): + for num_history in _STRESS_TEST_RANGE_NUM_HISTORIES: # Generate some super random history # Start some time from now to the past From a5d7dae8517bef8cd4df0099740e63c84a9dd3d9 Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 19:12:44 +0100 Subject: [PATCH 13/22] Fix indentation in matchy.py --- py/matchy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py/matchy.py b/py/matchy.py index ffd8c6e..f43d1f9 100755 --- a/py/matchy.py +++ b/py/matchy.py @@ -151,8 +151,8 @@ async def match(interaction: discord.Interaction, members_min: int = None): view.add_item(DynamicGroupButton(members_min)) else: # Let a non-matcher know why they don't have the button - msg += f"\n\nYou'll need the { - state.AuthScope.MATCHER} scope to post this to the channel, sorry!" + msg += f"\n\nYou'll need the {state.AuthScope.MATCHER}" + + " scope to post this to the channel, sorry!" await interaction.response.send_message(msg, ephemeral=True, silent=True, view=view) From 9043615498ba5e14b66743235f4e237d5b5be31a Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 22:07:43 +0100 Subject: [PATCH 14/22] Move core matching factors to the config file Bonus changes here were making the config a singleton, fixing some more tests and then re-writing the stress test because it was pissing me off. --- README.md | 17 +++- py/config.py | 66 ++++++++++++-- py/matching.py | 40 +++++---- py/matching_test.py | 206 +++++++++++++++++++++++++++++++++----------- py/matchy.py | 4 +- py/state.py | 7 +- 6 files changed, 254 insertions(+), 86 deletions(-) diff --git a/README.md b/README.md index f3f6dde..2d1e418 100644 --- a/README.md +++ b/README.md @@ -24,16 +24,25 @@ Only usable by `OWNER` users, reloads the config and syncs commands, or closes d ## Config Matchy is configured by a `config.json` file that takes this format: -``` +```json { - "version": 1, - "token": "<>", + "version" : 1, + "token" : "<>", + + "match" : { + "score_factors": { + "repeat_role" : 4, + "repeat_match" : 8, + "extra_member" : 32, + "upper_threshold" : 64 + } + } } ``` +Only token and version are required. See [`py/config.py`](for explanations for any of these) ## TODO * Write bot tests with [dpytest](https://dpytest.readthedocs.io/en/latest/tutorials/getting_started.html) -* Move more constants to the config * Add scheduling functionality * Fix logging in some sub files (doesn't seem to actually be output?) * Improve the weirdo \ No newline at end of file diff --git a/py/config.py b/py/config.py index 2719d73..c6a4ca9 100644 --- a/py/config.py +++ b/py/config.py @@ -1,5 +1,5 @@ """Very simple config loading library""" -from schema import Schema, And, Use +from schema import Schema, And, Use, Optional import files import os import logging @@ -13,10 +13,18 @@ _FILE = "config.json" _VERSION = 1 -class _Keys(): +class _Key(): TOKEN = "token" VERSION = "version" + MATCH = "match" + + SCORE_FACTORS = "score_factors" + REPEAT_ROLE = "repeat_role" + REPEAT_MATCH = "repeat_match" + EXTRA_MEMBER = "extra_member" + UPPER_THRESHOLD = "upper_threshold" + # Removed OWNERS = "owners" @@ -24,10 +32,21 @@ class _Keys(): _SCHEMA = Schema( { # The current version - _Keys.VERSION: And(Use(int)), + _Key.VERSION: And(Use(int)), # Discord bot token - _Keys.TOKEN: And(Use(str)), + _Key.TOKEN: And(Use(str)), + + # Settings for the match algorithmn, see matching.py for explanations on usage + Optional(_Key.MATCH): { + Optional(_Key.SCORE_FACTORS): { + + Optional(_Key.REPEAT_ROLE): And(Use(int)), + Optional(_Key.REPEAT_MATCH): And(Use(int)), + Optional(_Key.EXTRA_MEMBER): And(Use(int)), + Optional(_Key.UPPER_THRESHOLD): And(Use(int)), + } + } } ) @@ -35,7 +54,7 @@ _SCHEMA = Schema( def _migrate_to_v1(d: dict): # Owners moved to History in v1 # Note: owners will be required to be re-added to the state.json - owners = d.pop(_Keys.OWNERS) + owners = d.pop(_Key.OWNERS) logger.warn( "Migration removed owners from config, these must be re-added to the state.json") logger.warn("Owners: %s", owners) @@ -47,7 +66,29 @@ _MIGRATIONS = [ ] -class Config(): +class _ScoreFactors(): + def __init__(self, data: dict): + """Initialise and validate the config""" + self._dict = data + + @property + def repeat_role(self) -> int: + return self._dict.get(_Key.REPEAT_ROLE, None) + + @property + def repeat_match(self) -> int: + return self._dict.get(_Key.REPEAT_MATCH, None) + + @property + def extra_member(self) -> int: + return self._dict.get(_Key.EXTRA_MEMBER, None) + + @property + def upper_threshold(self) -> int: + return self._dict.get(_Key.UPPER_THRESHOLD, None) + + +class _Config(): def __init__(self, data: dict): """Initialise and validate the config""" _SCHEMA.validate(data) @@ -57,6 +98,10 @@ class Config(): def token(self) -> str: return self._dict["token"] + @property + def score_factors(self) -> _ScoreFactors: + return _ScoreFactors(self._dict.get(_Key.SCORE_FACTORS, {})) + def _migrate(dict: dict): """Migrate a dict through versions""" @@ -66,7 +111,7 @@ def _migrate(dict: dict): dict["version"] = _VERSION -def load_from_file(file: str = _FILE) -> Config: +def _load_from_file(file: str = _FILE) -> _Config: """ Load the state from a file Apply any required migrations @@ -74,4 +119,9 @@ def load_from_file(file: str = _FILE) -> Config: assert os.path.isfile(file) loaded = files.load(file) _migrate(loaded) - return Config(loaded) + return _Config(loaded) + + +# Core config for users to use +# Singleton as there should only be one, and it's global +Config = _load_from_file() diff --git a/py/matching.py b/py/matching.py index 4f9b31d..cef715e 100644 --- a/py/matching.py +++ b/py/matching.py @@ -3,20 +3,24 @@ import logging from datetime import datetime, timedelta from typing import Protocol, runtime_checkable import state - - -# Number of days to step forward from the start of history for each match attempt -_ATTEMPT_TIMESTEP_INCREMENT = timedelta(days=7) +import config class _ScoreFactors(int): - """Various eligability scoring factors for group meetups""" - REPEAT_ROLE = 2**2 - REPEAT_MATCH = 2**3 - EXTRA_MEMBER = 2**5 + """ + Score factors used when trying to build up "best fit" groups + Matchees are sequentially placed into the lowest scoring available group + """ - # Scores higher than this are fully rejected - UPPER_THRESHOLD = 2**6 + # Added for each role the matchee has that another group member has + REPEAT_ROLE = config.Config.score_factors.repeat_role or 2**2 + # Added for each member in the group that the matchee has already matched with + REPEAT_MATCH = config.Config.score_factors.repeat_match or 2**3 + # Added for each additional member over the set "per group" value + EXTRA_MEMBER = config.Config.score_factors.extra_member or 2**5 + + # Upper threshold, if the user scores higher than this they will not be placed in that group + UPPER_THRESHOLD = config.Config.score_factors.upper_threshold or 2**6 logger = logging.getLogger("matching") @@ -76,8 +80,8 @@ def get_member_group_eligibility_score(member: Member, return rating # Add score based on prior matchups of this user - rating += sum(m.id in prior_matches for m in group) * \ - _ScoreFactors.REPEAT_MATCH + num_prior = sum(m.id in prior_matches for m in group) + rating += num_prior * _ScoreFactors.REPEAT_MATCH # Calculate the number of roles that match all_role_ids = set(r.id for mr in [r.roles for r in group] for r in mr) @@ -159,7 +163,7 @@ def iterate_all_shifts(list: list): def members_to_groups(matchees: list[Member], - hist: state.State = state.State(), + st: state.State = state.State(), per_group: int = 3, allow_fallback: bool = False) -> list[list[Member]]: """Generate the groups from the set of matchees""" @@ -170,18 +174,16 @@ def members_to_groups(matchees: list[Member], if not matchees: return [] - # Grab the oldest timestamp - history_start = hist.get_oldest_timestamp() or datetime.now() - - # Walk from the start of time until now using the timestep increment - for oldest_relevant_datetime in datetime_range(history_start, _ATTEMPT_TIMESTEP_INCREMENT, datetime.now()): + # Walk from the start of history until now trying to match up groups + # Or if there's no + for oldest_relevant_datetime in st.get_history_timestamps() + [datetime.now()]: # Attempt with each starting matchee for shifted_matchees in iterate_all_shifts(matchees): attempts += 1 groups = attempt_create_groups( - shifted_matchees, hist, oldest_relevant_datetime, per_group) + shifted_matchees, st, oldest_relevant_datetime, per_group) # Fail the match if our groups aren't big enough if num_groups <= 1 or (groups and all(len(g) >= per_group for g in groups)): diff --git a/py/matching_test.py b/py/matching_test.py index 742d3a6..fa06560 100644 --- a/py/matching_test.py +++ b/py/matching_test.py @@ -6,6 +6,8 @@ import pytest import random import matching import state +import copy +import itertools from datetime import datetime, timedelta @@ -40,12 +42,16 @@ class Member(): def roles(self) -> list[Role]: return self._roles + @roles.setter + def roles(self, roles: list[Role]): + self._roles = roles + @property def id(self) -> int: return self._id -def inner_validate_members_to_groups(matchees: list[Member], tmp_state: state.State, per_group: int): +def members_to_groups_validate(matchees: list[Member], tmp_state: state.State, per_group: int): """Inner function to validate the main output of the groups function""" groups = matching.members_to_groups(matchees, tmp_state, per_group) @@ -83,7 +89,7 @@ def inner_validate_members_to_groups(matchees: list[Member], tmp_state: state.St def test_members_to_groups_no_history(matchees, per_group): """Test simple group matching works""" tmp_state = state.State() - inner_validate_members_to_groups(matchees, tmp_state, per_group) + members_to_groups_validate(matchees, tmp_state, per_group) def items_found_in_lists(list_of_lists, items): @@ -205,8 +211,113 @@ def items_found_in_lists(list_of_lists, items): [ # Nothing else ] + ), + # Another weird one pulled out of the stress test + ( + [ + # print([(str(h["ts"]), [[f"Member({gm.id})" for gm in g] for g in h["groups"]]) for h in history_data]) + {"ts": datetime.strptime(ts, r"%Y-%m-%d %H:%M:%S.%f"), "groups": [ + [Member(m) for m in group] for group in groups]} + for (ts, groups) in + [ + ( + '2024-07-07 20:25:56.313993', + [ + [1, 2, 3, 4, 5], + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], + [1], + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], + [1, 2, 3, 4, 5, 6, 7, 8] + ] + ), + ( + '2024-07-13 20:25:56.313993', + [ + [1, 2], + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 17, 18], + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + [1] + ] + ), + ( + '2024-06-29 20:25:56.313993', + [ + [1, 2, 3, 4, 5], + [1, 2, 3, 4, 5, 6, 7], + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 17, 18, 19, 20], + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17], + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 17, 18, 19, 20] + ] + ), + ( + '2024-06-25 20:25:56.313993', + [ + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 17, 18], + [1, 2], + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 17, 18, 19], + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], + [1, 2] + ] + ), + ( + '2024-07-04 20:25:56.313993', + [ + [1, 2, 3, 4, 5], + [1, 2, 3], + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], + [1, 2, 3, 4, 5, 6, 7] + ] + ), + ( + '2024-07-16 20:25:56.313993', + [ + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], + [1, 2, 3, 4, 5, 6, 7, 8, 9], + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 17, 18, 19, 20], + [1, 2, 3, 4, 5, 6], + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18] + ] + ) + ] + ], + [ + # print([(m.id, [r.id for r in m.roles]) for m in matchees]) to get the below + Member(i, [Role(r) for r in roles]) for (i, roles) in + [ + (10, [1, 2, 3]), + (4, [1, 2, 3]), + (5, [1, 2]), + (13, [1, 2]), + (3, [1, 2, 3, 4]), + (14, [1]), + (6, [1, 2, 3, 4]), + (11, [1]), + (9, [1]), + (1, [1, 2, 3]), + (16, [1, 2]), + (15, [1, 2]), + (2, [1, 2, 3]), + (7, [1, 2, 3]), + (12, [1, 2]), + (8, [1, 2, 3, 4]) + ] + ], + 5, + [ + # Nothing + ] ) -], ids=['simple_history', 'fallback', 'example_1', 'example_2']) +], ids=['simple_history', 'fallback', 'example_1', 'example_2', 'example_3']) def test_members_to_groups_with_history(history_data, matchees, per_group, checks): """Test more advanced group matching works""" tmp_state = state.State() @@ -215,66 +326,65 @@ def test_members_to_groups_with_history(history_data, matchees, per_group, check for d in history_data: tmp_state.log_groups(d["groups"], d["ts"]) - groups = inner_validate_members_to_groups(matchees, tmp_state, per_group) + groups = members_to_groups_validate(matchees, tmp_state, per_group) # Run the custom validate functions for check in checks: assert check(groups) -# Allows controling of the scale of the stress test -# Try and keep it under 10s when committed, but otherwise these numbers can be fudged -# Especially to test a wide range of weird situations -_STRESS_TEST_RANGE_PER_GROUP = range(2, 6) -_STRESS_TEST_RANGE_NUM_MEMBERS = range(1, 5) -_STRESS_TEST_RANGE_NUM_HISTORIES = range(8) +def random_chunk(li, min_chunk, max_chunk, rand): + """ + "Borrowed" from https://stackoverflow.com/questions/21439011/best-way-to-split-a-list-into-randomly-sized-chunks + """ + it = iter(li) + while True: + nxt = list(itertools.islice(it, rand.randint(min_chunk, max_chunk))) + if nxt: + yield nxt + else: + break -def test_members_to_groups_stress_test(): - """stress test firing significant random data at the code""" +# Generate a large set of "interesting" tests that replay a fake history onto random people +# Increase these numbers for some extreme programming +@pytest.mark.parametrize("per_group, num_members, num_history", ( + (per_group, num_members, num_history) + for per_group in range(2, 4) + for num_members in range(6, 24, 3) + for num_history in range(0, 4))) +def test_stess_random_groups(per_group, num_members, num_history): + """Run a randomised test based on the input""" - # Use a stable rand, feel free to adjust this if needed but this lets the test be stable - rand = random.Random(123) + # Seed the random based on the inputs paird with primes + # Ensures the test has interesting fake data, but is stable + rand = random.Random(per_group*3 + num_members*5 + num_history*7) - # Slowly ramp up the group size - for per_group in _STRESS_TEST_RANGE_PER_GROUP: + # Start with a list of all possible members + possible_members = [Member(i) for i in range(num_members*2)] + for member in possible_members: + # Give each member 3 random roles from 1-7 + member.roles = [Role(i) for i in rand.sample(range(1, 8), 3)] - # Slowly ramp a randomized shuffled list of members with randomised roles - for num_members in _STRESS_TEST_RANGE_NUM_MEMBERS: - matchees = [Member(i, [Role(i) for i in range(1, rand.randint(2, num_members*2 + 1))]) - for i in range(1, rand.randint(2, num_members*10 + 1))] - rand.shuffle(matchees) + # Grab a subset for our members + rand.shuffle(possible_members) + members = copy.deepcopy(possible_members[:num_members]) - for num_history in _STRESS_TEST_RANGE_NUM_HISTORIES: + history_data = {} + for i in range(num_history): + possible_members = copy.deepcopy(possible_members) + rand.shuffle(possible_members) + history_data[datetime.now() - timedelta(days=i)] = [ + chunk for chunk in random_chunk(possible_members, per_group, per_group+2, rand) + ] - # Generate some super random history - # Start some time from now to the past - time = datetime.now() - timedelta(days=rand.randint(0, num_history*5)) - history_data = [] - for _ in range(0, num_history): - run = { - "ts": time - } - groups = [] - for y in range(1, num_history): - groups.append([Member(i) - for i in range(1, max(num_members, rand.randint(2, num_members*10 + 1)))]) - run["groups"] = groups - history_data.append(run) + replay_state = state.State() - # Step some time backwards in time - time -= timedelta(days=rand.randint(1, num_history)) + # Replay the history + for ts, groups in history_data.items(): + replay_state.log_groups(groups, ts) - # No guarantees on history data order so make it a little harder for matchy - rand.shuffle(history_data) - - # Replay the history - tmp_state = state.State() - for d in history_data: - tmp_state.log_groups(d["groups"], d["ts"]) - - inner_validate_members_to_groups( - matchees, tmp_state, per_group) + members_to_groups_validate(members, replay_state, per_group) def test_auth_scopes(): diff --git a/py/matchy.py b/py/matchy.py index f43d1f9..fe5fd9c 100755 --- a/py/matchy.py +++ b/py/matchy.py @@ -12,9 +12,7 @@ import re STATE_FILE = "state.json" -CONFIG_FILE = "config.json" -Config = config.load_from_file(CONFIG_FILE) State = state.load_from_file(STATE_FILE) logger = logging.getLogger("matchy") @@ -229,4 +227,4 @@ def active_members_to_groups(channel: discord.channel, min_members: int): if __name__ == "__main__": handler = logging.StreamHandler() - bot.run(Config.token, log_handler=handler, root_logger=True) + bot.run(config.Config.token, log_handler=handler, root_logger=True) diff --git a/py/state.py b/py/state.py index 859b999..f75be41 100644 --- a/py/state.py +++ b/py/state.py @@ -130,10 +130,9 @@ class State(): dict = self._dict _SCHEMA.validate(dict) - def get_oldest_timestamp(self) -> datetime: - """Grab the oldest timestamp in history""" - times = (ts_to_datetime(dt) for dt in self._history.keys()) - return next(times, None) + def get_history_timestamps(self) -> list[datetime]: + """Grab all timestamps in the history""" + return sorted([ts_to_datetime(dt) for dt in self._history.keys()]) def get_user_matches(self, id: int) -> list[int]: return self._users.get(str(id), {}).get(_Key.MATCHES, {}) From 9cf74580e93072702eb65caa3fb9b7b226415259 Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 22:10:45 +0100 Subject: [PATCH 15/22] Fix the readme link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2d1e418..5d0015b 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Matchy is configured by a `config.json` file that takes this format: } } ``` -Only token and version are required. See [`py/config.py`](for explanations for any of these) +Only token and version are required. See [`py/config.py`](py/config.py) for explanations for any of these. ## TODO * Write bot tests with [dpytest](https://dpytest.readthedocs.io/en/latest/tutorials/getting_started.html) From 0e80937092a8b4e5a9f6ef8e37f66f395e00bf47 Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 22:11:56 +0100 Subject: [PATCH 16/22] Remove half-finished sentence --- py/matching.py | 1 - 1 file changed, 1 deletion(-) diff --git a/py/matching.py b/py/matching.py index cef715e..34bbe43 100644 --- a/py/matching.py +++ b/py/matching.py @@ -175,7 +175,6 @@ def members_to_groups(matchees: list[Member], return [] # Walk from the start of history until now trying to match up groups - # Or if there's no for oldest_relevant_datetime in st.get_history_timestamps() + [datetime.now()]: # Attempt with each starting matchee From 145907a0dfa25e181dc8594e7d56e99011f3b81f Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 22:31:20 +0100 Subject: [PATCH 17/22] Update the date format in the statefile --- py/config.py | 4 ++-- py/state.py | 45 ++++++++++++++++++++++++++++++++++++++------- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/py/config.py b/py/config.py index c6a4ca9..8adc7d6 100644 --- a/py/config.py +++ b/py/config.py @@ -26,7 +26,7 @@ class _Key(): UPPER_THRESHOLD = "upper_threshold" # Removed - OWNERS = "owners" + _OWNERS = "owners" _SCHEMA = Schema( @@ -54,7 +54,7 @@ _SCHEMA = Schema( def _migrate_to_v1(d: dict): # Owners moved to History in v1 # Note: owners will be required to be re-added to the state.json - owners = d.pop(_Key.OWNERS) + owners = d.pop(_Key._OWNERS) logger.warn( "Migration removed owners from config, these must be re-added to the state.json") logger.warn("Owners: %s", owners) diff --git a/py/state.py b/py/state.py index f75be41..453a76e 100644 --- a/py/state.py +++ b/py/state.py @@ -13,18 +13,48 @@ logger.setLevel(logging.INFO) # Warning: Changing any of the below needs proper thought to ensure backwards compatibility -_VERSION = 1 +_VERSION = 2 def _migrate_to_v1(d: dict): - logger.info("Renaming %s to %s", _Key.MATCHEES, _Key.USERS) - d[_Key.USERS] = d[_Key.MATCHEES] - del d[_Key.MATCHEES] + """v1 simply renamed matchees to users""" + logger.info("Renaming %s to %s", _Key._MATCHEES, _Key.USERS) + d[_Key.USERS] = d[_Key._MATCHEES] + del d[_Key._MATCHEES] + + +def _migrate_to_v2(d: dict): + """v2 swapped the date over to a less silly format""" + logger.info("Fixing up date format from %s to %s", + _TIME_FORMAT_OLD, _TIME_FORMAT) + + def old_to_new_ts(ts: str) -> str: + return datetime.strftime(datetime.strptime(ts, _TIME_FORMAT_OLD), _TIME_FORMAT) + + # Adjust all the history keys + d[_Key.HISTORY] = { + old_to_new_ts(ts): entry + for ts, entry in d[_Key.HISTORY].items() + } + # Adjust all the user parts + for user in d[_Key.USERS].values(): + # Update the match dates + matches = user.get(_Key.MATCHES, {}) + for id, ts in matches.items(): + matches[id] = old_to_new_ts(ts) + + # Update any reactivation dates + channels = user.get(_Key.CHANNELS, {}) + for id, channel in channels.items(): + old_ts = channel.get(_Key.REACTIVATE, None) + if old_ts: + channel[_Key.REACTIVATE] = old_to_new_ts(old_ts) # Set of migration functions to apply _MIGRATIONS = [ - _migrate_to_v1 + _migrate_to_v1, + _migrate_to_v2 ] @@ -48,10 +78,11 @@ class _Key(str): VERSION = "version" # Unused - MATCHEES = "matchees" + _MATCHEES = "matchees" -_TIME_FORMAT = "%a %b %d %H:%M:%S %Y" +_TIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f" +_TIME_FORMAT_OLD = "%a %b %d %H:%M:%S %Y" _SCHEMA = Schema( From 10ac46b7735d3a5b5c8b1730c30534ab9e190897 Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 22:31:34 +0100 Subject: [PATCH 18/22] Update the README * Add a note about using /join to rejoin * Fix a link * Remove a note about broken logging, seems to be working fine --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5d0015b..9307fde 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Matches groups of users in a channel and offers a button to pose those groups to Allows users to sign up and leave the group matching in the channel the command is used ### /pause [days: int(7)] -Allows users to pause their matching in a channel for a given number of days +Allows users to pause their matching in a channel for a given number of days. Users can use `/join` to re-join before the end of that time. ### $sync and $close Only usable by `OWNER` users, reloads the config and syncs commands, or closes down the bot. Only usable in DMs with the bot user. @@ -42,7 +42,6 @@ Matchy is configured by a `config.json` file that takes this format: Only token and version are required. See [`py/config.py`](py/config.py) for explanations for any of these. ## TODO -* Write bot tests with [dpytest](https://dpytest.readthedocs.io/en/latest/tutorials/getting_started.html) * Add scheduling functionality -* Fix logging in some sub files (doesn't seem to actually be output?) +* Write integration tests (maybe with [dpytest](https://dpytest.readthedocs.io/en/latest/tutorials/getting_started.html)?) * Improve the weirdo \ No newline at end of file From f7018e892d66c769664c98bb1898368af6c78003 Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 22:35:07 +0100 Subject: [PATCH 19/22] Allow not having a config file, otherwise pytest can't collect tests without it --- py/config.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/py/config.py b/py/config.py index 8adc7d6..9b7574d 100644 --- a/py/config.py +++ b/py/config.py @@ -116,9 +116,12 @@ def _load_from_file(file: str = _FILE) -> _Config: Load the state from a file Apply any required migrations """ - assert os.path.isfile(file) - loaded = files.load(file) - _migrate(loaded) + loaded = {} + if os.path.isfile(file): + loaded = files.load(file) + _migrate(loaded) + else: + logger.warn("No %s file found, bot cannot run!", file) return _Config(loaded) From 87da9b9673bd95331d8b53055508b5892646b80f Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 22:42:23 +0100 Subject: [PATCH 20/22] Ensure a valid (but empty) config dict --- py/config.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/py/config.py b/py/config.py index 9b7574d..5f6de44 100644 --- a/py/config.py +++ b/py/config.py @@ -50,6 +50,11 @@ _SCHEMA = Schema( } ) +_EMPTY_DICT = { + _Key.TOKEN: "", + _Key.VERSION: _VERSION +} + def _migrate_to_v1(d: dict): # Owners moved to History in v1 @@ -116,7 +121,7 @@ def _load_from_file(file: str = _FILE) -> _Config: Load the state from a file Apply any required migrations """ - loaded = {} + loaded = _EMPTY_DICT if os.path.isfile(file): loaded = files.load(file) _migrate(loaded) From a22701b480c2973506ca8f387320a9c7606591a1 Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 23:01:47 +0100 Subject: [PATCH 21/22] Improve the stress test Have it progressively match groups bit by bit --- py/matching_test.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/py/matching_test.py b/py/matching_test.py index fa06560..2ae4dca 100644 --- a/py/matching_test.py +++ b/py/matching_test.py @@ -350,9 +350,14 @@ def random_chunk(li, min_chunk, max_chunk, rand): # Increase these numbers for some extreme programming @pytest.mark.parametrize("per_group, num_members, num_history", ( (per_group, num_members, num_history) - for per_group in range(2, 4) - for num_members in range(6, 24, 3) - for num_history in range(0, 4))) + # Most of the time groups are gonna be from 2 to 5 + for per_group in range(2, 5) + # Going lower than 8 members doesn't give the bot much of a chance + # And it will fail to not fall back sometimes + # That's probably OK frankly + for num_members in range(8, 32, 5) + # Throw up to 7 histories at the algorithmn + for num_history in range(0, 8))) def test_stess_random_groups(per_group, num_members, num_history): """Run a randomised test based on the input""" @@ -366,25 +371,18 @@ def test_stess_random_groups(per_group, num_members, num_history): # Give each member 3 random roles from 1-7 member.roles = [Role(i) for i in rand.sample(range(1, 8), 3)] - # Grab a subset for our members - rand.shuffle(possible_members) - members = copy.deepcopy(possible_members[:num_members]) + # For each history item match up groups and log those + cumulative_state = state.State() + for i in range(num_history+1): - history_data = {} - for i in range(num_history): - possible_members = copy.deepcopy(possible_members) + # Grab the num of members and replay rand.shuffle(possible_members) - history_data[datetime.now() - timedelta(days=i)] = [ - chunk for chunk in random_chunk(possible_members, per_group, per_group+2, rand) - ] + members = copy.deepcopy(possible_members[:num_members]) - replay_state = state.State() - - # Replay the history - for ts, groups in history_data.items(): - replay_state.log_groups(groups, ts) - - members_to_groups_validate(members, replay_state, per_group) + groups = members_to_groups_validate( + members, cumulative_state, per_group) + cumulative_state.log_groups( + groups, datetime.now() - timedelta(days=num_history-i)) def test_auth_scopes(): From 34c8dc8ae28a4492be56e712e7b0b98240084092 Mon Sep 17 00:00:00 2001 From: Marc Di Luzio Date: Sun, 11 Aug 2024 23:03:39 +0100 Subject: [PATCH 22/22] Update the test name for regression tests --- py/matching_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py/matching_test.py b/py/matching_test.py index 2ae4dca..21f7c83 100644 --- a/py/matching_test.py +++ b/py/matching_test.py @@ -318,8 +318,8 @@ def items_found_in_lists(list_of_lists, items): ] ) ], ids=['simple_history', 'fallback', 'example_1', 'example_2', 'example_3']) -def test_members_to_groups_with_history(history_data, matchees, per_group, checks): - """Test more advanced group matching works""" +def test_unique_regressions(history_data, matchees, per_group, checks): + """Test a bunch of unqiue failures that happened in the past""" tmp_state = state.State() # Replay the history