matterbot.py

#!/usr/bin/env python3

import ast
import asyncio
import concurrent.futures
import copy
import fnmatch
import importlib.util
import json
import logging
import os
import pathlib
from pathlib import Path
import re
import sys
import time
import traceback
import configargparse
from mattermostdriver import Driver


class TokenAuth():
    def __call__(self, r):
        r.headers['Authorization'] = "Bearer %s" % options.Matterbot['password']
        r.headers['X-Requested-With'] = 'XMLHttpRequest'
        return r

class MattermostManager(object):
    def __init__(self):
        # Bounded thread pool so synchronous module.process() calls don't block
        # the asyncio event loop. See call_module(). Worker count is
        # configurable via Matterbot.command_workers — set to 1 if a module
        # turns out not to be thread-safe; raise for more parallelism.
        self._command_executor_workers = options.Matterbot.get('command_workers', 8)
        self._command_timeout = options.Matterbot.get('command_timeout', 30)
        self._command_executor = concurrent.futures.ThreadPoolExecutor(
            max_workers=self._command_executor_workers,
            thread_name_prefix='mb-cmd',
        )
        # Maximum number of in-flight module dispatches per user. Prevents a
        # single user from saturating the bot (or the command thread-pool)
        # with parallel commands and starving other users. Further requests
        # from that user are queued behind the same asyncio.timeout window
        # as the run itself, so a spammer can self-DoS their own queue but
        # cannot block anyone else. Tunable via Matterbot.user_concurrency.
        self._user_concurrency = options.Matterbot.get('user_concurrency', 2)
        # Map of userid -> asyncio.Semaphore, populated lazily on first command
        # from each user via dict.setdefault() so concurrent first-time
        # creations don't race. Entries are never evicted; size is bounded
        # by the number of distinct users who have ever talked to the bot
        # since process start (small for a single-team deployment).
        self._user_semaphores = {}
        self.mmDriver = Driver(options={
            'url'       : options.Matterbot['host'],
            'port'      : options.Matterbot['port'],
            'login_id'  : options.Matterbot['username'],
            'token'     : options.Matterbot['password'],
            'basepath'  : options.Matterbot['basepath'],
            'scheme'    : options.Matterbot['scheme'],
            'auth'      : TokenAuth,
            #'debug'     : options.debug,
            'keepalive' : True,
            'keepalive_delay': 30,
            'websocket_kw_args': {'ping_interval': 5},
        })
        try:
            self.mmDriver.login()
        except Exception:
            log.exception("Mattermost server is unreachable. Perhaps it is down, or you might have misconfigured one or more setting(s). Shutting down!")
            return False
        self.me = self.mmDriver.users.get_user(user_id='me')
        log.info("Who am I: %s" % (self.me,))
        self.my_id = self.me['id']
        self.my_team_name = options.Matterbot['teamname']
        self.my_team_id = self.mmDriver.teams.get_team_by_name(self.my_team_name)['id']
        # Load an existing module channel binding map if present
        modulepath = str(Path(options.Modules['commanddir']).expanduser().resolve())
        # Put the PARENT of the command directory on sys.path so modules can
        # be imported as `<dirname>.<command>.command` — the fully-qualified
        # dotted path avoids name collisions with installed PyPI packages
        # that share a directory name. Concrete case: `dfir-unfurl[all]`
        # installs a top-level `unfurl` package; `commands/unfurl/` would
        # otherwise resolve `import unfurl.command` against the installed
        # distribution and fail because it has no `command` submodule. Same
        # collision shape applies to commands/holehe/ vs the holehe pkg.
        _mp = Path(modulepath)
        _pkg_prefix = _mp.name
        if str(_mp.parent) not in sys.path:
            sys.path.insert(0, str(_mp.parent))
        # Keep the legacy commands-on-sys.path entry — some modules may rely
        # on it for sibling imports.
        sys.path.append(modulepath)
        self.commands = {}
        self.binds = []
        self.channelmapping = {'idtoname': {}, 'nametoid': {}}
        self.channels = self.mmDriver.channels.get_channels_for_user(self.my_id,self.my_team_id)
        self.feedmap = self.load_feedmap()
        self.bindmap = self.load_bindmap()
        self.welcome_channel_members = self.start_welcome_channel()
        # Load any new modules
        from commands import cmdutils  # commands/ is a namespace package on sys.path by now
        for root, dirs, files in os.walk(modulepath):
            for module in fnmatch.filter(files, "command.py"):
                module_name = root.split('/')[-1].lower()
                module = importlib.import_module(f"{_pkg_prefix}.{module_name}.command")
                if module_name not in self.commands:
                    module.settings.BINDS = None
                    module.settings.CHANS = None
                    defaults = importlib.import_module(f"{_pkg_prefix}.{module_name}.defaults")
                    if hasattr(defaults, 'BINDS'):
                        module.settings.BINDS = defaults.BINDS
                    if hasattr(defaults, 'CHANS'):
                        module.settings.CHANS = defaults.CHANS
                    if 'settings.py' in files:
                        overridesettings = importlib.import_module(f"{_pkg_prefix}.{module_name}.settings")
                        if hasattr(overridesettings, 'BINDS'):
                            module.settings.BINDS = overridesettings.BINDS
                        if hasattr(overridesettings, 'CHANS'):
                            module.settings.CHANS = overridesettings.CHANS
                    if not isinstance(module.settings.BINDS, list) or not isinstance(module.settings.CHANS, list):
                        log.error(f"Skipping command module {module_name}: BINDS and CHANS must both be lists")
                        continue
                    # The (optional) indicator-type filter is declared on the
                    # command's process() via @cmdutils.handles(...) -- on the
                    # handler, so it can't drift from what the code looks up.
                    # Absent or unusable -> "accepts anything" (cmdutils.accepts).
                    declared = getattr(getattr(module, 'process', None), 'accepts', None)
                    self.commands[module_name] = {
                        'binds': module.settings.BINDS,
                        'chans': module.settings.CHANS,
                        'accepts': cmdutils.normalise_accepts(declared),
                    }
                    self.binds.extend(module.settings.BINDS)
        try:
            with open(options.Matterbot['bindmap'],'w') as f:
                json.dump(self.commands,f)
        except Exception:
            log.exception("An error occurred writing the bindmap file: %s" % (options.Matterbot['bindmap'],))
        # Resolve function calls and update the module help
        for root, dirs, files in os.walk(modulepath):
            for module in fnmatch.filter(files, "command.py"):
                module_name = root.split('/')[-1].lower()
                if module_name not in self.commands:
                    continue
                module = importlib.import_module(f"{_pkg_prefix}.{module_name}.command")
                defaults = importlib.import_module(f"{_pkg_prefix}.{module_name}.defaults")
                HELP = {'DEFAULT': {'desc': 'No help available.'}}
                if hasattr(defaults, 'HELP'):
                    HELP = defaults.HELP
                if 'settings.py' in files:
                    overridesettings = importlib.import_module(f"{_pkg_prefix}.{module_name}.settings")
                    if hasattr(overridesettings, 'HELP'):
                        HELP = overridesettings.HELP
                process = getattr(module, 'process', None)
                if not callable(process):
                    log.error(f"Skipping command module {module_name}: process is missing or not callable")
                    del self.commands[module_name]
                    continue
                self.commands[module_name]['process'] = process
                self.commands[module_name]['help'] = HELP
        self.binds = sorted(list(set(self.binds)))

    def _recycle_command_executor(self):
        old_executor = self._command_executor
        self._command_executor = concurrent.futures.ThreadPoolExecutor(
            max_workers=self._command_executor_workers,
            thread_name_prefix='mb-cmd',
        )
        old_executor.shutdown(wait=False, cancel_futures=True)
        log.warning("Recycled command executor after a command timeout")

    def run_forever(self):
        """Drive the Mattermost websocket, reconnecting on disconnect with
        exponential backoff. Without this loop, any websocket termination
        (network blip, MM restart, idle timeout) causes init_websocket to
        return and the process to exit silently."""
        backoff = 1.0
        max_backoff = 60.0
        healthy_after = 60.0  # a connection that lasted this long resets backoff
        while True:
            connected_at = time.monotonic()
            try:
                # Welcome-module reconcile pass — catches anyone who joined a
                # configured channel while the bot was offline. Optional: if the
                # welcome module isn't loaded, skip without error.
                try:
                    from welcome.welcome import reconcile as _welcome_reconcile
                    _welcome_reconcile(self.mmDriver, self.my_id)
                except ImportError:
                    pass
                except Exception:
                    log.exception("welcome.reconcile failed (continuing)")
                log.info("Connecting Mattermost websocket")
                # Python 3.14 removed the implicit loop creation in
                # asyncio.get_event_loop(); mattermostdriver.init_websocket
                # still relies on it (driver.py:150). Give this thread a fresh
                # event loop per attempt and close it afterwards — the driver
                # never closes it, so reusing/leaking loops across reconnects
                # would leak the loop's self-pipe file descriptors.
                loop = asyncio.new_event_loop()
                asyncio.set_event_loop(loop)
                try:
                    self.mmDriver.init_websocket(self.handle_raw_message)
                finally:
                    loop.close()
                log.warning("Websocket loop returned (server closed connection?)")
            except KeyboardInterrupt:
                log.info("Interrupted — exiting")
                raise
            except Exception:
                log.exception("Websocket loop crashed")
            if time.monotonic() - connected_at >= healthy_after:
                backoff = 1.0
            else:
                backoff = min(backoff * 2, max_backoff)
            log.info(f"Reconnecting websocket in {backoff:.1f}s")
            time.sleep(backoff)

    def load_bindmap(self):
        try:
            bindmap = pathlib.Path(options.Matterbot['bindmap'])
            if bindmap.is_file():
                with open(options.Matterbot['bindmap']) as f:
                    self.commands = json.load(f)
                    for module in self.commands:
                        self.binds.extend(self.commands[module]['binds'])
                    log.info("Loaded existing bindmap file %s" % (options.Matterbot['bindmap']))
        except: # There is no existing command map, or it failed loading; create an empty map instead.
            raise

    def load_feedmap(self):
        try:
            feedmap = pathlib.Path(options.Matterbot['feedmap'])
            if feedmap.is_file():
                with open(options.Matterbot['feedmap']) as f:
                    log.info("Loaded existing feedmap file %s" % (options.Matterbot['feedmap']))
                    return json.load(f)
        except: # There is no existing feed map, or it failed loading; create an empty map instead.
            raise

    def start_welcome_channel(self):
        try:
            if options.Matterbot['welcome']:
                if options.Matterbot['welcome_channel']:
                    welcome_channel_id = options.Matterbot['welcome_channel']
                    if self.is_in_channel(welcome_channel_id):
                        channel_members = self.mmDriver.channels.get_channel_members(welcome_channel_id)
                        return [_['user_id'] for _ in channel_members]
        except Exception:
            log.exception("An error occurred updating the welcome channel state!")

    async def update_welcome_channel(self):
        try:
            if options.Matterbot['welcome']:
                if options.Matterbot['welcome_channel']:
                    welcome_channel_id = options.Matterbot['welcome_channel']
                    if self.is_in_channel(welcome_channel_id):
                        channel_members = self.mmDriver.channels.get_channel_members(welcome_channel_id)
                        return [_['user_id'] for _ in channel_members]
        except Exception:
            log.exception("An error occurred updating the welcome channel state!")

    async def update_bindmap(self):
        try:
            self.bindmap = copy.deepcopy(self.commands)
            for module in self.bindmap:
                del self.bindmap[module]['help']
                del self.bindmap[module]['process']
            with open(options.Matterbot['bindmap'],'w') as f:
                json.dump(self.bindmap,f)
        except Exception:
            log.exception("An error occurred updating the `%s` bindmap file; config changes were not successfully saved!" % (options.Matterbot['bindmap'],))

    async def update_feedmap(self):
        try:
            self.newfeedmap = copy.deepcopy(self.feedmap)
            with open(options.Matterbot['feedmap'],'w') as f:
                json.dump(self.newfeedmap,f)
        except Exception:
            log.exception("An error occurred updating the `%s` feedmap file; config changes were not successfully saved!" % (options.Matterbot['feedmap'],))

    async def handle_raw_message(self, raw_json: str):
        try:
            data = json.loads(raw_json)
            asyncio.create_task(self.handle_message(data))
        except json.JSONDecodeError as e:
            log.error(f"Could not handle raw JSON {raw_json}: {e}")

    async def handle_message(self, message: dict):
        try:
            if 'event' in message:
                post_data = message['data']
                # Welcome-module hook: 'user_added' events arrive here with the
                # target channel in `broadcast.channel_id`, which is not threaded
                # into handle_event. Dispatch the welcome on_join directly so the
                # channel id is preserved. ImportError = welcome module not in
                # this deployment; silently skip.
                if message.get('event') == 'user_added':
                    broadcast = message.get('broadcast') or {}
                    join_user_id = post_data.get('user_id')
                    join_channel_id = broadcast.get('channel_id') or post_data.get('channel_id')
                    if join_user_id and join_channel_id:
                        try:
                            from welcome.welcome import on_join as _welcome_on_join
                            loop = asyncio.get_running_loop()
                            loop.run_in_executor(
                                self._command_executor,
                                _welcome_on_join,
                                self.mmDriver, self.my_id, join_user_id, join_channel_id,
                            )
                        except ImportError:
                            pass
                        except Exception:
                            log.exception("welcome.on_join hook failed (user_added)")
                if 'post' in post_data: # We're handling some kind of post, e.g. a channel message
                    await self.handle_post(post_data)
                else: # We're probably handling something administrative, such as channel adds/removals
                    await self.handle_event(post_data)
        except json.JSONDecodeError as e:
            log.error(f"Could not handle message {message}: {e}")

    async def log_message(self, userid, command, params, chaninfo, rootid):
        try:
            logline = None
            channame = chaninfo['name']
            myname = self.userid_to_username(self.my_id)
            if '__' in channame and userid in channame and self.my_id in channame:
                channame = f'Direct Message with me ({myname})'
            username = self.userid_to_username(userid)
            if options.Matterbot['logcmd']:
                logline = f'Channel: {channame} - User: {username} - Command: {command}'
            if options.Matterbot['logcmdparams']:
                if len(params):
                    logline += f' - Params: {params}'
            if logline:
                log.info(f'Command Logging -> {logline}')
        except:
            raise

    async def send_message(self, chanid, text, postid=None, uploads=None):
        try:
            channame = self.chanid_to_chaninfo(chanid)['name']
            log.info(f'Channel:{channame} <- Message: ({len(text)} chars)')
            max_len = options.Matterbot["msglength"]
            lines = text.split("\n")
            blocks = []
            i = 0
            header_rx = re.compile(r'^\s*\|?\s*[^|]+(?:\s*\|\s*[^|]+)+\s*\|?\s*$')
            separator_rx = re.compile(r'^\s*\|?\s*[:\-]+(?:\s*\|\s*[:\-]+)+\s*\|?\s*$')
            while i < len(lines):
                line = lines[i]
                if header_rx.match(line) and i + 1 < len(lines) and separator_rx.match(lines[i + 1]):
                    header = line.rstrip()
                    separator = lines[i + 1].rstrip()
                    i += 2
                    table_block = f"{header}\n{separator}"
                    while i < len(lines) and lines[i].strip() != "":
                        table_block += f"\n{lines[i].rstrip()}"
                        i += 1
                    while len(table_block) > max_len:
                        split_point = table_block.rfind("\n", 0, max_len)
                        if split_point == -1:
                            split_point = max_len
                        part = table_block[:split_point]
                        blocks.append(part)
                        table_block = f"{header}\n{separator}\n{table_block[split_point:].lstrip()}"
                    blocks.append(table_block)
                else:
                    if not blocks or len(blocks[-1]) + len(line) + 1 > max_len:
                        blocks.append(line.rstrip())
                    else:
                        blocks[-1] += "\n" + line.rstrip()
                    i += 1
            for idx, block in enumerate(blocks):
                opts = {"channel_id": chanid, "message": block, "root_id": postid}
                if idx == len(blocks) - 1 and uploads:
                    opts["file_ids"] = uploads
                self.mmDriver.posts.create_post(options=opts)
        except Exception:
            log.exception("Failed to send message")
            raise

    def channame_to_chanid(self, channame, teamid=None):
        try:
            if not teamid:
                teamid = self.my_team_id
            return self.mmDriver.channels.get_channel_by_name(teamid,channame)['id']
        except Exception as e:
            log.error(f"Could not map {channame} to chanid: {e}")
            return None

    def chanid_to_channame(self, chanid):
        try:
            return self.mmDriver.channels.get_channel(chanid)['name']
        except Exception as e:
            log.error(f"Could not map {chanid} to channame: {e}")
            return None

    def chanid_to_chandisplayname(self, chanid):
        try:
            return self.mmDriver.channels.get_channel(chanid)['display_name']
        except Exception as e:
            log.error(f"Could not map {chanid} to chandisplayname: {e}")
            return None

    def channame_to_chandisplayname(self, channame):
        try:
            return self.chanid_to_chandisplayname(self.channame_to_chanid(channame))
        except Exception as e:
            log.error(f"Could not map {channame} to chandisplayname: {e}")
            return None

    def channame_to_chaninfo(self, channame):
        if channame in self.channelmapping['nametoid']:
            return self.channelmapping['nametoid'][channame]
        else:
            try:
                chaninfo = self.mmDriver.channels.get_channel_by_name(self.my_team_id, channame)
            except Exception as e:
                log.error(f"Could not map {channame} to chaninfo: {e}")
                return None
            else:
                self.channelmapping['nametoid'][chaninfo['name']] = chaninfo
                self.channelmapping['idtoname'][chaninfo['id']]   = chaninfo
                return chaninfo

    def chanid_to_chaninfo(self, chanid):
        if chanid in self.channelmapping['idtoname']:
            return self.channelmapping['idtoname'][chanid]
        else:
            try:
                chaninfo = self.mmDriver.channels.get_channel(chanid)
            except Exception as e:
                log.error(f"Could not map {chanid} to chaninfo: {e}")
                return None
            else:
                self.channelmapping['nametoid'][chaninfo['name']] = chaninfo
                self.channelmapping['idtoname'][chaninfo['id']]   = chaninfo
                return chaninfo

    def userid_to_username(self, userid):
        try:
            return self.mmDriver.users.get_user(userid)['username']
        except Exception as e:
            log.error(f"Could not map {userid} to username: {e}")
            return None

    def isadmin(self, userid):
        try:
            userinfo = self.mmDriver.users.get_user(userid)
            roles = [_.lower() for _ in userinfo['roles'].split()]
            # botadmins entries can be EITHER a Mattermost role name (e.g.
            # 'system_admin') OR a Mattermost user id (opaque token). Role
            # names need case-insensitive matching against `roles` (which is
            # lowercased above); user ids are case-sensitive and must match
            # exactly. We normalize the role-name comparison to lowercase
            # but keep the userid comparison as-is.
            botadmins = options.Matterbot['botadmins'] or []
            normalized_roles = [str(e).lower() for e in botadmins]
            if any(role in roles for role in normalized_roles) or userid in botadmins:
                return True
        except Exception:
            log.exception("isadmin check failed; treating as non-admin")
            return None

    def is_in_channel(self, chanid, userid=None):
        if not userid:
            userid = self.my_id
        self.channels = self.mmDriver.channels.get_channels_for_user(userid, self.my_team_id)
        return True if chanid in [_['id'] for _ in self.channels] else False

    def isallowed_module(self, userid, module, chaninfo):
        """
        Check if we are in a channel or in a private chat
        > There are four types of channels: public channels, private channels, direct messages, and group messages.
        source: https://docs.mattermost.com/collaborate/channel-types.html
        'O' for a public channel, 'P' for a private channel, "D": Direct message channel (1:1), "G": Group message channel (group direct message)
        """
        channame = chaninfo['name']
        username = self.userid_to_username(userid)
        if chaninfo['type'] in ('O', 'P'):
            log.debug(f"Channel name: {chaninfo['name']}")
            chans = self.commands[module]['chans']
            if 'any' in chans or channame in chans:
                return True
        elif chaninfo['type'] in ('D', 'G'):
            """
            Check if a user is in one of the channels that are configured in the modules 'chans'
            """
            memberlist = []
            for channame in self.commands[module]['chans']:
                try:
                    page=0
                    while True:
                        channel_info = self.mmDriver.channels.get_channel_members(self.channame_to_chanid(channame), params={'page': page, 'per_page':200})
                        for channel in channel_info:
                            memberlist.append(channel['user_id'])
                        page+=1
                        if len(channel_info) == 0:
                            break
                    if userid in memberlist:
                        return True
                except Exception as e:
                    # Apparently the channel does not exist; perhaps it is spelled incorrectly or otherwise a misconfiguration?
                    log.error("An error occurred during channel parsing: %s\nTraceback: %s" % (str(e),traceback.format_exc()))
        log.info(f"User {userid} is not allowed to use {module} in {channame}.")
        return False

    async def feed_message(self, userid, post, params, chaninfo, rootid):
        self.feedmap = self.load_feedmap()
        command = post['message'].split()[0]
        chanid = post['channel_id']
        channame = chaninfo['name']
        username = self.userid_to_username(userid)
        messages = []
        if not params:
            if command in ('!feeds', '@feeds'):
                if chaninfo['type'] == 'D':
                    text =  "**Feeds do not work in direct messages.**\n"
                    messages.append(text)
                else:
                    enabled_feeds = set()
                    unclassified_feeds = set()
                    if len(self.feedmap):
                        text =  "**List of available topics for channel: `%s`**\n" % (self.channame_to_chandisplayname(channame,))
                        text += "\n"
                        text += f"\n| **Topics** ({len(self.feedmap['TOPICS'])}) | **Available Feeds** ({len(self.feedmap['MODULES'])}) |"
                        text += "\n| :- | :- |"
                        if 'TOPICS' in self.feedmap:
                            for topic in sorted(self.feedmap['TOPICS']):
                                availablefeeds = list(self.feedmap['TOPICS'][topic])
                                for module_name in self.feedmap['MODULES']:
                                    if 'NAME' in self.feedmap['MODULES'][module_name]:
                                        if channame in self.feedmap['MODULES'][module_name]['CHANNELS']:
                                            if module_name in availablefeeds:
                                                availablefeeds.remove(module_name)
                                            enabled_feeds.add(module_name)
                                if len(availablefeeds):
                                    availablefeeds_displaynames = set()
                                    for availablefeed in availablefeeds:
                                        displayname = availablefeed
                                        if 'ADMIN_ONLY' in self.feedmap['MODULES'][availablefeed]:
                                            displayname = availablefeed+r'(*)' if self.feedmap['MODULES'][availablefeed]['ADMIN_ONLY'] else availablefeed
                                        availablefeeds_displaynames.add(displayname)
                                    text += f"\n| {topic} | `"+"`, `".join(sorted(availablefeeds_displaynames))+"` |"
                        for module_name in self.feedmap['MODULES']:
                            if 'NAME' in self.feedmap['MODULES'][module_name]:
                                if 'TOPICS' not in self.feedmap['MODULES'][module_name]:
                                    unclassified_feeds.add(module_name)
                                else:
                                    if not len(self.feedmap['MODULES'][module_name]['TOPICS']):
                                        unclassified_feeds.add(module_name)
                        if len(unclassified_feeds):
                            unclassified_feeds_displaynames = set()
                            for unclassified_feed in unclassified_feeds:
                                displayname = unclassified_feed
                                if 'ADMIN_ONLY' in self.feedmap['MODULES'][unclassified_feed]:
                                    displayname = unclassified_feed+r'(*)' if self.feedmap['MODULES'][unclassified_feed]['ADMIN_ONLY'] else unclassified_feed
                                unclassified_feeds_displaynames.add(displayname)
                            text += "\n| Unclassified | `"+"`, `".join(sorted(unclassified_feeds_displaynames))+"` |"
                        text += "\n\n"
                        text += "*An asterisk after a module name indicates the feed can only be enabled/disabled by a MatterBot admin.*\n"
                        messages.append(text)
                    if len(enabled_feeds):
                        text = "Enabled feeds: `"+"` ,`".join(sorted(enabled_feeds))+f"` ({len(enabled_feeds)})"
                        messages.append(text)
                    else:
                        text = "There are no feeds enabled in this channel.\n"
                        messages.append(text)
        else:
            if command in options.Matterbot['feedcmds']:
                if not self.isadmin(userid):
                    if options.Matterbot['feedmode'].lower() == 'admin':
                        logging.warning(f"User {username} ({userid}) attempted to use a feed (un)subscribe command without proper authorization.")
                        text = "@" + username + ", you do not have permission to (un)subscribe from/to feeds."
                        messages.append(text)
                if self.isadmin(userid) or options.Matterbot['feedmode'].lower() == 'user':
                    all_channel_types = [self.chanid_to_channame(_['id']) for _ in self.mmDriver.channels.get_channels_for_user(self.my_id,self.my_team_id) if self.is_in_channel(_['id'])]
                    my_channels = [_ for _ in all_channel_types if self.my_id not in _]
                    if channame not in my_channels:
                        text = f"@{username}, you cannot have feeds in a Direct Message window."
                        messages.append(text)
                    else:
                        feeds_to_consider = set()
                        if params[0] == '*':
                            feeds_to_consider = self.feedmap['MODULES']
                        else:
                            for param in params[0:]:
                                param = param.lower()
                                lowercase_topics = {_.lower(): _ for _ in self.feedmap['TOPICS']}
                                if param in lowercase_topics:
                                    topic_key = lowercase_topics[param]
                                    for module_name in self.feedmap['TOPICS'][topic_key]:
                                        feeds_to_consider.add(module_name)
                                else:
                                    feeds_to_consider.add(param)
                        switched_feeds = set()
                        blocked_feedchanges = set()
                        if len(params):
                            if command in ('!unsub', '!unsubscribe', '@unsub', '@unsubscribe'):
                                mode = 'disable'
                            elif command in ('!sub', '!subscribe', '@sub', '@subscribe'):
                                mode = 'enable'
                            for module_name in feeds_to_consider:
                                if module_name in self.feedmap['MODULES']:
                                    ADMIN_ONLY = self.feedmap['MODULES'][module_name]['ADMIN_ONLY'] if 'ADMIN_ONLY' in self.feedmap['MODULES'][module_name] else True
                                    if not ADMIN_ONLY or self.isadmin(userid):
                                        if mode == 'enable':
                                            if 'NAME' in self.feedmap['MODULES'][module_name]:
                                                if channame not in self.feedmap['MODULES'][module_name]['CHANNELS']:
                                                    self.feedmap['MODULES'][module_name]['CHANNELS'].append(channame)
                                                    switched_feeds.add(module_name)
                                        elif mode == 'disable':
                                            if 'NAME' in self.feedmap['MODULES'][module_name]:
                                                if channame in self.feedmap['MODULES'][module_name]['CHANNELS']:
                                                    self.feedmap['MODULES'][module_name]['CHANNELS'].remove(channame)
                                                    switched_feeds.add(module_name)
                                    else:
                                        blocked_feedchanges.add(module_name)
                            if len(blocked_feedchanges):
                                logging.warning(f"User {username} ({userid}) attempted an (un)subscribe from/to `"+"`, `".join(blocked_feedchanges)+f"` in `{channame}` without authorization.")
                                text = f"@{username}, you do not have permission to (un)subscribe from/to `"+"`, `".join(blocked_feedchanges)+f"` in `{channame}`."
                                messages.append(text)
                            if len(switched_feeds):
                                logging.info(f"User {username} ({userid}) (un)subscribed from/to in `{channame}`: `"+"`, `".join(switched_feeds)+"`.")
                                text = f"@{username}, the following feeds were {mode}d in `{channame}`: `"+"`, `".join(switched_feeds)+"`."
                                messages.append(text)
                            await self.update_feedmap()
                elif not self.isadmin(userid) and options.Matterbot['feedmode'].lower() == 'admin':
                    text = f"@{username}, feed (un)subscription is restricted to administrators in the current bot configuration."
                    messages.append(text)
                else:
                    text = f"@{username}, how did you end up here?"
                    messages.append(text)
        if len(messages):
            for message in messages:
                await self.send_message(chanid, message, rootid)

    async def bind_message(self, userid, post, params, chaninfo, rootid):
        command = post['message'].split()[0]
        chanid = post['channel_id']
        channame = chaninfo['name']
        username = self.userid_to_username(userid)
        messages = []
        if not params:
            if command in ('!map', '@map'):
                if len(self.commands):
                    chans = set()
                    if chaninfo['type'] == 'D':
                        text =  "**List of modules in direct message:**\n"
                    else:
                        text =  "**List of modules for channel: `%s`**\n" % (self.channame_to_chandisplayname(channame,))
                    text += "\n"
                    text += "\n| **Module Name** | **Available** | **Binds** | **Description** |"
                    text += "\n| :- | :- | :- | :- |"
                    for module in sorted(self.commands):
                        if self.isallowed_module(userid,module,chaninfo):
                            chans.add(module)
                            if 'binds' in self.commands[module] and 'help' in self.commands[module]:
                                text += "\n| %s | **YES** | `%s` | %s |" % (module,'`, `'.join(sorted(self.commands[module]['binds'])),self.commands[module]['help']['DEFAULT']['desc'].replace('|','/'))
                        elif self.isadmin(userid):
                            chans.add(module)
                            if 'binds' in self.commands[module] and 'help' in self.commands[module]:
                                text += "\n| %s | **NO** | `%s` | %s |" % (module,'`, `'.join(sorted(self.commands[module]['binds'])),self.commands[module]['help']['DEFAULT']['desc'].replace('|','/'))
                    text += "\n\n"
                if not len(chans):
                    text = '@' + username + ", I don't know about any commands here.\n"
                text += "*Remember that not every command works everywhere: this depends on the configuration. Modules may offer additional help if you add the subcommand.*"
                messages.append(text)
        else:
            if not self.isadmin(userid):
                logging.warning(f"User {username} ({userid}) attempted to use a bind command without proper authorization.")
                text = "@" + username + ", you do not have permission to bind commands."
            else:
                all_channel_types = [self.chanid_to_channame(_['id']) for _ in self.mmDriver.channels.get_channels_for_user(self.my_id,self.my_team_id) if self.is_in_channel(_['id'])]
                my_channels = [_ for _ in all_channel_types if self.my_id not in _]
                if channame not in my_channels:
                    text = "@" + username + ", you cannot bind commands to direct message windows."
                else:
                    if params[0] == '*':
                        params = self.commands.keys() # Attempt to enable/disable all modules
                    for modulename in params:
                        if modulename not in self.commands:
                            text = "@" + username + ", there is no `%s` module loaded. Use one of the help commands (`%s`) to see a list of available modules." % (modulename,"`, `".join(options.Matterbot['helpcmds']))
                        elif command in ('!bind', '@bind'):
                            if channame in self.commands[modulename]['chans']:
                                text = "The `%s` module is already available in the `%s` channel." % (modulename,self.channame_to_chandisplayname(channame))
                            else:
                                self.commands[modulename]['chans'].append(channame)
                                text = "The `%s` module is now available in the `%s` channel." % (modulename,self.channame_to_chandisplayname(channame))
                        elif command in ('!unbind', '@unbind'):
                            if channame not in self.commands[modulename]['chans']:
                                text = "The `%s` module is not loaded in the `%s` channel." % (modulename,self.channame_to_chandisplayname(channame))
                            else:
                                self.commands[modulename]['chans'].remove(channame)
                                text = "The `%s` module has been removed from the `%s` channel." % (modulename,self.channame_to_chandisplayname(channame))
                        messages.append(text)
                await self.update_bindmap()
        if len(messages):
            for message in messages:
                await self.send_message(chanid, message, rootid)

    async def help_message(self, userid, params, chaninfo, rootid):
        chanid = chaninfo['id']
        commands = set()
        params = [_.lower() for _ in params]
        if not params:
            for module in self.commands:
                if self.isallowed_module(userid, module, chaninfo):
                    for bind in self.commands[module]['binds']:
                        commands.add('`' + bind + '`')
            text =  "I know about: `"+'`, `'.join(sorted(options.Matterbot['helpcmds']))+"`, " + ', '.join(sorted(commands)) + " here.\n"
            text += "Every command has its own specific help. For example: `!help @dice` will show you how to use the `@dice` command.\n"
            text += "*Remember: not every command works in every channel: this depends on a module's configuration*"
            await self.send_message(chanid, text, rootid)
        else:
            # User is asking for specific module help
            for module in self.commands:
                if self.isallowed_module(userid, module, chaninfo):
                    if set(params) & set(self.commands[module]['binds']): # for future use
                        try:
                            text = ''
                            HELP = self.commands[module]['help']
                            paramsubcommands = set(params) & set(HELP)
                            if len(paramsubcommands) == 0:
                                if 'DEFAULT' in HELP:
                                    # Trigger the default help message
                                    args = HELP['DEFAULT']['args'] if HELP['DEFAULT']['args'] else None
                                    desc = HELP['DEFAULT']['desc']
                                    text += '**Module**: `' + module + '`'
                                    text += '\n**Description**: '
                                    text += desc
                                    if args:
                                        text += '\n**Arguments**: `' + args + '`'
                                    subcommands = set()
                                    if len(HELP)>1:
                                        text += '\n**Subcommmands**: '
                                    for subcommand in HELP:
                                        if subcommand != 'DEFAULT':
                                            subcommands.add(subcommand)
                                    if len(subcommands)>0:
                                        text += '`' + '`, `'.join(subcommands) + '`'
                            else: # paramsubcommands >= 1
                                for subcommand in paramsubcommands:
                                    args = HELP[subcommand]['args'] if HELP[subcommand]['args'] else None
                                    desc = HELP[subcommand]['desc']
                                    text += '**Module**: `' + module + '`/`' + subcommand + '`'
                                    text += '\n**Description**: '
                                    text += desc
                                    if args:
                                        text += '\n**Arguments**: `' + args + '`'
                            if len(text)>0:
                                await self.send_message(chanid, text, rootid)
                        except NameError:
                            await self.send_message(chanid, text, rootid)

    async def handle_event(self, event: dict):
        eventtype = event['type'] if 'type' in event else None
        chanid = event['channel_id'] if 'channel_id' in event else None
        if chanid:
            channame = self.channelmapping['idtoname'][chanid]['name'] if chanid in self.channelmapping['idtoname'] else None
        else:
            channame = None
        userid = event['user_id'] if 'user_id' in event else None
        if userid:
            username = self.userid_to_username(userid)
        if not eventtype: # Not a regular type of event, check for the various types
            if 'remover_id' in event and 'user_id' in event: # Removed from a channel!
                if 'user_id' == self.my_id:
                    username = self.userid_to_username(userid)
                    if channame:
                        for modulename in self.commands:
                            if channame in self.commands[modulename]['chans']:
                                self.commands[modulename]['chans'].remove(channame)
                                log.info(f"I was just removed from the '{channame}' ({chanid}) channel by '{username}' ({userid}). Existing module bindings for the channel were removed the config file.")
                        await self.update_bindmap()
            if (options.Matterbot['welcome'] and len(event) == 2 and 'team_id' in event and 'user_id' in event) or \
                (options.Matterbot['welcome'] and len(event) == 2 and 'remover_id' in event and 'user_id' in event):
                old_members = self.welcome_channel_members
                self.welcome_channel_members = await self.update_welcome_channel()
                if len(self.welcome_channel_members) > len(old_members):
                    difference = list(set(self.welcome_channel_members) - set(old_members))
                    channel_change = 'joined'
                elif len(old_members) > len(self.welcome_channel_members):
                    difference = list(set(old_members) - set(self.welcome_channel_members))
                    channel_change = 'left'
                else:
                    difference = None
                if difference:
                    if options.Matterbot['welcome_channel']:
                        welcome_channel = options.Matterbot['welcome_channel']
                    if options.Matterbot['welcome_banner']:
                        welcome_banner = options.Matterbot['welcome_banner']
                    if welcome_channel and welcome_banner:
                        if channel_change == 'joined':
                            usernames = ['@'+self.userid_to_username(_) for _ in difference]
                            text = ", ".join(usernames)+": "+welcome_banner+"\n"
                            if options.Matterbot['welcome_file']:
                                welcome_file = options.Matterbot['welcome_file']
                                welcome_message = pathlib.Path(welcome_file)
                                if welcome_message.is_file():
                                    try:
                                        with open(welcome_file) as f:
                                            text += f.read()
                                    except Exception:
                                        log.exception(f"The welcome message file {welcome_file} could not be read!")
                            await self.send_message(welcome_channel, text)


    def _semaphore_for(self, userid):
        # setdefault is atomic in CPython for the dict op, so two coroutines
        # racing on the same first-time userid get the same Semaphore back
        # (one freshly-constructed instance may be discarded — harmless).
        return self._user_semaphores.setdefault(
            userid, asyncio.Semaphore(self._user_concurrency)
        )

    async def call_module(self, module, command, channame, rootid, username, params, files, conn):
        try:
            chanid = self.channame_to_chanid(channame)
            # Run the (synchronous) module handler in a thread so it cannot block
            # the asyncio event loop. The outer asyncio.timeout in handle_post
            # is what bounds wall-clock duration; this await is the yield point
            # that lets it fire.
            loop = asyncio.get_running_loop()
            result = await loop.run_in_executor(
                self._command_executor,
                self.commands[module]['process'],
                command, channame, username, params, files, conn,
            )
            # Command logging: see config.defaults.yaml for clarification
            if result and 'messages' in result:
                for message in result['messages']:
                    uploads = []
                    if 'text' in message:
                        text = message['text']
                    if 'uploads' in message:
                        if message['uploads'] is not None:
                            uploads = []
                            for upload in message['uploads']:
                                filename = upload['filename']
                                payload = upload['bytes']
                                if not isinstance(payload, (bytes, bytearray)):
                                    payload = payload.encode()
                                file_id = conn.files.upload_file(
                                    channel_id=chanid,
                                    files={'files': (filename, payload)}
                                )['file_infos'][0]['id']
                                uploads.append(file_id)
                    await self.send_message(chanid, text, rootid, uploads or None)
        except Exception as e:
            log.exception(f"call_module: error dispatching module={module}")
            text = "An error occurred during the %s module call: `%s`" % (str(module), str(e))
            await self.send_message(chanid, text, rootid)

    async def handle_post(self, data: dict):
        if 'sender_name' in data:
            username = data['sender_name']
        else:
            # We're currently not handling users editing messages
            return
        post = json.loads(data['post'])
        # Welcome-module hook: system_join_channel posts indicate a user joined
        # a (typically public) channel. Defer to the welcome module and short-
        # circuit — system posts have empty `message` and would otherwise drop
        # through the command-dispatch loop harmlessly, but explicit early
        # return is cheaper and clearer.
        if post.get('type') == 'system_join_channel':
            try:
                from welcome.welcome import on_join as _welcome_on_join
                loop = asyncio.get_running_loop()
                loop.run_in_executor(
                    self._command_executor,
                    _welcome_on_join,
                    self.mmDriver, self.my_id,
                    post.get('user_id'), post.get('channel_id'),
                )
            except ImportError:
                pass
            except Exception:
                log.exception("welcome.on_join hook failed (system_join_channel)")
            return
        userid = post['user_id']
        chanid = post['channel_id']
        chaninfo = self.chanid_to_chaninfo(chanid)
        channame = chaninfo['name']
        rootid = post['root_id'] if len(post['root_id']) else post['id']
        messagelines = post['message'].splitlines()
        # We're probably handling a regular message; make sure to check we're allowed to respond our own messages too (see config file)
        # Additionally, check if we're not self-triggering on the display of the bind map
        if options.Matterbot['recursion'] or userid != self.my_id:
            # Watch-module hook: scan the post against active keyword watches
            # and DM matched watchers. Fire-and-forget on the command pool so
            # we don't stall the event loop or command dispatch below.
            # ImportError = watch module not in this deployment; silently skip.
            try:
                from watch.command import scan_message as _watch_scan
                loop = asyncio.get_running_loop()
                loop.run_in_executor(
                    self._command_executor,
                    _watch_scan,
                    self.mmDriver, self.my_id,
                    userid, chanid, channame, username,
                    post.get('message') or '',
                )
            except ImportError:
                pass
            except Exception:
                log.exception("watch.scan_message hook failed")
            messages = list()
            for mline in messagelines:
                addparams = False
                message = mline.split()
                for idx,word in enumerate(message):
                    if ((word.lower() in self.binds) \
                        and (message[idx-1] not in options.Matterbot['helpcmds'] and message[idx-1] not in options.Matterbot['mapcmds'] \
                        and message[idx-1] not in options.Matterbot['feedcmds'] ) \
                        or (word in options.Matterbot['helpcmds']) \
                        or ((word in options.Matterbot['mapcmds']) and (message[idx-1] not in options.Matterbot['helpcmds'] )) \
                        or ((word in options.Matterbot['feedcmds']) and (message[idx-1] not in options.Matterbot['helpcmds'] ))  ):
                        messages.append({'command':word.lower(),'parameters':[]})
                        addparams = True
                    elif addparams:
                        messages[-1]['parameters'].append(word)
            log.debug(f"Messages: {messages}")
            for messagedict in messages:
                command = messagedict['command']
                params = messagedict['parameters']
                if command in options.Matterbot['helpcmds']:
                    await self.log_message(userid, command, params, chaninfo, rootid)
                    await self.help_message(userid, params, chaninfo, rootid)
                elif command in options.Matterbot['mapcmds']:
                    await self.log_message(userid, command, params, chaninfo, rootid)
                    await self.bind_message(userid, post, params, chaninfo, rootid)
                elif command in options.Matterbot['feedcmds']:
                    await self.log_message(userid, command, params, chaninfo, rootid)
                    await self.feed_message(userid, post, params, chaninfo, rootid)
                else:
                    await self.log_message(userid, command, params, chaninfo, rootid)
                    if not any(_ in post['message'] for _ in ('| **YES** |', '| **NO** |', 'I know about `!help')):
                        # Bindings like @ioc subscribe many modules to a single command.
                        # Collect them up front, then fan out concurrently via gather
                        # so the slowest module sets the wall-clock floor, not the sum.
                        bound = []
                        for module in self.commands:
                            if command in self.commands[module]['binds']:
                                if self.isallowed_module(userid, module, chaninfo):
                                    if module not in bound:
                                        bound.append(module)
                        # Route by indicator type. A shared bind like @ioc reaches
                        # domain-only, hash-only and IP-only modules alike; calling
                        # a module with an input it cannot handle makes it answer
                        # with an error in the channel. Classify the argument once
                        # and only run modules that declare they accept that type.
                        # Modules that declare nothing accept anything (free-text
                        # commands, or not-yet-annotated), so this only ever narrows
                        # an @ioc-style fanout, never a plain command.
                        from commands import cmdutils
                        if params:
                            _, indicator_type = cmdutils.classify(params[0])
                            modules_to_run = [m for m in bound
                                              if cmdutils.accepts(self.commands[m], indicator_type)]
                            if bound and not modules_to_run:
                                # Something was bound to this command, but nothing
                                # accepts what the user typed. Say so once, instead
                                # of staying silent or letting N modules each error.
                                if indicator_type is None:
                                    text = (f"`{params[0]}` doesn't look like {cmdutils.TYPES_HUMAN}, "
                                            f"so I didn't query anything for `{command}`.")
                                else:
                                    text = (f"No `{command}` module is configured to look up "
                                            f"{indicator_type} indicators.")
                                await self.send_message(chanid, text, rootid)
                        else:
                            # No argument to classify: preserve prior behaviour
                            # (each module handles its own empty-input case).
                            modules_to_run = bound
                        if modules_to_run:
                            files = []
                            if 'metadata' in post:
                                if 'files' in post['metadata']:
                                    if len(post['metadata']['files']):
                                        files = post['metadata']['files']

                            async def _run_module(module):
                                try:
                                    async with asyncio.timeout(self._command_timeout):
                                        await self.call_module(module, command, channame, rootid, username, params, files, self.mmDriver)
                                except asyncio.TimeoutError:
                                    self._recycle_command_executor()
                                    log.warning(
                                        f"Command timed out: module={module} command={command} "
                                        f"user={username} channel={channame}"
                                    )
                                    text = f"Error: the command to the {module} module timed out while processing/waiting for a response."
                                    await self.send_message(chanid, text, rootid)

                            # Per-user fairness gate is acquired ONCE per command
                            # invocation, not per subscribed module — otherwise an
                            # @ioc-style fanout to N modules would serialize behind
                            # user_concurrency=2 and lose most of the gather win.
                            async with self._semaphore_for(userid):
                                await asyncio.gather(*(_run_module(m) for m in modules_to_run))

if __name__ == '__main__' :
    '''
    Interactive run from the command-line
    '''
    parser = configargparse.ArgParser(
        config_file_parser_class=configargparse.YAMLConfigFileParser,
        description='Matterbot loads modules and sends their output to Mattermost.',
        default_config_files=['config.yaml'])
    parser.add('--Matterbot', type=str, help='MatterBot configuration, as a dictionary (see YAML config)')
    parser.add('--Modules', type=str, help='Modules configuration, as a dictionary (see YAML config)')
    parser.add('--debug', default=False, action='store_true', help='Enable debug mode and log to foreground')
    options, unknown = parser.parse_known_args()
    options.Matterbot = ast.literal_eval(options.Matterbot)
    options.Modules = ast.literal_eval(options.Modules)
    if not options.debug:
        logging.basicConfig(level=logging.INFO, filename=options.Matterbot['logfile'], format='%(levelname)s - %(name)s - %(asctime)s - %(message)s')
    else:
        logging.basicConfig(level=logging.DEBUG,format='%(levelname)s - %(name)s - %(asctime)s - %(message)s')
    log = logging.getLogger('MatterBot')
    log.info('Starting MatterBot')
    mm = MattermostManager()
    mm.run_forever()