From 856e6f429b31cf9999fee7077292381757933b96 Mon Sep 17 00:00:00 2001 From: Michael Woods Date: Tue, 18 Mar 2025 22:29:29 -0400 Subject: [PATCH] CLI messages seems mostly feature complete. Added some examples to the readme. --- Readme.md | 106 ++++++++++++++++++-- src/packetserver/client/cli/__init__.py | 2 + src/packetserver/client/cli/message.py | 128 +++++++++++++++++++++++- src/packetserver/client/cli/util.py | 10 +- src/packetserver/client/messages.py | 41 +++++++- src/packetserver/common/util.py | 4 +- src/packetserver/server/messages.py | 6 +- 7 files changed, 282 insertions(+), 15 deletions(-) diff --git a/Readme.md b/Readme.md index 73da42f..44cdff1 100644 --- a/Readme.md +++ b/Readme.md @@ -36,27 +36,121 @@ UI packets later on.. - protocol using connected mode sessions to provide request/response architecture - Object CRUD operations -- Podman containerized job orchestrator +- Server-side Podman containerized job orchestrator - automatic compression for all RF communication +- Creating and retrieving bulletins +- Sending and receiving and searching messages to/from other users + ## Features in-progress and working to some extent: -- Sending and receiving and searching messages to/from other users -- Posting, retrieving, and editing public bulletins -- Python client wrapper library for the complete RF 'API' +- corresponding Python client wrapper library for most elements of the server-side (RF) API (enough for basic usage anyway) +- Python CLI client supporting: + - listing registered users on the server + - setting and retrieving personal user profile details + - uploading files as objects, searching objects on server, downloading object data + - sending and retrieving messages to other users including attached arbitrary string and binary data + - running basic scripts/commands as jobs on the server inside containers + +## Planned features not yet implemented: + +- client API and CLI capability to fully interact with the job system including getting artifacts back from jobs +- client API and CLI capability to set and retrieve bulletins +- editing public bulletins, once created +- client/server API capability to modify objects + ## I'm considering several other features like: - Useful documentation of any variety.. - RF beacon - service administration over RF -- cli administration tools - - Right now, just edit the zope database with a python interpreter +- cli server administration tools + - Right now, just edit the server's zope database with a python interpreter (included example scripts to help) - possibly a cron system (again in containers for safety) - maybe an e-mail or an sms gateway (though clever user uploaded scripts could do this instead) - maybe APRS integration through APRS-IS - Kubernetes or possibly simple shell job execution. + +## Examples + +### Main help dialog: +```commandline +(venv) [user@host]$ packetcli +Usage: packetcli [OPTIONS] COMMAND [ARGS]... + + Command line interface for the PacketServer client and server API. + +Options: + --conf TEXT path to configfile + -s, --server TEXT server radio callsign to connect to (required) + -a, --agwpe TEXT AGWPE TNC server address to connect to (config file) + -p, --port INTEGER AGWPE TNC server port to connect to (config file) + -c, --callsign TEXT radio callsign[+ssid] of this client station (config + file) + -k, --keep-log Save local copy of request log after session ends? + -v, --version Show the version and exit. + --help Show this message and exit. + +Commands: + job Runs commands on the BBS server if jobs are enabled on it. + message Send, search, and filter messages to and from other users... + object Manages objects stored on the BBS. + query-server Query the server for basic info. + set Set your user profile settings on the BBS. + user Query users on the BBS. +``` + +### Working with objects: +```commandline +(venv) [user@host]$ packetcli object list +name size_bytes binary private created_at modified_at uuid +--------------- ------------ -------- --------- -------------------------------- -------------------------------- ------------------------------------ +testdb.txt 13 False True 2025-03-16T22:26:05.049173+00:00 2025-03-16T22:26:05.051375+00:00 fbbd4527-a5f0-447f-9fc9-55b7b263c458 + +(venv) [user@host]$ packetcli object upload-file +Usage: packetcli object upload-file [OPTIONS] FILE_PATH +Try 'packetcli object upload-file --help' for help. + +Error: Missing argument 'FILE_PATH'. + +(venv) [user@host]$ packetcli object upload-file /tmp/hello-world.txt +35753577-21e3-4f64-8776-e3f86f1bb0e0 + +(venv) [user@host]$ packetcli object list +name size_bytes binary private created_at modified_at uuid +--------------- ------------ -------- --------- -------------------------------- -------------------------------- ------------------------------------ +testdb.txt 13 False True 2025-03-16T22:26:05.049173+00:00 2025-03-16T22:26:05.051375+00:00 fbbd4527-a5f0-447f-9fc9-55b7b263c458 +hello-world.txt 13 False True 2025-03-19T02:25:41.501833+00:00 2025-03-19T02:25:41.503502+00:00 35753577-21e3-4f64-8776-e3f86f1bb0e0 + +(venv) packetcli object get 35753577-21e3-4f64-8776-e3f86f1bb0e0 +Hello world. + +``` + +### Retrieving messages: +```commandline +(venv) [user@host]$ packetcli message get +from to id text sent_at attachments +------ ------ ------------------------------------ ------------------------------- -------------------------------- ------------- +KQ4PEC KQ4PEC df7493d7-5880-4c24-9e3c-1d3987a5203e testing.. again with attachment 2025-03-18T03:41:36.597371+00:00 random.txt +KQ4PEC KQ4PEC e3056cdf-1f56-4790-8aef-dfea959bfa13 from stdin 2025-03-18T03:40:36.051667+00:00 +KQ4PEC KQ4PEC 992c3e81-005a-49e2-81d7-8bf3026a2c46 testing.. again 2025-03-18T03:40:05.025017+00:00 +KQ4PEC KQ4PEC 05684b13-40f8-40aa-ab7a-f50a3c22261e testing.. 1.. 2.. 3 2025-03-18T03:39:50.510164+00:00 +KQ4PEC KQ4PEC ad513075-e50f-4f84-8a87-a1217b43bef3 testing.. 1.. 2.. 3 2025-03-18T03:38:01.634498+00:00 +``` + +### Listing users: +```commandline +(venv) [user@host]$ packetcli user -l +username status bio socials created last_seen email location +---------- --------------------- ----- --------- -------------------------------- -------------------------------- --------------- ---------- +KQ4PEC just happy to be here 2025-03-16 04:29:52.044216+00:00 2025-03-19 02:22:21.413896+00:00 user@domain.com +``` + +### + ## Final Thoughts I may also add a TCP/IP interface to this later, since that shouldn't be too difficult. We'll see. diff --git a/src/packetserver/client/cli/__init__.py b/src/packetserver/client/cli/__init__.py index 082fb90..ffb57ef 100644 --- a/src/packetserver/client/cli/__init__.py +++ b/src/packetserver/client/cli/__init__.py @@ -55,6 +55,8 @@ def cli(ctx, conf, server, agwpe, port, callsign, keep_log): else: ctx.obj['callsign'] = click.prompt('Please enter your station callsign (with ssid if needed)', type=str) + ctx.obj['directory'] = cfg['cli']['directory'] + if not ax25.Address.valid_call(ctx.obj['callsign']): click.echo(f"Provided client callsign '{ctx.obj['callsign']}' is invalid.", err=True) sys.exit(1) diff --git a/src/packetserver/client/cli/message.py b/src/packetserver/client/cli/message.py index d1cca98..b544151 100644 --- a/src/packetserver/client/cli/message.py +++ b/src/packetserver/client/cli/message.py @@ -4,11 +4,18 @@ import os.path from email.policy import default import click -from packetserver.client.cli.util import exit_client, format_list_dicts +from zodbpickle.pickle_3 import FALSE + +from packetserver.client.cli.util import exit_client, format_list_dicts, unit_seconds from copy import deepcopy from uuid import UUID +import datetime +import re +import json from packetserver.client.messages import * +rel_date = '^-(\\d+)([dhms])$' + @click.group() @click.pass_context def message(ctx): @@ -81,6 +88,125 @@ def send(ctx, recipients, body, body_filename, attachment): click.echo(f"Error sending message: {str(e)}", err=True) exit_client(ctx.obj, 53) + +@click.command() +@click.option("--number", "-n", type=int, default=0,help="Retrieve the first N messages matching filters/sort. 0 for all.") +@click.option('--sent', '-S', is_flag=True, default=False, help="Include sent messages in results.") +@click.option("--not-received", "-R", is_flag=True, default=False, help="Don't include received messages.") +@click.option("--ascending", "-A", is_flag=True, default=False, help="Show older/smaller results first after sorting.") +@click.option("--no-attachments", "-N", is_flag=True, default=False, help="Don't fetch attachment data.") +@click.option("--uuid", "-u", type=str, default=None, help="If specified, ignore other filters and retrieve only messages matching uuid.") +@click.option("--since-date", "-d", type=str, default=None, help="Only include messages since date (iso format), or '-' ex: -5d") +@click.option("--output-format", "-f", default="table", help="Print data as table[default], list, or JSON", + type=click.Choice(['table', 'json', 'list'], case_sensitive=False)) +@click.option("--save-copy", "-C", is_flag=True, default=False, help="Save a full copy of each message to fs.") +@click.option("--search", "-F", type=str, default="", help="Return only messages containing search string.") +@click.option("--no-text", "-T", is_flag=True, default=False, help="Don't return the message text.") +@click.option("--sort-by", "-B", default="date", help="Choose to sort by 'date', 'from', or 'to'", + type=click.Choice(['date', 'from', 'to'], case_sensitive=False)) +@click.pass_context +def get(ctx, number, sent, not_received, ascending, no_attachments, uuid, since_date, output_format, save_copy, + search, no_text, sort_by): + client = ctx.obj['client'] + bbs = ctx.obj['bbs'] + messages = [] + get_attach = not no_attachments + get_text = not no_text + reverse = not ascending + if uuid is not None: + try: + uuid = UUID(uuid) + except: + exit_client(ctx.obj, 52, message="Must provide a valid UUID.") + + if type(search) is str and (search.strip() == ""): + search = None + if not_received: + if sent: + source='sent' + else: + exit_client(ctx.obj, 23, "Can't exclude both sent and received messages.") + else: + if sent: + source='all' + else: + source='received' + + if number == 0: + limit = None + else: + limit = number + + if since_date is not None: + if len(since_date) < 3: + exit_client(ctx.obj, 41, "Invalid date specification.") + + if since_date[0] == "-": + m = re.match(rel_date, since_date) + if m is None: + exit_client(ctx.obj, 41, "Invalid date specification.") + else: + unit = m.group(2).lower() + multiplier = int(m.group(1)) + if unit not in unit_seconds: + exit_client(ctx.obj, 41, "Invalid date specification.") + total_seconds = int(multiplier * unit_seconds[unit]) + cutoff_date = datetime.datetime.now() - datetime.timedelta(seconds=total_seconds) + else: + try: + cutoff_date = datetime.datetime.fromisoformat(since_date) + except: + exit_client(ctx.obj, 41, "Invalid date specification.") + + if type(uuid) is UUID: + try: + messages.append(get_message_uuid(client, bbs, uuid, get_attachments=get_attach)) + except Exception as e: + exit_client(ctx.obj, 40, message=f"Couldn't get message specified: {str(e)}") + elif since_date is not None: + try: + messages = get_messages_since(client, bbs, cutoff_date, get_text=get_text, limit=limit, sort_by=sort_by, + reverse=reverse, search=search, get_attachments=get_attach, source=source) + except Exception as e: + exit_client(ctx.obj, 40, message=f"Couldn't fetch messages: {str(e)}") + else: + try: + messages = get_messages(client, bbs, get_text=get_text, limit=limit, sort_by=sort_by, reverse=reverse, + search=search, get_attachments=get_attach, source=source) + except Exception as e: + exit_client(ctx.obj, 40, message=f"Couldn't fetch messages: {str(e)}") + + save_dir = os.path.join(ctx.obj['directory'], 'message_cache') + if save_copy: + if not os.path.isdir(save_dir): + os.mkdir(save_dir) + + message_display = [] + for msg in messages: + json_filename = f"{msg.sent.strftime("%Y%m%d%H%M%s")}-{msg.from_user}.json" + json_path = os.path.join(save_dir, json_filename) + if save_copy: + json.dump(msg.to_dict(json=True), open(json_path, 'w')) + d = { + 'from': msg.from_user, + 'to': ",".join(msg.to_users), + 'id': str(msg.msg_id), + 'text': msg.text, + 'sent_at': msg.sent.isoformat(), + 'attachments': "", + } + if len(msg.attachments) > 0: + d['attachments'] = ",".join([a.name for a in msg.attachments]) + + if save_copy: + d['saved_path'] = json_path + message_display.append(d) + exit_client(ctx.obj, 0, format_list_dicts(message_display, output_format=output_format)) + + + + +message.add_command(get) message.add_command(send) diff --git a/src/packetserver/client/cli/util.py b/src/packetserver/client/cli/util.py index 683cdee..c91e40b 100644 --- a/src/packetserver/client/cli/util.py +++ b/src/packetserver/client/cli/util.py @@ -52,7 +52,9 @@ def exit_client(context: dict, return_code: int, message=""): click.echo(message, err=is_err) sys.exit(return_code) - - - - +unit_seconds ={ + 'h': 3600, + 'm': 60, + 's': 1, + 'd': 86400 +} \ No newline at end of file diff --git a/src/packetserver/client/messages.py b/src/packetserver/client/messages.py index 7135874..a4cbf34 100644 --- a/src/packetserver/client/messages.py +++ b/src/packetserver/client/messages.py @@ -6,6 +6,7 @@ from packetserver.common.util import to_date_digits from typing import Union, Optional from uuid import UUID, uuid4 import os.path +import base64 class AttachmentWrapper: @@ -33,6 +34,21 @@ class AttachmentWrapper: else: return self._data['data'].decode() + def to_dict(self, json: bool = True) -> dict: + d = { + "name": self.name, + "binary": self.binary, + } + + if not self.binary: + d['data'] = self.data + else: + if json: + d['data'] = base64.b64encode(self.data).decode() + else: + d['data'] = self.data + return d + class MessageWrapper: def __init__(self, data: dict): for i in ['attachments', 'to', 'from', 'id', 'sent_at', 'text']: @@ -67,6 +83,26 @@ class MessageWrapper: a_list.append(AttachmentWrapper(a)) return a_list + def to_dict(self, json: bool = True) -> dict: + d = { + 'text': self.text, + 'sent': self.sent, + 'id': self.msg_id, + 'to': self.to_users, + 'from': self.from_user, + 'attachments': [] + } + + if json: + d['id'] = str(d['id']) + d['sent'] = d['sent'].isoformat() + for a in self.attachments: + d['attachments'].append(a.to_dict(json=True)) + else: + for a in self.attachments: + d['attachments'].append(a.to_dict(json=False)) + return d + class MsgAttachment: def __init__(self, name: str, data: Union[bytes,str]): self.binary = True @@ -113,11 +149,12 @@ def send_message(client: Client, bbs_callsign: str, text: str, to: list[str], raise RuntimeError(f"POST message failed: {response.status_code}: {response.payload}") return response.payload -def get_message_uuid(client: Client, bbs_callsign: str, msg_id: UUID, ) -> MessageWrapper: +def get_message_uuid(client: Client, bbs_callsign: str, msg_id: UUID, get_attachments: bool = True) -> MessageWrapper: req = Request.blank() req.path = "message" req.method = Request.Method.GET req.set_var('id', msg_id.bytes) + req.set_var('fetch_attachments', get_attachments) response = client.send_receive_callsign(req, bbs_callsign) if response.status_code != 200: raise RuntimeError(f"GET message failed: {response.status_code}: {response.payload}") @@ -141,6 +178,7 @@ def get_messages_since(client: Client, bbs_callsign: str, since: datetime.dateti req.set_var('limit', limit) req.set_var('fetch_text', get_text) req.set_var('reverse', reverse) + req.set_var('fetch_attachments', get_attachments) if sort_by.strip().lower() not in ['date', 'from', 'to']: raise ValueError("sort_by must be in ['date', 'from', 'to']") @@ -175,6 +213,7 @@ def get_messages(client: Client, bbs_callsign: str, get_text: bool = True, limit req.set_var('limit', limit) req.set_var('fetch_text', get_text) req.set_var('reverse', reverse) + req.set_var('fetch_attachments', get_attachments) if sort_by.strip().lower() not in ['date', 'from', 'to']: raise ValueError("sort_by must be in ['date', 'from', 'to']") diff --git a/src/packetserver/common/util.py b/src/packetserver/common/util.py index e9e7243..3b36e38 100644 --- a/src/packetserver/common/util.py +++ b/src/packetserver/common/util.py @@ -19,7 +19,7 @@ def email_valid(email: str) -> bool: def to_date_digits(index: datetime.datetime) -> str: return f"{str(index.year).zfill(4)}{str(index.month).zfill(2)}{str(index.day).zfill(2)}{str(index.hour).zfill(2)}{str(index.minute).zfill(2)}{str(index.second).zfill(2)}" -def from_date_digits(index: str) -> datetime: +def from_date_digits(index: str, tz: datetime.timezone = datetime.UTC) -> datetime: ind = str(index) if not ind.isdigit(): raise ValueError("Received invalid date digit string, containing non-digit chars.") @@ -46,7 +46,7 @@ def from_date_digits(index: str) -> datetime: if len(ind) >= 14: second = int(ind[12:14]) - return datetime.datetime(year, month, day ,hour, minute, second) + return datetime.datetime(year, month, day ,hour, minute, second, tzinfo=tz) def tar_bytes(file: Union[str, Iterable]) -> bytes: """Creates a tar archive in a temporary file with the specified files at root level. diff --git a/src/packetserver/server/messages.py b/src/packetserver/server/messages.py index 7da96e1..10898cf 100644 --- a/src/packetserver/server/messages.py +++ b/src/packetserver/server/messages.py @@ -338,12 +338,16 @@ def handle_messages_since(req: Request, conn: PacketServerConnection, db: ZODB.D with db.transaction() as db: mailbox_create(username, db.root()) mb = db.root.messages[username] + logging.debug(f"Only grabbing messages since {since_date}") new_mb = [msg for msg in mb if msg.sent_at >= since_date] + if len(new_mb) > 0: + logging.debug(f"First message in new list: {new_mb[0].sent_at}") + logging.debug(f"Last message in new list: {new_mb[-1].sent_at}") if opts.search: messages = [msg for msg in new_mb if (opts.search in msg.text.lower()) or (opts.search in msg.msg_to[0].lower()) or (opts.search in msg.msg_from.lower())] else: - messages = [msg for msg in mb] + messages = [msg for msg in new_mb] if opts.sort_by == "from": messages.sort(key=lambda x: x.msg_from, reverse=opts.reverse)