From 338afc7a63681d1528a7a73b1e4a700ad9a75565 Mon Sep 17 00:00:00 2001 From: Michael Woods Date: Tue, 18 Mar 2025 22:50:24 -0400 Subject: [PATCH] Reorganized repo a bit --- Readme.md | 22 +- packetserver/__init__.py | 1 + .../build/lib}/client/__init__.py | 0 .../build/lib}/client/bulletins.py | 0 .../build/lib}/client/cli/__init__.py | 0 .../build/lib}/client/cli/bulletin.py | 0 .../build/lib}/client/cli/config.py | 0 .../build/lib}/client/cli/constants.py | 0 .../build/lib}/client/cli/db.py | 0 .../build/lib}/client/cli/job.py | 0 .../build/lib}/client/cli/message.py | 0 .../build/lib}/client/cli/object.py | 0 .../build/lib}/client/cli/server.py | 0 .../build/lib}/client/cli/util.py | 0 .../build/lib}/client/jobs.py | 0 .../build/lib}/client/messages.py | 0 .../build/lib}/client/objects.py | 0 .../build/lib}/client/testing.py | 0 .../build/lib}/client/users.py | 0 .../build/lib}/common/__init__.py | 0 .../build/lib}/common/constants.py | 0 .../build/lib}/common/testing.py | 0 .../build/lib}/common/util.py | 0 .../build/lib}/runner/__init__.py | 0 .../build/lib}/runner/constants.py | 0 .../build/lib}/runner/podman.py | 0 .../build/lib}/server/__init__.py | 2 +- .../build/lib}/server/bulletin.py | 0 .../build/lib}/server/constants.py | 0 .../build/lib}/server/db.py | 0 .../build/lib}/server/jobs.py | 0 .../build/lib}/server/messages.py | 0 .../build/lib}/server/objects.py | 0 .../build/lib}/server/requests.py | 0 .../build/lib}/server/testserver.py | 0 .../build/lib}/server/users.py | 0 packetserver/client/__init__.py | 172 +++++++ packetserver/client/bulletins.py | 73 +++ packetserver/client/cli/__init__.py | 193 +++++++ packetserver/client/cli/bulletin.py | 0 packetserver/client/cli/config.py | 24 + packetserver/client/cli/constants.py | 5 + packetserver/client/cli/db.py | 0 packetserver/client/cli/job.py | 103 ++++ packetserver/client/cli/message.py | 213 ++++++++ packetserver/client/cli/object.py | 127 +++++ packetserver/client/cli/server.py | 0 packetserver/client/cli/util.py | 60 +++ packetserver/client/jobs.py | 179 +++++++ packetserver/client/messages.py | 231 +++++++++ packetserver/client/objects.py | 161 ++++++ packetserver/client/testing.py | 85 ++++ packetserver/client/users.py | 122 +++++ packetserver/common/__init__.py | 343 +++++++++++++ packetserver/common/constants.py | 2 + packetserver/common/testing.py | 268 ++++++++++ packetserver/common/util.py | 151 ++++++ packetserver/packetserver.egg-info/PKG-INFO | 12 + .../packetserver.egg-info/SOURCES.txt | 41 ++ .../dependency_links.txt | 1 + .../packetserver.egg-info/entry_points.txt | 2 + .../packetserver.egg-info/requires.txt | 8 + .../packetserver.egg-info/top_level.txt | 1 + packetserver/runner/__init__.py | 191 +++++++ packetserver/runner/constants.py | 48 ++ packetserver/runner/podman.py | 463 +++++++++++++++++ packetserver/server/__init__.py | 329 ++++++++++++ packetserver/server/bulletin.py | 159 ++++++ packetserver/server/constants.py | 6 + packetserver/server/db.py | 60 +++ packetserver/server/jobs.py | 382 ++++++++++++++ packetserver/server/messages.py | 478 ++++++++++++++++++ packetserver/server/objects.py | 435 ++++++++++++++++ packetserver/server/requests.py | 66 +++ packetserver/server/testserver.py | 115 +++++ packetserver/server/users.py | 321 ++++++++++++ src/packetserver/setup.py => setup.py | 6 +- src/packetserver/__init__.py | 1 - 78 files changed, 5647 insertions(+), 15 deletions(-) create mode 100644 packetserver/__init__.py rename {src/packetserver => packetserver/build/lib}/client/__init__.py (100%) rename {src/packetserver => packetserver/build/lib}/client/bulletins.py (100%) rename {src/packetserver => packetserver/build/lib}/client/cli/__init__.py (100%) rename {src/packetserver => packetserver/build/lib}/client/cli/bulletin.py (100%) rename {src/packetserver => packetserver/build/lib}/client/cli/config.py (100%) rename {src/packetserver => packetserver/build/lib}/client/cli/constants.py (100%) rename {src/packetserver => packetserver/build/lib}/client/cli/db.py (100%) rename {src/packetserver => packetserver/build/lib}/client/cli/job.py (100%) rename {src/packetserver => packetserver/build/lib}/client/cli/message.py (100%) rename {src/packetserver => packetserver/build/lib}/client/cli/object.py (100%) rename {src/packetserver => packetserver/build/lib}/client/cli/server.py (100%) rename {src/packetserver => packetserver/build/lib}/client/cli/util.py (100%) rename {src/packetserver => packetserver/build/lib}/client/jobs.py (100%) rename {src/packetserver => packetserver/build/lib}/client/messages.py (100%) rename {src/packetserver => packetserver/build/lib}/client/objects.py (100%) rename {src/packetserver => packetserver/build/lib}/client/testing.py (100%) rename {src/packetserver => packetserver/build/lib}/client/users.py (100%) rename {src/packetserver => packetserver/build/lib}/common/__init__.py (100%) rename {src/packetserver => packetserver/build/lib}/common/constants.py (100%) rename {src/packetserver => packetserver/build/lib}/common/testing.py (100%) rename {src/packetserver => packetserver/build/lib}/common/util.py (100%) rename {src/packetserver => packetserver/build/lib}/runner/__init__.py (100%) rename {src/packetserver => packetserver/build/lib}/runner/constants.py (100%) rename {src/packetserver => packetserver/build/lib}/runner/podman.py (100%) rename {src/packetserver => packetserver/build/lib}/server/__init__.py (99%) rename {src/packetserver => packetserver/build/lib}/server/bulletin.py (100%) rename {src/packetserver => packetserver/build/lib}/server/constants.py (100%) rename {src/packetserver => packetserver/build/lib}/server/db.py (100%) rename {src/packetserver => packetserver/build/lib}/server/jobs.py (100%) rename {src/packetserver => packetserver/build/lib}/server/messages.py (100%) rename {src/packetserver => packetserver/build/lib}/server/objects.py (100%) rename {src/packetserver => packetserver/build/lib}/server/requests.py (100%) rename {src/packetserver => packetserver/build/lib}/server/testserver.py (100%) rename {src/packetserver => packetserver/build/lib}/server/users.py (100%) create mode 100644 packetserver/client/__init__.py create mode 100644 packetserver/client/bulletins.py create mode 100644 packetserver/client/cli/__init__.py create mode 100644 packetserver/client/cli/bulletin.py create mode 100644 packetserver/client/cli/config.py create mode 100644 packetserver/client/cli/constants.py create mode 100644 packetserver/client/cli/db.py create mode 100644 packetserver/client/cli/job.py create mode 100644 packetserver/client/cli/message.py create mode 100644 packetserver/client/cli/object.py create mode 100644 packetserver/client/cli/server.py create mode 100644 packetserver/client/cli/util.py create mode 100644 packetserver/client/jobs.py create mode 100644 packetserver/client/messages.py create mode 100644 packetserver/client/objects.py create mode 100644 packetserver/client/testing.py create mode 100644 packetserver/client/users.py create mode 100644 packetserver/common/__init__.py create mode 100644 packetserver/common/constants.py create mode 100644 packetserver/common/testing.py create mode 100644 packetserver/common/util.py create mode 100644 packetserver/packetserver.egg-info/PKG-INFO create mode 100644 packetserver/packetserver.egg-info/SOURCES.txt create mode 100644 packetserver/packetserver.egg-info/dependency_links.txt create mode 100644 packetserver/packetserver.egg-info/entry_points.txt create mode 100644 packetserver/packetserver.egg-info/requires.txt create mode 100644 packetserver/packetserver.egg-info/top_level.txt create mode 100644 packetserver/runner/__init__.py create mode 100644 packetserver/runner/constants.py create mode 100644 packetserver/runner/podman.py create mode 100644 packetserver/server/__init__.py create mode 100644 packetserver/server/bulletin.py create mode 100644 packetserver/server/constants.py create mode 100644 packetserver/server/db.py create mode 100644 packetserver/server/jobs.py create mode 100644 packetserver/server/messages.py create mode 100644 packetserver/server/objects.py create mode 100644 packetserver/server/requests.py create mode 100644 packetserver/server/testserver.py create mode 100644 packetserver/server/users.py rename src/packetserver/setup.py => setup.py (85%) delete mode 100644 src/packetserver/__init__.py diff --git a/Readme.md b/Readme.md index 44cdff1..4dede32 100644 --- a/Readme.md +++ b/Readme.md @@ -77,8 +77,8 @@ UI packets later on.. ### Main help dialog: ```commandline -(venv) [user@host]$ packetcli -Usage: packetcli [OPTIONS] COMMAND [ARGS]... +(venv) [user@host]$ packcli +Usage: packcli [OPTIONS] COMMAND [ARGS]... Command line interface for the PacketServer client and server API. @@ -104,34 +104,34 @@ Commands: ### Working with objects: ```commandline -(venv) [user@host]$ packetcli object list +(venv) [user@host]$ packcli object list name size_bytes binary private created_at modified_at uuid --------------- ------------ -------- --------- -------------------------------- -------------------------------- ------------------------------------ testdb.txt 13 False True 2025-03-16T22:26:05.049173+00:00 2025-03-16T22:26:05.051375+00:00 fbbd4527-a5f0-447f-9fc9-55b7b263c458 -(venv) [user@host]$ packetcli object upload-file -Usage: packetcli object upload-file [OPTIONS] FILE_PATH -Try 'packetcli object upload-file --help' for help. +(venv) [user@host]$ packcli object upload-file +Usage: packcli object upload-file [OPTIONS] FILE_PATH +Try 'packcli object upload-file --help' for help. Error: Missing argument 'FILE_PATH'. -(venv) [user@host]$ packetcli object upload-file /tmp/hello-world.txt +(venv) [user@host]$ packcli object upload-file /tmp/hello-world.txt 35753577-21e3-4f64-8776-e3f86f1bb0e0 -(venv) [user@host]$ packetcli object list +(venv) [user@host]$ packcli object list name size_bytes binary private created_at modified_at uuid --------------- ------------ -------- --------- -------------------------------- -------------------------------- ------------------------------------ testdb.txt 13 False True 2025-03-16T22:26:05.049173+00:00 2025-03-16T22:26:05.051375+00:00 fbbd4527-a5f0-447f-9fc9-55b7b263c458 hello-world.txt 13 False True 2025-03-19T02:25:41.501833+00:00 2025-03-19T02:25:41.503502+00:00 35753577-21e3-4f64-8776-e3f86f1bb0e0 -(venv) packetcli object get 35753577-21e3-4f64-8776-e3f86f1bb0e0 +(venv) packcli object get 35753577-21e3-4f64-8776-e3f86f1bb0e0 Hello world. ``` ### Retrieving messages: ```commandline -(venv) [user@host]$ packetcli message get +(venv) [user@host]$ packcli message get from to id text sent_at attachments ------ ------ ------------------------------------ ------------------------------- -------------------------------- ------------- KQ4PEC KQ4PEC df7493d7-5880-4c24-9e3c-1d3987a5203e testing.. again with attachment 2025-03-18T03:41:36.597371+00:00 random.txt @@ -143,7 +143,7 @@ KQ4PEC KQ4PEC ad513075-e50f-4f84-8a87-a1217b43bef3 testing.. 1.. 2.. 3 ### Listing users: ```commandline -(venv) [user@host]$ packetcli user -l +(venv) [user@host]$ packcli user -l username status bio socials created last_seen email location ---------- --------------------- ----- --------- -------------------------------- -------------------------------- --------------- ---------- KQ4PEC just happy to be here 2025-03-16 04:29:52.044216+00:00 2025-03-19 02:22:21.413896+00:00 user@domain.com diff --git a/packetserver/__init__.py b/packetserver/__init__.py new file mode 100644 index 0000000..a7ae8b9 --- /dev/null +++ b/packetserver/__init__.py @@ -0,0 +1 @@ +VERSION="0.4.1" \ No newline at end of file diff --git a/src/packetserver/client/__init__.py b/packetserver/build/lib/client/__init__.py similarity index 100% rename from src/packetserver/client/__init__.py rename to packetserver/build/lib/client/__init__.py diff --git a/src/packetserver/client/bulletins.py b/packetserver/build/lib/client/bulletins.py similarity index 100% rename from src/packetserver/client/bulletins.py rename to packetserver/build/lib/client/bulletins.py diff --git a/src/packetserver/client/cli/__init__.py b/packetserver/build/lib/client/cli/__init__.py similarity index 100% rename from src/packetserver/client/cli/__init__.py rename to packetserver/build/lib/client/cli/__init__.py diff --git a/src/packetserver/client/cli/bulletin.py b/packetserver/build/lib/client/cli/bulletin.py similarity index 100% rename from src/packetserver/client/cli/bulletin.py rename to packetserver/build/lib/client/cli/bulletin.py diff --git a/src/packetserver/client/cli/config.py b/packetserver/build/lib/client/cli/config.py similarity index 100% rename from src/packetserver/client/cli/config.py rename to packetserver/build/lib/client/cli/config.py diff --git a/src/packetserver/client/cli/constants.py b/packetserver/build/lib/client/cli/constants.py similarity index 100% rename from src/packetserver/client/cli/constants.py rename to packetserver/build/lib/client/cli/constants.py diff --git a/src/packetserver/client/cli/db.py b/packetserver/build/lib/client/cli/db.py similarity index 100% rename from src/packetserver/client/cli/db.py rename to packetserver/build/lib/client/cli/db.py diff --git a/src/packetserver/client/cli/job.py b/packetserver/build/lib/client/cli/job.py similarity index 100% rename from src/packetserver/client/cli/job.py rename to packetserver/build/lib/client/cli/job.py diff --git a/src/packetserver/client/cli/message.py b/packetserver/build/lib/client/cli/message.py similarity index 100% rename from src/packetserver/client/cli/message.py rename to packetserver/build/lib/client/cli/message.py diff --git a/src/packetserver/client/cli/object.py b/packetserver/build/lib/client/cli/object.py similarity index 100% rename from src/packetserver/client/cli/object.py rename to packetserver/build/lib/client/cli/object.py diff --git a/src/packetserver/client/cli/server.py b/packetserver/build/lib/client/cli/server.py similarity index 100% rename from src/packetserver/client/cli/server.py rename to packetserver/build/lib/client/cli/server.py diff --git a/src/packetserver/client/cli/util.py b/packetserver/build/lib/client/cli/util.py similarity index 100% rename from src/packetserver/client/cli/util.py rename to packetserver/build/lib/client/cli/util.py diff --git a/src/packetserver/client/jobs.py b/packetserver/build/lib/client/jobs.py similarity index 100% rename from src/packetserver/client/jobs.py rename to packetserver/build/lib/client/jobs.py diff --git a/src/packetserver/client/messages.py b/packetserver/build/lib/client/messages.py similarity index 100% rename from src/packetserver/client/messages.py rename to packetserver/build/lib/client/messages.py diff --git a/src/packetserver/client/objects.py b/packetserver/build/lib/client/objects.py similarity index 100% rename from src/packetserver/client/objects.py rename to packetserver/build/lib/client/objects.py diff --git a/src/packetserver/client/testing.py b/packetserver/build/lib/client/testing.py similarity index 100% rename from src/packetserver/client/testing.py rename to packetserver/build/lib/client/testing.py diff --git a/src/packetserver/client/users.py b/packetserver/build/lib/client/users.py similarity index 100% rename from src/packetserver/client/users.py rename to packetserver/build/lib/client/users.py diff --git a/src/packetserver/common/__init__.py b/packetserver/build/lib/common/__init__.py similarity index 100% rename from src/packetserver/common/__init__.py rename to packetserver/build/lib/common/__init__.py diff --git a/src/packetserver/common/constants.py b/packetserver/build/lib/common/constants.py similarity index 100% rename from src/packetserver/common/constants.py rename to packetserver/build/lib/common/constants.py diff --git a/src/packetserver/common/testing.py b/packetserver/build/lib/common/testing.py similarity index 100% rename from src/packetserver/common/testing.py rename to packetserver/build/lib/common/testing.py diff --git a/src/packetserver/common/util.py b/packetserver/build/lib/common/util.py similarity index 100% rename from src/packetserver/common/util.py rename to packetserver/build/lib/common/util.py diff --git a/src/packetserver/runner/__init__.py b/packetserver/build/lib/runner/__init__.py similarity index 100% rename from src/packetserver/runner/__init__.py rename to packetserver/build/lib/runner/__init__.py diff --git a/src/packetserver/runner/constants.py b/packetserver/build/lib/runner/constants.py similarity index 100% rename from src/packetserver/runner/constants.py rename to packetserver/build/lib/runner/constants.py diff --git a/src/packetserver/runner/podman.py b/packetserver/build/lib/runner/podman.py similarity index 100% rename from src/packetserver/runner/podman.py rename to packetserver/build/lib/runner/podman.py diff --git a/src/packetserver/server/__init__.py b/packetserver/build/lib/server/__init__.py similarity index 99% rename from src/packetserver/server/__init__.py rename to packetserver/build/lib/server/__init__.py index 154f2fa..96a135c 100644 --- a/src/packetserver/server/__init__.py +++ b/packetserver/build/lib/server/__init__.py @@ -25,7 +25,7 @@ from threading import Thread from packetserver.server.jobs import get_orchestrator_from_config, Job, JobStatus from packetserver.runner import RunnerStatus, RunnerFile, Orchestrator, Runner -VERSION="0.4.0-alpha" +VERSION="0.4.1" def init_bulletins(root: PersistentMapping): if 'bulletins' not in root: diff --git a/src/packetserver/server/bulletin.py b/packetserver/build/lib/server/bulletin.py similarity index 100% rename from src/packetserver/server/bulletin.py rename to packetserver/build/lib/server/bulletin.py diff --git a/src/packetserver/server/constants.py b/packetserver/build/lib/server/constants.py similarity index 100% rename from src/packetserver/server/constants.py rename to packetserver/build/lib/server/constants.py diff --git a/src/packetserver/server/db.py b/packetserver/build/lib/server/db.py similarity index 100% rename from src/packetserver/server/db.py rename to packetserver/build/lib/server/db.py diff --git a/src/packetserver/server/jobs.py b/packetserver/build/lib/server/jobs.py similarity index 100% rename from src/packetserver/server/jobs.py rename to packetserver/build/lib/server/jobs.py diff --git a/src/packetserver/server/messages.py b/packetserver/build/lib/server/messages.py similarity index 100% rename from src/packetserver/server/messages.py rename to packetserver/build/lib/server/messages.py diff --git a/src/packetserver/server/objects.py b/packetserver/build/lib/server/objects.py similarity index 100% rename from src/packetserver/server/objects.py rename to packetserver/build/lib/server/objects.py diff --git a/src/packetserver/server/requests.py b/packetserver/build/lib/server/requests.py similarity index 100% rename from src/packetserver/server/requests.py rename to packetserver/build/lib/server/requests.py diff --git a/src/packetserver/server/testserver.py b/packetserver/build/lib/server/testserver.py similarity index 100% rename from src/packetserver/server/testserver.py rename to packetserver/build/lib/server/testserver.py diff --git a/src/packetserver/server/users.py b/packetserver/build/lib/server/users.py similarity index 100% rename from src/packetserver/server/users.py rename to packetserver/build/lib/server/users.py diff --git a/packetserver/client/__init__.py b/packetserver/client/__init__.py new file mode 100644 index 0000000..b59ddaa --- /dev/null +++ b/packetserver/client/__init__.py @@ -0,0 +1,172 @@ +import datetime +import pe.app +from ZEO.asyncio.server import new_connection +from packetserver.common.testing import SimpleDirectoryConnection +from packetserver.common import Response, Message, Request, PacketServerConnection, send_response, send_blank_response +import ax25 +import logging +import signal +import time +from threading import Lock +from msgpack import Unpacker +from msgpack.exceptions import OutOfData +from typing import Callable, Self, Union, Optional +from traceback import format_exc +from os import linesep +from shutil import rmtree +from threading import Thread + +class Client: + def __init__(self, pe_server: str, port: int, client_callsign: str, keep_log=False): + if not ax25.Address.valid_call(client_callsign): + raise ValueError(f"Provided callsign '{client_callsign}' is invalid.") + self.pe_server = pe_server + self.pe_port = port + self.callsign = client_callsign + self.app = pe.app.Application() + self.started = False + self._connection_locks = {} + self.lock_locker = Lock() + self.keep_log = keep_log + self.request_log = [] + signal.signal(signal.SIGINT, self.exit_gracefully) + signal.signal(signal.SIGTERM, self.exit_gracefully) + + def exit_gracefully(self, signum, frame): + self.stop() + + def __del__(self): + self.stop() + + @property + def connections(self) -> dict: + if not self.started: + return {} + else: + return self.app._engine._active_handler._handlers[1]._connection_map._connections + + def connection_exists(self, callsign: str): + if not ax25.Address.valid_call(callsign): + raise ValueError("Must supply a valid callsign.") + callsign = callsign.upper().strip() + for key in self.connections.keys(): + if key.split(":")[1] == callsign: + return True + return False + + def connection_callsign(self, callsign: str): + if not ax25.Address.valid_call(callsign): + raise ValueError("Must supply a valid callsign.") + callsign = callsign.upper().strip() + for key in self.connections.keys(): + if key.split(":")[1] == callsign: + return self.connections[key] + return None + + def connection_for(self, callsign: str) -> Union[PacketServerConnection, SimpleDirectoryConnection]: + if not ax25.Address.valid_call(callsign): + raise ValueError("Must supply a valid callsign.") + callsign = callsign.upper().strip() + if self.connection_exists(callsign): + return self.connection_callsign(callsign) + else: + return self.new_connection(callsign) + + def stop(self): + self.started = False + self.clear_connections() + self.app.stop() + self.connection_map = None + + def start(self): + self.app.start(self.pe_server, self.pe_port) + self.app.register_callsigns(self.callsign) + self.connection_map = self.app._engine._active_handler._handlers[1]._connection_map + self.started = True + + def clear_connections(self): + if self.app._engine is not None: + cm = self.app._engine._active_handler._handlers[1]._connection_map + for key in cm._connections.keys(): + cm._connections[key].close() + + def new_connection(self, dest: str) -> Union[PacketServerConnection, SimpleDirectoryConnection]: + if not self.started: + raise RuntimeError("Must start client before creating connections.") + if not ax25.Address.valid_call(dest): + raise ValueError(f"Provided destination callsign '{dest}' is invalid.") + with self.lock_locker: + if dest.upper() not in self._connection_locks: + self._connection_locks[dest.upper()] = Lock() + with self._connection_locks[dest.upper()]: + conn = self.connection_callsign(dest.upper()) + if conn is not None: + return conn + + conn = self.app.open_connection(0, self.callsign, dest.upper()) + while conn.state.name != "CONNECTED": + if conn.state.name in ['DISCONNECTED', 'DISCONNECTING']: + raise RuntimeError("Connection disconnected unexpectedly.") + time.sleep(.1) + logging.debug(f"Connection to {dest} ready.") + logging.debug("Allowing connection to stabilize for 8 seconds") + time.sleep(8) + return conn + + def receive(self, req: Request, conn: Union[PacketServerConnection,SimpleDirectoryConnection], timeout: int = 300): + cutoff_date = datetime.datetime.now() + datetime.timedelta(seconds=timeout) + logging.debug(f"{datetime.datetime.now()}: Request timeout date is {cutoff_date}") + while datetime.datetime.now() < cutoff_date: + if conn.state.name != "CONNECTED": + logging.error(f"Connection {conn} disconnected.") + if self.keep_log: + self.request_log.append((req, None)) + return None + try: + unpacked = conn.data.unpack() + except: + time.sleep(.1) + continue + msg = Message.partial_unpack(unpacked) + resp = Response(msg) + return resp + logging.warning(f"{datetime.datetime.now()}: Request {req} timed out.") + return None + + def send_and_receive(self, req: Request, conn: Union[PacketServerConnection,SimpleDirectoryConnection], + timeout: int = 300) -> Optional[Response]: + if conn.state.name != "CONNECTED": + raise RuntimeError("Connection is not connected.") + logging.debug(f"Sending request {req}") + dest = conn.remote_callsign.upper() + with self.lock_locker: + if dest not in self._connection_locks: + self._connection_locks[dest] = Lock() + with self._connection_locks[dest]: + conn.data = Unpacker() + conn.send_data(req.pack()) + resp = self.receive(req, conn, timeout=timeout) + self.request_log.append((req, resp)) + return resp + + def send_receive_callsign(self, req: Request, callsign: str, timeout: int = 300) -> Optional[Response]: + return self.send_and_receive(req, self.connection_for(callsign), timeout=timeout) + + def single_connect_send_receive(self, dest: str, req: Request, timeout: int = 300) -> Optional[Response]: + conn = self.new_connection(dest) + logging.debug("Waiting for connection to be ready.") + cutoff_date = datetime.datetime.now() + datetime.timedelta(seconds=timeout) + + while (datetime.datetime.now() < cutoff_date) and (conn.state.name != "CONNECTED"): + if conn.state.name in ["DISCONNECTED", "DISCONNECTING"]: + logging.error(f"Connection {conn} disconnected.") + return None + + remaining_time = int((cutoff_date - datetime.datetime.now()).total_seconds()) + 1 + if remaining_time <= 0: + logging.debug("Connection attempt timed out.") + conn.close() + return None + response = self.send_and_receive(req, conn, timeout=int(remaining_time)) + conn.close() + return response diff --git a/packetserver/client/bulletins.py b/packetserver/client/bulletins.py new file mode 100644 index 0000000..e378704 --- /dev/null +++ b/packetserver/client/bulletins.py @@ -0,0 +1,73 @@ +from packetserver.client import Client +from packetserver.common import Request, Response, PacketServerConnection +from typing import Union, Optional +import datetime +import time + +class BulletinWrapper: + def __init__(self, data: dict): + for i in ['author', 'id', 'subject', 'body', 'created_at', 'updated_at']: + if i not in data: + raise ValueError("Was not given a bulletin dictionary.") + self.data = data + + def __repr__(self): + return f"" + + @property + def id(self) -> int: + return self.data['id'] + + @property + def created(self) -> datetime.datetime: + return datetime.datetime.fromisoformat(self.data['created_at']) + + @property + def updated(self) -> datetime.datetime: + return datetime.datetime.fromisoformat(self.data['updated_at']) + + @property + def author(self) -> str: + return self.data['author'] + + @property + def subject(self) -> str: + return self.data['subject'] + + @property + def body(self) -> str: + return self.data['body'] + +def post_bulletin(client: Client, bbs_callsign: str, subject: str, body: str) -> int: + req = Request.blank() + req.path = "bulletin" + req.payload = {'subject': subject, 'body': body} + req.method = Request.Method.POST + response = client.send_receive_callsign(req, bbs_callsign) + if response.status_code != 201: + raise RuntimeError(f"Posting bulletin failed: {response.status_code}: {response.payload}") + return response.payload['bulletin_id'] + +def get_bulletin_by_id(client: Client, bbs_callsign: str, bid: int) -> BulletinWrapper: + req = Request.blank() + req.path = "bulletin" + req.set_var('id', bid) + req.method = Request.Method.GET + response = client.send_receive_callsign(req, bbs_callsign) + if response.status_code != 200: + raise RuntimeError(f"GET bulletin {bid} failed: {response.status_code}: {response.payload}") + return BulletinWrapper(response.payload) + +def get_bulletins_recent(client: Client, bbs_callsign: str, limit: int = None) -> list[BulletinWrapper]: + req = Request.blank() + req.path = "bulletin" + req.method = Request.Method.GET + if limit is not None: + req.set_var('limit', limit) + response = client.send_receive_callsign(req, bbs_callsign) + if response.status_code != 200: + raise RuntimeError(f"Listing bulletins failed: {response.status_code}: {response.payload}") + out_list = [] + for b in response.payload: + out_list.append(BulletinWrapper(b)) + return out_list \ No newline at end of file diff --git a/packetserver/client/cli/__init__.py b/packetserver/client/cli/__init__.py new file mode 100644 index 0000000..ffb57ef --- /dev/null +++ b/packetserver/client/cli/__init__.py @@ -0,0 +1,193 @@ +import click +from packetserver.client.cli.config import get_config, default_app_dir, config_path +from packetserver.client.cli.constants import DEFAULT_DB_FILE +from packetserver.client import Client +from packetserver.common.constants import yes_values +from packetserver.common import Request, Response +from packetserver.client.cli.util import format_list_dicts, exit_client +from packetserver.client.cli.job import job +from packetserver.client.cli.object import objects +from packetserver.client.cli.message import message +import ZODB +import ZODB.FileStorage +import ax25 +import sys +import os +import json +import os.path +from pathlib import Path +from packetserver.client import Client +from packetserver.client import users +from packetserver.client.users import get_user_by_username, UserWrapper + +VERSION="0.1.0-alpha" + +@click.group() +@click.option('--conf', default=config_path(), help='path to configfile') +@click.option('--server', '-s', default='', help="server radio callsign to connect to (required)", + envvar='PSCLIENT_SERVER') +@click.option('--agwpe', '-a', default='', help="AGWPE TNC server address to connect to (config file)", + envvar='PSCLIENT_AGWPE') +@click.option('--port', '-p', default=0, help="AGWPE TNC server port to connect to (config file)", + envvar='PSCLIENT_PORT') +@click.option('--callsign', '-c', default='', help="radio callsign[+ssid] of this client station (config file)", + envvar='PSCLIENT_CALLSIGN') +@click.option('--keep-log', '-k', is_flag=True, default=False, help="Save local copy of request log after session ends?") +@click.version_option(VERSION,"--version", "-v") +@click.pass_context +def cli(ctx, conf, server, agwpe, port, callsign, keep_log): + """Command line interface for the PacketServer client and server API.""" + ctx.ensure_object(dict) + cfg = get_config(config_file_path=conf) + + ctx.obj['keep_log'] = False + if keep_log: + ctx.obj['keep_log'] = True + else: + if cfg['cli'].get('keep_log', fallback='n') in yes_values: + ctx.obj['keep_log'] = True + + if callsign.strip() != '': + ctx.obj['callsign'] = callsign.strip().upper() + else: + if 'callsign' in cfg['cli']: + ctx.obj['callsign'] = cfg['cli']['callsign'] + else: + ctx.obj['callsign'] = click.prompt('Please enter your station callsign (with ssid if needed)', type=str) + + ctx.obj['directory'] = cfg['cli']['directory'] + + if not ax25.Address.valid_call(ctx.obj['callsign']): + click.echo(f"Provided client callsign '{ctx.obj['callsign']}' is invalid.", err=True) + sys.exit(1) + + if server.strip() != '': + ctx.obj['server'] = server.strip().upper() + else: + if 'server' in cfg['cli']: + ctx.obj['server'] = cfg['cli']['server'] + else: + ctx.obj['server'] = click.prompt('Please enter the bbs station callsign (with ssid if needed)', type=str) + + if not ax25.Address.valid_call(ctx.obj['server']): + click.echo(f"Provided remote server callsign '{ctx.obj['server']}' is invalid.", err=True) + sys.exit(1) + + if agwpe.strip() != '': + ctx.obj['agwpe_server'] = agwpe.strip() + else: + if 'agwpe_server' in cfg['cli']: + ctx.obj['agwpe_server'] = cfg['cli']['agwpe_server'] + else: + ctx.obj['agwpe_server'] = 'localhost' + + if port != 0: + ctx.obj['port'] = port + else: + if 'port' in cfg['cli']: + ctx.obj['port'] = int(cfg['cli']['port']) + else: + ctx.obj['port'] = 8000 + + storage = ZODB.FileStorage.FileStorage(os.path.join(cfg['cli']['directory'], DEFAULT_DB_FILE)) + db = ZODB.DB(storage) + if 'TEST_SERVER_DIR' in os.environ: + from packetserver.client.testing import TestClient + client = TestClient(os.environ['TEST_SERVER_DIR'], ctx.obj['callsign']) + else: + client = Client(ctx.obj['agwpe_server'], ctx.obj['port'], ctx.obj['callsign'], keep_log=ctx.obj['keep_log']) + try: + client.start() + except Exception as e: + click.echo(f"Error connecting to TNC: {str(e)}", err=True) + sys.exit(1) + + ctx.obj['client'] = client + ctx.obj['CONFIG'] = cfg + ctx.obj['bbs'] = server + ctx.obj['db'] = db + +@click.command() +@click.pass_context +def query_server(ctx): + """Query the server for basic info.""" + client = ctx.obj['client'] + req = Request.blank() + req.path = "" + req.method = Request.Method.GET + resp = client.send_receive_callsign(req, ctx.obj['bbs']) + if resp is None: + click.echo(f"No response from {ctx.obj['bbs']}") + exit_client(ctx.obj, 1) + else: + if resp.status_code != 200: + exit_client(ctx.obj, 1, message=f"Error contacting server: {resp.payload}") + else: + click.echo(json.dumps(resp.payload, indent=2)) + exit_client(ctx.obj, 0) + + +@click.command() +@click.argument('username', required=False, default='') +@click.option('--list-users', '-l', is_flag=True, default=False, help="If set, downloads list of all users.") +@click.option("--output-format", "-f", default="table", help="Print data as table[default], list, or JSON", + type=click.Choice(['table', 'json', 'list'], case_sensitive=False)) +@click.pass_context +def user(ctx, list_users, output_format, username): + """Query users on the BBS. Either listing multiple users or looking up information of USERNAME""" + client = ctx.obj['client'] + # validate args + if list_users and (username.strip() != ""): + exit_client(ctx.obj,1, "Can't specify a username while listing all users.") + + if not list_users and (username.strip() == ""): + exit_client(ctx.obj,1, message="Must provide either a username or --list-users flag.") + + output_objects = [] + try: + if list_users: + output_objects = users.get_users(client, ctx.obj['bbs']) + else: + output_objects.append(users.get_user_by_username(client, ctx.obj['bbs'], username)) + except Exception as e: + exit_client(ctx.obj,1, str(e)) + finally: + client.stop() + + click.echo(format_list_dicts([x.pretty_dict() for x in output_objects], output_format=output_format.lower())) + exit_client(ctx.obj, 0) + +@click.command() +@click.option('--email', '-e', type=str, default=None, help="Sets your e-mail address in your profile.") +@click.option('--bio', '-b', type=str, default=None, help="Sets your about you text in your profile.") +@click.option('--status', '-S', type=str, default=None, + help="Sets your status (happy,sad, gone fishing, etc) in your profile.") +@click.option('--location', '-l', type=str, default=None, + help="Sets your physical location (in whatever form you want) in your profile.") +@click.option('--socials', '-m', type=str, default=None, + help="Comma (,) separated list of social media or websites you are known by.") +@click.pass_context +def set_user(ctx, email, bio, status, location, socials): + """Set your user profile settings on the BBS.""" + social_list = None + client = ctx.obj['client'] + if type(socials) is str: + social_list = socials.split(',') + + try: + users.update_self(client, ctx.obj['bbs'], email=email, bio=bio, socials=social_list, + location=location, status=status) + exit_client(ctx.obj, 0) + except Exception as e: + click.echo(str(e), err=True) + exit_client(ctx.obj, 98) + +cli.add_command(user) +cli.add_command(query_server) +cli.add_command(job, name='job') +cli.add_command(objects, name='object') +cli.add_command(set_user, name='set') +cli.add_command(message) + +if __name__ == '__main__': + cli() diff --git a/packetserver/client/cli/bulletin.py b/packetserver/client/cli/bulletin.py new file mode 100644 index 0000000..e69de29 diff --git a/packetserver/client/cli/config.py b/packetserver/client/cli/config.py new file mode 100644 index 0000000..510b70c --- /dev/null +++ b/packetserver/client/cli/config.py @@ -0,0 +1,24 @@ +import os +import os.path +from configparser import ConfigParser +from pathlib import Path +from packetserver.client.cli.constants import DEFAULT_APP_DIR, DEFAULT_CONFIG_FILE + +def default_app_dir() -> str: + return os.path.join(str(Path.home()), DEFAULT_APP_DIR) + +def config_path(app_path=default_app_dir()) -> str: + return os.path.join(app_path, DEFAULT_CONFIG_FILE) + +def get_config(config_file_path=config_path()) -> ConfigParser: + config = ConfigParser() + if os.path.isfile(config_file_path): + config.read(config_file_path) + + if not 'cli' in config.sections(): + config.add_section('cli') + + if 'directory' not in config['cli']: + config['cli']['directory'] = default_app_dir() + + return config \ No newline at end of file diff --git a/packetserver/client/cli/constants.py b/packetserver/client/cli/constants.py new file mode 100644 index 0000000..73fc190 --- /dev/null +++ b/packetserver/client/cli/constants.py @@ -0,0 +1,5 @@ +import os.path + +DEFAULT_APP_DIR = ".packetserver" +DEFAULT_CONFIG_FILE = "cli.ini" +DEFAULT_DB_FILE = "cli-client.zopedb" \ No newline at end of file diff --git a/packetserver/client/cli/db.py b/packetserver/client/cli/db.py new file mode 100644 index 0000000..e69de29 diff --git a/packetserver/client/cli/job.py b/packetserver/client/cli/job.py new file mode 100644 index 0000000..5b8798a --- /dev/null +++ b/packetserver/client/cli/job.py @@ -0,0 +1,103 @@ +"""CLI client for dealing with jobs.""" +import os + +import click +from persistent.mapping import default +from packetserver.client import Client +from packetserver.client.jobs import JobSession, get_job_id, get_user_jobs, send_job, send_job_quick, JobWrapper +import datetime +from packetserver.client.cli.util import exit_client, format_list_dicts + +@click.group() +@click.pass_context +def job(ctx): + """Runs commands on the BBS server if jobs are enabled on it.""" + pass + +@click.command() +@click.pass_context +def start(ctx): + """Start a job on the BBS server.""" + pass + + +@click.command() +@click.argument('job_id', required=False, type=int) +@click.option("--all-jobs", "-a", is_flag=True, default=False, help="Get all of your jobs.") +@click.option("--no-data", '-n', is_flag=True, default=True, + help="Don't fetch job result data, just metadata.") +@click.pass_context +def get(ctx, job_id, all_jobs, no_data): # TODO decide what to do with output and artifacts in a cli tool force full JSON? + """Retrieve your jobs. Pass either '-a' or a job_id.""" + + fetch_data = not no_data + if job_id is None: + job_id = "" + job_id = job_id.strip() + if all_jobs and (job_id != ""): + click.echo("Can't use --all and specify a job_id.") + + client = ctx.obj['client'] + try: + if all_jobs: + jobs_out = get_user_jobs(client, ctx.obj['bbs'], get_data=fetch_data) + else: + jobs_out = [get_job_id(client,ctx.obj['bbs'], get_data=fetch_data)] + dicts_out = [] + for j in jobs_out: + pass + + except Exception as e: + click.echo(str(e), err=True) + exit_client(ctx.obj, 1) + + + + +@click.command() +@click.option("--transcript", "-T", default="", help="File to write command transcript to if desired.") +@click.pass_context +def quick_session(ctx, transcript): + """Start a session to submit multiple commands and receive responses immediately""" + session_transcript = [] + client = ctx.obj['client'] + bbs = ctx.obj['bbs'] + js = JobSession(client, bbs, stutter=2) + db_enabled = True + while True: + cmd = click.prompt("CMD", prompt_suffix=" >") + cmd = cmd.strip() + session_transcript.append((datetime.datetime.now(),"c",cmd)) + next_db = False + if db_enabled: + next_db = True + db_enabled = False + if cmd == "": + continue + if cmd == "/exit": + break + elif cmd == "/db": + click.echo("DB requested for next command.") + db_enabled = True + else: + try: + job_result = js.send_quick(['bash', '-c', cmd], db=next_db) + output = job_result.output_str + "\n" + if job_result.errors_str != "": + output = output + "Errors: " + job_result.errors_str + session_transcript.append((datetime.datetime.now(), "r", output)) + click.echo(output) + except Exception as e: + session_transcript.append((datetime.datetime.now(), "e", e)) + click.echo(f"ERROR! {str(e)}", err=True) + continue + try: + if transcript.strip() != "": + with open(transcript.strip(), 'w') as tran_file: + for l in session_transcript: + tran_file.write(f"{l[1]}:{l[0].isoformat()}: {l[2]}{os.linesep}") + finally: + exit_client(ctx.obj, 0) + + +job.add_command(quick_session) \ No newline at end of file diff --git a/packetserver/client/cli/message.py b/packetserver/client/cli/message.py new file mode 100644 index 0000000..b544151 --- /dev/null +++ b/packetserver/client/cli/message.py @@ -0,0 +1,213 @@ +import os +import sys +import os.path +from email.policy import default + +import click +from zodbpickle.pickle_3 import FALSE + +from packetserver.client.cli.util import exit_client, format_list_dicts, unit_seconds +from copy import deepcopy +from uuid import UUID +import datetime +import re +import json +from packetserver.client.messages import * + +rel_date = '^-(\\d+)([dhms])$' + +@click.group() +@click.pass_context +def message(ctx): + """Send, search, and filter messages to and from other users on the BBS system.""" + pass + +@click.command() +@click.argument("recipients", type=str) +@click.argument("body", type=str) +@click.option("--body-filename", '-f', is_flag=True, default=False, help="Treat body argument as a filename to read body text from. '-' to read from stdin.") +@click.option("--attachment", "-A", multiple=True, default=[], + help="Files to attach to message in form '[:]' use 't' for text (default), 'b' to interpret file as binary data.") +@click.pass_context +def send(ctx, recipients, body, body_filename, attachment): + """Send a message to one or more recipients. + + should be a comma-separated list of recipients to send the message to + + should be either body text, or a filename (or '-' for stdin) to read body text from + """ + client = ctx.obj['client'] + bbs = ctx.obj['bbs'] + + recips = [x.strip() for x in recipients.split(",") if x.strip() != ""] + + if len(recips) == 0: + click.echo("You must specify at least one recipient.", err=True) + exit_client(ctx.obj, 89) + + attachments = [] + for a in attachment: + is_text = True + filename = a + if len(a) > 1: + if a[1] == ":": + filename = a[2:] + if a[0].lower() == "b": + is_text = False + try: + attachments.append(attachment_from_file(filename, binary=not is_text)) + except Exception as e: + click.echo(str(e), err=True) + exit_client(ctx.obj, 89) + + if len(attachments) == 0: + attachments = None + + if body_filename: + if body == "-": + body_text = sys.stdin.read() + else: + if not os.path.isfile(body): + click.echo(f"{body} is not a file that can be read for body text.", err=True) + exit_client(ctx.obj, 92) + sys.exit(92) + try: + body_text = open(body, "r").read() + except: + click.echo(f"{body} is not a file that can be read for body text.", err=True) + exit_client(ctx.obj, 92) + sys.exit(92) + else: + body_text = body + + try: + resp = send_message(client, bbs, body_text, recips, attachments=attachments) + click.echo(f"Message received by server: {resp}") + exit_client(ctx.obj, 0) + except Exception as e: + click.echo(f"Error sending message: {str(e)}", err=True) + exit_client(ctx.obj, 53) + + +@click.command() +@click.option("--number", "-n", type=int, default=0,help="Retrieve the first N messages matching filters/sort. 0 for all.") +@click.option('--sent', '-S', is_flag=True, default=False, help="Include sent messages in results.") +@click.option("--not-received", "-R", is_flag=True, default=False, help="Don't include received messages.") +@click.option("--ascending", "-A", is_flag=True, default=False, help="Show older/smaller results first after sorting.") +@click.option("--no-attachments", "-N", is_flag=True, default=False, help="Don't fetch attachment data.") +@click.option("--uuid", "-u", type=str, default=None, help="If specified, ignore other filters and retrieve only messages matching uuid.") +@click.option("--since-date", "-d", type=str, default=None, help="Only include messages since date (iso format), or '-' ex: -5d") +@click.option("--output-format", "-f", default="table", help="Print data as table[default], list, or JSON", + type=click.Choice(['table', 'json', 'list'], case_sensitive=False)) +@click.option("--save-copy", "-C", is_flag=True, default=False, help="Save a full copy of each message to fs.") +@click.option("--search", "-F", type=str, default="", help="Return only messages containing search string.") +@click.option("--no-text", "-T", is_flag=True, default=False, help="Don't return the message text.") +@click.option("--sort-by", "-B", default="date", help="Choose to sort by 'date', 'from', or 'to'", + type=click.Choice(['date', 'from', 'to'], case_sensitive=False)) +@click.pass_context +def get(ctx, number, sent, not_received, ascending, no_attachments, uuid, since_date, output_format, save_copy, + search, no_text, sort_by): + client = ctx.obj['client'] + bbs = ctx.obj['bbs'] + messages = [] + get_attach = not no_attachments + get_text = not no_text + reverse = not ascending + if uuid is not None: + try: + uuid = UUID(uuid) + except: + exit_client(ctx.obj, 52, message="Must provide a valid UUID.") + + if type(search) is str and (search.strip() == ""): + search = None + if not_received: + if sent: + source='sent' + else: + exit_client(ctx.obj, 23, "Can't exclude both sent and received messages.") + else: + if sent: + source='all' + else: + source='received' + + if number == 0: + limit = None + else: + limit = number + + if since_date is not None: + if len(since_date) < 3: + exit_client(ctx.obj, 41, "Invalid date specification.") + + if since_date[0] == "-": + m = re.match(rel_date, since_date) + if m is None: + exit_client(ctx.obj, 41, "Invalid date specification.") + else: + unit = m.group(2).lower() + multiplier = int(m.group(1)) + if unit not in unit_seconds: + exit_client(ctx.obj, 41, "Invalid date specification.") + total_seconds = int(multiplier * unit_seconds[unit]) + cutoff_date = datetime.datetime.now() - datetime.timedelta(seconds=total_seconds) + else: + try: + cutoff_date = datetime.datetime.fromisoformat(since_date) + except: + exit_client(ctx.obj, 41, "Invalid date specification.") + + if type(uuid) is UUID: + try: + messages.append(get_message_uuid(client, bbs, uuid, get_attachments=get_attach)) + except Exception as e: + exit_client(ctx.obj, 40, message=f"Couldn't get message specified: {str(e)}") + elif since_date is not None: + try: + messages = get_messages_since(client, bbs, cutoff_date, get_text=get_text, limit=limit, sort_by=sort_by, + reverse=reverse, search=search, get_attachments=get_attach, source=source) + except Exception as e: + exit_client(ctx.obj, 40, message=f"Couldn't fetch messages: {str(e)}") + else: + try: + messages = get_messages(client, bbs, get_text=get_text, limit=limit, sort_by=sort_by, reverse=reverse, + search=search, get_attachments=get_attach, source=source) + except Exception as e: + exit_client(ctx.obj, 40, message=f"Couldn't fetch messages: {str(e)}") + + save_dir = os.path.join(ctx.obj['directory'], 'message_cache') + if save_copy: + if not os.path.isdir(save_dir): + os.mkdir(save_dir) + + message_display = [] + for msg in messages: + json_filename = f"{msg.sent.strftime("%Y%m%d%H%M%s")}-{msg.from_user}.json" + json_path = os.path.join(save_dir, json_filename) + if save_copy: + json.dump(msg.to_dict(json=True), open(json_path, 'w')) + d = { + 'from': msg.from_user, + 'to': ",".join(msg.to_users), + 'id': str(msg.msg_id), + 'text': msg.text, + 'sent_at': msg.sent.isoformat(), + 'attachments': "", + } + if len(msg.attachments) > 0: + d['attachments'] = ",".join([a.name for a in msg.attachments]) + + if save_copy: + d['saved_path'] = json_path + message_display.append(d) + exit_client(ctx.obj, 0, format_list_dicts(message_display, output_format=output_format)) + + + + +message.add_command(get) +message.add_command(send) + + + diff --git a/packetserver/client/cli/object.py b/packetserver/client/cli/object.py new file mode 100644 index 0000000..724a8b8 --- /dev/null +++ b/packetserver/client/cli/object.py @@ -0,0 +1,127 @@ +import os +import os.path +import click +from packetserver.client.objects import (ObjectWrapper, post_object, post_file, + get_user_objects, get_object_by_uuid, delete_object_by_uuid) +from packetserver.client.cli.util import exit_client, format_list_dicts +from copy import deepcopy +from uuid import UUID + +@click.group() +@click.pass_context +def objects(ctx): + """Manages objects stored on the BBS.""" + pass + +@click.command() +@click.argument('file_path', required=True, type=str) +@click.option("--public", "-P", is_flag=True, default=False, help="Mark the object public for all users.") +@click.option("--binary", '-b', is_flag=True, default=False, help="Treat the file as binary instead of text.") +@click.option('--name', '-n', type=str, default=None, help="Name of object instead of source filename.") +@click.pass_context +def upload_file(ctx, file_path, public, name, binary): + """Upload file to object store. Return the assigned UUID.""" + + private = not public + client = ctx.obj['client'] + if not os.path.isfile(file_path): + click.echo(f"'{file_path}' is not a file.", err=True) + exit_client(ctx.obj, 15) + + uuid = post_file(client, ctx.obj['bbs'], file_path, private=private, name=name, binary=binary) + click.echo(str(uuid)) + exit_client(ctx.obj, 0) + +@click.command() +@click.argument('uuid', required=True, type=str) +@click.pass_context +def get(ctx, uuid): + """Get an object's data by its UUID.""" + client = ctx.obj['client'] + u = "" + try: + u = UUID(uuid) + except ValueError as e: + click.echo(f"'{uuid}' is not a valid UUID.", err=True) + exit_client(ctx.obj, 13) + + try: + obj = get_object_by_uuid(client, ctx.obj['bbs'], u, include_data=True) + click.echo(obj.data, nl=False) + exit_client(ctx.obj, 0) + except Exception as e: + click.echo(e, err=True) + exit_client(ctx.obj, 19) + + +@click.command() +@click.argument('uuid', required=True, type=str) +@click.pass_context +def delete(ctx, uuid): + """Delete the object identified by its UUID.""" + client = ctx.obj['client'] + u = "" + try: + u = UUID(uuid) + except ValueError as e: + click.echo(f"'{uuid}' is not a valid UUID.", err=True) + exit_client(ctx.obj, 13) + + try: + delete_object_by_uuid(client, ctx.obj['bbs'], u) + exit_client(ctx.obj, 0) + except Exception as e: + click.echo(e, err=True) + exit_client(ctx.obj, 19) + + +@click.command() +@click.option('--number', '-n', type=int, default=0, help="Number of objects to list. Default 0 for all.") +@click.option('--search', '-S', type=str, default=None, help="Search string to filter objects with.") +@click.option('--reverse', '-r', is_flag=True, default=False, help="Return results in reverse order.") +@click.option('--sort-by', '-B', default='date', help="Sort objects by size, date(default), or name", + type=click.Choice(['size', 'name', 'date'], case_sensitive=False)) +@click.option("--output-format", "-f", default="table", help="Print data as table[default], list, or JSON", + type=click.Choice(['table', 'json', 'list'], case_sensitive=False)) +@click.pass_context +def list_objects(ctx, number, search, reverse, sort_by, output_format): + """Get a list of user objects without the data.""" + # def get_user_objects(client: Client, bbs_callsign: str, limit: int = 10, include_data: bool = True, search: str = None, +# reverse: bool = False, sort_date: bool = False, sort_name: bool = False, sort_size: bool = False)\ +# -> list[ObjectWrapper]: + + client = ctx.obj['client'] + sort_date = False + sort_name = False + sort_size = False + + if sort_by == "size": + sort_size = True + elif sort_by == "name": + sort_name = True + else: + sort_date = True + + object_list = get_user_objects(client, ctx.obj['bbs'], limit=number, include_data=False, search=search, + reverse=reverse, sort_date=sort_date, sort_name=sort_name, sort_size=sort_size) + + obj_dicts = [] + for x in object_list: + d = deepcopy(x.obj_data) + d['uuid'] = "" + if 'uuid_bytes' in d: + d['uuid'] = str(UUID(bytes=d['uuid_bytes'])) + del d['uuid_bytes'] + if 'data' in d: + del d['data'] + if 'includes_data' in d: + del d['includes_data'] + obj_dicts.append(d) + + click.echo(format_list_dicts(obj_dicts, output_format=output_format.lower())) + exit_client(ctx.obj, 0) + +objects.add_command(upload_file) +objects.add_command(list_objects, name='list') +objects.add_command(get) +objects.add_command(delete) \ No newline at end of file diff --git a/packetserver/client/cli/server.py b/packetserver/client/cli/server.py new file mode 100644 index 0000000..e69de29 diff --git a/packetserver/client/cli/util.py b/packetserver/client/cli/util.py new file mode 100644 index 0000000..c91e40b --- /dev/null +++ b/packetserver/client/cli/util.py @@ -0,0 +1,60 @@ +from tabulate import tabulate +import json +import click +from packetserver.client import Client +import sys +import ZODB +from persistent.mapping import PersistentMapping +import datetime + +def format_list_dicts(dicts: list[dict], output_format: str = "table") -> str: + if output_format == "table": + return tabulate(dicts, headers="keys") + + elif output_format == "json": + return json.dumps(dicts, indent=2) + + elif output_format == "list": + output = "" + for i in dicts: + t = [] + for key in i: + t.append([str(key), str(i[key])]) + output = output + tabulate(t) + "\n" + return output + else: + raise ValueError("Unsupported format type.") + +def write_request_log(db: ZODB.DB, client: Client): + with db.transaction() as db_trans: + if not 'request_log' in db_trans.root(): + db_trans['request_log'] = PersistentMapping() + now = datetime.datetime.now() + db_trans['request_log'][now.isoformat()] = client.request_log + + + +def exit_client(context: dict, return_code: int, message=""): + client = context['client'] + db = context['db'] + client.stop() + + if context['keep_log']: + write_request_log(db, client) + + db.close() + client.stop() + if return_code == 0: + is_err = False + else: + is_err = True + if message.strip() != "": + click.echo(message, err=is_err) + sys.exit(return_code) + +unit_seconds ={ + 'h': 3600, + 'm': 60, + 's': 1, + 'd': 86400 +} \ No newline at end of file diff --git a/packetserver/client/jobs.py b/packetserver/client/jobs.py new file mode 100644 index 0000000..f9aea6e --- /dev/null +++ b/packetserver/client/jobs.py @@ -0,0 +1,179 @@ +from packetserver.client import Client +from packetserver.common import Request, Response, PacketServerConnection +from typing import Union, Optional +import datetime +import time + +class JobWrapper: + def __init__(self, data: dict): + for i in ['output', 'errors', 'artifacts', 'return_code', 'status']: + if i not in data: + raise ValueError("Was not given a job dictionary.") + self.data = data + self.artifacts = {} + for i in data['artifacts']: + self.artifacts[i[0]] = i[1] + + @property + def return_code(self) -> int: + return self.data['return_code'] + + @property + def output_raw(self) -> bytes: + return self.data['output'] + + @property + def output_str(self) -> str: + return self.data['output'].decode() + + @property + def errors_raw(self) -> bytes: + return self.data['errors'] + + @property + def errors_str(self) -> str: + return self.data['errors'].decode() + + @property + def status(self) -> str: + return self.data['status'] + + @property + def owner(self) -> str: + return self.data['owner'] + + @property + def cmd(self) -> Union[str, list]: + return self.data['cmd'] + + @property + def created(self) -> datetime.datetime: + return datetime.datetime.fromisoformat(self.data['created_at']) + + @property + def started(self) -> Optional[datetime.datetime]: + if not self.data['created_at']: + return None + return datetime.datetime.fromisoformat(self.data['created_at']) + + @property + def finished(self) -> Optional[datetime.datetime]: + if not self.data['finished_at']: + return None + return datetime.datetime.fromisoformat(self.data['finished_at']) + + @property + def is_finished(self) -> bool: + if self.finished is not None: + return True + return False + + @property + def id(self) -> int: + return self.data['id'] + + def __repr__(self): + return f"" + +def send_job(client: Client, bbs_callsign: str, cmd: Union[str, list], db: bool = False, env: dict = None, + files: dict = None) -> int: + """Send a job using client to bbs_callsign with args cmd. Return remote job_id.""" + req = Request.blank() + req.path = "job" + req.payload = {'cmd': cmd} + if db: + req.payload['db'] = '' + if env is not None: + req.payload['env']= env + if files is not None: + req.payload['files'] = files + req.method = Request.Method.POST + response = client.send_receive_callsign(req, bbs_callsign) + if response.status_code != 201: + raise RuntimeError(f"Sending job failed: {response.status_code}: {response.payload}") + return response.payload['job_id'] + +def send_job_quick(client: Client, bbs_callsign: str, cmd: Union[str, list], db: bool = False, env: dict = None, + files: dict = None) -> JobWrapper: + """Send a job using client to bbs_callsign with args cmd. Wait for quick job to return job results.""" + req = Request.blank() + req.path = "job" + req.payload = {'cmd': cmd} + req.set_var('quick', True) + if db: + req.payload['db'] = '' + if env is not None: + req.payload['env']= env + if files is not None: + req.payload['files'] = files + req.method = Request.Method.POST + response = client.send_receive_callsign(req, bbs_callsign) + if response.status_code == 200: + return JobWrapper(response.payload) + elif response.status_code == 202: + raise RuntimeError(f"Quick Job timed out. Job ID: {response.payload}") + else: + raise RuntimeError(f"Waiting for quick job failed: {response.status_code}: {response.payload}") + + +def get_job_id(client: Client, bbs_callsign: str, job_id: int, get_data=True) -> JobWrapper: + req = Request.blank() + req.path = f"job/{job_id}" + req.set_var('data', get_data) + req.method = Request.Method.GET + response = client.send_receive_callsign(req, bbs_callsign) + if response.status_code != 200: + raise RuntimeError(f"GET job {job_id} failed: {response.status_code}: {response.payload}") + return JobWrapper(response.payload) + +def get_user_jobs(client: Client, bbs_callsign: str, get_data=True) -> list[JobWrapper]: + req = Request.blank() + req.path = f"job/user" + req.set_var('data', get_data) + req.method = Request.Method.GET + response = client.send_receive_callsign(req, bbs_callsign) + if response.status_code != 200: + raise RuntimeError(f"GET user jobs failed: {response.status_code}: {response.payload}") + jobs = [] + for j in response.payload: + jobs.append(JobWrapper(j)) + return jobs + +class JobSession: + def __init__(self, client: Client, bbs_callsign: str, default_timeout: int = 300, stutter: int = 2): + self.client = client + self.bbs = bbs_callsign + self.timeout = default_timeout + self.stutter = stutter + self.job_log = [] + + def connect(self) -> PacketServerConnection: + return self.client.new_connection(self.bbs) + + def send(self, cmd: Union[str, list], db: bool = False, env: dict = None, files: dict = None) -> int: + return send_job(self.client, self.bbs, cmd, db=db, env=env, files=files) + + def send_quick(self, cmd: Union[str, list], db: bool = False, env: dict = None, files: dict = None) -> JobWrapper: + return send_job_quick(self.client, self.bbs, cmd, db=db, env=env, files=files) + + def get_id(self, jid: int) -> JobWrapper: + return get_job_id(self.client, self.bbs, jid) + + def run_job(self, cmd: Union[str, list], db: bool = False, env: dict = None, files: dict = None, + quick: bool = False) -> JobWrapper: + if quick: + j = self.send_quick(cmd, db=db, env=env, files=files) + self.job_log.append(j) + return j + else: + jid = self.send(cmd, db=db, env=env, files=files) + time.sleep(self.stutter) + j = self.get_id(jid) + while not j.is_finished: + time.sleep(self.stutter) + j = self.get_id(jid) + self.job_log.append(j) + return j + + + diff --git a/packetserver/client/messages.py b/packetserver/client/messages.py new file mode 100644 index 0000000..a4cbf34 --- /dev/null +++ b/packetserver/client/messages.py @@ -0,0 +1,231 @@ +import datetime + +from packetserver.client import Client +from packetserver.common import Request, Response, PacketServerConnection +from packetserver.common.util import to_date_digits +from typing import Union, Optional +from uuid import UUID, uuid4 +import os.path +import base64 + + +class AttachmentWrapper: + def __init__(self, data: dict): + for i in ['name', 'binary', 'data']: + if i not in data.keys(): + raise ValueError("Data dict was not an attachment dictionary.") + self._data = data + + def __repr__(self): + return f"" + + @property + def name(self) -> str: + return self._data['name'] + + @property + def binary(self) -> bool: + return self._data['binary'] + + @property + def data(self) -> Union[str,bytes]: + if self.binary: + return self._data['data'] + else: + return self._data['data'].decode() + + def to_dict(self, json: bool = True) -> dict: + d = { + "name": self.name, + "binary": self.binary, + } + + if not self.binary: + d['data'] = self.data + else: + if json: + d['data'] = base64.b64encode(self.data).decode() + else: + d['data'] = self.data + return d + +class MessageWrapper: + def __init__(self, data: dict): + for i in ['attachments', 'to', 'from', 'id', 'sent_at', 'text']: + if i not in data.keys(): + raise ValueError("Data dict was not a message dictionary.") + self.data = data + + @property + def text(self) -> str: + return self.data['text'] + + @property + def sent(self) -> datetime.datetime: + return datetime.datetime.fromisoformat(self.data['sent_at']) + + @property + def msg_id(self) -> UUID: + return UUID(self.data['id']) + + @property + def from_user(self) -> str: + return self.data['from'] + + @property + def to_users(self) -> list[str]: + return self.data['to'] + + @property + def attachments(self) -> list[AttachmentWrapper]: + a_list = [] + for a in self.data['attachments']: + a_list.append(AttachmentWrapper(a)) + return a_list + + def to_dict(self, json: bool = True) -> dict: + d = { + 'text': self.text, + 'sent': self.sent, + 'id': self.msg_id, + 'to': self.to_users, + 'from': self.from_user, + 'attachments': [] + } + + if json: + d['id'] = str(d['id']) + d['sent'] = d['sent'].isoformat() + for a in self.attachments: + d['attachments'].append(a.to_dict(json=True)) + else: + for a in self.attachments: + d['attachments'].append(a.to_dict(json=False)) + return d + +class MsgAttachment: + def __init__(self, name: str, data: Union[bytes,str]): + self.binary = True + self.name = name + if type(data) in [bytes, bytearray]: + self.data = data + else: + self.data = str(data).encode() + self.binary = False + + def __repr__(self) -> str: + return f"" + + def to_dict(self) -> dict: + return { + "name": self.name, + "data": self.data, + "binary": self.binary + } + +def attachment_from_file(filename: str, binary: bool = True) -> MsgAttachment: + a = MsgAttachment(os.path.basename(filename), open(filename, 'rb').read()) + if not binary: + a.binary = False + return a + +def send_message(client: Client, bbs_callsign: str, text: str, to: list[str], + attachments: list[MsgAttachment] = None) -> dict: + payload = { + "text": text, + "to": to, + "attachments": [] + } + if attachments is not None: + for a in attachments: + payload["attachments"].append(a.to_dict()) + + req = Request.blank() + req.path = "message" + req.method = Request.Method.POST + req.payload = payload + response = client.send_receive_callsign(req, bbs_callsign) + if response.status_code != 201: + raise RuntimeError(f"POST message failed: {response.status_code}: {response.payload}") + return response.payload + +def get_message_uuid(client: Client, bbs_callsign: str, msg_id: UUID, get_attachments: bool = True) -> MessageWrapper: + req = Request.blank() + req.path = "message" + req.method = Request.Method.GET + req.set_var('id', msg_id.bytes) + req.set_var('fetch_attachments', get_attachments) + response = client.send_receive_callsign(req, bbs_callsign) + if response.status_code != 200: + raise RuntimeError(f"GET message failed: {response.status_code}: {response.payload}") + return MessageWrapper(response.payload) + +def get_messages_since(client: Client, bbs_callsign: str, since: datetime.datetime, get_text: bool = True, limit: int = None, + sort_by: str = 'date', reverse: bool = False, search: str = None, get_attachments: bool = True, + source: str = 'received') -> list[MessageWrapper]: + req = Request.blank() + req.path = "message" + req.method = Request.Method.GET + + # put vars together + req.set_var('since', to_date_digits(since)) + + source = source.lower().strip() + if source not in ['sent', 'received', 'all']: + raise ValueError("Source variable must be ['sent', 'received', 'all']") + req.set_var('source', source) + + req.set_var('limit', limit) + req.set_var('fetch_text', get_text) + req.set_var('reverse', reverse) + req.set_var('fetch_attachments', get_attachments) + + if sort_by.strip().lower() not in ['date', 'from', 'to']: + raise ValueError("sort_by must be in ['date', 'from', 'to']") + req.set_var('sort', sort_by) + + if type(search) is str: + req.set_var('search', search) + + response = client.send_receive_callsign(req, bbs_callsign) + if response.status_code != 200: + raise RuntimeError(f"GET message failed: {response.status_code}: {response.payload}") + msg_list = [] + for m in response.payload: + msg_list.append(MessageWrapper(m)) + return msg_list + +def get_messages(client: Client, bbs_callsign: str, get_text: bool = True, limit: int = None, + sort_by: str = 'date', reverse: bool = True, search: str = None, get_attachments: bool = True, + source: str = 'received') -> list[MessageWrapper]: + + req = Request.blank() + req.path = "message" + req.method = Request.Method.GET + + # put vars together + + source = source.lower().strip() + if source not in ['sent', 'received', 'all']: + raise ValueError("Source variable must be ['sent', 'received', 'all']") + req.set_var('source', source) + + req.set_var('limit', limit) + req.set_var('fetch_text', get_text) + req.set_var('reverse', reverse) + req.set_var('fetch_attachments', get_attachments) + + if sort_by.strip().lower() not in ['date', 'from', 'to']: + raise ValueError("sort_by must be in ['date', 'from', 'to']") + req.set_var('sort', sort_by) + + if type(search) is str: + req.set_var('search', search) + + response = client.send_receive_callsign(req, bbs_callsign) + if response.status_code != 200: + raise RuntimeError(f"GET message failed: {response.status_code}: {response.payload}") + msg_list = [] + for m in response.payload: + msg_list.append(MessageWrapper(m)) + return msg_list diff --git a/packetserver/client/objects.py b/packetserver/client/objects.py new file mode 100644 index 0000000..c00f070 --- /dev/null +++ b/packetserver/client/objects.py @@ -0,0 +1,161 @@ +import datetime + +from packetserver.client import Client +from packetserver.common import Request, Response, PacketServerConnection +from typing import Union, Optional +from uuid import UUID, uuid4 +import os.path + + + +class ObjectWrapper: + def __init__(self, data: dict): + for i in ['name', 'uuid_bytes', 'binary', 'private', 'created_at', 'modified_at', 'data']: + if i not in data.keys(): + raise ValueError("Data dict was not an object dictionary.") + self.obj_data = data + + @property + def name(self) -> Optional[str]: + return self.obj_data['name'] + + @property + def size(self) -> int: + return len(self.obj_data['data']) + + @property + def created(self) -> datetime.datetime: + return datetime.datetime.fromisoformat(self.obj_data['created_at']) + + @property + def modified(self) -> datetime.datetime: + return datetime.datetime.fromisoformat(self.obj_data['modified_at']) + + @property + def uuid(self) -> UUID: + return UUID(bytes=self.obj_data['uuid_bytes']) + + @property + def private(self) -> bool: + return self.obj_data['private'] + + @property + def binary(self) -> bool: + return self.obj_data['binary'] + + @property + def data(self) -> Union[str, bytes]: + dat = self.obj_data['data'] + if type(dat) is str: + dat = dat.encode() + if self.binary: + return dat + else: + return dat.decode() + + +def post_object(client: Client, bbs_callsign: str, name:str, data: Union[str, bytes, bytearray], private=True) -> UUID: + if type(data) in [bytes, bytearray]: + data = bytes(data) + binary = True + else: + binary = False + data = str(data).encode() + + req = Request.blank() + req.path = "object" + req.payload = {'name': name, 'data': data, 'binary': binary, 'private': private} + req.method = Request.Method.POST + response = client.send_receive_callsign(req, bbs_callsign) + if response.status_code != 201: + raise RuntimeError(f"Posting object failed: {response.status_code}: {response.payload}") + return UUID(response.payload) + +def post_file(client: Client, bbs_callsign: str, file_path: str, private=True, name: str = None, binary=True) -> UUID: + if name is None: + obj_name = os.path.basename(file_path) + else: + obj_name = os.path.basename(str(name)) + if binary: + mode = 'rb' + else: + mode = 'r' + data = open(file_path, mode).read() + return post_object(client, bbs_callsign, obj_name, data, private=private) + +def get_object_by_uuid(client: Client, bbs_callsign: str, uuid: Union[str, bytes, UUID, int], + include_data=True) -> ObjectWrapper: + if type(uuid) is str: + uid = UUID(uuid) + elif type(uuid) is bytes: + uid = UUID(bytes=uuid) + elif type(uuid) is UUID: + uid = uuid + elif type(uuid) is int: + uid = UUID(int=uuid) + else: + raise ValueError("uuid must represent a UUID object") + + req = Request.blank() + if include_data: + req.set_var('fetch', 1) + req.path = "object" + req.set_var('uuid', uid.bytes) + req.method = Request.Method.GET + response = client.send_receive_callsign(req, bbs_callsign) + if response.status_code != 200: + raise RuntimeError(f"GET object {uid} failed: {response.status_code}: {response.payload}") + return ObjectWrapper(response.payload) + +def get_user_objects(client: Client, bbs_callsign: str, limit: int = 10, include_data: bool = True, search: str = None, + reverse: bool = False, sort_date: bool = False, sort_name: bool = False, sort_size: bool = False)\ + -> list[ObjectWrapper]: + + req = Request.blank() + if include_data: + req.set_var('fetch', 1) + if sort_date: + req.set_var('sort', 'date') + if sort_size: + req.set_var('sort', 'size') + if sort_name: + req.set_var('sort', 'name') + req.set_var('reverse', reverse) + if limit != 0: + req.set_var('limit', limit) + if search is not None: + req.set_var('search', str(search)) + req.path = "object" + req.method = Request.Method.GET + response = client.send_receive_callsign(req, bbs_callsign) + if response.status_code != 200: + raise RuntimeError(f"Listing objects failed: {response.status_code}: {response.payload}") + out_list = [] + for o in response.payload: + out_list.append(ObjectWrapper(o)) + return out_list + +def delete_object_by_uuid(client: Client, bbs_callsign: str, uuid: Union[str, bytes, UUID, int]) -> bool: + if type(uuid) is str: + uid = UUID(uuid) + elif type(uuid) is bytes: + uid = UUID(bytes=uuid) + elif type(uuid) is UUID: + uid = uuid + elif type(uuid) is int: + uid = UUID(int=uuid) + else: + raise ValueError("uuid must represent a UUID object") + + req = Request.blank() + req.path = "object" + req.set_var('uuid', uid.bytes) + req.method = Request.Method.DELETE + response = client.send_receive_callsign(req, bbs_callsign) + if response.status_code != 200: + raise RuntimeError(f"Deleting object {uid} failed: {response.status_code}: {response.payload}") + return True + +def update_object_by_uuid(): + # TODO update object by uuid client + pass \ No newline at end of file diff --git a/packetserver/client/testing.py b/packetserver/client/testing.py new file mode 100644 index 0000000..cd5a240 --- /dev/null +++ b/packetserver/client/testing.py @@ -0,0 +1,85 @@ +import os.path +import time +from typing import Union + +from packetserver.common import Request, PacketServerConnection +from packetserver.common.testing import SimpleDirectoryConnection +from packetserver.client import Client +import ax25 +from threading import Lock +import logging +import os.path +import datetime +from shutil import rmtree + +class TestClient(Client): + def __init__(self, conn_dir: str, callsign: str, keep_log: bool = True): + super().__init__('', 0, callsign, keep_log=keep_log) + self._connections = {} + if not os.path.isdir(conn_dir): + raise NotADirectoryError(f"Conn dir {conn_dir} does not exist.") + self._connection_directory = os.path.abspath(conn_dir) + + @property + def connections(self) -> dict: + return self._connections + + def connection_exists(self, callsign: str): + if not ax25.Address.valid_call(callsign): + raise ValueError("Must supply a valid callsign.") + callsign = callsign.upper().strip() + for key in self.connections.keys(): + if key.split(":")[1] == callsign: + return True + return False + + def new_connection(self, dest: str) -> SimpleDirectoryConnection: + if not self.started: + raise RuntimeError("Must start client before creating connections.") + if not ax25.Address.valid_call(dest): + raise ValueError(f"Provided destination callsign '{dest}' is invalid.") + with self.lock_locker: + if dest.upper() not in self._connection_locks: + self._connection_locks[dest.upper()] = Lock() + with self._connection_locks[dest.upper()]: + conn = self.connection_callsign(dest.upper()) + if conn is not None: + return conn + + conn_dir = os.path.join(self._connection_directory, f"{self.callsign.upper()}--{dest.upper()}") + if not os.path.isdir(conn_dir): + os.mkdir(conn_dir) + conn = SimpleDirectoryConnection.create_directory_connection(self.callsign, conn_dir) + self.connections[f"{dest.upper()}:{self.callsign.upper()}"] = conn + logging.debug(f"Connection to {dest} ready.") + return conn + + def receive(self, req: Request, conn: Union[PacketServerConnection,SimpleDirectoryConnection], timeout: int = 300): + if type(conn) is SimpleDirectoryConnection: + time.sleep(1) + cutoff_date = datetime.datetime.now() + datetime.timedelta(seconds=timeout) + while datetime.datetime.now() < cutoff_date: + logging.debug(f"Client {self.callsign} checking for connection conn {conn}") + if conn.check_for_data(): + break + return super().receive(req, conn, timeout=timeout) + + def clear_connections(self): + closing = [x for x in self.connections] + for key in closing: + conn = self.connections[key] + conn.closing = True + conn.check_closed() + while os.path.exists(conn.directory): + try: + rmtree(conn.directory) + except: + time.sleep(.5) + pass + + def start(self): + self.started = True + + def stop(self): + self.clear_connections() + self.started = False \ No newline at end of file diff --git a/packetserver/client/users.py b/packetserver/client/users.py new file mode 100644 index 0000000..f479559 --- /dev/null +++ b/packetserver/client/users.py @@ -0,0 +1,122 @@ +import datetime + +from packetserver.client import Client +from packetserver.common import Request, Response, PacketServerConnection +from typing import Union, Optional +from packetserver.common.util import email_valid, random_string +from uuid import UUID, uuid4 +import os.path + +class UserWrapper: + + def __init__(self, data: dict): + for i in ['username', 'status', 'bio', 'socials', 'created_at', 'last_seen', 'email', 'location']: + if i not in data.keys(): + raise ValueError("Data dict was not an object dictionary.") + self.data = data + + def pretty_dict(self) -> dict: + out_dict = {} + for a in ['username', 'status', 'bio', 'socials', 'created', 'last_seen', 'email', 'location']: + if a != 'socials': + out_dict[a] = str(getattr(self, a)) + else: + social_str = "\n".join(self.socials) + out_dict['socials'] = social_str + + return out_dict + + + def __repr__(self): + return f"" + + @property + def socials(self) -> list[str]: + return self.data['socials'] + + @property + def created(self) -> datetime.datetime: + return datetime.datetime.fromisoformat(self.data['created_at']) + + @property + def last_seen(self) -> Optional[datetime.datetime]: + if self.data['last_seen'] is not None: + return datetime.datetime.fromisoformat(self.data['last_seen']) + else: + return None + + @property + def username(self) -> str: + return self.data['username'] + + @property + def status(self) -> str: + return self.data['status'] + + @property + def bio(self) -> str: + return self.data['bio'] + + @property + def email(self) -> str: + return self.data['email'] + + @property + def location(self) -> str: + return self.data['location'] + + +def get_user_by_username(client: Client, bbs_callsign: str, username: str) -> UserWrapper: + req = Request.blank() + req.path = "user" + req.set_var('username', username.strip().upper()) + req.method = Request.Method.GET + response = client.send_receive_callsign(req, bbs_callsign) + if response.status_code != 200: + raise RuntimeError(f"GET user {username} failed: {response.status_code}: {response.payload}") + return UserWrapper(response.payload) + +def get_users(client: Client, bbs_callsign: str, limit=None): + req = Request.blank() + req.path = "user" + if limit is not None: + req.set_var('limit', limit) + req.method = Request.Method.GET + response = client.send_receive_callsign(req, bbs_callsign) + if response.status_code != 200: + raise RuntimeError(f"GET userlist failed: {response.status_code}: {response.payload}") + user_list = [] + for u in response.payload: + user_list.append(UserWrapper(u)) + return user_list + +def update_self(client: Client, bbs_callsign: str, email: str = None, bio: str = None, + socials: Union[list[str],str] = None, location: str = None, status: str = None) -> bool: + + payload = {} + + if email is not None: + if not email_valid(email): + raise ValueError(f"{email} is not a valid e-mail address") + payload['email'] = email + + if socials is not None: + payload['social'] = socials + + if status is not None: + payload['status'] = str(status) + + if location is not None: + payload['location'] = str(location) + + if bio is not None: + payload['bio'] = str(bio) + + req = Request.blank() + req.path = "user" + req.method = Request.Method.UPDATE + req.payload = payload + response = client.send_receive_callsign(req, bbs_callsign) + if response.status_code != 200: + raise RuntimeError(f"Updating profile failed: {response.status_code}: {response.payload}") + return True \ No newline at end of file diff --git a/packetserver/common/__init__.py b/packetserver/common/__init__.py new file mode 100644 index 0000000..f291fa7 --- /dev/null +++ b/packetserver/common/__init__.py @@ -0,0 +1,343 @@ +from pe.connect import Connection, ConnectionState +from threading import Lock +from msgpack import Unpacker +from msgpack import packb, unpackb +from enum import Enum +import bz2 +from typing import Union, Self +import datetime +import logging +import ax25 + + +class PacketServerConnection(Connection): + + connection_subscribers = [] + receive_subscribers = [] + max_send_size = 2000 + + def __init__(self, port, call_from, call_to, incoming=False): + super().__init__(port, call_from, call_to, incoming=incoming) + # Now perform any initialization of your own that you might need + self.data = Unpacker() + self.data_lock = Lock() + self.connection_created = datetime.datetime.now(datetime.UTC) + self.connection_last_activity = datetime.datetime.now(datetime.UTC) + self.closing = False + + + @property + def local_callsign(self): + if self.incoming: + return self.call_to + else: + return self.call_from + + @property + def remote_callsign(self): + if self.incoming: + return self.call_from + else: + return self.call_to + + def connected(self): + logging.debug("connected") + logging.debug(f"new connection from {self.call_from} to {self.call_to}") + for fn in PacketServerConnection.connection_subscribers: + fn(self) + + def disconnected(self): + logging.debug(f"connection disconnected: {self.call_from} -> {self.call_to}") + + def data_received(self, pid, data): + self.connection_last_activity = datetime.datetime.now(datetime.UTC) + logging.debug(f"received data: {data}") + with self.data_lock: + logging.debug(f"fed received data to unpacker {data}") + self.data.feed(data) + for fn in PacketServerConnection.receive_subscribers: + logging.debug("found function to notify about received data") + fn(self) + logging.debug("notified function about received data") + + def send_data(self, data: Union[bytes, bytearray]): + logging.debug(f"sending data: {data}") + self.connection_last_activity = datetime.datetime.now(datetime.UTC) + if len(data) > self.max_send_size: + logging.debug(f"Large frame detected {len(data)} breaking it up into chunks") + index = 0 + counter = 0 + while index <= len(data): + logging.debug(f"Sending chunk {counter}") + if (len(data) - index) < self.max_send_size: + super().send_data(data[index:]) + break + super().send_data(data[index:index + self.max_send_size]) + index = index + self.max_send_size + counter = counter + 1 + else: + super().send_data(data) + + @classmethod + def query_accept(cls, port, call_from, call_to): + return True + + +class Message: + """Base class for communication encapsulated in msgpack objects.""" + + class CompressionType(Enum): + NONE = 0 + BZIP2 = 1 + GZIP = 2 + DEFLATE = 3 + + class MessageType(Enum): + REQUEST = 0 + RESPONSE = 1 + + def __init__(self, msg_type: MessageType, compression: CompressionType, payload: dict): + self.type = Message.MessageType(msg_type) + self.compression = Message.CompressionType(compression) + self.data = payload + + @property + def vars(self) -> dict: + if 'v' in self.data: + if type(self.data['v']) is dict: + return self.data['v'] + return {} + + def get_var(self, key: str): + if 'v' not in self.data: + raise KeyError(f"Variable '{key}' not found.") + if str(key) not in self.data['v']: + raise KeyError(f"Variable '{key}' not found.") + return self.data['v'][str(key)] + + def set_var(self, key: str, value): + if 'v' not in self.data: + self.data['v'] = {} + self.data['v'][str(key)] = value + + @property + def data_bytes(self): + return packb(self.data) + + def pack(self) -> bytes: + output = {'t': self.type.value, 'c': self.compression.value} + data_bytes = self.data_bytes + logging.debug("Packing Message") + if (self.compression is self.CompressionType.NONE) or (len(data_bytes) < 30): + output['d'] = data_bytes + output['c'] = self.CompressionType.NONE.value + return packb(output) + + if self.compression is self.CompressionType.BZIP2: + compressed = bz2.compress(packb(self.data)) + else: + raise NotImplementedError(f"Compression type {self.compression.name} is not implemented yet.") + + if len(compressed) < len(data_bytes): + output['d'] = compressed + else: + output['d'] = data_bytes + output['c'] = self.CompressionType.NONE.value + return packb(output) + + @property + def payload(self): + if 'd' in self.data: + pl = self.data['d'] + if type(pl) in (dict, str, bytes, list): + return pl + else: + return str(pl) + else: + return "" + + @payload.setter + def payload(self, payload: Union[str, bytes, dict, list]): + logging.debug(f"Setting a message payload: {type(payload)}: {payload}") + if type(payload) in (str, bytes, dict, list): + logging.debug(f"Payload type is {type(payload)}, conversion to string unnecessary") + self.data['d'] = payload + else: + logging.debug("payload type is not in (str, bytes, dict, list); converting to string") + self.data['d'] = str(payload) + logging.debug(f"Final payload is: {type(payload)}: {payload}") + + @classmethod + def partial_unpack(cls, msg: dict) -> Self: + unpacked = msg + comp = Message.CompressionType(unpacked['c']) + msg_type = Message.MessageType(unpacked['t']) + raw_data = unpacked['d'] + + if comp is Message.CompressionType.NONE: + data = unpackb(raw_data) + elif comp is Message.CompressionType.BZIP2: + data = unpackb(bz2.decompress(raw_data)) + else: + raise NotImplementedError(f"Compression type {comp.name} is not implemented yet.") + + return Message(msg_type, comp, data) + + @classmethod + def unpack(cls, msg_bytes: bytes) -> Self: + try: + unpacked = unpackb(msg_bytes) + except Exception as e: + raise ValueError("ERROR: msg_bytes didn't contain a valid msgpack object.\n" + str(e)) + if type(unpacked) is not dict: + raise ValueError("ERROR: unpacked message was not a packetserver message.") + for i in ('t', 'c', 'd'): + if i not in unpacked: + raise ValueError("ERROR: unpacked message was not a packetserver message.") + return Message.partial_unpack(unpacked) + +class Request(Message): + class Method(Enum): + GET = 0 + POST = 1 + UPDATE = 2 + DELETE = 3 + + def __init__(self, msg: Message): + if msg.type is not Message.MessageType.REQUEST: + raise ValueError(f"Can't create a Request Object from a {msg.type} Message object.") + + super().__init__(msg.type, msg.compression, msg.data) + + if ('p' in msg.data) and (type(msg.data['p']) is not str): + raise ValueError("Path of Request must be a string.") + + if 'p' in self.data: + self.data['p'] = str(self.data['p']).strip().lower() + + if 'm' in msg.data: + if type(msg.data['m']) is not bytes: + raise ValueError("Method of Request must be bytes.") + self.Method(int(self.data['m'][0])) + + @property + def path(self): + if 'p' in self.data: + return str(self.data['p']).lower().strip() + else: + return "" + + @path.setter + def path(self, path: str): + self.data['p'] = str(path).strip().lower() + + @property + def method(self) -> Method: + if 'm' in self.data: + return self.Method(int(self.data['m'][0])) + else: + return self.Method.GET + + @method.setter + def method(self, meth: Method): + meth = self.Method(meth) + self.data['m'] = meth.value.to_bytes(1) + + @classmethod + def unpack(cls, msg_bytes: bytes) -> Self: + msg = super().unpack(msg_bytes) + return Request(msg) + + @classmethod + def blank(cls) -> Self: + msg = Message(Message.MessageType.REQUEST, Message.CompressionType.NONE, {}) + return Request(msg) + + def __repr__(self): + return f"" + +class Response(Message): + def __init__(self, msg: Message): + if msg.type is not Message.MessageType.RESPONSE: + raise ValueError(f"Can't create a Response Object from a {msg.type} Message object.") + + super().__init__(msg.type, msg.compression, msg.data) + if 'c' in msg.data: + status_bytes = self.data['c'] + if type(status_bytes) is not bytes: + raise ValueError("Invalid Response data") + status_code = int.from_bytes(status_bytes) + if status_code >= 600: + raise ValueError("Invalid status code.") + + @classmethod + def unpack(cls, msg_bytes: bytes) -> Self: + msg = super().unpack(msg_bytes) + return Response(msg) + + @classmethod + def blank(cls) -> Self: + msg = Message(Message.MessageType.RESPONSE, Message.CompressionType.NONE, {}) + return Response(msg) + + @property + def status_code(self) -> int: + if 'c' in self.data: + status_bytes = self.data['c'] + if type(status_bytes) is not bytes: + raise ValueError("Invalid Response data") + status_code = int.from_bytes(status_bytes) + if status_code >= 600: + raise ValueError("Invalid status code.") + return status_code + else: + return 200 + + @status_code.setter + def status_code(self, code: int): + if (code <= 0) or (code >= 600): + raise ValueError("Status must be a positive integer <= 600") + self.data['c'] = code.to_bytes(2) + + def __repr__(self): + return f"" + +def send_response(conn: PacketServerConnection, response: Response, original_request: Request, + compression: Message.CompressionType = Message.CompressionType.BZIP2): + if conn.state.name == "CONNECTED" and not conn.closing: + + # figure out compression setting based on request + logging.debug("Determining compression of response") + comp = compression + logging.debug(f"Default comp: {comp}") + logging.debug(f"Original vars: {original_request.vars}") + if 'C' in original_request.vars: + logging.debug(f"Detected compression header in original request: {original_request.vars['C']}") + val = original_request.vars['C'] + for i in Message.CompressionType: + logging.debug(f"Checking type: {i}") + if str(val).strip().upper() == i.name: + comp = i + logging.debug(f"matched compression with var to {comp}") + break + try: + if int(val) == i.value: + comp = i + logging.debug(f"matched compression with var to {comp}") + except ValueError: + pass + response.compression = comp + logging.debug(f"Final compression: {response.compression}") + + logging.debug(f"sending response: {response}, {response.compression}, {response.payload}") + conn.send_data(response.pack()) + logging.debug("response sent successfully") + else: + logging.warning(f"Attempted to send data, but connection state is {conn.state.name}") + +def send_blank_response(conn: PacketServerConnection, original_request: Request, status_code: int = 200, + payload: Union[bytes, bytearray, str, dict, list] = ""): + response = Response.blank() + response.status_code = status_code + response.payload = payload + send_response(conn, response, original_request) \ No newline at end of file diff --git a/packetserver/common/constants.py b/packetserver/common/constants.py new file mode 100644 index 0000000..e0cf8c1 --- /dev/null +++ b/packetserver/common/constants.py @@ -0,0 +1,2 @@ +no_values = [0, '0', 'n', 'N', 'f', 'F', 'no', 'NO', False] +yes_values = [1, '1', 'y', 'Y', 't', 'T', 'yes', 'YES', True] diff --git a/packetserver/common/testing.py b/packetserver/common/testing.py new file mode 100644 index 0000000..1cbb7c1 --- /dev/null +++ b/packetserver/common/testing.py @@ -0,0 +1,268 @@ +import msgpack + +from . import PacketServerConnection +from pe.connect import ConnectionState +from msgpack import Unpacker +from typing import Union, Self, Optional +import os.path +import logging +import ax25 + +class DummyPacketServerConnection(PacketServerConnection): + + def __init__(self, call_from: str, call_to: str, incoming=False): + super().__init__(0, call_from, call_to, incoming=incoming) + self.sent_data = Unpacker() + self._state = ConnectionState.CONNECTED + + @property + def state(self): + return self._state + + def send_data(self, data: Union[bytes, bytearray]): + self.sent_data.feed(data) + logging.debug(f"Sender added {data} to self.sent_data.feed") + +class DirectoryTestServerConnection(PacketServerConnection): + """Monitors a directory for messages in msgpack format.""" + def __init__(self, call_from: str, call_to: str, directory: str, incoming=False): + super().__init__(0, call_from, call_to, incoming=incoming) + self._state = ConnectionState.CONNECTED + if not os.path.isdir(directory): + raise FileNotFoundError(f"No such directory as {directory}") + self._directory = directory + self._sent_data = Unpacker() + self._pid = 1 + self.closing = False + + @classmethod + def create_directory_connection(cls, self_callsign: str, directory: str) -> Self: + + if not ax25.Address.valid_call(self_callsign): + raise ValueError("self_callsign must be a valid callsign.") + + if not os.path.isdir(directory): + raise NotADirectoryError(f"{directory} is not a directory or doesn't exist.") + + spl = os.path.basename(directory).split('--') + if len(spl) != 2: + raise ValueError(f"Directory {directory} has the wrong name to be a connection dir.") + + src = spl[0] + dst = spl[1] + + if not ax25.Address.valid_call(src): + raise ValueError(f"Directory {directory} has the wrong name to be a connection dir.") + + if not ax25.Address.valid_call(dst): + raise ValueError(f"Directory {directory} has the wrong name to be a connection dir.") + + if dst.upper() == self_callsign.upper(): + incoming = True + else: + incoming = False + + return DirectoryTestServerConnection(src, dst, directory, incoming=incoming) + + @property + def pid(self) -> int: + old = self._pid + self._pid = self._pid + 1 + return old + + @property + def directory(self) -> str: + return self._directory + + @property + def state(self): + return self._state + + @property + def file_path(self) -> str: + file_name = f"{self.local_callsign}.msg" + file_path = os.path.join(self._directory, file_name) + return file_path + + @property + def remote_file_path(self) -> str: + file_name = f"{self.remote_callsign}.msg" + file_path = os.path.join(self._directory, file_name) + return file_path + + def check_closed(self): + if self.closing: + self._state = ConnectionState.DISCONNECTED + if self._state is not ConnectionState.CONNECTED: + return True + if not os.path.isdir(self._directory): + self._state = ConnectionState.DISCONNECTED + self.disconnected() + return True + return False + + def write_out(self, data: bytes): + if self.check_closed(): + raise RuntimeError("Connection is closed. Cannot send.") + + if os.path.exists(self.file_path): + raise RuntimeError("The outgoing message file already exists. State is wrong for sending.") + + if os.path.exists(self.file_path+".tmp"): + os.remove(self.file_path+".tmp") + + open(self.file_path+".tmp", 'wb').write(data) + os.rename(self.file_path+".tmp", self.file_path) + + def send_data(self, data: Union[bytes, bytearray]): + if self.check_closed(): + raise RuntimeError("Connection is closed. Cannot send.") + self._sent_data.feed(data) + logging.debug(f"Sender added {data} to self.sent_data.feed") + try: + obj = self._sent_data.unpack() + self.write_out(msgpack.packb(obj)) + logging.debug(f"Wrote complete binary message to {self.file_path}") + except msgpack.OutOfData as e: + pass + + def check_for_data(self): + """Monitors connection directory for data.""" + if self.closing: + self._state = ConnectionState.DISCONNECTED + if self.check_closed(): + return + + if os.path.isfile(self.remote_file_path): + logging.debug(f"{self.local_callsign} Found that the remote file path '{self.remote_file_path}' exists now.") + data = open(self.remote_file_path, 'rb').read() + self.data_received(self.pid, bytearray(data)) + os.remove(self.remote_file_path) + logging.debug(f"{self.local_callsign} detected data from {self.remote_callsign}: {msgpack.unpackb(data)}") + + +class SimpleDirectoryConnection: + def __init__(self, call_from: str, call_to: str, directory: str, incoming=False): + self._state = ConnectionState.CONNECTED + if not os.path.isdir(directory): + raise FileNotFoundError(f"No such directory as {directory}") + self._directory = directory + self._sent_data = Unpacker() + self.data = Unpacker() + self._pid = 1 + self.call_to = call_to + self.call_from = call_from + self.incoming = incoming + self._incoming = incoming + self.closing = False + if incoming: + self.local_callsign = call_to + self.remote_callsign = call_from + else: + self.local_callsign = call_from + self.remote_callsign = call_to + + @classmethod + def create_directory_connection(cls, self_callsign: str, directory: str) -> Self: + + if not ax25.Address.valid_call(self_callsign): + raise ValueError("self_callsign must be a valid callsign.") + + if not os.path.isdir(directory): + raise NotADirectoryError(f"{directory} is not a directory or doesn't exist.") + + spl = os.path.basename(directory).split('--') + if len(spl) != 2: + raise ValueError(f"Directory {directory} has the wrong name to be a connection dir.") + + src = spl[0] + dst = spl[1] + + if not ax25.Address.valid_call(src): + raise ValueError(f"Directory {directory} has the wrong name to be a connection dir.") + + if not ax25.Address.valid_call(dst): + raise ValueError(f"Directory {directory} has the wrong name to be a connection dir.") + + if dst.upper() == self_callsign.upper(): + incoming = True + else: + incoming = False + + return SimpleDirectoryConnection(src, dst, directory, incoming=incoming) + + @property + def pid(self) -> int: + old = self._pid + self._pid = self._pid + 1 + return old + + @property + def directory(self) -> str: + return self._directory + + @property + def state(self): + return self._state + + @property + def file_path(self) -> str: + file_name = f"{self.local_callsign}.msg" + file_path = os.path.join(self._directory, file_name) + return file_path + + @property + def remote_file_path(self) -> str: + file_name = f"{self.remote_callsign}.msg" + file_path = os.path.join(self._directory, file_name) + return file_path + + def check_closed(self): + if self.closing: + self._state = ConnectionState.DISCONNECTED + if self._state is not ConnectionState.CONNECTED: + return True + if not os.path.isdir(self._directory): + self._state = ConnectionState.DISCONNECTED + return True + return False + + def write_out(self, data: bytes): + if self.check_closed(): + raise RuntimeError("[SIMPLE] Connection is closed. Cannot send.") + + if os.path.exists(self.file_path): + raise RuntimeError("[SIMPLE] The outgoing message file already exists. State is wrong for sending.") + + if os.path.exists(self.file_path+".tmp"): + os.remove(self.file_path+".tmp") + + open(self.file_path+".tmp", 'wb').write(data) + os.rename(self.file_path+".tmp", self.file_path) + + def send_data(self, data: Union[bytes, bytearray]): + if self.check_closed(): + raise RuntimeError("[SIMPLE] Connection is closed. Cannot send.") + self._sent_data.feed(data) + logging.debug(f"[SIMPLE] Sender added {data} to self.sent_data.feed") + try: + obj = self._sent_data.unpack() + self.write_out(msgpack.packb(obj)) + logging.debug(f"[SIMPLE] Wrote complete binary message to {self.file_path}") + except msgpack.OutOfData as e: + pass + + def check_for_data(self) -> bool: + """Monitors connection directory for data.""" + if self.closing: + self._state = ConnectionState.DISCONNECTED + if self.check_closed(): + return False + if os.path.isfile(self.remote_file_path): + data = open(self.remote_file_path, 'rb').read() + os.remove(self.remote_file_path) + logging.debug(f"[SIMPLE] {self.local_callsign} detected data from {self.remote_callsign}: {data}") + self.data.feed(data) + return True + else: + return False diff --git a/packetserver/common/util.py b/packetserver/common/util.py new file mode 100644 index 0000000..3b36e38 --- /dev/null +++ b/packetserver/common/util.py @@ -0,0 +1,151 @@ +import re +import datetime +import tempfile +import tarfile +from typing import Union, Iterable, Tuple, Optional, IO +import os.path +from io import BytesIO, BufferedReader +import random +import string + +def email_valid(email: str) -> bool: + """Taken from https://www.geeksforgeeks.org/check-if-email-address-valid-or-not-in-python/""" + regex = r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,7}\b' + if re.fullmatch(regex, email): + return True + else: + return False + +def to_date_digits(index: datetime.datetime) -> str: + return f"{str(index.year).zfill(4)}{str(index.month).zfill(2)}{str(index.day).zfill(2)}{str(index.hour).zfill(2)}{str(index.minute).zfill(2)}{str(index.second).zfill(2)}" + +def from_date_digits(index: str, tz: datetime.timezone = datetime.UTC) -> datetime: + ind = str(index) + if not ind.isdigit(): + raise ValueError("Received invalid date digit string, containing non-digit chars.") + if len(ind) < 4: + raise ValueError("Received invalid date digit string, needs to at least by four digits for a year") + year = int(ind[:4]) + month = 1 + day = 1 + hour = 0 + minute = 0 + second = 0 + if len(ind) >= 6: + month = int(ind[4:6]) + + if len(ind) >= 8: + day = int(ind[6:8]) + + if len(ind) >= 10: + hour = int(ind[8:10]) + + if len(ind) >= 12: + minute = int(ind[10:12]) + + if len(ind) >= 14: + second = int(ind[12:14]) + + return datetime.datetime(year, month, day ,hour, minute, second, tzinfo=tz) + +def tar_bytes(file: Union[str, Iterable]) -> bytes: + """Creates a tar archive in a temporary file with the specified files at root level. + Returns the bytes of the archive.""" + files = [] + if type(file) is str: + files.append(file) + else: + for i in file: + files.append(str(i)) + + with tempfile.TemporaryFile() as temp: + tar_obj = tarfile.TarFile(fileobj=temp, mode="w") + for i in files: + tar_obj.add(i, arcname=os.path.basename(i)) + tar_obj.close() + temp.seek(0) + return temp.read() + +def bytes_to_tar_bytes(name: str, data: bytes) -> bytes: + """Creates a tar archive with a single file of name with bytes as the contents""" + with tempfile.TemporaryFile() as temp: + tar_obj = tarfile.TarFile(fileobj=temp, mode="w") + bio = BytesIO(data) + tar_info = tarfile.TarInfo(name=name) + tar_info.size = len(data) + tar_obj.addfile(tar_info, bio) + tar_obj.close() + temp.seek(0) + return temp.read() + +def bytes_tar_has_files(data: Union[bytes, IO]): + if type(data) is bytes: + bio = BytesIO(data) + else: + bio = data + tar_obj = tarfile.TarFile(fileobj=bio, mode="r") + files = [m for m in tar_obj.getmembers() if m.isfile()] + if len(files) > 0: + return True + else: + return False + +def multi_bytes_to_tar_bytes(objects: dict) -> bytes: + """Creates a tar archive with a single file of name with bytes as the contents""" + with tempfile.TemporaryFile() as temp: + tar_obj = tarfile.TarFile(fileobj=temp, mode="w") + for name in objects: + data = bytes(objects[name]) + bio = BytesIO(data) + tar_info = tarfile.TarInfo(name=name) + tar_info.size = len(data) + tar_obj.addfile(tar_info, bio) + tar_obj.close() + temp.seek(0) + return temp.read() + +def extract_tar_bytes(tarfile_bytes: bytes) -> Tuple[str, bytes]: + """Takes the bytes of a tarfile, and returns the name and bytes of the first file in the archive.""" + out_bytes = b'' + bio = BytesIO(tarfile_bytes) + tar_obj = tarfile.TarFile(fileobj=bio, mode="r") + members = tar_obj.getmembers() + for i in range(0, len(members)): + if members[i].isfile(): + return members[i].name, tar_obj.extractfile(members[i]).read() + raise FileNotFoundError("No files found to extract from archive") + +def random_string(length=8) -> str: + rand_str = ''.join(random.choices(string.ascii_letters + string.digits, k=length)) + return rand_str + + +class TarFileExtractor(object): + """Generator created from file like object pointing to tar data""" + def __init__(self, fileobj: IO): + self.fileobj = fileobj + try: + self.tar_file = tarfile.TarFile(fileobj=self.fileobj) + self._raw_members = [m for m in self.tar_file.getmembers() if m.isfile()] + except: + self._raw_members = [] + self._count = 0 + + def __iter__(self): + return self + + # Python 3 compatibility + def __next__(self): + return self.next() + + def next(self) -> Tuple[str, IO]: + if (self._count + 1) > len(self._raw_members): + raise StopIteration() + else: + member = self._raw_members[self._count] + name = member.name + if type(name) is bytes: + name = name.decode() + name = str(name) + self._count = self._count + 1 + return os.path.basename(name), self.tar_file.extractfile(member) diff --git a/packetserver/packetserver.egg-info/PKG-INFO b/packetserver/packetserver.egg-info/PKG-INFO new file mode 100644 index 0000000..fff0895 --- /dev/null +++ b/packetserver/packetserver.egg-info/PKG-INFO @@ -0,0 +1,12 @@ +Metadata-Version: 2.2 +Name: packetserver +Version: 0.4.1 +Requires-Dist: click +Requires-Dist: pyham_pe +Requires-Dist: msgpack +Requires-Dist: pyham_ax25 +Requires-Dist: ZODB +Requires-Dist: ZEO +Requires-Dist: podman +Requires-Dist: tabulate +Dynamic: requires-dist diff --git a/packetserver/packetserver.egg-info/SOURCES.txt b/packetserver/packetserver.egg-info/SOURCES.txt new file mode 100644 index 0000000..2fec869 --- /dev/null +++ b/packetserver/packetserver.egg-info/SOURCES.txt @@ -0,0 +1,41 @@ +setup.py +client/__init__.py +client/bulletins.py +client/jobs.py +client/messages.py +client/objects.py +client/testing.py +client/users.py +client/cli/__init__.py +client/cli/bulletin.py +client/cli/config.py +client/cli/constants.py +client/cli/db.py +client/cli/job.py +client/cli/message.py +client/cli/object.py +client/cli/server.py +client/cli/util.py +common/__init__.py +common/constants.py +common/testing.py +common/util.py +packetserver.egg-info/PKG-INFO +packetserver.egg-info/SOURCES.txt +packetserver.egg-info/dependency_links.txt +packetserver.egg-info/entry_points.txt +packetserver.egg-info/requires.txt +packetserver.egg-info/top_level.txt +runner/__init__.py +runner/constants.py +runner/podman.py +server/__init__.py +server/bulletin.py +server/constants.py +server/db.py +server/jobs.py +server/messages.py +server/objects.py +server/requests.py +server/testserver.py +server/users.py \ No newline at end of file diff --git a/packetserver/packetserver.egg-info/dependency_links.txt b/packetserver/packetserver.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/packetserver/packetserver.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/packetserver/packetserver.egg-info/entry_points.txt b/packetserver/packetserver.egg-info/entry_points.txt new file mode 100644 index 0000000..f3c57c3 --- /dev/null +++ b/packetserver/packetserver.egg-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +packcli = packetserver.client.cli:cli diff --git a/packetserver/packetserver.egg-info/requires.txt b/packetserver/packetserver.egg-info/requires.txt new file mode 100644 index 0000000..f46210a --- /dev/null +++ b/packetserver/packetserver.egg-info/requires.txt @@ -0,0 +1,8 @@ +click +pyham_pe +msgpack +pyham_ax25 +ZODB +ZEO +podman +tabulate diff --git a/packetserver/packetserver.egg-info/top_level.txt b/packetserver/packetserver.egg-info/top_level.txt new file mode 100644 index 0000000..52ad438 --- /dev/null +++ b/packetserver/packetserver.egg-info/top_level.txt @@ -0,0 +1 @@ +packetserver diff --git a/packetserver/runner/__init__.py b/packetserver/runner/__init__.py new file mode 100644 index 0000000..01854e9 --- /dev/null +++ b/packetserver/runner/__init__.py @@ -0,0 +1,191 @@ +"""Package runs arbitrary commands/jobs via different mechanisms.""" +from typing import Union,Optional,Iterable,Self +from enum import Enum +import datetime +from uuid import UUID, uuid4 +from threading import Lock +import os.path +from packetserver.runner.constants import job_setup_script, job_end_script, container_setup_script, container_run_script +from packetserver.common.util import multi_bytes_to_tar_bytes, bytes_to_tar_bytes, TarFileExtractor + + +def scripts_tar() -> bytes: + return multi_bytes_to_tar_bytes({ + 'job_setup_script.sh': job_setup_script.encode(), + 'job_end_script.sh': job_end_script.encode(), + 'container_run_script.sh': container_run_script.encode(), + 'container_setup_script.sh': container_setup_script.encode() + }) + +class RunnerFile: + def __init__(self, destination_path: str, source_path: str = None, data: bytes = b'', root_owned: bool = False): + self._data = data + self._source_path = "" + + if source_path is not None: + if source_path.strip() != "": + if not os.path.isfile(source_path.strip()): + raise ValueError("Source Path must point to a file.") + self._source_path = source_path.strip() + + self.destination_path = destination_path.strip() + if self.destination_path == "": + raise ValueError("Destination path cannot be empty.") + + self.root_owned = root_owned + + def __repr__(self): + return f"" + + @property + def basename(self) -> str: + return os.path.basename(self.destination_path) + + @property + def dirname(self) -> str: + return os.path.dirname(self.destination_path) + + @property + def isabs(self) -> bool: + return os.path.isabs(self.destination_path) + + @property + def data(self) -> bytes: + if self._source_path == "": + return self._data + else: + return open(self._source_path, "rb").read() + + def tar_data(self) -> bytes: + return bytes_to_tar_bytes(self.basename, self.data) + +class RunnerStatus(Enum): + CREATED = 1 + QUEUED = 2 + STARTING = 3 + RUNNING = 4 + STOPPING = 5 + SUCCESSFUL = 6 + FAILED = 7 + TIMED_OUT = 8 + +class Runner: + """Abstract class to take arguments and run a job and track the status and results.""" + def __init__(self, username: str, args: Union[str, list[str]], job_id: int, environment: Optional[dict] = None, + timeout_secs: str = 300, labels: Optional[list] = None, + files: list[RunnerFile] = None): + self.files = [] + if files is not None: + for f in files: + self.files.append(f) + self.status = RunnerStatus.CREATED + self.username = username.strip().lower() + self.args = args + self.job_id = int(job_id) + self.env = {} + self.started_at = datetime.datetime.now() + self.finished_at = None + self._result = (0,(b'', b'')) + self._artifact_archive = b'' + if environment: + for key in environment: + self.env[key] = environment[key] + self.labels = [] + if type(labels) is list: + for l in labels: + self.labels.append(l) + + self.timeout_seconds = timeout_secs + self.created_at = datetime.datetime.now(datetime.UTC) + + def __repr__(self): + return f"<{type(self).__name__}: {self.username}[{self.job_id}] - {self.status.name}>" + + def is_finished(self) -> bool: + if self.status in [RunnerStatus.TIMED_OUT, RunnerStatus.SUCCESSFUL, RunnerStatus.FAILED]: + return True + return False + + def is_in_process(self) -> bool: + if self.status in [RunnerStatus.QUEUED, RunnerStatus.RUNNING, RunnerStatus.STARTING, RunnerStatus.STOPPING]: + return True + return False + + def start(self): + self.started = datetime.datetime.now() + + def stop(self): + raise RuntimeError("Attempting to stop an abstract class.") + + @property + def output(self) -> bytes: + raise RuntimeError("Attempting to interact with an abstract class.") + + def output_str(self) -> str: + raise RuntimeError("Attempting to interact with an abstract class.") + + @property + def errors(self) -> bytes: + raise RuntimeError("Attempting to interact with an abstract class.") + + @property + def errors_str(self) -> str: + raise RuntimeError("Attempting to interact with an abstract class.") + + @property + def return_code(self) -> Optional[int]: + raise RuntimeError("Attempting to interact with an abstract class.") + + @property + def artifacts(self) -> TarFileExtractor: + raise RuntimeError("Attempting to interact with an abstract class.") + + @property + def has_artifacts(self) -> bool: + raise RuntimeError("Abstract method called.") + +class Orchestrator: + """Abstract class holds configuration and also tracks runners through their lifecycle. Prepares environments to + run jobs in runners.""" + def __init__(self): + self.runners = [] + self.runner_lock = Lock() + + def get_finished_runners(self) -> list[Runner]: + return [r for r in self.runners if r.is_finished()] + + def remove_runner(self, job_id: int): + runner_object = None + for r in self.runners: + if r.job_id == job_id: + runner_object = r + break + + if runner_object is not None: + self.runners.remove(runner_object) + + def get_runner_by_id(self, job_id: int) -> Optional[Runner]: + for r in self.runners: + if r.job_id == job_id: + return r + + def runners_available(self) -> bool: + """Abstract. True if a runner can be started. False, if queue is full or orchestrator not ready.""" + pass + + def new_runner(self, username: str, args: Iterable[str], job_id: int, environment: Optional[dict] = None, + timeout_secs: str = 300, refresh_db: bool = True, labels: Optional[list] = None, + files: list[RunnerFile] = None) -> Runner: + pass + + def manage_lifecycle(self): + """When called, updates runner statuses and performs any housekeeping.""" + pass + + def start(self): + """Do any setup and then be ready to operate""" + pass + + def stop(self): + """Do any cleanup needed.""" + pass \ No newline at end of file diff --git a/packetserver/runner/constants.py b/packetserver/runner/constants.py new file mode 100644 index 0000000..90add08 --- /dev/null +++ b/packetserver/runner/constants.py @@ -0,0 +1,48 @@ +from packetserver.common.util import multi_bytes_to_tar_bytes + +container_setup_script = """#!/bin/bash +set -e +echo "Place holder for now." +""" + +container_run_script = """#!/bin/bash +set -e +echo "Creating user ${PACKETSERVER_USER}" +useradd -m -s /bin/bash "${PACKETSERVER_USER}" -u 1000 +echo "Creating directories." +mkdir -pv "/home/${PACKETSERVER_USER}/.packetserver" +mkdir -pv /artifact_output +chown -Rv ${PACKETSERVER_USER} "/home/${PACKETSERVER_USER}" +echo +echo "Looping. Waiting for /root/ENDNOW to exist before stopping." +while ! [ -f "/root/ENDNOW" ]; do + sleep 1 +done +echo "Ending now.." +""" + +job_setup_script = """#!/bin/bash +set -e +PACKETSERVER_JOB_DIR="/home/${PACKETSERVER_USER}/.packetserver/${PACKETSERVER_JOBID}" +mkdir -pv "${PACKETSERVER_JOB_DIR}/artifacts" +chown ${PACKETSERVER_USER} "/home/${PACKETSERVER_USER}" +chown -R ${PACKETSERVER_USER} "${PACKETSERVER_JOB_DIR}" +""" + +job_end_script = """#!/bin/bash +set -e +PACKETSERVER_JOB_DIR="/home/$PACKETSERVER_USER/.packetserver/${PACKETSERVER_JOBID}" +PACKETSERVER_ARTIFACT_DIR="${PACKETSERVER_JOB_DIR}/artifacts" +PACKETSERVER_ARTIFACT_TAR="/artifact_output/${PACKETSERVER_JOBID}.tar.gz" +tar -czvf "${PACKETSERVER_ARTIFACT_TAR}" -C "${PACKETSERVER_ARTIFACT_DIR}" . +rm -rfv "${PACKETSERVER_JOB_DIR}" +""" + +podman_bash_start = """ echo 'waiting for /root/scripts/container_run_script.sh to exist' +while ! [ -f '/root/scripts/container_run_script.sh' ]; do + sleep .1 +done +echo 'entering /root/scripts/container_run_script.sh ...' +bash /root/scripts/container_run_script.sh +""" +podman_run_command = ["bash", "-c", podman_bash_start] \ No newline at end of file diff --git a/packetserver/runner/podman.py b/packetserver/runner/podman.py new file mode 100644 index 0000000..2b74e97 --- /dev/null +++ b/packetserver/runner/podman.py @@ -0,0 +1,463 @@ +"""Uses podman to run jobs in containers.""" +import time + +from ZEO import client + +from . import Runner, Orchestrator, RunnerStatus, RunnerFile, scripts_tar +from packetserver.runner.constants import podman_run_command +from urllib.parse import urlparse +from collections import namedtuple +from typing import Optional, Iterable, Union +from traceback import format_exc +import podman +import gzip +from podman.domain.containers import Container +import podman.errors +import os +import os.path +import logging +import datetime +from os.path import basename, dirname +from packetserver.common.util import bytes_to_tar_bytes, random_string, extract_tar_bytes, bytes_tar_has_files, \ + TarFileExtractor +from packetserver import VERSION as packetserver_version +import re +from threading import Thread +from io import BytesIO + +env_splitter_rex = '''([a-zA-Z0-9]+)=([a-zA-Z0-9]*)''' + +PodmanOptions = namedtuple("PodmanOptions", ["default_timeout", "max_timeout", "image_name", + "max_active_jobs", "container_keepalive", "name_prefix"]) + +class PodmanRunner(Runner): + def __init__(self, username: str, args: Union[str, list[str]], job_id: int, container: Container, + environment: Optional[dict] = None, timeout_secs: str = 300, labels: Optional[list] = None, + files: list[RunnerFile] = None): + super().__init__(username, args, job_id, environment=environment, timeout_secs=timeout_secs, + labels=labels, files=files) + self._artifact_archive = b'' + if not container.inspect()['State']['Running']: + raise ValueError(f"Container {container} is not in state Running.") + self.container = container + self._thread = None + self.env['PACKETSERVER_JOBID'] = str(job_id) + self.job_path = os.path.join("/home", self.username, ".packetserver", str(job_id)) + self.archive_path = os.path.join("/artifact_output", f"{str(job_id)}.tar.gz") + + def thread_runner(self): + self.status = RunnerStatus.RUNNING + logging.debug(f"Thread for runner {self.job_id} started. Command for {(type(self.args))}:\n{self.args}") + # run the exec call + if type(self.args) is str: + logging.debug(f"Running string: {self.args}") + res = self.container.exec_run(cmd=self.args, environment=self.env, user=self.username, demux=True, + workdir=self.job_path) + else: + logging.debug(f"Running iterable: {list(self.args)}") + res = self.container.exec_run(cmd=list(self.args), environment=self.env, user=self.username, demux=True, + workdir=self.job_path) + logging.debug(str(res)) + # cleanup housekeeping + self.status = RunnerStatus.STOPPING + self._result = res + # run cleanup script + logging.debug(f"Running cleanup script for {self.job_id}") + end_res = self.container.exec_run("bash /root/scripts/job_end_script.sh", + environment=self.env, user="root", tty=True) + logging.debug(f"End result: {end_res}") + if end_res[0] != 0: + logging.error(f"End Job script failed:\n{end_res[1].decode()}") + # collect any artifacts + try: + retrieved_tar_bytes = b''.join(self.container.get_archive(self.archive_path)[0]) + art_tar_bytes = extract_tar_bytes(retrieved_tar_bytes)[1] + logging.debug(f"bytes retrieved: {retrieved_tar_bytes}") + if bytes_tar_has_files(gzip.GzipFile(fileobj=BytesIO(art_tar_bytes))): + logging.debug("found artifacts; attaching to runner object") + self._artifact_archive = art_tar_bytes + else: + logging.debug(f"no artifacts returned for job {self.job_id}") + except: + logging.warning(f"Error retrieving artifacts for {self.job_id}:\n{format_exc()}") + self._artifact_archive = b'' + self.finished_at = datetime.datetime.now() + # set final status to FAILED or SUCCEEDED + if self.return_code == 0: + self.status = RunnerStatus.SUCCESSFUL + else: + self.status = RunnerStatus.FAILED + + @property + def has_artifacts(self) -> bool: + if self._artifact_archive == b'': + return False + else: + return True + + @property + def artifacts(self) -> TarFileExtractor: + if self._artifact_archive == b'': + return TarFileExtractor(BytesIO(b'')) + else: + return TarFileExtractor(gzip.GzipFile(fileobj=BytesIO(self._artifact_archive))) + + @property + def output(self) -> bytes: + return self._result[1][0] + + @property + def output_str(self) -> str: + try: + output = self.output.decode() + except: + output = str(self.output) + return output + + @property + def errors(self) -> str: + return self._result[1][1] + + @property + def errors_str(self) -> str: + return self._result[1][1].decode() + + @property + def return_code(self) -> int: + return self._result[0] + + def start(self): + logging.debug(f"Starting runner {self.job_id} for {self.username} with command:\n({type(self.args)}){self.args}") + self.status = RunnerStatus.STARTING + # Run job setup script + logging.debug(f"Running job setup script for {self.job_id} runner") + setup_res = self.container.exec_run("bash /root/scripts/job_setup_script.sh", + environment=self.env, user="root", tty=True) + logging.debug(f"Job {self.job_id} setup script:\n{str(setup_res[1])}") + if setup_res[0] != 0: + self.status = RunnerStatus.FAILED + raise RuntimeError(f"Couldn't run setup scripts for {self.job_id}:\n{setup_res[1]}") + # put files where they need to be + for f in self.files: + logging.debug(f"Adding file {f} for job {self.job_id}") + if not f.isabs: + dest = os.path.join(self.job_path, f.destination_path) + dirn = os.path.dirname(dest) + else: + dest = f.destination_path + dirn = f.dirname + if self.container.put_archive(dirn, f.tar_data()): + logging.debug(f"Placed file {dest} for job {self.job_id}") + else: + logging.warning(f"Failed to place file {dest} for job {self.job_id}!!") + if not f.root_owned: + self.container.exec_run(f"chown -R {self.username} {dest}") + + # start thread + logging.debug(f"Starting runner thread for {self.job_id}") + self._thread = Thread(target=self.thread_runner) + super().start() + self._thread.start() + +class PodmanOrchestrator(Orchestrator): + def __init__(self, uri: Optional[str] = None, options: Optional[PodmanOptions] = None): + super().__init__() + self.started = False + self.user_containers = {} + self.manager_thread = None + self._client = None + self._five_min_ticker = 600 + + if uri: + self.uri = uri + else: + self.uri = f"unix:///run/user/{os.getuid()}/podman/podman.sock" + uri_parsed = urlparse(self.uri) + if uri_parsed.scheme == "unix": + if not os.path.exists(uri_parsed.path): + raise FileNotFoundError(f"Podman socket not found: {self.uri}") + test_client = self.new_client() + logging.debug(f"Testing podman socket. Version: {test_client.info()}") + self._client = None + + self.username_containers = {} + if options: + self.opts = options + else: + self.opts = PodmanOptions(default_timeout=300, max_timeout=3600, image_name="debian", max_active_jobs=5, + container_keepalive=300, name_prefix="packetserver_") + + @property + def client(self) -> Optional[podman.PodmanClient]: + return self._client + + def new_client(self) -> podman.PodmanClient: + cli = podman.PodmanClient(base_url=self.uri) + self._client = cli + return cli + + def add_file_to_user_container(self, username: str, data: bytes, path: str, root_owned=False): + cli = self.client + file_dir = dirname(path) + tar_data_bytes = bytes_to_tar_bytes(basename(path), data) + con = cli.containers.get(self.get_container_name(username)) + res = con.exec_run(cmd=["mkdir", "-p", file_dir], user="root") + if res[0] != 1: + raise RuntimeError("Couldn't create directory") + con.put_archive(file_dir, tar_data_bytes) + + def get_file_from_user_container(self, username: str, path: str) -> bytes : + cli = self.client + con = cli.containers.get(self.get_container_name(username)) + tar_result = con.get_archive(path) + bytes_tar = b"".join(list(tar_result[0])) + return extract_tar_bytes(bytes_tar)[1] + + def podman_container_env(self, container_name: str) -> dict: + cli = self.client + try: + con = cli.containers.get(container_name) + splitter = re.compile(env_splitter_rex) + env = {} + for i in con.inspect()['Config']['Env']: + m = splitter.match(i) + if m: + env[m.groups()[0]] = m.groups()[1] + return env + except podman.errors.exceptions.NotFound as e: + return {} + + + def podman_user_container_env(self, username: str) -> dict: + container_name = self.get_container_name(username) + return self.podman_container_env(container_name) + + + def podman_start_user_container(self, username: str) -> Container: + container_env = { + "PACKETSERVER_VERSION": packetserver_version, + "PACKETSERVER_USER": username.strip().lower() + } + logging.debug(f"Starting user container for {username} with command {podman_run_command}") + con = self.client.containers.create(self.opts.image_name, name=self.get_container_name(username), + command=podman_run_command, + environment=container_env, user="root") + con.start() + logging.debug(f"Container started for {username} from image {self.opts.image_name}") + started_at = datetime.datetime.now() + logging.debug(f"Container state: \n{con.inspect()['State']}") + while con.inspect()['State']['Status'] not in ['exited', 'running']: + logging.debug("Container state not in ['exited', 'running']") + now = datetime.datetime.now() + if (now - started_at).total_seconds() > 300: + con.stop() + con.remove() + time.sleep(.1) + time.sleep(.5) + if con.inspect()['State']['Status'] != 'running': + logging.debug(f"Container for {username} isn't running. Cleaning it up.") + try: + con.stop() + except: + pass + try: + con.rename(f"{self.get_container_name(username)}_old") + con.remove() + except: + pass + raise RuntimeError(f"Couldn't start container for user {username}") + if not con.put_archive('/root/scripts', scripts_tar()): + con.stop() + con.remove() + raise RuntimeError("Failed to upload job scripts to container.") + res = con.exec_run(cmd=["bash", "/root/scripts/container_setup_script.sh"], tty=True, user="root") + logging.debug(f"Container setup script run:\n{res[1].decode()}\nExit Code: {res[0]}") + if res[0] != 0: + logging.error(f"Container setup script failed:\n{res[1].decode()}\nExit Code: {res[0]}") + con.stop() + con.remove() + raise RuntimeError(f"Container setup script failed:\n{res[1].decode()}\nExit Code: {res[0]}") + self.touch_user_container(username) + return con + + def podman_remove_container_name(self, container_name: str): + cli = self.client + logging.debug(f"Attempting to remove container named {container_name}") + try: + con = cli.containers.get(container_name) + if con.inspect()['State']['Status'] == 'running': + con.exec_run(cmd="touch /root/ENDNOW", user="root") + time.sleep(1) + except podman.errors.exceptions.NotFound as e: + logging.warning(f"Didn't find container named {container_name}") + return + try: + con.rename(f"{container_name}_{random_string()}") + except: + logging.error(f"Couldn't rename container:\n{format_exc()}") + if con.inspect()['State']['Status'] != 'exited': + try: + con.stop(timeout=10) + except: + logging.error(f"Couldn't stop container:\n{format_exc()}") + try: + con.remove() + except: + logging.error(f"Couldn't remove container:\n{format_exc()}") + return + + def podman_stop_user_container(self, username: str): + self.podman_remove_container_name(self.get_container_name(username)) + if self.get_container_name(username) in self.user_containers: + del self.user_containers[self.get_container_name(username)] + + def podman_user_container_exists(self, username: str) -> bool: + try: + self.client.containers.get(self.get_container_name(username)) + return True + except podman.errors.exceptions.NotFound: + return False + + def podman_run_command_simple(self, username: str, command: Iterable[str], as_root: bool = True) -> int: + """Runs command defined by arguments iterable in container. As root by default. Returns exit code.""" + container_name = self.get_container_name(username) + un = username.lower().strip() + con = self.client.containers.get(container_name) + if as_root: + un = 'root' + return con.exec_run(list(command), user=un)[0] + + def clean_orphaned_containers(self): + cli = self.client + for i in cli.containers.list(all=True): + if self.opts.name_prefix in str(i.name): + if str(i.name) not in self.user_containers: + self.podman_remove_container_name(str(i.name)) + + def get_container_name(self, username: str) -> str: + return self.opts.name_prefix + username.lower().strip() + + def get_username_from_container_name(self, container_name: str) -> str: + if not self.opts.name_prefix in container_name: + raise ValueError(f"{container_name} is not a user container") + return container_name.replace(self.opts.name_prefix, "") + + + def touch_user_container(self, username: str): + self.user_containers[self.get_container_name(username)] = datetime.datetime.now() + + def start_user_container(self, username: str) -> Container: + if not self.podman_user_container_exists(username): + con = self.podman_start_user_container(username) + else: + con = self.client.containers.get(self.get_container_name(username)) + return con + + def clean_containers(self): + """Checks running containers and stops them if they have been running too long.""" + containers_to_clean = set() + for c in self.user_containers: + if (datetime.datetime.now() - self.user_containers[c]).total_seconds() > self.opts.container_keepalive: + logging.debug(f"Container {c} no activity for {self.opts.container_keepalive} seconds. Clearing.") + containers_to_clean.add(c) + for c in list(containers_to_clean): + self.podman_remove_container_name(c) + del self.user_containers[c] + + + def user_runners_in_process(self, username: str) -> int: + un = username.strip().lower() + count = 0 + for r in self.runners: + if r.is_in_process: + if r.username == un: + count = count + 1 + return count + + def user_running(self, username: str) -> bool: + if self.user_runners_in_process(username) > 0: + return True + else: + return False + + def runners_in_process(self) -> int: + count = 0 + for r in self.runners: + if not r.is_finished(): + count = count + 1 + return count + + def runners_available(self) -> bool: + if not self.started: + return False + + if self.runners_in_process() < self.opts.max_active_jobs: + return True + + return False + + def new_runner(self, username: str, args: Union[str, list[str]], job_id: int, environment: Optional[dict] = None, + timeout_secs: str = 300, refresh_db: bool = True, labels: Optional[list] = None, + files: list[RunnerFile] = None) -> Optional[PodmanRunner]: + if not self.started: + logging.warning("Attempted to queue a runner when not started") + return None + with self.runner_lock: + if not self.runners_available(): + logging.warning("Attempted to queue a runner when no runner slots available.") + return None + con = self.start_user_container(username) + logging.debug(f"Started a container for {username} successfully.") + self.touch_user_container(username) + logging.debug(f"Queuing a runner on container {con}, with command '{args}' of type '{type(args)}'") + runner = PodmanRunner(username, args, job_id, con, environment=environment, timeout_secs=timeout_secs, + labels=labels, files=files) + self.runners.append(runner) + runner.start() + return runner + + def manage_lifecycle(self): + if not self.started: + return + with self.runner_lock: + for r in self.runners: + if not r.is_finished(): + self.touch_user_container(r.username) + self.clean_containers() + + if self._five_min_ticker >= 600: + self.clean_orphaned_containers() + self._five_min_ticker = 0 + + def manager(self): + logging.debug("Starting podman orchestrator thread.") + while self.started: + self.manage_lifecycle() + time.sleep(.5) + logging.debug("Stopping podman orchestrator thread.") + + def start(self): + if not self.started: + self.new_client() + self.clean_orphaned_containers() + self._five_min_ticker = 0 + self.started = True + self.manager_thread = Thread(target=self.manager) + self.manager_thread.start() + + def __del__(self): + if self.started: + self.stop() + + def stop(self): + logging.debug("Stopping podman orchestrator.") + self.started = False + cli = self.client + self.user_containers = {} + self.clean_orphaned_containers() + if self.manager_thread is not None: + logging.debug("Joining orchestrator manager thread.") + self.manager_thread.join(timeout=15) + logging.debug("Orchestrator manager thread stopped") + self.manager_thread = None + self._client = None \ No newline at end of file diff --git a/packetserver/server/__init__.py b/packetserver/server/__init__.py new file mode 100644 index 0000000..96a135c --- /dev/null +++ b/packetserver/server/__init__.py @@ -0,0 +1,329 @@ +import datetime +import tempfile + +import pe.app +from packetserver.common import Response, Message, Request, PacketServerConnection, send_response, send_blank_response +from packetserver.server.constants import default_server_config +from packetserver.server.users import User +from copy import deepcopy +import ax25 +from pathlib import Path +import ZODB, ZODB.FileStorage +from BTrees.OOBTree import OOBTree +from persistent.mapping import PersistentMapping +from persistent.list import PersistentList +from packetserver.server.requests import standard_handlers +import logging +import signal +import time +from msgpack.exceptions import OutOfData +from typing import Callable, Self, Union +from traceback import format_exc +from os import linesep +from shutil import rmtree +from threading import Thread +from packetserver.server.jobs import get_orchestrator_from_config, Job, JobStatus +from packetserver.runner import RunnerStatus, RunnerFile, Orchestrator, Runner + +VERSION="0.4.1" + +def init_bulletins(root: PersistentMapping): + if 'bulletins' not in root: + root['bulletins'] = PersistentList() + if 'bulletin_counter' not in root: + root['bulletin_counter'] = 0 + +class Server: + def __init__(self, pe_server: str, port: int, server_callsign: str, data_dir: str = None, zeo: bool = True): + if not ax25.Address.valid_call(server_callsign): + raise ValueError(f"Provided callsign '{server_callsign}' is invalid.") + self.callsign = server_callsign + self.pe_server = pe_server + self.pe_port = port + self.handlers = deepcopy(standard_handlers) + self.zeo_addr = None + self.zeo_stop = None + self.zeo = zeo + self.started = False + self.orchestrator = None + self.worker_thread = None + self.check_job_queue = True + self.last_check_job_queue = datetime.datetime.now() + self.job_check_interval = 60 + self.quick_job = False + if data_dir: + data_path = Path(data_dir) + else: + data_path = Path.home().joinpath(".packetserver") + if data_path.is_dir(): + if data_path.joinpath("data.zopedb").exists(): + if not data_path.joinpath("data.zopedb").is_file(): + raise FileExistsError("data.zopedb exists as non-file in specified path") + self.home_dir = data_path + else: + if data_path.exists(): + raise FileExistsError(f"Non-Directory path '{data_dir}' already exists.") + else: + data_path.mkdir() + self.home_dir = data_path + self.storage = ZODB.FileStorage.FileStorage(self.data_file) + self.db = ZODB.DB(self.storage) + with self.db.transaction() as conn: + logging.debug(f"checking for datastructures: conn.root.keys(): {list(conn.root().keys())}") + if 'config' not in conn.root(): + logging.debug("no config, writing blank default config") + conn.root.config = PersistentMapping(deepcopy(default_server_config)) + conn.root.config['blacklist'] = PersistentList() + if 'SYSTEM' not in conn.root.config['blacklist']: + logging.debug("Adding 'SYSTEM' to blacklist in case someone feels like violating FCC rules.") + conn.root.config['blacklist'].append('SYSTEM') + if 'users' not in conn.root(): + logging.debug("users missing, creating bucket") + conn.root.users = PersistentMapping() + if 'messages' not in conn.root(): + logging.debug("messages container missing, creating bucket") + conn.root.messages = PersistentMapping() + if 'SYSTEM' not in conn.root.users: + logging.debug("Creating system user for first time.") + User('SYSTEM', hidden=True, enabled=False).write_new(conn.root()) + if 'objects' not in conn.root(): + logging.debug("objects bucket missing, creating") + conn.root.objects = OOBTree() + if 'jobs' not in conn.root(): + logging.debug("jobss bucket missing, creating") + conn.root.jobs = OOBTree() + if 'job_queue' not in conn.root(): + conn.root.job_queue = PersistentList() + if 'user_jobs' not in conn.root(): + conn.root.user_jobs = PersistentMapping() + init_bulletins(conn.root()) + if ('jobs_enabled' in conn.root.config) and conn.root.config['jobs_enabled']: + logging.debug(conn.root.config['jobs_enabled']) + logging.debug(conn.root.config['jobs_config']) + if 'runner' in conn.root.config['jobs_config']: + val = str(conn.root.config['jobs_config']['runner']).lower().strip() + if val in ['podman']: + logging.debug(f"Enabling {val} orchestrator") + self.orchestrator = get_orchestrator_from_config(conn.root.config['jobs_config']) + + self.app = pe.app.Application() + PacketServerConnection.receive_subscribers.append(lambda x: self.server_receiver(x)) + PacketServerConnection.connection_subscribers.append(lambda x: self.server_connection_bouncer(x)) + signal.signal(signal.SIGINT, self.exit_gracefully) + signal.signal(signal.SIGTERM, self.exit_gracefully) + self.db.close() + self.storage.close() + + @property + def data_file(self) -> str: + return str(Path(self.home_dir).joinpath('data.zopedb')) + + def ping_job_queue(self): + self.check_job_queue = True + self.last_check_job_queue = datetime.datetime.now() + if self.quick_job: + logging.debug("Setting the final quick job timer.") + self.job_check_interval = 5 + self.quick_job = False + else: + self.job_check_interval = 60 + + def server_connection_bouncer(self, conn: PacketServerConnection): + logging.debug("new connection bouncer checking user status") + # blacklist check + blacklisted = False + base = ax25.Address(conn.remote_callsign).call + with self.db.transaction() as storage: + if 'blacklist' in storage.root.config: + bl = storage.root.config['blacklist'] + logging.debug(f"A blacklist exists: {bl}") + logging.debug(f"Checking callsign {base.upper()}") + if base.upper() in bl: + logging.debug(f"Connection from blacklisted callsign {base}") + conn.closing = True + blacklisted = True + + # user object check + logging.debug(f"checking user existence for {base}") + logging.debug(f"users in db right now: {list(storage.root.users.keys())}") + if base in storage.root.users: + logging.debug(f"User {base} exists in db.") + u = storage.root.users[base] + u.seen() + else: + logging.debug(f"User {base} doesn't exist in db") + logging.info(f"Creating new user {base}") + u = User(base.upper().strip()) + u.write_new(storage.root()) + if blacklisted: + count = 0 + while count < 10: + time.sleep(.5) + if conn.state.name == "CONNECTED": + break + conn.close() + + def handle_request(self, req: Request, conn: PacketServerConnection): + """Handles a proper request by handing off to the appropriate function depending on method and Path.""" + logging.debug(f"asked to handle request: {req}") + if conn.closing: + logging.debug("Connection marked as closing. Ignoring it.") + return + req_root_path = req.path.split("/")[0] + if ('quick' in req.vars) or (req_root_path == "job"): + logging.debug("Setting quick job timer for a quick job.") + self.job_check_interval = 8 + self.quick_job = True + if req_root_path in self.handlers: + logging.debug(f"found handler for req {req}") + self.handlers[req_root_path](req, conn, self.db) + return + logging.warning(f"unhandled request found: {req}") + send_blank_response(conn, req, status_code=404) + + def process_incoming_data(self, connection: PacketServerConnection): + """Handles incoming data.""" + logging.debug("Running process_incoming_data on connection") + with connection.data_lock: + logging.debug("Data lock acquired") + while True: + try: + msg = Message.partial_unpack(connection.data.unpack()) + logging.debug(f"parsed a Message from data received") + except OutOfData: + logging.debug("no complete message yet, done until more data arrives") + break + except ValueError: + connection.send_data(b"BAD REQUEST. COULD NOT PARSE INCOMING DATA AS PACKETSERVER MESSAGE") + try: + request = Request(msg) + logging.debug(f"parsed Message into request {request}") + except ValueError: + connection.send_data(b"BAD REQUEST. DID NOT RECEIVE A REQUEST MESSAGE.") + logging.debug(f"attempting to handle request {request}") + self.handle_request(request, connection) + self.ping_job_queue() + logging.debug("request handled") + + def server_receiver(self, conn: PacketServerConnection): + logging.debug("running server receiver") + try: + self.process_incoming_data(conn) + except Exception: + logging.debug(f"Unhandled exception while processing incoming data:\n{format_exc()}") + + def register_path_handler(self, path_root: str, fn: Callable): + self.handlers[path_root.strip().lower()] = fn + + def server_worker(self): + """When called, do things. Should get called every so often.""" + if not self.started: + return + # Add things to do here: + now = datetime.datetime.now() + if (now - self.last_check_job_queue).total_seconds() > self.job_check_interval: + self.ping_job_queue() + if (self.orchestrator is not None) and self.orchestrator.started and self.check_job_queue: + with self.db.transaction() as storage: + # queue as many jobs as possible + while self.orchestrator.runners_available(): + if len(storage.root.job_queue) > 0: + jid = storage.root.job_queue[0] + try: + logging.info(f"Starting job {jid}") + job = Job.get_job_by_id(jid, storage.root()) + except: + logging.error(f"Error retrieving job {jid}") + break + runner = self.orchestrator.new_runner(job.owner, job.cmd, jid, environment=job.env, files=job.files) + if runner is not None: + storage.root.job_queue.remove(jid) + job.status = JobStatus.RUNNING + job.started_at = datetime.datetime.now() + logging.info(f"Started job {job}") + else: + break + if len(storage.root.job_queue) == 0: + self.check_job_queue = False + else: + self.ping_job_queue() + + finished_runners = [] + for runner in self.orchestrator.runners: + if runner.is_finished(): + logging.debug(f"Finishing runner {runner}") + with self.db.transaction() as storage: + try: + if Job.update_job_from_runner(runner, storage.root()): + finished_runners.append(runner) + logging.info(f"Runner {runner} successfully synced with jobs.") + else: + logging.error(f"update_job_from_runner returned False.") + logging.error(f"Error while finishing runner and updating job status {runner}") + except: + logging.error(f"Error while finishing runner and updating job status {runner}\n:{format_exc()}") + for runner in finished_runners: + logging.info(f"Removing completed runner {runner}") + with self.orchestrator.runner_lock: + self.orchestrator.runners.remove(runner) + + def run_worker(self): + """Intended to be running as a thread.""" + logging.info("Starting worker thread.") + while self.started: + self.server_worker() + time.sleep(.5) + + def __del__(self): + self.stop() + + def start_db(self): + if not self.zeo: + self.storage = ZODB.FileStorage.FileStorage(self.data_file) + self.db = ZODB.DB(self.storage) + else: + import ZEO + address, stop = ZEO.server(path=self.data_file) + self.zeo_addr = address + self.zeo_stop = stop + self.db = ZEO.DB(self.zeo_addr) + logging.info(f"Starting ZEO server with address {self.zeo_addr}") + try: + zeo_address_file = str(self.home_dir.joinpath("zeo-address.txt")) + open(zeo_address_file, 'w').write(f"{self.zeo_addr[0]}:{self.zeo_addr[1]}{linesep}") + logging.info(f"Wrote ZEO server info to '{zeo_address_file}'") + except: + logging.warning(f"Couldn't write ZEO server info to '{zeo_address_file}'\n{format_exc()}") + + def start(self): + self.start_db() + self.app.start(self.pe_server, self.pe_port) + self.app.register_callsigns(self.callsign) + self.started = True + if self.orchestrator is not None: + logging.info(f"Starting orchestrator {self.orchestrator}") + self.orchestrator.start() + self.worker_thread = Thread(target=self.run_worker) + self.worker_thread.start() + + def exit_gracefully(self, signum, frame): + self.stop() + + def stop_db(self): + self.storage.close() + self.db.close() + if self.zeo: + logging.info("Stopping ZEO.") + self.zeo_stop() + + def stop(self): + self.started = False + cm = self.app._engine._active_handler._handlers[1]._connection_map + for key in cm._connections.keys(): + cm._connections[key].close() + if self.orchestrator is not None: + self.orchestrator.stop() + self.app.stop() + self.stop_db() + + diff --git a/packetserver/server/bulletin.py b/packetserver/server/bulletin.py new file mode 100644 index 0000000..76f91f6 --- /dev/null +++ b/packetserver/server/bulletin.py @@ -0,0 +1,159 @@ +import ax25 +import persistent +import persistent.list +from persistent.mapping import PersistentMapping +import datetime +from typing import Self,Union,Optional +from packetserver.common import PacketServerConnection, Request, Response, Message, send_response, send_blank_response +import ZODB +import logging +from packetserver.server.users import user_authorized + +def get_new_bulletin_id(root: PersistentMapping) -> int: + if 'bulletin_counter' not in root: + root['bulletin_counter'] = 1 + return 0 + else: + current = root['bulletin_counter'] + root['bulletin_counter'] = current + 1 + return current + +class Bulletin(persistent.Persistent): + @classmethod + def get_bulletin_by_id(cls, bid: int, db_root: PersistentMapping) -> Optional[Self]: + for bull in db_root['bulletins']: + if bull.id == bid: + return bull + return None + + @classmethod + def get_recent_bulletins(cls, db_root: PersistentMapping, limit: int = None) -> list: + all_bulletins = sorted(db_root['bulletins'], key=lambda bulletin: bulletin.updated_at, reverse=True) + if not limit: + return all_bulletins + else: + if len(all_bulletins) < limit: + return all_bulletins + else: + return all_bulletins[:limit] + + def __init__(self, author: str, subject: str, text: str): + self.author = author + self.subject = subject + self.body = text + self.created_at = datetime.datetime.now(datetime.UTC) + self.updated_at = datetime.datetime.now(datetime.UTC) + self.id = None + + @classmethod + def from_dict(cls, bulletin_dict: dict) -> Self: + return Bulletin(bulletin_dict['author'], bulletin_dict['subject'], bulletin_dict['body']) + + def write_new(self, db_root: PersistentMapping) -> int: + if self.id is None: + self.id = get_new_bulletin_id(db_root) + self.created_at = datetime.datetime.now(datetime.UTC) + self.updated_at = datetime.datetime.now(datetime.UTC) + db_root['bulletins'].append(self) + return self.id + + def update_subject(self, new_text: str): + self.subject = new_text + self.updated_at = datetime.datetime.now(datetime.UTC) + + def update_body(self, new_text: str): + self.body = new_text + self.updated_at = datetime.datetime.now(datetime.UTC) + + def to_dict(self): + return { + "id": self.id, + "author": self.author, + "subject": self.subject, + "body": self.body, + "created_at": self.created_at.isoformat(), + "updated_at": self.updated_at.isoformat() + } + + +def handle_bulletin_get(req: Request, conn: PacketServerConnection, db: ZODB.DB): + response = Response.blank() + sp = req.path.split("/") + logging.debug(f"bulletin get path: {sp}") + bid = None + limit = None + if 'limit' in req.vars: + try: + limit = int(req.vars['limit']) + except ValueError: + pass + if 'id' in req.vars: + try: + bid = int(req.vars['id']) + except ValueError: + pass + if len(sp) > 1: + logging.debug(f"checking path for bulletin id") + try: + logging.debug(f"{sp[1]}") + bid = int(sp[1].strip()) + except ValueError: + pass + logging.debug(f"bid is {bid}") + + with db.transaction() as db: + if bid is not None: + logging.debug(f"retrieving bulletin: {bid}") + bull = Bulletin.get_bulletin_by_id(bid, db.root()) + if bull: + response.payload = bull.to_dict() + response.status_code = 200 + else: + response.status_code = 404 + else: + logging.debug(f"retrieving all bulletins") + bulls = Bulletin.get_recent_bulletins(db.root(), limit=limit) + response.payload = [bulletin.to_dict() for bulletin in bulls] + response.status_code = 200 + + send_response(conn, response, req) + +def handle_bulletin_post(req: Request, conn: PacketServerConnection, db: ZODB.DB): + author = ax25.Address(conn.remote_callsign).call + if type(req.payload) is not dict: + send_blank_response(conn, req, 400, payload="Include dict in payload with subject and body") + if 'subject' not in req.payload: + send_blank_response(conn, req, 400, payload="Include dict in payload with subject and body") + if 'body' not in req.payload: + send_blank_response(conn, req, 400, payload="Include dict in payload with subject and body") + b = Bulletin(author, str(req.payload['subject']), str(req.payload['body'])) + response = Response.blank() + with db.transaction() as db: + bid = b.write_new(db.root()) + send_blank_response(conn, req, status_code=201, payload={'bulletin_id': bid}) + +def handle_bulletin_update(req: Request, conn: PacketServerConnection, db: ZODB.DB): # TODO + response = Response.blank() + with db.transaction() as db: + pass + send_response(conn, response, req) + +def handle_bulletin_delete(req: Request, conn: PacketServerConnection, db: ZODB.DB): # TODO + response = Response.blank() + with db.transaction() as db: + pass + send_response(conn, response, req) + +def bulletin_root_handler(req: Request, conn: PacketServerConnection, db: ZODB.DB): + logging.debug(f"{req} being processed by bulletin_root_handler") + if not user_authorized(conn, db): + logging.debug(f"user {conn.remote_callsign} not authorized") + send_blank_response(conn, req, status_code=401) + return + logging.debug("user is authorized") + if req.method is Request.Method.GET: + handle_bulletin_get(req, conn, db) + elif req.method is Request.Method.POST: + handle_bulletin_post(req, conn, db) + else: + send_blank_response(conn, req, status_code=404) diff --git a/packetserver/server/constants.py b/packetserver/server/constants.py new file mode 100644 index 0000000..a80ca1e --- /dev/null +++ b/packetserver/server/constants.py @@ -0,0 +1,6 @@ + +default_server_config = { + "motd": "Welcome to this PacketServer BBS!", + "operator": "placeholder", + "max_message_length": 2000 +} \ No newline at end of file diff --git a/packetserver/server/db.py b/packetserver/server/db.py new file mode 100644 index 0000000..fe3b0ea --- /dev/null +++ b/packetserver/server/db.py @@ -0,0 +1,60 @@ +import ZODB +import json +import gzip +import base64 +from io import BytesIO +from uuid import UUID + +def get_user_db(username: str, db: ZODB.DB) -> dict: + udb = { + "objects": [], + "messages": [], + "user": {}, + "bulletins": [], + "jobs": [] + } + username = username.strip().upper() + with (db.transaction() as db_conn): + user = db_conn.root.users[username] + udb['user'] = user.to_safe_dict() + for o in user.object_uuids: + obj = {} + tmp = db_conn.root.objects[o].to_dict() + + obj['name'] = tmp['name'] + obj['private'] = tmp['private'] + obj['uuid'] = str(UUID(bytes=tmp['uuid_bytes'])) + obj['created_at'] = tmp['created_at'] + obj['modified_at'] = tmp['modified_at'] + + if type(tmp['data']) is bytes: + obj['data'] = base64.b64encode(tmp['data']).decode() + else: + obj['data'] = str(tmp['data']) + + udb['objects'].append(obj) + + if user in db_conn.root.messages: + for m in db_conn.root.messages[username]: + for a in m.attachments: + if type(a.data) is bytes: + a.data = base64.b64encode(a.data).decode() + else: + a.data = base64.b64encode(a.data.encode()).decode() + udb['messages'].append(m.to_dict()) + for b in db_conn.root.bulletins: + udb['bulletins'].append(b.to_dict()) + + if username in db_conn.root.user_jobs: + for jid in db_conn.root.user_jobs[username]: + udb['jobs'].append(db_conn.root.jobs[jid].to_dict(binary_safe=True)) + + return udb + +def get_user_db_json(username: str, db: ZODB.DB, gzip_output=True) -> bytes: + udb = get_user_db(username, db) + j = json.dumps(udb).encode() + if gzip_output: + return gzip.compress(j) + else: + return j diff --git a/packetserver/server/jobs.py b/packetserver/server/jobs.py new file mode 100644 index 0000000..0cdf6f6 --- /dev/null +++ b/packetserver/server/jobs.py @@ -0,0 +1,382 @@ +import re + +import ax25 +import persistent +import persistent.list +from persistent.mapping import PersistentMapping +import datetime +from typing import Self,Union,Optional,Tuple +from traceback import format_exc +from packetserver.common import PacketServerConnection, Request, Response, Message, send_response, send_blank_response +from packetserver.common.constants import no_values, yes_values +from packetserver.server.db import get_user_db_json +import ZODB +from persistent.list import PersistentList +import logging +from packetserver.server.users import user_authorized +import gzip +import tarfile +import time +import json +from packetserver.common.util import TarFileExtractor +from packetserver.runner import Orchestrator, Runner, RunnerStatus, RunnerFile +from enum import Enum +from io import BytesIO +import base64 + +class JobStatus(Enum): + CREATED = 1 + QUEUED = 2 + STARTING = 3 + RUNNING = 4 + STOPPING = 5 + SUCCESSFUL = 6 + FAILED = 7 + TIMED_OUT = 8 + +def get_orchestrator_from_config(cfg: dict) -> Orchestrator: + if 'runner' in cfg: + val = cfg['runner'].lower().strip() + if val == "podman": + from packetserver.runner.podman import PodmanOrchestrator, PodmanOptions + image = cfg.get('image', 'debian') + opts = PodmanOptions(default_timeout=300, max_timeout=3600, image_name=image, max_active_jobs=5, + container_keepalive=300, name_prefix="packetserver_") + orch = PodmanOrchestrator(options=opts) + return orch + else: + raise RuntimeError("Other orchestrators not implemented yet.") + else: + raise RuntimeError("Runners not configured in root.config.jobs_config") + +def get_new_job_id(root: PersistentMapping) -> int: + if 'job_counter' not in root: + root['job_counter'] = 1 + return 0 + else: + current = root['job_counter'] + root['job_counter'] = current + 1 + return current + +class Job(persistent.Persistent): + @classmethod + def update_job_from_runner(cls, runner: Runner, db_root: PersistentMapping) -> True: + job = Job.get_job_by_id(runner.job_id, db_root) + if job is None: + logging.warning(f"Couldn't match runner {runner} with a job by id.") + return False + if not runner.is_finished(): + return False + job.finished_at = datetime.datetime.now() + job.output = runner.output + job.errors = runner.errors + job.return_code = runner.return_code + job._artifact_archive = runner._artifact_archive + if runner.status == RunnerStatus.SUCCESSFUL: + job.status = JobStatus.SUCCESSFUL + else: + job.status = JobStatus.FAILED + return True + + @classmethod + def get_job_by_id(cls, jid: int, db_root: PersistentMapping) -> Optional[Self]: + if jid in db_root['jobs']: + return db_root['jobs'][jid] + return None + + @classmethod + def get_jobs_by_username(cls, username:str, db_root: PersistentMapping) -> list[Self]: + un = username.strip().upper() + if un in db_root['user_jobs']: + l = [] + for j in db_root['user_jobs'][un]: + l.append(Job.get_job_by_id(j, db_root)) + return l + else: + return [] + + @classmethod + def num_jobs_queued(cls, db_root: PersistentMapping) -> int: + return len(db_root['job_queue']) + + @classmethod + def jobs_in_queue(cls, db_root: PersistentMapping) -> bool: + if Job.num_jobs_queued(db_root) > 0: + return True + else: + return False + + @classmethod + def get_next_queued_job(cls, db_root: PersistentMapping) -> Self: + return db_root['job_queue'][0] + + def __init__(self, cmd: Union[list[str], str], owner: Optional[str] = None, timeout: int = 300, + env: dict = None, files: list[RunnerFile] = None): + self.owner = None + if owner is not None: + self.owner = str(owner).upper().strip() + self.cmd = cmd + self.env = {} + if env is not None: + for key in env: + self.env[key] = env[key] + self.files = [] + if files is not None: + self.files = files + self.created_at = datetime.datetime.now(datetime.UTC) + self.started_at = None + self.finished_at = None + self._artifact_archive = b'' + self.output = b'' + self.errors = b'' + self.return_code = 0 + self.id = None + self.status = JobStatus.CREATED + + @property + def is_finished(self) -> bool: + if self.finished_at is None: + return False + else: + return True + + @property + def output_str(self) -> str: + return self.output.decode() + + @property + def errors_str(self) -> str: + return self.errors.decode() + + @property + def artifacts(self) -> TarFileExtractor: + if self._artifact_archive == b'': + return TarFileExtractor(BytesIO(b'')) + else: + return TarFileExtractor(gzip.GzipFile(fileobj=BytesIO(self._artifact_archive))) + + @property + def num_artifacts(self) -> int: + return len(list(self.artifacts)) + + def __repr__(self) -> str: + return f"" + + def artifact(self, index: int) -> Tuple[str, bytes]: + artifacts = list(self.artifacts) + if (index + 1) > len(artifacts): + raise IndexError(f"Index {index} out of bounds.") + else: + return artifacts[index][0], artifacts[index][1].read() + + def queue(self, db_root: PersistentMapping) -> int: + logging.debug(f"Attempting to queue job {self}") + if self.owner is None or (str(self.owner).strip() == ""): + raise ValueError("Job must have an owner to be queued.") + + if self.id is None: + self.id = get_new_job_id(db_root) + owner = self.owner.upper().strip() + if owner not in db_root['user_jobs']: + db_root['user_jobs'][owner] = PersistentList() + db_root['user_jobs'][owner].append(self.id) + db_root['jobs'][self.id] = self + db_root['job_queue'].append(self.id) + return self.id + + def to_dict(self, include_data: bool = True, binary_safe: bool = False): + started_at = None + finished_at = None + if self.started_at is not None: + started_at = self.started_at.isoformat() + if self.finished_at is not None: + finished_at = self.finished_at.isoformat() + output = { + "cmd": self.cmd, + "owner": self.owner, + "created_at": self.created_at.isoformat(), + "started_at": started_at, + "finished_at": finished_at, + "output": b'', + "errors": b'', + "return_code": self.return_code, + "artifacts": [], + "status": self.status.name, + "id": self.id + } + if include_data: + if binary_safe: + output['output'] = base64.b64encode(self.output).decode() + output['errors'] = base64.b64encode(self.errors).decode() + else: + output['output'] = self.output + output['errors'] = self.errors + + for a in self.artifacts: + if binary_safe: + output['artifacts'].append((a[0], base64.b64encode(a[1].read()).decode())) + else: + output['artifacts'].append((a[0], a[1].read())) + return output + + def json(self, include_data: bool = True) -> str: + return json.dumps(self.to_dict(include_data=include_data, binary_safe=True)) + +def handle_job_get_id(req: Request, conn: PacketServerConnection, db: ZODB.DB, jid: int): + username = ax25.Address(conn.remote_callsign).call.upper().strip() + value = "y" + include_data = True + for key in req.vars: + if key.lower().strip() == "data": + value = req.vars[key].lower().strip() + if value in no_values: + include_data = False + + with db.transaction() as storage: + try: + job = Job.get_job_by_id(jid, storage.root()) + if job is None: + send_blank_response(conn, req, 404) + return + if job.owner != username: + send_blank_response(conn, req, 401) + return + send_blank_response(conn, req, 200, job.to_dict(include_data=include_data)) + return + except: + logging.error(f"Error looking up job {jid}:\n{format_exc()}") + send_blank_response(conn, req, 500, payload="unknown server error") + +def handle_job_get_user(req: Request, conn: PacketServerConnection, db: ZODB.DB): + username = ax25.Address(conn.remote_callsign).call.upper().strip() + jobs = [] + value = "y" + include_data = True + for key in req.vars: + if key.lower().strip() == "data": + value = req.vars[key].lower().strip() + if value in no_values: + include_data = False + id_only = False + if 'id_only' in req.vars: + if req.vars['id_only'] in yes_values: + id_only = True + with db.transaction() as storage: + for jid in storage.root()['user_jobs'][username]: + jobs.append(Job.get_job_by_id(jid, storage.root()).to_dict(include_data=include_data)) + + if id_only: + send_blank_response(conn, req, status_code=200, payload=[x['id'] for x in jobs]) + else: + send_blank_response(conn, req, status_code=200, payload=jobs) + +def handle_job_get(req: Request, conn: PacketServerConnection, db: ZODB.DB): + spl = [x for x in req.path.split("/") if x.strip() != ""] + if (len(spl) == 2) and (spl[1].isdigit()): + handle_job_get_id(req, conn, db, int(spl[1])) + elif (len(spl) == 2) and (spl[1].lower() == "user"): + handle_job_get_user(req, conn, db) + else: + send_blank_response(conn, req, status_code=404) + +def handle_new_job_post(req: Request, conn: PacketServerConnection, db: ZODB.DB): + username = ax25.Address(conn.remote_callsign).call.upper().strip() + quick = False + if 'quick' in req.vars: + quick_val = req.vars['quick'] + if type(quick_val) is str: + quick_val = quick_val.lower() + if quick_val in yes_values: + quick = True + if 'cmd' not in req.payload: + logging.info(f"request {req} did not contain job command (cmd) key") + send_blank_response(conn, req, 401, "job post must contain cmd key containing str or list[str]") + return + if type(req.payload['cmd']) not in [str, list]: + send_blank_response(conn, req, 401, "job post must contain cmd key containing str or list[str]") + return + files = [] + if 'db' in req.payload: + logging.debug(f"Fetching a user db as requested.") + try: + dbf = RunnerFile('user-db.json.gz', data=get_user_db_json(username.lower(), db)) + except: + logging.error(format_exc()) + send_blank_response(conn, req, 500) + return + files.append(dbf) + if 'files' in req.payload: + if type(files) is dict: + for key in req.payload['files']: + val = req.payload['files'][key] + if type(val) is bytes: + files.append(RunnerFile(key, data=val)) + env = {} + if 'env' in req.payload: + if type(req.payload['env']) is dict: + for key in req.payload['env']: + env[key] = req.payload['env'][key] + job = Job(req.payload['cmd'], owner=username, env=env, files=files) + with db.transaction() as storage: + try: + new_jid = job.queue(storage.root()) + logging.info(f"New job created with id {new_jid}") + except: + logging.error(f"Failed to queue new job {job}:\n{format_exc()}") + send_blank_response(conn, req, 500, "unknown server error while queuing job") + return + if quick: + start_time = datetime.datetime.now() + now = datetime.datetime.now() + job_done = False + quick_job = None + logging.debug(f"{start_time}: Waiting for a quick job for 30 seconds") + while (now - start_time).total_seconds() < 30: + with db.transaction() as storage: + try: + j = Job.get_job_by_id(new_jid, storage.root()) + if j.is_finished: + job_done = True + quick_job = j + break + except: + pass + time.sleep(1) + now = datetime.datetime.now() + if job_done and (type(quick_job) is Job): + send_blank_response(conn, req, 200, job.to_dict(include_data=True)) + else: + logging.warning(f"Quick job {new_jid} timed out.") + send_blank_response(conn, req, status_code=202, payload={'job_id': new_jid, 'msg': 'queued'}) + else: + send_blank_response(conn, req, 201, {'job_id': new_jid}) + +def handle_job_post(req: Request, conn: PacketServerConnection, db: ZODB.DB): + spl = [x for x in req.path.split("/") if x.strip() != ""] + + if len(spl) == 1: + handle_new_job_post(req, conn, db) + else: + send_blank_response(conn, req, status_code=404) + +def job_root_handler(req: Request, conn: PacketServerConnection, db: ZODB.DB): + logging.debug(f"{req} being processed by job_root_handler") + if not user_authorized(conn, db): + logging.debug(f"user {conn.remote_callsign} not authorized") + send_blank_response(conn, req, status_code=401) + return + logging.debug("user is authorized") + with db.transaction() as storage: + if 'jobs_enabled' in storage.root.config: + jobs_enabled = storage.root.config['jobs_enabled'] + else: + jobs_enabled = False + if not jobs_enabled: + send_blank_response(conn, req, 400, payload="jobs not enabled on this server") + return + if req.method is Request.Method.GET: + handle_job_get(req, conn, db) + elif req.method is Request.Method.POST: + handle_job_post(req, conn, db) + else: + send_blank_response(conn, req, status_code=404) \ No newline at end of file diff --git a/packetserver/server/messages.py b/packetserver/server/messages.py new file mode 100644 index 0000000..10898cf --- /dev/null +++ b/packetserver/server/messages.py @@ -0,0 +1,478 @@ +"""BBS private message system""" +import ax25 +import persistent +import persistent.list +from persistent.mapping import PersistentMapping +import datetime +from typing import Self,Union,Optional,Iterable,Sequence +from packetserver.common import PacketServerConnection, Request, Response, send_response, send_blank_response +from packetserver.common import Message as PacketMessage +from packetserver.common.constants import yes_values, no_values +from packetserver.common.util import from_date_digits, to_date_digits +import ZODB +import logging +import uuid +from uuid import UUID +from packetserver.common.util import email_valid +from packetserver.server.objects import Object +from packetserver.server.users import User +from BTrees.OOBTree import TreeSet +from packetserver.server.users import User, user_authorized +from traceback import format_exc +from collections import namedtuple +import re + +since_regex = """^message\\/since\\/(\\d+)$""" + +def mailbox_create(username: str, db_root: PersistentMapping): + un = username.upper().strip() + u = User.get_user_by_username(un, db_root) + if u is None: + raise KeyError(f"Username {username} does not exist.") + if not u.enabled: + raise KeyError(f"Username {username} does not exist.") + if un not in db_root['messages']: + db_root['messages'][un] = persistent.list.PersistentList() + + +def global_unique_message_uuid(db_root: PersistentMapping) -> UUID: + if "message_uuids" not in db_root: + db_root['message_uuids'] = TreeSet() + logging.debug("Created message_uuid set for global message ids.") + uid = uuid.uuid4() + while uid in db_root['message_uuids']: + uid = uuid.uuid4() + return uid + +class Attachment: + """Name and data that is sent with a message.""" + def __init__(self, name: str, data: Union[bytes,bytearray,str]): + self._name = "" + self._data = b"" + self._binary = True + self.data = data + self.name = name + + @property + def name(self) -> str: + return self._name + + @name.setter + def name(self, name: str): + if name.strip() != self._name: + if len(name.strip()) > 300: + raise ValueError("Object names must be no more than 300 characters.") + self._name = name.strip() + + @property + def binary(self): + return self._binary + + @property + def data(self) -> Union[str,bytes]: + if self.binary: + return self._data + else: + return self._data.decode() + + @data.setter + def data(self, data: Union[bytes,bytearray,str]): + if type(data) in (bytes,bytearray): + if bytes(data) != self._data: + self._data = bytes(data) + self._binary = True + else: + if str(data).encode() != self._data: + self._data = str(data).encode() + self._binary = False + + @property + def size(self) -> int: + return len(self.data) + + def copy(self): + return Attachment(self.name, self.data) + + @classmethod + def from_dict(cls, attachment: dict): + name = attachment.get("name") + data = attachment.get("data") + return Attachment(name, data) + + def to_dict(self, include_data: bool = True): + d = { + "name": self.name, + "binary": self.binary, + "size_bytes": self.size, + "data": b'' + } + if include_data: + d['data'] = self.data + return d + +class ObjectAttachment(Attachment): + def __init__(self, name: str, obj: Object): + self.object = obj + super().__init__(name, "") + + @property + def size(self) -> int: + return self.object.size + + @property + def data(self) -> Union[str,bytes]: + return self.object.data + + @property + def binary(self) -> bool: + return self.object.binary + + +class MessageTextTooLongError(Exception): + """Raised when the message text exceeds the length allowed in the server config.""" + pass + +class MessageAlreadySentError(Exception): + """Raised when the message text exceeds the length allowed in the server config.""" + pass + +class Message(persistent.Persistent): + def __init__(self, text: str, msg_to: Optional[Iterable[str]]= None, msg_from: Optional[str] = None, + attachments: Optional[Iterable[Attachment]] = None): + self.retrieved = False + self.sent_at = datetime.datetime.now(datetime.UTC) + self.text = text + self.attachments = () + self.msg_to = (None,) + self.msg_from = None + self.msg_id = uuid.uuid4() + self.msg_delivered = False + if msg_to: + if type(msg_to) is str: + msg_to = msg_to.upper().strip() + self.msg_to = (msg_to,) + else: + msg_to_tmp = [] + for i in msg_to: + i = str(i).strip().upper() + if i == "ALL": + msg_to_tmp = ["ALL"] + break + else: + msg_to_tmp.append(i) + self.msg_to = tuple(msg_to_tmp) + if msg_from: + self.msg_from = str(msg_from).upper().strip() + + if attachments: + attch = [] + for i in attachments: + if type(i) is Attachment: + attch.append(i) + elif type(i) is dict: + attch.append(Attachment.from_dict(i)) + elif not isinstance(i,Attachment): + attch.append(Attachment("",str(i))) + else: + attch.append(i) + self.attachments = tuple(attch) + def __repr__(self): + return f"" + + def to_dict(self, get_text: bool = True, get_attachments: bool = True) -> dict: + attachments = [] + for attachment in self.attachments: + attachments.append(attachment.to_dict(include_data=get_attachments)) + d = { + "attachments": attachments, + "to": self.msg_to, + "from": self.msg_from, + "id": str(self.msg_id), + "sent_at": self.sent_at.isoformat(), + "text": "" + } + if get_text: + d['text'] = self.text + + return d + + @classmethod + def from_dict(cls, data: dict) -> Self: + return Message(data['text'],msg_to=data.get('to'), attachments=data.get("attachments")) + + def send(self, db: ZODB.DB) -> tuple: + if self.msg_delivered: + raise MessageAlreadySentError("Cannot send a private message that has already been sent.") + if self.msg_from is None: + raise ValueError("Message sender (message_from) cannot be None.") + new_attachments = [] + for i in self.attachments: + if isinstance(i,ObjectAttachment): + logging.debug("Skpping object attachments for now. Resolve db queries for them at send time.") + # new_attachments.append(Attachment(i.name, i.data)) TODO send object attachments + pass + else: + new_attachments.append(i) + send_counter = 0 + recipients = [] + failed = [] + to_all = False + with db.transaction() as db: + mailbox_create(self.msg_from, db.root()) + self.msg_id = global_unique_message_uuid(db.root()) + for recipient in self.msg_to: + recipient = recipient.upper().strip() + if recipient is None: + continue + if recipient == "ALL": + recipients = [x for x in db.root.users if db.root.users[x].enabled] + to_all = True + break + recipients.append(recipient) + if self.msg_from.upper().strip() in recipients: + recipients.remove(self.msg_from.upper().strip()) + send_counter = send_counter + 1 + for recipient in recipients: + msg = Message(self.text, recipient, self.msg_from, attachments=[x.copy() for x in new_attachments]) + msg.msg_id = self.msg_id + try: + mailbox_create(recipient, db.root()) + msg.msg_delivered = True + msg.sent_at = datetime.datetime.now(datetime.UTC) + if to_all: + msg.msg_to = 'ALL' + db.root.messages[recipient].append(msg) + send_counter = send_counter + 1 + except: + logging.error(f"Error sending message to {recipient}:\n{format_exc()}") + failed.append(recipient) + self.msg_delivered = True + msg = Message(self.text, recipient, self.msg_from, attachments=[x.copy() for x in new_attachments]) + msg.msg_id = self.msg_id + msg.msg_to = self.msg_to + db.root.messages[self.msg_from.upper().strip()].append(msg) + return send_counter, failed, self.msg_id + +DisplayOptions = namedtuple('DisplayOptions', ['get_text', 'limit', 'sort_by', 'reverse', 'search', + 'get_attachments', 'sent_received_all']) + +def parse_display_options(req: Request) -> DisplayOptions: + logging.debug(f"Parsing request vars for message get: {req.vars}") + sent_received_all = "received" + d = req.vars.get("source") + if type(d) is str: + d.lower().strip() + if d == "sent": + sent_received_all = "sent" + elif d == "all": + sent_received_all = "all" + + limit = req.vars.get('limit') + try: + limit = int(limit) + except: + limit = None + + d = req.vars.get('fetch_text') + if type(d) is str: + d.lower().strip() + if d in no_values: + get_text = False + else: + get_text = True + + d = req.vars.get('fetch_attachments') + logging.debug(f"Parsing fetch_attachment var: {d}") + if type(d) is str: + d.lower().strip() + if d in yes_values: + logging.debug("fetch_attachment is yes") + get_attachments = True + else: + get_attachments = False + logging.debug("fetch_attachment is no") + + r = req.vars.get('reverse') + if type(r) is str: + r.lower().strip() + if r in yes_values: + reverse = True + else: + reverse = False + + sort = req.vars.get('sort') + sort_by = "date" + if type(sort) is str: + sort = sort.lower().strip() + if sort == "from": + sort_by = "from" + elif sort == "to": + sort_by = "to" + + s = req.vars.get('search') + search = None + if type(s) is str: + s = s.lower() + if s: + search = str(s).lower() + + return DisplayOptions(get_text, limit, sort_by, reverse, search, get_attachments, sent_received_all) + +def handle_messages_since(req: Request, conn: PacketServerConnection, db: ZODB.DB): + if req.method is not Request.Method.GET: + send_blank_response(conn, req, 400, "method not implemented") + logging.warning(f"Received req with wrong message for path {req.path}.") + return + try: + since_date = from_date_digits(req.vars['since']) + except ValueError as v: + send_blank_response(conn, req, 400, "invalid date string") + return + except: + send_blank_response(conn, req, 500, "unknown error") + logging.error(f"Unhandled exception: {format_exc()}") + return + opts = parse_display_options(req) + username = ax25.Address(conn.remote_callsign).call.upper().strip() + msg_return = [] + with db.transaction() as db: + mailbox_create(username, db.root()) + mb = db.root.messages[username] + logging.debug(f"Only grabbing messages since {since_date}") + new_mb = [msg for msg in mb if msg.sent_at >= since_date] + if len(new_mb) > 0: + logging.debug(f"First message in new list: {new_mb[0].sent_at}") + logging.debug(f"Last message in new list: {new_mb[-1].sent_at}") + if opts.search: + messages = [msg for msg in new_mb if (opts.search in msg.text.lower()) or (opts.search in msg.msg_to[0].lower()) + or (opts.search in msg.msg_from.lower())] + else: + messages = [msg for msg in new_mb] + + if opts.sort_by == "from": + messages.sort(key=lambda x: x.msg_from, reverse=opts.reverse) + elif opts.sort_by == "to": + messages.sort(key=lambda x: x.msg_to, reverse=opts.reverse) + else: + messages.sort(key=lambda x: x.sent_at, reverse=opts.reverse) + + for i in range(0, len(messages)): + if opts.limit and (len(msg_return) >= opts.limit): + break + + msg = messages[i] + msg.retrieved = True + msg_return.append(msg.to_dict(get_text=opts.get_text, get_attachments=opts.get_attachments)) + + response = Response.blank() + response.status_code = 200 + response.payload = msg_return + send_response(conn, response, req) + +def handle_message_get_id(req: Request, conn: PacketServerConnection, db: ZODB.DB): + uuid_val = req.vars['id'] + obj_uuid = None + try: + if type(uuid_val) is bytes: + obj_uuid = UUID(bytes=uuid_val) + elif type(uuid_val) is int: + obj_uuid = UUID(int=uuid_val) + elif type(uuid_val) is str: + obj_uuid = UUID(uuid_val) + except: + pass + if obj_uuid is None: + send_blank_response(conn, req, 400) + return + opts = parse_display_options(req) + username = ax25.Address(conn.remote_callsign).call.upper().strip() + msg = None + with db.transaction() as db: + mailbox_create(username, db.root()) + for m in db.root.messages[username]: + if m.msg_id == obj_uuid: + msg = m + break + if msg is None: + send_blank_response(conn, req, status_code=404) + return + else: + send_blank_response(conn, req, + payload=msg.to_dict(get_text=opts.get_text, get_attachments=opts.get_attachments)) + +def handle_message_get(req: Request, conn: PacketServerConnection, db: ZODB.DB): + if 'id' in req.vars: + return handle_message_get_id(req, conn, db) + + if 'since' in req.vars: + return handle_messages_since(req, conn, db) + + opts = parse_display_options(req) + username = ax25.Address(conn.remote_callsign).call.upper().strip() + msg_return = [] + with db.transaction() as db: + mailbox_create(username, db.root()) + mb = db.root.messages[username] + if opts.search: + messages = [msg for msg in mb if (opts.search in msg.text.lower()) or (opts.search in msg.msg_to[0].lower()) + or (opts.search in msg.msg_from.lower())] + else: + messages = [msg for msg in mb] + + if opts.sort_by == "from": + messages.sort(key=lambda x: x.msg_from, reverse=opts.reverse) + elif opts.sort_by == "to": + messages.sort(key=lambda x: x.msg_to, reverse=opts.reverse) + else: + messages.sort(key=lambda x: x.sent_at, reverse=opts.reverse) + + for i in range(0, len(messages)): + if opts.limit and (len(msg_return) >= opts.limit): + break + + msg = messages[i] + msg.retrieved = True + msg_return.append(msg.to_dict(get_text=opts.get_text, get_attachments=opts.get_attachments)) + + response = Response.blank() + response.status_code = 200 + response.payload = msg_return + send_response(conn, response, req) + +def handle_message_post(req: Request, conn: PacketServerConnection, db: ZODB.DB): + username = ax25.Address(conn.remote_callsign).call.upper().strip() + try: + msg = Message.from_dict(req.payload) + except: + send_blank_response(conn, req, status_code=400) + logging.warning(f"User '{username}' attempted to post message with invalid payload: {req.payload}") + return + msg.msg_from = username + try: + send_counter, failed, msg_id = msg.send(db) + except: + send_blank_response(conn, req, status_code=500) + logging.error(f"Error while attempting to send message:\n{format_exc()}") + return + + send_blank_response(conn, req, status_code=201, payload={ + "successes": send_counter, + "failed": failed, + 'msg_id': str(msg_id)}) + +def message_root_handler(req: Request, conn: PacketServerConnection, db: ZODB.DB): + logging.debug(f"{req} being processed by message_root_handler") + if not user_authorized(conn, db): + logging.debug(f"user {conn.remote_callsign} not authorized") + send_blank_response(conn, req, status_code=401) + return + logging.debug("user is authorized") + if req.method is Request.Method.GET: + handle_message_get(req, conn, db) + elif req.method is Request.Method.POST: + handle_message_post(req, conn, db) + else: + send_blank_response(conn, req, status_code=404) + + + diff --git a/packetserver/server/objects.py b/packetserver/server/objects.py new file mode 100644 index 0000000..596f963 --- /dev/null +++ b/packetserver/server/objects.py @@ -0,0 +1,435 @@ +"""Server object storage system.""" +from copy import deepcopy + +import persistent +import ax25 +import persistent.list +from persistent.mapping import PersistentMapping +import datetime +from typing import Self,Union,Optional +from packetserver.common import PacketServerConnection, Request, Response, Message, send_response, send_blank_response +import ZODB +import logging +import uuid +from uuid import UUID +from packetserver.server.users import User, user_authorized +from collections import namedtuple +from traceback import format_exc +import base64 + +class Object(persistent.Persistent): + def __init__(self, name: str = "", data: Union[bytes,bytearray,str] = None): + self.private = False + self._binary = False + self._data = b'' + self._name = "" + self._owner = None + if data: + self.data = data + if name: + self._name = name + self._uuid = None + self.created_at = datetime.datetime.now(datetime.UTC) + self.modified_at = datetime.datetime.now(datetime.UTC) + + + @property + def name(self) -> str: + return self._name + + @name.setter + def name(self, name: str): + if name.strip() != self._name: + if len(name.strip()) > 300: + raise ValueError("Object names must be no more than 300 characters.") + self._name = name.strip() + self.touch() + + def touch(self): + self.modified_at = datetime.datetime.now(datetime.UTC) + + @property + def size(self) -> int: + return len(self.data) + + @property + def binary(self): + return self._binary + + @property + def data(self) -> Union[str,bytes]: + if self.binary: + return self._data + else: + return self._data.decode() + + @data.setter + def data(self, data: Union[bytes,bytearray,str]): + if type(data) in (bytes,bytearray): + if bytes(data) != self._data: + self._data = bytes(data) + self._binary = True + self.touch() + else: + if str(data).encode() != self._data: + self._data = str(data).encode() + self._binary = False + self.touch() + + @property + def owner(self) -> Optional[UUID]: + return self._owner + + @owner.setter + def owner(self, owner_uuid: UUID): + if owner_uuid: + if type(owner_uuid) is UUID: + self._owner = owner_uuid + self.touch() + else: + raise ValueError("Owner must be a UUID") + else: + self._owner = None + self.touch() + + def chown(self, username: str, db: ZODB.DB): + logging.debug(f"chowning object {self} to user {username}") + un = username.strip().upper() + old_owner_uuid = self._owner + with db.transaction() as db: + user = User.get_user_by_username(username, db.root()) + old_owner = User.get_user_by_uuid(old_owner_uuid, db.root()) + if user: + logging.debug(f"new owner user exists: {user}") + db.root.objects[self.uuid].owner = user.uuid + if old_owner_uuid: + if old_owner: + logging.debug(f"The object has an old owner user: {old_owner}") + old_owner.remove_obj_uuid(self.uuid) + logging.debug(f"adding this object uuid to user objects set ({self.uuid})") + logging.debug(f"user {user} objects before: {user.object_uuids}") + user.add_obj_uuid(self.uuid) + logging.debug(f"user objects now: {user.object_uuids}") + else: + raise KeyError(f"User '{un}' not found.") + + @classmethod + def get_object_by_uuid(cls, obj: UUID, db_root: PersistentMapping): + return db_root['objects'].get(obj) + + @classmethod + def get_objects_by_username(cls, username: str, db: ZODB.DB) -> list[Self]: + un = username.strip().upper() + objs = [] + with db.transaction() as db: + user = User.get_user_by_username(username, db.root()) + if user: + uuids = user.object_uuids + for u in uuids: + try: + obj = cls.get_object_by_uuid(u, db) + if obj: + objs.append(obj) + except: + pass + return objs + + @property + def uuid(self) -> Optional[UUID]: + return self._uuid + + def write_new(self, db: ZODB.DB) -> UUID: + if self.uuid: + raise KeyError("Object already has UUID. Manually clear it to write it again.") + self._uuid = uuid.uuid4() + with db.transaction() as db: + while self.uuid in db.root.objects: + self._uuid = uuid.uuid4() + db.root.objects[self.uuid] = self + self.touch() + return self.uuid + + def to_dict(self, include_data: bool = True) -> dict: + data = b'' + if include_data: + data = self.data + if self.uuid: + uuid_bytes = self.uuid.bytes + else: + uuid_bytes = None + + return { + "name": self.name, + "uuid_bytes": uuid_bytes, + "size_bytes": self.size, + "binary": self.binary, + "private": self.private, + "created_at": self.created_at.isoformat(), + "modified_at": self.modified_at.isoformat(), + "includes_data": include_data, + "data": data + } + + @classmethod + def from_dict(cls, obj: dict) -> Self: + o = Object(name=obj['name']) + if 'uuid_bytes' in obj: + if obj['uuid_bytes']: + o._uuid = UUID(bytes=obj['uuid_bytes']) + o.private = obj['private'] + o.data = obj['data'] + o._binary = obj['binary'] + return o + + def authorized_write(self, username: str, db: ZODB.DB): + un = username.strip().upper() + with db.transaction() as db: + user = User.get_user_by_username(username, db.root()) + if user: + if user.uuid == self.owner: + return True + else: + return False + + def authorized_get(self, username: str, db: ZODB.DB): + if not self.private: + return True + un = username.strip().upper() + with db.transaction() as db: + user = User.get_user_by_username(username, db.root()) + if user: + if user.uuid == self.owner: + return True + else: + return False + + def __repr__(self): + return f"" + +DisplayOptions = namedtuple('DisplayOptions', ['get_data', 'limit', 'sort_by', 'reverse', 'search']) + +def parse_display_options(req: Request) -> DisplayOptions: + limit = req.vars.get('limit') + try: + limit = int(limit) + except: + limit = None + + d = req.vars.get('fetch') + if type(d) is str: + d.lower().strip() + if d in [1, 'y', True, 'yes', 'true', 't']: + get_data = True + else: + get_data = False + + r = req.vars.get('reverse') + if type(r) is str: + r.lower().strip() + if r in [1, 'y', True, 'yes', 'true', 't']: + reverse = True + else: + reverse = False + + sort = req.vars.get('sort') + sort_by = "name" + if type(sort) is str: + sort = sort.lower().strip() + if sort == "date": + sort_by = "date" + elif sort == "size": + sort_by = "size" + + s = req.vars.get('search') + search = None + if type(s) is str: + s = s.lower() + if s: + search = str(s) + + return DisplayOptions(get_data, limit, sort_by, reverse, search) + +def object_display_filter(source: list[Object], opts: DisplayOptions) -> list[dict]: + if opts.search: + objs = [x for x in source if str(opts.search) in x.name.lower()] + else: + objs = deepcopy(source) + + if opts.sort_by == "size": + objs.sort(key=lambda x: x.size, reverse=opts.reverse) + + elif opts.sort_by == "date": + objs.sort(key=lambda x: x.modified_at, reverse=opts.reverse) + else: + objs.sort(key=lambda x: x.name, reverse=opts.reverse) + + if opts.limit: + if len(objs) >= opts.limit: + objs = objs[:opts.limit] + + return [o.to_dict(include_data=opts.get_data) for o in objs] + +def handle_get_no_path(req: Request, conn: PacketServerConnection, db: ZODB.DB): + opts = parse_display_options(req) + logging.debug(f"Handling a GET 'object' request: {opts}") + response = Response.blank() + response.status_code = 404 + username = ax25.Address(conn.remote_callsign).call.upper().strip() + with db.transaction() as db: + user = User.get_user_by_username(username, db.root()) + if not user: + send_blank_response(conn, req, status_code=500, payload="Unknown user account problem") + return + if 'uuid' in req.vars: + logging.debug(f"uuid req.var: {req.vars['uuid']}") + uid = req.vars['uuid'] + if type(uid) is bytes: + obj = Object.get_object_by_uuid(UUID(bytes=uid), db.root()) + if obj: + if not obj.owner == user.uuid: + if not obj.private: + send_blank_response(conn, req, status_code=401) + return + if opts.get_data: + response.payload = obj.to_dict() + response.status_code = 200 + else: + response.payload = obj.to_dict(include_data=False) + response.status_code = 200 + else: + uuids = user.object_uuids + objs = [] + logging.debug(f"No uuid var, all user object_uuids: {uuids}") + for i in uuids: + obj = Object.get_object_by_uuid(i, db.root()) + logging.debug(f"Checking {obj}") + if not obj.private: + logging.debug("object not private") + objs.append(obj) + else: + logging.debug("object private") + if obj.owner == user.uuid: + logging.debug("user uuid matches object uuid") + objs.append(obj) + response.payload = object_display_filter(objs, opts) + logging.debug(f"object payload: {response.payload}") + response.status_code = 200 + + send_response(conn, response, req) + +def handle_object_get(req: Request, conn: PacketServerConnection, db: ZODB.DB): + # Case: User searching their own objects -> list + # or passes specific UUID as var -> Object + handle_get_no_path(req, conn, db) + + +def handle_object_post(req: Request, conn: PacketServerConnection, db: ZODB.DB): + if type(req.payload) is not dict: + send_blank_response(conn, req, 400, payload="object payload must be 'dict'") + + try: + obj = Object.from_dict(req.payload) + except: + logging.debug(f"Error parsing new object:\n{format_exc()}") + send_blank_response(conn, req, status_code=400) + return + username = ax25.Address(conn.remote_callsign).call.upper().strip() + with db.transaction() as db_conn: + logging.debug(f"User {username}'s objects: {db_conn.root.users[username].object_uuids}") + logging.debug(f"writing new object: {obj}") + obj.write_new(db) + logging.debug("chowning new object") + obj.chown(username, db) + send_blank_response(conn, req, status_code=201, payload=str(obj.uuid)) + +def handle_object_update(req: Request, conn: PacketServerConnection, db: ZODB.DB): + username = ax25.Address(conn.remote_callsign).call.upper().strip() + if type(req.payload) is not dict: + send_blank_response(conn, req, status_code=400) + return + if 'uuid' in req.vars: + uid = req.vars['uuid'] + if type(uid) is bytes: + u_obj = UUID(bytes=uid) + elif type(uid) is int: + u_obj = UUID(int=uid) + else: + try: + u_obj = UUID(str(uid)) + except ValueError: + send_blank_response(conn, req, status_code=400) + new_name = req.payload.get("name") + new_data = req.payload.get("data") + if new_data: + if type(new_data) not in (bytes, bytearray, str): + send_blank_response(conn, req, status_code=400) + return + with db.transaction() as db: + obj = Object.get_object_by_uuid(uid, db.root()) + user = User.get_user_by_username(username, db.root()) + if user.uuid != obj.owner: + send_blank_response(conn, req, status_code=401) + return + if obj is None: + send_blank_response(conn, req, status_code=404) + return + if new_name: + obj.name = new_name + if new_data: + obj.data = new_data + send_blank_response(conn, req, status_code=200) + else: + send_blank_response(conn, req, status_code=400) + return + +def handle_object_delete(req: Request, conn: PacketServerConnection, db: ZODB.DB): + username = ax25.Address(conn.remote_callsign).call.upper().strip() + if 'uuid' in req.vars: + uid = req.vars['uuid'] + if type(uid) is bytes: + u_obj = UUID(bytes=uid) + elif type(uid) is int: + u_obj = UUID(int=uid) + else: + try: + u_obj = UUID(str(uid)) + except ValueError: + send_blank_response(conn, req, status_code=400, payload='badly formatted uuid') + return + with db.transaction() as db: + obj = Object.get_object_by_uuid(u_obj, db.root()) + user = User.get_user_by_username(username, db.root()) + if user.uuid != obj.owner: + send_blank_response(conn, req, status_code=401) + return + if obj is None: + send_blank_response(conn, req, status_code=404) + return + try: + user.remove_obj_uuid(u_obj) + del db.root.objects[u_obj] + except: + send_blank_response(conn, req, status_code=500) + logging.error(f"Error handling delete:\n{format_exc()}") + send_blank_response(conn, req, status_code=200) + else: + send_blank_response(conn, req, status_code=400) + return + +def object_root_handler(req: Request, conn: PacketServerConnection, db: ZODB.DB): + logging.debug(f"{req} being processed by user_root_handler") + if not user_authorized(conn, db): + logging.debug(f"user {conn.remote_callsign} not authorized") + send_blank_response(conn, req, status_code=401) + return + logging.debug("user is authorized") + if req.method is Request.Method.GET: + handle_object_get(req, conn, db) + elif req.method is Request.Method.POST: + handle_object_post(req, conn, db) + elif req.method is Request.Method.UPDATE: + handle_object_update(req, conn, db) + elif req.method is Request.Method.DELETE: + handle_object_delete(req, conn, db) + else: + send_blank_response(conn, req, status_code=404) diff --git a/packetserver/server/requests.py b/packetserver/server/requests.py new file mode 100644 index 0000000..42f2c14 --- /dev/null +++ b/packetserver/server/requests.py @@ -0,0 +1,66 @@ +"""Module for handling requests as they arrive to connection objects and servers.""" +import ax25 +from msgpack.exceptions import OutOfData +from packetserver.common import Message, Request, Response, PacketServerConnection, send_response, send_blank_response +from .bulletin import bulletin_root_handler +from .users import user_root_handler, user_authorized +from .objects import object_root_handler +from .messages import message_root_handler +from .jobs import job_root_handler +import logging +from typing import Union +import ZODB + +def handle_root_get(req: Request, conn: PacketServerConnection, + db: ZODB.DB): + logging.debug(f"Root get handler received request: {req}") + response = Response.blank() + response.compression = Message.CompressionType.BZIP2 + operator = "" + motd = "" + jobs_enabled = False + with db.transaction() as storage: + if 'motd' in storage.root.config: + motd = storage.root.config['motd'] + if 'operator' in storage.root.config: + operator = storage.root.config['operator'] + if 'jobs_enabled' in storage.root.config: + jobs_enabled = storage.root.config['jobs_enabled'] + logging.debug(f"Root handler retrieved config. {operator} - {motd}") + logging.debug("Running user_authorized") + base = ax25.Address(conn.remote_callsign).call + if user_authorized(conn, db): + user_message = f"User {base} is enabled." + else: + user_message = f"User {base} is not enabled." + logging.debug(f"User authorized: {user_message}") + response.payload = { + 'operator': operator, + 'motd': motd, + 'user': user_message, + 'accepts_jobs': jobs_enabled + } + + logging.debug(f"Sending response {response}") + send_response(conn, response, req) + logging.debug("Sent reesponse.") + +def root_root_handler(req: Request, conn: PacketServerConnection, + db: ZODB.DB): + logging.debug(f"{req} got to root_root_handler") + if req.method is Request.Method.GET: + handle_root_get(req, conn, db) + else: + logging.warning(f"unhandled request found: {req}") + send_blank_response(conn, req, status_code=404) + +standard_handlers = { + "": root_root_handler, + "bulletin": bulletin_root_handler, + "user": user_root_handler, + "object": object_root_handler, + "message": message_root_handler, + "job": job_root_handler +} + + diff --git a/packetserver/server/testserver.py b/packetserver/server/testserver.py new file mode 100644 index 0000000..767f61b --- /dev/null +++ b/packetserver/server/testserver.py @@ -0,0 +1,115 @@ +import tempfile +from packetserver.common import Response, Message, Request, send_response, send_blank_response +from packetserver.common.testing import DirectoryTestServerConnection, DummyPacketServerConnection +from pe.connect import ConnectionState +from shutil import rmtree +from threading import Thread +from . import Server +import os +import os.path +import time +import logging +from traceback import format_exc + +class TestServer(Server): + def __init__(self, server_callsign: str, data_dir: str = None, zeo: bool = True): + super().__init__('localhost', 8000, server_callsign, data_dir=data_dir, zeo=zeo) + self._data_pid = 1 + self._file_traffic_dir = tempfile.mkdtemp() + self._file_traffic_thread = None + + def start(self): + if self.orchestrator is not None: + self.orchestrator.start() + self.start_db() + self.started = True + self.worker_thread = Thread(target=self.run_worker) + self.worker_thread.start() + + def stop(self): + self.started = False + if self.orchestrator is not None: + self.orchestrator.stop() + self.stop_db() + rmtree(self._file_traffic_dir) + + def data_pid(self) -> int: + old = self._data_pid + self._data_pid = self._data_pid + 1 + return old + + def send_test_data(self, conn: DummyPacketServerConnection, data: bytearray): + conn.data_received(self.data_pid(), data) + self.server_receiver(conn) + + +class DirectoryTestServer(Server): + def __init__(self, server_callsign: str, connection_directory: str, data_dir: str = None, zeo: bool = True): + super().__init__('localhost', 8000, server_callsign, data_dir=data_dir, zeo=zeo) + if not os.path.isdir(connection_directory): + raise NotADirectoryError(f"{connection_directory} is not a directory or doesn't exist.") + self._file_traffic_dir = os.path.abspath(connection_directory) + self._dir_connections = [] + + def check_connection_directories(self): + logging.debug(f"Server checking connection directory {self._file_traffic_dir}") + if not os.path.isdir(self._file_traffic_dir): + raise NotADirectoryError(f"{self._file_traffic_dir} is not a directory or doesn't exist.") + + for path in os.listdir(self._file_traffic_dir): + dir_path = os.path.join(self._file_traffic_dir, path) + logging.debug(f"Checking directory {dir_path}") + if not os.path.isdir(dir_path): + logging.debug(f"Server: {dir_path} is not a directory; skipping") + continue + + conn_exists = False + for conn in self._dir_connections: + if os.path.abspath(conn.directory) == dir_path: + conn_exists = True + break + + if conn_exists: + continue + + try: + conn = DirectoryTestServerConnection.create_directory_connection(self.callsign, dir_path) + logging.debug(f"New connection detected from {conn.remote_callsign}") + self._dir_connections.append(conn) + self.server_connection_bouncer(conn) + except ValueError: + logging.debug(format_exc()) + pass + + closed = [] + + for conn in self._dir_connections: + conn.check_for_data() + if conn.state is not ConnectionState.CONNECTED: + closed.append(conn) + + for conn in closed: + if conn in self._dir_connections: + self._dir_connections.remove(conn) + + def dir_worker(self): + """Intended to be running as a thread.""" + logging.info("Starting worker thread.") + while self.started: + self.server_worker() + self.check_connection_directories() + time.sleep(.5) + + def start(self): + if self.orchestrator is not None: + self.orchestrator.start() + self.start_db() + self.started = True + self.worker_thread = Thread(target=self.dir_worker) + self.worker_thread.start() + + def stop(self): + self.started = False + if self.orchestrator is not None: + self.orchestrator.stop() + self.stop_db() diff --git a/packetserver/server/users.py b/packetserver/server/users.py new file mode 100644 index 0000000..512ff80 --- /dev/null +++ b/packetserver/server/users.py @@ -0,0 +1,321 @@ +"""Module containing code related to users.""" + +import ax25 +import persistent +import persistent.list +from persistent.list import PersistentList +from persistent.mapping import PersistentMapping +import datetime +from typing import Self,Union,Optional +from packetserver.common import PacketServerConnection, Request, Response, Message, send_response, send_blank_response +import ZODB +import logging +import uuid +from traceback import format_exc +from uuid import UUID +from packetserver.common.util import email_valid +from BTrees.OOBTree import TreeSet + +class User(persistent.Persistent): + def __init__(self, username: str, enabled: bool = True, hidden: bool = False, bio: str = "", status: str = "", + email: str = None, location: str = "", socials: list[str] = None): + self._username = username.upper().strip() + self.enabled = enabled + self.hidden = hidden + self.created_at = datetime.datetime.now(datetime.UTC) + self.last_seen = self.created_at + self._email = "" + if email: + self.email = email + self._location = "" + self.location = location + self._socials = [] + if socials: + self.socials = socials + self._uuid = None + self.bio = bio + self._status = "" + self.status = status + self._objects = TreeSet() + + def write_new(self, db_root: PersistentMapping): + all_uuids = [db_root['users'][x].uuid for x in db_root['users']] + self._uuid = uuid.uuid4() + while self.uuid in all_uuids: + self._uuid = uuid.uuid4() + logging.debug(f"Creating new user account {self.username} - {self.uuid}") + if self.username not in db_root['users']: + db_root['users'][self.username] = self + + @property + def object_uuids(self) -> list[UUID]: + return list(self._objects) + + def remove_obj_uuid(self, obj: UUID): + self._objects.remove(obj) + + def add_obj_uuid(self, obj: UUID): + self._objects.add(obj) + + def user_has_obj(self, obj: UUID) -> bool: + if obj in self._objects: + return True + return False + + @property + def location(self) -> str: + return self._location + + @location.setter + def location(self, location: str): + if len(location) > 1000: + self._location = location[:1000] + else: + self._location = location + + @property + def email(self) -> str: + return self._email + + @email.setter + def email(self, email: str): + if email_valid(email.strip().lower()): + self._email = email.strip().lower() + else: + raise ValueError(f"Invalid e-mail given: {email}") + + @property + def socials(self) -> list[str]: + return [] + + @socials.setter + def socials(self, socials: list[str]): + for social in socials: + if len(social) > 300: + social = social[:300] + self._socials.append(social) + + def add_social(self, social: str): + if len(social) > 300: + social = social[:300] + self._socials.append(social) + + def remove_social(self, social: str): + self.socials.remove(social) + + @property + def uuid(self): + return self._uuid + + @classmethod + def get_user_by_username(cls, username: str, db_root: PersistentMapping) -> Self: + try: + if username.upper().strip() in db_root['users']: + return db_root['users'][username.upper().strip()] + except Exception: + return None + return None + + @classmethod + def get_user_by_uuid(cls, user_uuid: Union[UUID, bytes, int, str], db_root: PersistentMapping) -> Self: + try: + if type(uuid) is uuid.UUID: + uid = user_uuid + elif type(uuid) is bytes: + uid = uuid.UUID(bytes=user_uuid) + elif type(uuid) is int: + uid = uuid.UUID(int=user_uuid) + else: + uid = uuid.UUID(str(user_uuid)) + for user in db_root['users']: + if uid == db_root['users'][user].uuid: + return db_root['users'][user].uuid + except Exception: + return None + return None + + @classmethod + def get_all_users(cls, db_root: PersistentMapping, limit: int = None) -> list: + all_users = sorted(db_root['users'].values(), key=lambda user: user.username) + if not limit: + return all_users + else: + if len(all_users) < limit: + return all_users + else: + return all_users[:limit] + + @classmethod + def is_authorized(cls, username: str, db_root: PersistentMapping) -> bool: + user = User.get_user_by_username(username, db_root) + if user: + if user.enabled: + return True + return False + + def seen(self): + self.last_seen = datetime.datetime.now(datetime.UTC) + + @property + def username(self) -> str: + return self._username.upper().strip() + + @property + def bio(self) -> str: + return self._bio + + @bio.setter + def bio(self, bio: str): + if len(bio) > 4000: + self._bio = bio[:4000] + else: + self._bio = bio + + @property + def status(self) -> str: + return self._status + + @status.setter + def status(self, status: str): + if len(status) > 300: + self._status = status[:300] + else: + self._status = status + + def to_safe_dict(self) -> dict: + return { + "username": self.username, + "status": self.status, + "bio": self.bio, + "socials": self.socials, + "email": self.email, + "location": self.location, + "last_seen": self.last_seen.isoformat(), + "created_at": self.created_at.isoformat() + } + + def __repr__(self): + return f"" + +def user_authorized(conn: PacketServerConnection, db: ZODB.DB) -> bool: + username = ax25.Address(conn.remote_callsign).call + logging.debug(f"Running authcheck for user {username}") + result = False + with db.transaction() as db: + result = User.is_authorized(username, db.root()) + logging.debug(f"User is authorized? {result}") + return result + +def handle_user_get(req: Request, conn: PacketServerConnection, db: ZODB.DB): + sp = req.path.split("/") + logging.debug("handle_user_get working") + user = None + user_var = req.vars.get('username') + response = Response.blank() + response.status_code = 404 + limit = None + if 'limit' in req.vars: + try: + limit = int(req.vars['limit']) + except ValueError: + pass + with db.transaction() as db: + if len(sp) > 1: + logging.debug(f"trying to get the username from the path {sp[1].strip().upper()}") + user = User.get_user_by_username(sp[1].strip().upper(), db.root()) + logging.debug(f"user holds: {user}") + if user and not user.hidden: + response.status_code = 200 + response.payload = user.to_safe_dict() + else: + if user_var: + user = User.get_user_by_username(user_var.upper().strip(), db.root()) + if user and not user.hidden: + response.status_code = 200 + response.payload = user.to_safe_dict() + else: + if user_var: + user = User.get_user_by_username(user_var.upper().strip(), db.root()) + if user and not user.hidden: + response.status_code = 200 + response.payload = user.to_safe_dict() + else: + response.status_code = 200 + response.payload = [x.to_safe_dict() for x in User.get_all_users(db.root(), limit=limit) if not x.hidden] + send_response(conn, response, req) + +def handle_user_update(req: Request, conn: PacketServerConnection, db: ZODB.DB): + """ + "status": str 300 cutoff + "bio": str 4k cutoff + "socials": list[str] each 300 cutoff + "email": str (must be an e-mail) validate with valid_email function from util + "location": str 1000 char cutoff + """ + username = ax25.Address(conn.remote_callsign).call.upper().strip() + logging.debug(f"Handling user update request for {username}: {req.payload}") + + email = None + bio = None + socials = None + location = None + status = None + + # set vars + + if 'bio' in req.payload: + bio = str(req.payload['bio']) + + if 'location' in req.payload: + location = str(req.payload['location']) + + if 'status' in req.payload: + status = str(req.payload['status']) + + if 'email' in req.payload: + email = req.payload['email'] + if not email_valid(email): + send_blank_response(conn, req, status_code=400, payload="email must be valid format") + return + + if 'socials' in req.payload: + var_socials = req.payload['socials'] + socials = [] + if type(var_socials) is list: + for s in var_socials: + socials.append(str(s)) + else: + socials.append(str(var_socials)) + try: + with db.transaction() as db: + user = User.get_user_by_username(username, db.root()) + if email is not None: + user.email = email + if bio is not None: + user.bio = bio + if socials is not None: + user.socials = socials + if location is not None: + user.location = location + if status is not None: + user.status = status + except: + logging.error(f"Error while updating user {username}:\n{format_exc()}") + send_blank_response(conn, req, status_code=500) + return + + send_blank_response(conn, req, status_code=200) + +def user_root_handler(req: Request, conn: PacketServerConnection, db: ZODB.DB): + logging.debug(f"{req} being processed by user_root_handler") + if not user_authorized(conn, db): + logging.debug(f"user {conn.remote_callsign} not authorized") + send_blank_response(conn, req, status_code=401) + return + logging.debug("user is authorized") + if req.method is Request.Method.GET: + handle_user_get(req, conn, db) + elif req.method is Request.Method.UPDATE: + handle_user_update(req, conn ,db) + else: + send_blank_response(conn, req, status_code=404) diff --git a/src/packetserver/setup.py b/setup.py similarity index 85% rename from src/packetserver/setup.py rename to setup.py index f0d4bf0..470dc5c 100644 --- a/src/packetserver/setup.py +++ b/setup.py @@ -2,8 +2,10 @@ from setuptools import setup, find_packages setup( name='packetserver', - version='VERSION="0.4.0-alpha', - packages=find_packages(), + version='0.4.1', + packages=[ + 'packetserver', + ], include_package_data=True, install_requires=[ 'click', diff --git a/src/packetserver/__init__.py b/src/packetserver/__init__.py deleted file mode 100644 index a0832b3..0000000 --- a/src/packetserver/__init__.py +++ /dev/null @@ -1 +0,0 @@ -VERSION="0.4.0-alpha" \ No newline at end of file