Some client improvements and some improvements to testing classes for non-rf testing.

examples/client-settings.env

@@ -0,0 +1,5 @@
+export PSCLIENT_SERVER=KQ4PEC
+export PSCLIENT_AGWPE=localhost
+export PSCLIENT_PORT=1
+export PSCLIENT_CALLSIGN=KQ4PEC-7
+#export TEST_SERVER_DIR=/tmp/ts_conn_dir

packetserver/client/__init__.py

@@ -16,6 +16,10 @@ from os import linesep
 from shutil import rmtree
 from threading import Thread
 
+class ConnectionClosedError(Exception):
+    """Raised when a connection closes unexpectedly."""
+    pass
+
 class Client:
     def __init__(self, pe_server: str, port: int, client_callsign: str, keep_log=False):
         if not ax25.Address.valid_call(client_callsign):
@@ -106,7 +110,7 @@ class Client:
         conn =  self.app.open_connection(0, self.callsign, dest.upper())
         while conn.state.name != "CONNECTED":
             if conn.state.name in ['DISCONNECTED', 'DISCONNECTING']:
-                raise RuntimeError("Connection disconnected unexpectedly.")
+                raise ConnectionClosedError(f"Connection to {conn.remote_callsign} closed unexpectedly.")
             time.sleep(.1)
         logging.debug(f"Connection to {dest} ready.")
         logging.debug("Allowing connection to stabilize for 8 seconds")
@@ -121,7 +125,7 @@ class Client:
                 logging.error(f"Connection {conn} disconnected.")
                 if self.keep_log:
                     self.request_log.append((req, None))
-                return None
+                raise ConnectionClosedError(f"Connection to {conn.remote_callsign} closed unexpectedly.")
             try:
                 unpacked = conn.data.unpack()
             except:
@@ -136,7 +140,7 @@ class Client:
     def send_and_receive(self, req: Request, conn: Union[PacketServerConnection,SimpleDirectoryConnection],
                          timeout: int = 300) -> Optional[Response]:
         if conn.state.name != "CONNECTED":
-            raise RuntimeError("Connection is not connected.")
+            raise ConnectionClosedError(f"Connection to {conn.remote_callsign} closed unexpectedly.")
         logging.debug(f"Sending request {req}")
         dest = conn.remote_callsign.upper()
         with self.lock_locker:
@@ -160,7 +164,7 @@ class Client:
         while (datetime.datetime.now() < cutoff_date) and (conn.state.name != "CONNECTED"):
             if conn.state.name in ["DISCONNECTED", "DISCONNECTING"]:
                 logging.error(f"Connection {conn} disconnected.")
-                return None
+                raise ConnectionClosedError(f"Connection to {conn.remote_callsign} closed unexpectedly.")
 
         remaining_time = int((cutoff_date - datetime.datetime.now()).total_seconds()) + 1
         if remaining_time <= 0:

packetserver/client/cli/__init__.py

@@ -1,7 +1,7 @@
 import click
 from packetserver.client.cli.config import get_config, default_app_dir, config_path
 from packetserver.client.cli.constants import DEFAULT_DB_FILE
-from packetserver.client import Client
+from packetserver.client import Client, ConnectionClosedError
 from packetserver.common.constants import yes_values
 from packetserver.common import Request, Response
 from packetserver.client.cli.util import format_list_dicts, exit_client
@@ -115,7 +115,10 @@ def query_server(ctx):
     req = Request.blank()
     req.path = ""
     req.method = Request.Method.GET
+    try:
         resp = client.send_receive_callsign(req, ctx.obj['bbs'])
+    except Exception as e:
+        exit_client(ctx.obj, 2, str(e))
     if resp is None:
         click.echo(f"No response from {ctx.obj['bbs']}")
         exit_client(ctx.obj, 1)

packetserver/client/cli/job.py

@@ -74,7 +74,14 @@ def quick_session(ctx, transcript):
         db_enabled = False
         if cmd == "":
             continue
-        if cmd == "/exit":
+        elif cmd in ["/h", "/?", '/H', "/help", "/HELP"]:
+            click.echo("""Enter a command to run in a container, or enter one of the following special commands:
+            '/h' | '/?' to get this help message
+            '/exit' | '/q' to exit
+            '/db' to have the remote job put a copy of your user's server db (messages/objects/etc) in a json file
+                in the remote container in the working directory.
+            """)
+        elif cmd in ["/exit", '/q', '/quit']:
             break
         elif cmd == "/db":
             click.echo("DB requested for next command.")

packetserver/client/cli/object.py

@@ -101,9 +101,11 @@ def list_objects(ctx, number, search, reverse, sort_by, output_format):
         sort_name = True
     else:
         sort_date = True
-
+    try:
         object_list = get_user_objects(client, ctx.obj['bbs'], limit=number, include_data=False, search=search,
                                    reverse=reverse, sort_date=sort_date, sort_name=sort_name, sort_size=sort_size)
+    except Exception as e:
+        exit_client(ctx.obj, 19, message=str(e))
 
     obj_dicts = []
     for x in object_list:

packetserver/client/testing.py

@@ -4,7 +4,8 @@ from typing import Union
 
 from packetserver.common import Request, PacketServerConnection
 from packetserver.common.testing import SimpleDirectoryConnection
-from packetserver.client import Client
+from packetserver.client import Client, ConnectionClosedError
+from pe.connect import ConnectionState
 import ax25
 from threading import Lock
 import logging
@@ -59,6 +60,8 @@ class TestClient(Client):
             time.sleep(1)
             cutoff_date = datetime.datetime.now() + datetime.timedelta(seconds=timeout)
             while datetime.datetime.now() < cutoff_date:
+                if conn.state != ConnectionState.CONNECTED:
+                    raise ConnectionClosedError(f"Connection to {conn.remote_callsign} closed unexpectedly.")
                 logging.debug(f"Client {self.callsign} checking for connection conn {conn}")
                 if conn.check_for_data():
                     break

packetserver/common/testing.py

@@ -7,6 +7,7 @@ from typing import Union, Self, Optional
 import os.path
 import logging
 import ax25
+from shutil import rmtree
 
 class DummyPacketServerConnection(PacketServerConnection):
 
@@ -90,9 +91,17 @@ class DirectoryTestServerConnection(PacketServerConnection):
         file_path = os.path.join(self._directory, file_name)
         return file_path
 
+    def close(self):
+        self.closing = True
+        self._state = ConnectionState.DISCONNECTED
+        if os.path.exists(self._directory):
+            rmtree(self._directory)
+
     def check_closed(self):
         if self.closing:
             self._state = ConnectionState.DISCONNECTED
+            if os.path.exists(self._directory):
+                rmtree(self._directory)
         if self._state is not ConnectionState.CONNECTED:
             return True
         if not os.path.isdir(self._directory):
@@ -256,6 +265,7 @@ class SimpleDirectoryConnection:
         """Monitors connection directory for data."""
         if self.closing:
             self._state = ConnectionState.DISCONNECTED
+            logging.debug(f"Connection {self} closed.")
         if self.check_closed():
             return False
         if os.path.isfile(self.remote_file_path):

packetserver/common/util.py

@@ -7,6 +7,8 @@ import os.path
 from io import BytesIO, BufferedReader
 import random
 import string
+from persistent.mapping import PersistentMapping
+from persistent.list import PersistentList
 
 def email_valid(email: str) -> bool:
     """Taken from https://www.geeksforgeeks.org/check-if-email-address-valid-or-not-in-python/"""
@@ -149,3 +151,25 @@ class TarFileExtractor(object):
             name = str(name)
             self._count = self._count + 1
             return os.path.basename(name), self.tar_file.extractfile(member)
+
+def convert_to_persistent(data: Union[list,dict]):
+    if isinstance(data, dict):
+        persistent_dict = PersistentMapping()
+        for key, value in data.items():
+            persistent_dict[key] = convert_to_persistent(value)
+        return persistent_dict
+    elif isinstance(data, list):
+        return PersistentList([convert_to_persistent(item) for item in data])
+    else:
+        return data
+
+def convert_from_persistent(data):
+    if isinstance(data, PersistentMapping):
+        nonpersistent_dict = {}
+        for key, value in data.items():
+            nonpersistent_dict[key] = convert_from_persistent(value)
+        return nonpersistent_dict
+    elif isinstance(data, PersistentList):
+        return [convert_from_persistent(item) for item in data]
+    else:
+        return data

packetserver/server/__init__.py

@@ -90,7 +90,7 @@ class Server:
                 logging.debug("objects bucket missing, creating")
                 conn.root.objects = OOBTree()
             if 'jobs' not in conn.root():
-                logging.debug("jobss bucket missing, creating")
+                logging.debug("jobs bucket missing, creating")
                 conn.root.jobs = OOBTree()
             if 'job_queue' not in conn.root():
                 conn.root.job_queue = PersistentList()

packetserver/server/cli/__init__.py

@@ -0,0 +1,72 @@
+import click
+import ZODB, ZODB.FileStorage
+import ZEO
+import json
+import os
+import os.path
+import sys
+from persistent.mapping import PersistentMapping
+from persistent.list import PersistentList
+from pathlib import Path
+from packetserver.common.util import convert_from_persistent, convert_to_persistent
+
+
+
+@click.group()
+@click.option("--database", "-d", type=str, default="",
+              help="DATABASE is either the path to the database file, or a tcp host:port string to a zeo server that is running."
+              )
+@click.option("--zeo", "-z", is_flag=True, default=False, help="<database> is a zeo address:port string.")
+@click.pass_context
+def config(ctx, database, zeo):
+    """Dump or set the packetserver configuration."""
+
+    ctx.ensure_object(dict)
+    if zeo:
+        if database is None:
+            raise ValueError("Database must be at least a port to a zeo server, or an address:port string.")
+        spl = database.split(":")
+        if len(spl) == 1:
+            host = 'localhost'
+            port = int(spl[0])
+        else:
+            host = spl[0]
+            port = int(spl[1])
+        db = ZEO.DB((host, port))
+    else:
+        if type(database) is str and (database != "") :
+            data_file = Path(database)
+        else:
+            data_file = Path.home().joinpath(".packetserver").joinpath("data.zopedb")
+        if not data_file.is_file():
+                raise FileExistsError(f"Database file {str(data_file)} is not a file, or it doesn't exit.")
+
+        storage = ZODB.FileStorage.FileStorage(str(data_file))
+        db = ZODB.DB(storage)
+
+    ctx.obj['db'] = db
+
+
+@click.command()
+@click.pass_context
+def dump(ctx):
+    with ctx.obj['db'].transaction() as conn:
+        click.echo(json.dumps(convert_from_persistent(conn.root.config), indent=2))
+
+@click.command()
+@click.option("--json-data", "-j", type=str, required=True, help="Filename to json or '-' for stdin.")
+@click.pass_context
+def load(ctx, json_data):
+    if json_data == "-":
+        data = json.load(sys.stdin)
+    else:
+        data = json.load(open(json_data, 'r'))
+
+    with ctx.obj['db'].transaction() as conn:
+        conn.root.config = convert_to_persistent(data)
+
+config.add_command(dump)
+config.add_command(load)
+
+if __name__ == '__main__':
+    config()

packetserver/server/testserver.py

@@ -3,7 +3,7 @@ from packetserver.common import Response, Message, Request, send_response, send_
 from packetserver.common.testing import DirectoryTestServerConnection, DummyPacketServerConnection
 from pe.connect import ConnectionState
 from shutil import rmtree
-from threading import Thread
+from threading import Thread, Lock
 from . import Server
 import os
 import os.path
@@ -50,10 +50,13 @@ class DirectoryTestServer(Server):
             raise NotADirectoryError(f"{connection_directory} is not a directory or doesn't exist.")
         self._file_traffic_dir = os.path.abspath(connection_directory)
         self._dir_connections = []
+        self._conn_lock = Lock()
+        self._conn_thread_running = False
 
     def check_connection_directories(self):
         logging.debug(f"Server checking connection directory {self._file_traffic_dir}")
         if not os.path.isdir(self._file_traffic_dir):
+            self._conn_thread_running = False
             raise NotADirectoryError(f"{self._file_traffic_dir} is not a directory or doesn't exist.")
 
         for path in os.listdir(self._file_traffic_dir):
@@ -91,13 +94,18 @@ class DirectoryTestServer(Server):
         for conn in closed:
             if conn in self._dir_connections:
                 self._dir_connections.remove(conn)
+        self._conn_thread_running = False
 
     def dir_worker(self):
         """Intended to be running as a thread."""
         logging.info("Starting worker thread.")
         while self.started:
             self.server_worker()
-            self.check_connection_directories()
+            with self._conn_lock:
+                if not self._conn_thread_running:
+                    self._conn_thread_running = True
+                    conn_thread = Thread(target=self.check_connection_directories)
+                    conn_thread.start()
             time.sleep(.5)
 
     def start(self):

setup.py

@@ -3,9 +3,7 @@ from setuptools import setup, find_packages
 setup(
     name='packetserver',
     version='0.4.1',
-    packages=[
+    packages=find_packages(),
-        'packetserver',
-        ],
     include_package_data=True,
     install_requires=[
         'click',
@@ -20,6 +18,7 @@ setup(
     entry_points={
         'console_scripts': [
             'packcli = packetserver.client.cli:cli',
+            'packcfg = packetserver.server.cli:config',
         ],
     },
 )