diff --git a/Readme.md b/Readme.md index 4dede32..fa6c9aa 100644 --- a/Readme.md +++ b/Readme.md @@ -77,6 +77,10 @@ UI packets later on.. ### Main help dialog: ```commandline +(venv) [user@host]$ export PSCLIENT_SERVER=KQ4PEC +(venv) [user@host]$ export PSCLIENT_AGWPE=localhost +(venv) [user@host]$ export PSCLIENT_PORT=8000 +(venv) [user@host]$ export PSCLIENT_CALLSIGN=KQ4PEC-7 (venv) [user@host]$ packcli Usage: packcli [OPTIONS] COMMAND [ARGS]... @@ -102,6 +106,47 @@ Commands: user Query users on the BBS. ``` +### Running jobs on the remote server: +```commandline +(venv) [user@host]$ packcli job start -- ls -l / +67 +(venv) [user@host]$ packcli job get +You must either supply a job id, or --all-jobs +(venv) [user@host]$ packcli job get 67 +----------- ------------------------------------------------------------------ +id 67 +return_code 0 +status SUCCESSFUL +created 2025-03-21T00:15:41.142544+00:00 +finished 2025-03-20T20:15:46.257489 +cmd ['ls', '-l', '/'] +owner KQ4PEC +artifacts +output total 4 + drwxr-xr-x 2 root root 40 Mar 21 00:14 artifact_output + lrwxrwxrwx 1 root root 7 Mar 11 2024 bin -> usr/bin + drwxr-xr-x 2 root root 6 Jan 28 2024 boot + drwxr-xr-x 5 root root 340 Mar 21 00:14 dev + drwxr-xr-x 1 root root 4096 Mar 21 00:14 etc + drwxr-xr-x 1 root root 20 Mar 21 00:14 home + lrwxrwxrwx 1 root root 7 Mar 11 2024 lib -> usr/lib + lrwxrwxrwx 1 root root 9 Mar 11 2024 lib64 -> usr/lib64 + drwxr-xr-x 2 root root 6 Mar 11 2024 media + drwxr-xr-x 2 root root 6 Mar 11 2024 mnt + drwxr-xr-x 2 root root 6 Mar 11 2024 opt + dr-xr-xr-x 534 nobody nogroup 0 Mar 21 00:14 proc + drwx------ 1 root root 21 Mar 21 00:14 root + drwxr-xr-x 1 root root 27 Mar 21 00:14 run + lrwxrwxrwx 1 root root 8 Mar 11 2024 sbin -> usr/sbin + drwxr-xr-x 2 root root 6 Mar 11 2024 srv + dr-xr-xr-x 13 nobody nogroup 0 Mar 21 00:14 sys + drwxrwxrwt 1 root root 6 Feb 16 02:08 tmp + drwxr-xr-x 1 root root 66 Mar 11 2024 usr + drwxr-xr-x 1 root root 41 Mar 11 2024 var +errors +----------- ------------------------------------------------------------------ +``` + ### Working with objects: ```commandline (venv) [user@host]$ packcli object list diff --git a/packetserver/client/cli/bulletin.py b/packetserver/client/cli/bulletin.py index b66ea04..5ebc7ae 100644 --- a/packetserver/client/cli/bulletin.py +++ b/packetserver/client/cli/bulletin.py @@ -10,7 +10,7 @@ import os.path @click.group() @click.pass_context def bulletin(ctx): - """List and create bulletins on the BBS.""" + """List, create, and delete bulletins on the BBS.""" pass @@ -18,9 +18,10 @@ def bulletin(ctx): @click.argument("subject", type=str) @click.argument("body", type=str) @click.option('--from-file', '-f', is_flag=True, default=False, - help="Get body text from file or stdin ('-').") + help="Get body text from specified file or stdin ('-').") @click.pass_context def post(ctx, subject, body, from_file): + """Post a bulletin with a subject and body text.""" client = ctx.obj['client'] bbs = ctx.obj['bbs'] if subject.strip() == "": @@ -51,6 +52,7 @@ def post(ctx, subject, body, from_file): type=click.Choice(['table', 'json', 'list'], case_sensitive=False)) @click.pass_context def list_bulletin(ctx, number, only_subject, output_format): + """List recent bulletins.""" client = ctx.obj['client'] bbs = ctx.obj['bbs'] if number == 0: @@ -71,6 +73,7 @@ def list_bulletin(ctx, number, only_subject, output_format): type=click.Choice(['table', 'json', 'list'], case_sensitive=False)) @click.pass_context def get(ctx, bid, only_subject, output_format): + """Fetch the bulletin specified by bulletin ID.""" client = ctx.obj['client'] bbs = ctx.obj['bbs'] @@ -81,6 +84,23 @@ def get(ctx, bid, only_subject, output_format): except Exception as e: exit_client(ctx.obj, 2, message=str(e)) + +@click.command() +@click.argument("bid", metavar="", type=int) +@click.pass_context +def delete(ctx, bid): + """Delete the bulletin (that you authored) specified with bulletin id.""" + client = ctx.obj['client'] + bbs = ctx.obj['bbs'] + + try: + delete_bulletin_by_id(client, bbs, bid) + exit_client(ctx.obj, 0) + except Exception as e: + exit_client(ctx.obj, 2, message=str(e)) + + bulletin.add_command(post) bulletin.add_command(list_bulletin, name = "list") bulletin.add_command(get) +bulletin.add_command(delete) diff --git a/packetserver/client/cli/job.py b/packetserver/client/cli/job.py index c3aa9b8..1e58aea 100644 --- a/packetserver/client/cli/job.py +++ b/packetserver/client/cli/job.py @@ -1,7 +1,9 @@ """CLI client for dealing with jobs.""" import os - +import os.path +import json import click +import traceback from persistent.mapping import default from packetserver.client import Client from packetserver.client.jobs import JobSession, get_job_id, get_user_jobs, send_job, send_job_quick, JobWrapper @@ -15,45 +17,130 @@ def job(ctx): pass @click.command() +@click.argument("cmd", nargs=-1) +@click.option('--bash', '-B', is_flag=True, default=False, help="Run command with /bin/bash -c {}") +@click.option('--quick', '-q', is_flag=True, default=False, help="Wait for fast job results in the response.") +@click.option("--database", "-D", is_flag=True, default=False, help="Request copy of user db for job.") +@click.option("--env", '-e', multiple=True, default=[], help="'=' pairs for environment of job.") +@click.option("--file", '-F', multiple=True, default=[], help="Upload given file to sit in job directory.") +@click.option("--output-format", "-f", default="list", help="Print data as table[default], list, or JSON", + type=click.Choice(['table', 'json', 'list'], case_sensitive=False)) +@click.option("--save-copy", "-C", is_flag=True, default=False, help="Save a full copy of each job to fs.") @click.pass_context -def start(ctx): - """Start a job on the BBS server.""" - pass +def start(ctx, bash, quick, database, env, file, cmd, output_format, save_copy): + """Start a job on the BBS server with '$packcli job start [opts] -- '""" + client = ctx.obj['client'] + bbs = ctx.obj['bbs'] + environ = {} + files = {} + for i in env: + split = i.find("=") + if (split == -1) or ((split + 1) >= len(i)): + exit_client(ctx.obj, 4, message=f"'{i}' is invalid env string") + key = i[:split] + val = i[split + 1:] + environ[key] = val + + for f in file: + if not os.path.isfile(f): + exit_client(ctx.obj, 5, message=f"{f} doesn't exit.") + files[f] = open(f,'rb').read() + + if len(environ) == 0: + environ = None + + if len(files) == 0: + files = None + + if bash: + cmd = ['/bin/bash', '-c', ' '.join(cmd)] + else: + cmd = list(cmd) + + save_dir = os.path.join(ctx.obj['directory'], 'job_cache') + if save_copy: + if not os.path.exists(save_dir): + os.mkdir(save_dir) + try: + if quick: + j = send_job_quick(client, bbs, cmd, db=database, env=environ, files=files) + dicts_out = [] + d = j.to_dict(json=True) + if save_copy: + file_path = os.path.join(save_dir, f"job-{j.id}.json") + json.dump(d, open(file_path, 'w')) + d['saved_path'] = file_path + d['artifacts'] = ",".join([x[0] for x in d['artifacts']]) + del d['output_bytes'] + dicts_out.append(d) + exit_client(ctx.obj, 0, message=format_list_dicts(dicts_out, output_format=output_format)) + else: + resp = send_job(client, bbs, cmd, db=database, env=environ, files=files) + exit_client(ctx.obj, 0, message=resp) + except Exception as e: + exit_client(ctx.obj, 40, message=f"Couldn't queue job: {str(e)}") @click.command() -@click.argument('job_id', required=False, type=int) -@click.option("--all-jobs", "-a", is_flag=True, default=False, help="Get all of your jobs.") -@click.option("--no-data", '-n', is_flag=True, default=True, +@click.argument('job_id', required=False, type=int, default=None) +@click.option("--all-jobs", "-A", is_flag=True, default=False, help="Get all of your jobs.") +@click.option("--id-only", "-I", is_flag=True, default=False, help="Only retrieve list of job ids.") +@click.option("--save-copy", "-C", is_flag=True, default=False, help="Save a full copy of each job to fs.") +@click.option("--no-data", '-n', is_flag=True, default=False, help="Don't fetch job result data, just metadata.") +@click.option("--output-format", "-f", default="list", help="Print data as table[default], list, or JSON", + type=click.Choice(['table', 'json', 'list'], case_sensitive=False)) @click.pass_context -def get(ctx, job_id, all_jobs, no_data): # TODO decide what to do with output and artifacts in a cli tool force full JSON? +def get(ctx, job_id, all_jobs, no_data, id_only, save_copy, output_format): """Retrieve your jobs. Pass either '-a' or a job_id.""" fetch_data = not no_data - if job_id is None: - job_id = "" - job_id = job_id.strip() - if all_jobs and (job_id != ""): - click.echo("Can't use --all and specify a job_id.") + + if (job_id is None) and not all_jobs: + exit_client(ctx.obj, 3, message="You must either supply a job id, or --all-jobs") + + if all_jobs and (job_id is not None): + exit_client(ctx.obj, 3, message="Can't use --all and specify a job_id.") + + if job_id is not None and id_only: + exit_client(ctx.obj, 3, message="Can't use --id-only and specify a job_id. You already know it.") + + if save_copy and id_only: + exit_client(ctx.obj, 3, message="Can't use --id-only and save_copy. There's no data to save.") client = ctx.obj['client'] try: if all_jobs: - jobs_out = get_user_jobs(client, ctx.obj['bbs'], get_data=fetch_data) + jobs_out = get_user_jobs(client, ctx.obj['bbs'], get_data=fetch_data, id_only=id_only) else: - jobs_out = [get_job_id(client,ctx.obj['bbs'], get_data=fetch_data)] - dicts_out = [] - for j in jobs_out: - pass + jobs_out = [get_job_id(client,ctx.obj['bbs'], job_id, get_data=fetch_data)] + if id_only: + output = ",".join([str(x) for x in jobs_out]) + if output_format == "json": + output = json.dumps([x for x in jobs_out]) + exit_client(ctx.obj, 0, message=output) + else: + save_dir = os.path.join(ctx.obj['directory'], 'job_cache') + if save_copy: + if not os.path.exists(save_dir): + os.mkdir(save_dir) + dicts_out = [] + for j in jobs_out: + d = j.to_dict(json=True) + if save_copy: + file_path = os.path.join(save_dir, f"job-{j.id}.json") + json.dump(d, open(file_path, 'w')) + d['saved_path'] = file_path + d['artifacts'] = ",".join([x[0] for x in d['artifacts']]) + del d['output_bytes'] + dicts_out.append(d) + exit_client(ctx.obj, 0, message=format_list_dicts(dicts_out, output_format=output_format)) except Exception as e: click.echo(str(e), err=True) exit_client(ctx.obj, 1) - - @click.command() @click.option("--transcript", "-T", default="", help="File to write command transcript to if desired.") @click.pass_context @@ -107,4 +194,6 @@ def quick_session(ctx, transcript): exit_client(ctx.obj, 0) -job.add_command(quick_session) \ No newline at end of file +job.add_command(quick_session) +job.add_command(get) +job.add_command(start) \ No newline at end of file diff --git a/packetserver/client/cli/util.py b/packetserver/client/cli/util.py index c91e40b..238be37 100644 --- a/packetserver/client/cli/util.py +++ b/packetserver/client/cli/util.py @@ -48,6 +48,7 @@ def exit_client(context: dict, return_code: int, message=""): is_err = False else: is_err = True + message = str(message) if message.strip() != "": click.echo(message, err=is_err) sys.exit(return_code) diff --git a/packetserver/client/jobs.py b/packetserver/client/jobs.py index f9aea6e..f6ddc0c 100644 --- a/packetserver/client/jobs.py +++ b/packetserver/client/jobs.py @@ -3,10 +3,12 @@ from packetserver.common import Request, Response, PacketServerConnection from typing import Union, Optional import datetime import time +from base64 import b64encode class JobWrapper: def __init__(self, data: dict): - for i in ['output', 'errors', 'artifacts', 'return_code', 'status']: + for i in ['output', 'errors', 'artifacts', 'return_code', 'status', 'created_at', 'finished_at', 'id', 'cmd', + 'owner']: if i not in data: raise ValueError("Was not given a job dictionary.") self.data = data @@ -72,6 +74,38 @@ class JobWrapper: def id(self) -> int: return self.data['id'] + # ['output', 'errors', 'artifacts', 'return_code', 'status', 'created_at', 'finished_at', 'id', 'cmd', + # 'owner'] + + def to_dict(self, json=True): + d = { + 'id': self.id, + 'return_code': self.return_code, + 'status': self.status, + 'created': self.created, + 'finished': self.finished, + 'cmd': self.cmd, + 'owner': self.owner, + 'artifacts': [], + 'output': self.output_str, + 'output_bytes': self.output_raw, + 'errors': self.errors_str, + } + if json: + d['output_bytes'] = b64encode(self.output_raw).decode() + if self.created is not None: + d['created'] = self.created.isoformat() + if self.finished is not None: + d['finished'] = self.finished.isoformat() + + for a in self.artifacts: + if json: + d['artifacts'].append((a[0], b64encode(a[1]).decode())) + else: + d['artifacts'].append(a) + + return d + def __repr__(self): return f"" @@ -126,17 +160,22 @@ def get_job_id(client: Client, bbs_callsign: str, job_id: int, get_data=True) -> raise RuntimeError(f"GET job {job_id} failed: {response.status_code}: {response.payload}") return JobWrapper(response.payload) -def get_user_jobs(client: Client, bbs_callsign: str, get_data=True) -> list[JobWrapper]: +def get_user_jobs(client: Client, bbs_callsign: str, get_data=True, id_only=False) -> list[Union[JobWrapper,int]]: req = Request.blank() req.path = f"job/user" req.set_var('data', get_data) + if id_only: + req.set_var('id_only', True) req.method = Request.Method.GET response = client.send_receive_callsign(req, bbs_callsign) if response.status_code != 200: raise RuntimeError(f"GET user jobs failed: {response.status_code}: {response.payload}") jobs = [] for j in response.payload: - jobs.append(JobWrapper(j)) + if id_only: + jobs.append(j) + else: + jobs.append(JobWrapper(j)) return jobs class JobSession: diff --git a/packetserver/common/__init__.py b/packetserver/common/__init__.py index f291fa7..88edff7 100644 --- a/packetserver/common/__init__.py +++ b/packetserver/common/__init__.py @@ -111,14 +111,14 @@ class Message: def get_var(self, key: str): if 'v' not in self.data: raise KeyError(f"Variable '{key}' not found.") - if str(key) not in self.data['v']: + if str(key).lower() not in self.data['v']: raise KeyError(f"Variable '{key}' not found.") - return self.data['v'][str(key)] + return self.data['v'][str(key).lower()] def set_var(self, key: str, value): if 'v' not in self.data: self.data['v'] = {} - self.data['v'][str(key)] = value + self.data['v'][str(key).lower()] = value @property def data_bytes(self): diff --git a/packetserver/server/__init__.py b/packetserver/server/__init__.py index 79d7ee5..1c42257 100644 --- a/packetserver/server/__init__.py +++ b/packetserver/server/__init__.py @@ -170,7 +170,7 @@ class Server: logging.debug("Connection marked as closing. Ignoring it.") return req_root_path = req.path.split("/")[0] - if ('quick' in req.vars) or (req_root_path == "job"): + if ('quick' in req.vars) or ((req_root_path == "job") and (req.method == Request.Method.POST)): logging.debug("Setting quick job timer for a quick job.") self.job_check_interval = 8 self.quick_job = True diff --git a/packetserver/server/jobs.py b/packetserver/server/jobs.py index 0cdf6f6..ec86f2c 100644 --- a/packetserver/server/jobs.py +++ b/packetserver/server/jobs.py @@ -226,10 +226,9 @@ def handle_job_get_id(req: Request, conn: PacketServerConnection, db: ZODB.DB, j username = ax25.Address(conn.remote_callsign).call.upper().strip() value = "y" include_data = True - for key in req.vars: - if key.lower().strip() == "data": - value = req.vars[key].lower().strip() - if value in no_values: + data_val = req.vars['data'] + if data_val in no_values: + logging.debug(f"Not including job data per variable setting in request. 'data': {req.vars['data']} ") include_data = False with db.transaction() as storage: @@ -254,7 +253,9 @@ def handle_job_get_user(req: Request, conn: PacketServerConnection, db: ZODB.DB) include_data = True for key in req.vars: if key.lower().strip() == "data": - value = req.vars[key].lower().strip() + value = req.vars[key] + if type(value) is str: + value = value.lower().strip() if value in no_values: include_data = False id_only = False