diff --git a/README.md b/README.md index 4c8ac0a..e1ad9b0 100644 --- a/README.md +++ b/README.md @@ -1 +1,139 @@ -RDA python package to hold miscellaneous utility programs. +# rda_python_miscs + +RDA Python package to hold miscellaneous utility programs for the +[NSF NCAR Geoscience Data Exchange (GDEX)](https://gdex.ucar.edu). + +## Programs + +The package provides two categories of programs: + +**Run as current user (no setuid required):** + +| Command | Alias | Description | +|---------|-------|-------------| +| `bashqsub` | | Submit a job as a bash batch job on a PBS node via qsub | +| `tcshqsub` | | Submit a job as a tcsh batch job on a PBS node via qsub | +| `rdasub` | `gdexsub` | Submit a command as a nohup background process on the local machine | +| `pgwget` | | Download remote files by root name pattern, combining downloaded parts into a single file | +| `gdexls` | | List local files and directories with matching metadata from the GDEX database | +| `rdaps` | `gdexps` | Show process status for local or PBS batch processes, with filtering by PID, owner, or name | +| `rdazip` | `gdexzip` | Compress or uncompress files using a supported format | +| `rdaown` | `gdexown` | Change ownership of files and directories to rdadata (must be run as root) | +| `pgrst` | | Convert .usg files to RST and push to gdex-docs-* repos on GitHub for readthedocs.io | + +**Run as gdexdata via setuid (requires setup below):** + +| Command | Alias | Connector script | Description | +|---------|-------|-----------------|-------------| +| `rdacp` | `gdexcp` | `setuid_rdacp` / `setuid_gdexcp` | Copy files and directories across local, remote, Object Store, or Globus endpoints | +| `rdakill` | `gdexkill` | `setuid_rdakill` / `setuid_gdexkill` | Kill local processes and their children, or cancel PBS batch jobs | +| `rdamod` | `gdexmod` | `setuid_rdamod` / `setuid_gdexmod` | Change permission modes for files and directories owned by rdadata | + +## Environment setup + +Create a Python environment first; package installs in the next section run +inside whichever environment you activate here. + +### Option A — Python venv (DECS machines) + +```bash +python3 -m venv $ENVHOME # e.g. /glade/u/home/gdexdata/gdexmsenv +source $ENVHOME/bin/activate +``` + +### Option B — Conda (DAV/Casper) + +```bash +conda create --prefix $ENVHOME python=3.12 # e.g. /glade/work/gdexdata/conda-envs/pg-gdex +conda activate $ENVHOME +``` + +## Installing rda-python-miscs + +Pick whichever install mode fits your workflow. All four pull in the +transitive dependencies (`rda_python_common`, `rda_python_setuid`) +automatically. + +For local development, clone this repo alongside your project and install it +in editable mode so that changes are picked up without re-installing: + +```bash +git clone https://github.com/NCAR/rda-python-miscs.git +cd rda-python-miscs +pip install -e . +``` + +To test a specific branch (e.g. an in-progress feature or fix branch), pass +`-b/--branch` to `git clone`: + +```bash +git clone -b https://github.com/NCAR/rda-python-miscs.git +cd rda-python-miscs +pip install -e . +``` + +For a regular (non-editable) install from a checkout: + +```bash +pip install /path/to/rda-python-miscs +``` + +For a production install on a system that uses the published distribution: + +```bash +pip install rda_python_miscs +``` + +## Setuid Setup + +The setuid programs (`rdacp`, `rdakill`, `rdamod` and their `gdex*` aliases) +execute as the common user `PGLOG['COMMONUSER']` (default `gdexdata`) via +the `rda_python_setuid` mechanism, which is pulled in automatically as a +dependency. After `pip install` above, choose one of the wiring options +below. + +### Full setuid install (requires sudo access to COMMONUSER) + +Run these steps once per environment: + +```bash +# 1. Compile the pywrapper C binary (once per environment): +pywrapper-install -c|--compile -n|--username gdexdata + +# 2. Wire up all setuid programs in one pass: +pywrapper-install -l|--link all + +# 3. Optionally, install a pgstart_ binary so (any +# user in the same group as PGLOG['COMMONUSER']) can run commands as +# themselves. Run either by PGLOG['ADMINUSER'] (default zji, if it has +# 'sudo -u '), or by directly: +pywrapper-install -p|--pgstart -n|--username +``` + +`pywrapper-install` with no arguments displays the full user guide. + +### Simple install (no sudo required, runs as current user) + +Users who do not need the setuid mechanism can create direct symlinks instead: + +```bash +pywrapper-install -l|--link all -s|--simple +``` + +This creates `bin/ -> bin/setuid_` for every setuid program and +they run as the current user with no privilege change. + +### Update an existing installation (no sudo required) + +When the package is upgraded and a new `pywrapper.c` is bundled, recompile and +reinstall all setuid binaries using the existing `pgstart_*` binaries: + +```bash +pywrapper-install -u|--update +``` + +### Setup guide + +The shared setuid setup guide is shown automatically if any `setuid_*` +connector script is invoked directly before the setuid wrapper has been +configured. diff --git a/pyproject.toml b/pyproject.toml index 3913d41..9daf974 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "rda_python_miscs" -version = "2.0.16" +version = "3.0.0" authors = [ { name="Zaihua Ji", email="zji@ucar.edu" }, ] @@ -38,7 +38,6 @@ pythonpath = [ "rdasub" = "rda_python_miscs.rdasub:main" "gdexsub" = "rda_python_miscs.rdasub:main" "pgwget" = "rda_python_miscs.pgwget:main" -"rdals" = "rda_python_miscs.rdals:main" "gdexls" = "rda_python_miscs.gdexls:main" "rdaps" = "rda_python_miscs.rdaps:main" "gdexps" = "rda_python_miscs.rdaps:main" @@ -46,10 +45,10 @@ pythonpath = [ "gdexzip" = "rda_python_miscs.rdazip:main" "rdaown" = "rda_python_miscs.rdaown:main" "gdexown" = "rda_python_miscs.rdaown:main" -"rdacp.py" = "rda_python_miscs.rdacp:main" -"gdexcp.py" = "rda_python_miscs.rdacp:main" -"rdakill.py" = "rda_python_miscs.rdakill:main" -"gdexkill.py" = "rda_python_miscs.rdakill:main" -"rdamod.py" = "rda_python_miscs.rdamod:main" -"gdexmod.py" = "rda_python_miscs.rdamod:main" +"setuid_rdacp" = "rda_python_miscs.rdacp:main" +"setuid_gdexcp" = "rda_python_miscs.rdacp:main" +"setuid_rdakill" = "rda_python_miscs.rdakill:main" +"setuid_gdexkill" = "rda_python_miscs.rdakill:main" +"setuid_rdamod" = "rda_python_miscs.rdamod:main" +"setuid_gdexmod" = "rda_python_miscs.rdamod:main" "pgrst" = "rda_python_miscs.pg_rst:main" diff --git a/src/rda_python_miscs/pg_rst.py b/src/rda_python_miscs/pg_rst.py index dd7934d..164fc6d 100755 --- a/src/rda_python_miscs/pg_rst.py +++ b/src/rda_python_miscs/pg_rst.py @@ -1237,6 +1237,11 @@ def load_opts_alias_from_pyfile(self, pyfile): def main(): """Entry point for command-line usage of pg_rst.py.""" + import sys + if len(sys.argv) == 1 or any(a in sys.argv[1:] for a in ('-h', '--help', '-?')): + pg = PgRST() + pg.show_usage("pg_rst") + parser = argparse.ArgumentParser( description=( "Convert a .usg help document to reStructuredText (.rst) using RST templates. " diff --git a/src/rda_python_miscs/pg_rst.usg b/src/rda_python_miscs/pg_rst.usg new file mode 100644 index 0000000..6e45298 --- /dev/null +++ b/src/rda_python_miscs/pg_rst.usg @@ -0,0 +1,60 @@ + Convert a text-based program usage document (.usg file) into reStructuredText + (.rst) files using RST template files, for publication to readthedocs.io via + a gdex-docs-* GitHub repository. + + OPTS and ALIAS are loaded from rda_python_/.py (or from + --pyfile if given): the module is searched first for a class that carries + both as class attributes, then for module-level OPTS/ALIAS variables. + + Usage: pgrst [docname] [-u FILE] [-p FILE] [-d DIR] [-h] + + - docname + Short document name, e.g. 'dsarch' or 'dsupdt'. Required unless + --usgfile is given, in which case the name is derived from the .usg + filename by removing the extension. + + - Option -u or --usgfile FILE + Path to the .usg source document. When given, docname is derived + from the filename by removing the .usg extension, and the source + directory is set to the directory containing the file. + + - Option -p or --pyfile FILE + Path to a Python file that defines OPTS (and optionally ALIAS) either + at module level or as class attributes. When given, the default + module-import convention (rda_python_/.py) is + bypassed. + + - Option -d or --docdir DIR + Root directory under which the per-document RST output is written. + Defaults to the current working directory. + + - Option -h, display this help document. + + The .usg source document must be structured with a summary paragraph at the + top, followed by option descriptions and an examples section. OPTS defines + the option types (mode, single-value, multi-value, or action) used to + categorise each option in the RST output. + + Output files are written to DOCDIR using RST template files bundled with + this package under rda_python_miscs/rst_templates/. + + Examples: + + 1. Convert dsarch.usg to RST using the default module-import convention. + OPTS and ALIAS are loaded from rda_python_dsarch/dsarch.py. + RST output is written under the current directory: + + pgrst dsarch + + 2. Convert dsarch.usg from a specific path, writing RST to /tmp/docs/: + + pgrst dsarch -u /path/to/dsarch.usg -d /tmp/docs/ + + 3. Convert using a custom Python file for OPTS/ALIAS instead of the + installed package module: + + pgrst dsarch -p /path/to/dsarch.py + + 4. Derive the document name from the .usg filename (no positional arg): + + pgrst -u /path/to/dsupdt.usg -d /tmp/docs/ diff --git a/src/rda_python_miscs/rdacp.py b/src/rda_python_miscs/rdacp.py index e227e80..7f9eb09 100644 --- a/src/rda_python_miscs/rdacp.py +++ b/src/rda_python_miscs/rdacp.py @@ -233,7 +233,9 @@ def copy_file(self, fromfile, isfile): # main function to execute this script def main(): """Entry point: instantiate RdaCp, parse arguments, run, and exit.""" + from rda_python_setuid.setup_guide import show_setup_guide object = RdaCp() + show_setup_guide(object, 'rda_python_miscs', ['rdacp', 'rdakill', 'rdamod']) object.read_parameters() object.start_actions() object.pgexit(0) diff --git a/src/rda_python_miscs/rdakill.py b/src/rda_python_miscs/rdakill.py index b8da4a4..a7fd231 100644 --- a/src/rda_python_miscs/rdakill.py +++ b/src/rda_python_miscs/rdakill.py @@ -258,7 +258,9 @@ def record_dscheck_interrupt(self, pid, host): # main function to execute this script def main(): """Entry point: instantiate RdaKill, parse arguments, run, and exit.""" + from rda_python_setuid.setup_guide import show_setup_guide object = RdaKill() + show_setup_guide(object, 'rda_python_miscs', ['rdacp', 'rdakill', 'rdamod']) object.read_parameters() object.start_actions() object.pgexit(0) diff --git a/src/rda_python_miscs/rdamod.py b/src/rda_python_miscs/rdamod.py index 257a815..c668da1 100644 --- a/src/rda_python_miscs/rdamod.py +++ b/src/rda_python_miscs/rdamod.py @@ -194,7 +194,9 @@ def change_mode(self, file, info): # main function to execute this script def main(): """Entry point: instantiate RdaMod, parse arguments, run, and exit.""" + from rda_python_setuid.setup_guide import show_setup_guide object = RdaMod() + show_setup_guide(object, 'rda_python_miscs', ['rdacp', 'rdakill', 'rdamod']) object.read_parameters() object.start_actions() object.pgexit(0)