diff --git a/README.md b/README.md index b608c36..8b627a5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,31 @@ # lpc-scripts scripts of use on the cmslpc cluster +Table of Contents +================= + +* [call_host.sh](#call_hostsh) + * [Note](#note) + * [Usage](#usage) + * [Manual](#manual) + * [Automatic](#automatic) + * [Details](#details) + * [Options](#options) + * [Caveats](#caveats) +* [bind_condor.sh](#bind_condorsh) + * [Usage](#usage-1) + * [Setting up bindings](#setting-up-bindings) +* [tunn](#tunn) + * [Detailed usage](#detailed-usage) + * [Web browser usage](#web-browser-usage) +* [Unit and Integration testing](#unit-and-integration-testing) + * [Automated](#automated) + * [Manual](#manual-1) + * [Bats for Bash scripts](#bats-for-bash-scripts) + * [Pytest for Python modules](#pytest-for-python-modules) + + + ## `call_host.sh` Many commands are installed on interactive nodes but are not accessible inside containers. @@ -189,6 +214,75 @@ In this particular case, it is necessary to upgrade `pip` because the Python ver **NOTE**: These recipes only install the bindings for Python3. (Python2 was still the default in `CMSSW_10_6_X`.) You will need to make sure any scripts using the bindings are compatible with Python3. +## `tunn` + +A simple utility to create and manage SSH tunnels. + +The basic usage of `tunn` follows this pattern: +1. `tunn make user@cmslpc-sl9.fnal.gov` +2. `tunn list`: + ``` + index: socket port command + 0: "/home/[user]/.tsock_xyz" 8XXX "user@cmslpc-sl9.fnal.gov" + ``` +3. `tunn kill 0`: + ``` + Exit request sent. + ``` + +If you have host aliases defined in your `~/.ssh/config` file, you can use them with `tunn`. + +### Detailed usage + +The configuration and command-line options for `tunn` are described in its usage message: +``` +tunn [operation] [options] [arguments] + +Default settings are obtained from the config file at /home/[user]/.tunnconfig. +To override the config file location, put this in your .bashrc or other login file: + export TUNN_CONFIG=/my/preferred/file +The available config variables are: TUNN_PREFIX, TUNN_PORT, TUNN_VERBOSE. +Their values should be specified in the config file using bash syntax, e.g.: + TUNN_PORT=8XXX +(If TUNN_PORT is not specified in the config file or via the command line option, +the default value is taken from the last three digits of your UID.) + +Operations: + +make make new tunnel + -n [name] tunnel socket name prefix (default: /home/[user]/.tsock) + -p [port] tunnel port (default: 8XXX) + [destination] ssh destination for tunnel (required) + +list list open tunnels + +kill kill specified tunnel + [index] index of tunnel (required) + +Common options: +-u (unclean) do not auto-remove closed tunnels from list +-v toggle verbosity (default: false) +-h print this message and exit +``` + +### Web browser usage + +There are addons available for web browsers to route traffic through ssh tunnels. +[FoxyProxy](https://getfoxyproxy.org/downloads/) is recommended for most browsers (Chrome, Firefox, and derivatives; Safari has [equivalent internal settings](https://help.getfoxyproxy.org/index.php/knowledge-base/how-to-use-your-proxy-services-with-safari/).) + +You can add LPC as a proxy server in the "Proxies" tab, with settings as follows: +* Title: LPC +* Hostname: localhost +* Type: SOCKS5 +* Port: [your TUNN_PORT value 8XXX] +* Proxy by patterns: [blank], "Include", "Wildcard", Title: FNAL, Pattern: \*://\*.fnal.gov/ + +
+Firefox example screenshot: + +![Foxyproxy settings screenshot](./docs/foxyproxy_lpc_generic.png) +
+ ## Unit and Integration testing ### Automated diff --git a/docs/foxyproxy_lpc_generic.png b/docs/foxyproxy_lpc_generic.png new file mode 100755 index 0000000..063d489 Binary files /dev/null and b/docs/foxyproxy_lpc_generic.png differ diff --git a/tunn b/tunn new file mode 100755 index 0000000..d269427 --- /dev/null +++ b/tunn @@ -0,0 +1,222 @@ +#!/bin/bash + +# helper functions + +tunn_check_op() { + if [ "$TUNN_OP" != "$1" ]; then + usage 1 + fi +} + +tunn_echo() { + if [ "$TUNN_VERBOSE" == "true" ]; then + echo "$@" + fi +} + +tunn_read_impl() { + # get list of tunnels + readarray -t TUNN_LIST < "$TUNN_LISTFILE" + export TUNN_LIST +} + +tunn_update() { + tunn_read_impl + # clear list of tunnels + : > "$TUNN_LISTFILE" + # write open tunnels back into list + for ((i=0; i < ${#TUNN_LIST[@]}; i++)); do + tunn_entry $i + # shellcheck disable=SC2086 + if ssh $TUNN_DEST -S "$TUNN_SOCKETNAME" -O check >&/dev/null; then + echo "${TUNN_LIST[$i]}" >> "$TUNN_LISTFILE" + else + tunn_echo "Removing dead tunnel: ${TUNN_LIST[$i]}" + rm "$TUNN_SOCKETNAME" >&/dev/null + fi + done + unset TUNN_LIST +} + +tunn_read() { + if [ -z "$TUNN_UNCLEAN" ]; then + tunn_read_impl + tunn_update + fi + tunn_read_impl +} + +tunn_entry() { + ENTRY="${TUNN_LIST[$1]}" + # split, respecting quotes + declare -a "ENTRY_ARR=($ENTRY)" + + # reconstruct command based on output format + export TUNN_SOCKETNAME="${ENTRY_ARR[0]}" + export TUNN_PORT="${ENTRY_ARR[1]}" + export TUNN_DEST="${ENTRY_ARR[2]}" +} + +# operations + +tunn_make() { + TUNN_DEST="$*" + if [ -z "$TUNN_DEST" ]; then + usage 1 + fi + + # generate a random name for socket + TUNN_SOCKETNUM=$(uuidgen) + TUNN_SOCKETNAME="$TUNN_PREFIX"_"$TUNN_SOCKETNUM" + + # execute tunnel command + # shellcheck disable=SC2086 + ssh $TUNN_DEST -N -D "$TUNN_PORT" -f -M -S "$TUNN_SOCKETNAME" 2>/dev/null + TUNN_EXIT=$? + + # add to list of tunnels + if [ $TUNN_EXIT -eq 0 ]; then + echo "\"$TUNN_SOCKETNAME\" $TUNN_PORT \"$TUNN_DEST\"" >> "$TUNN_LISTFILE" + tunn_echo "Tunnel ready, port: $TUNN_PORT" + else + echo "Could not create tunnel (exit code $TUNN_EXIT)" + exit $TUNN_EXIT + fi +} + +tunn_list() { + if [ -n "$1" ]; then + usage 1 + fi + + tunn_read + if [ ${#TUNN_LIST[@]} -eq 0 ]; then + return + fi + echo "index: socket port destination" + for ((i=0; i < ${#TUNN_LIST[@]}; i++)); do + echo "$i: ${TUNN_LIST[$i]}" + done +} + +tunn_kill() { + INDEX="$1" + if [ -z "$INDEX" ]; then + usage 1 + fi + + # get this tunnel + tunn_read + if [ "$INDEX" -lt ${#TUNN_LIST[@]} ]; then + tunn_entry "$INDEX" + else + echo "Tunnel index $INDEX not found" + exit 2 + fi + + # execute kill command + # shellcheck disable=SC2086 + ssh $TUNN_DEST -S "$TUNN_SOCKETNAME" -O exit + tunn_echo "Killed tunnel: ${TUNN_LIST[$INDEX]}" + + # update list + sed -i "$((INDEX+1))"d "$TUNN_LISTFILE" +} + +# defaults +TUNN_LISTFILE=~/.tunnlist +: "${TUNN_CONFIG:=~/.tunnconfig}" +TUNN_SOCKETNAME="" +TUNN_DEST="" +TUNN_UNCLEAN="" + +# get config defaults +if [ -e "$TUNN_CONFIG" ]; then + # shellcheck source=/dev/null + source "$TUNN_CONFIG" +fi +: "${TUNN_PREFIX:=~/.tsock}" +UTMP=$(id -u) +UTMP=${UTMP:0-3} +: "${TUNN_PORT:=8${UTMP}}" +: "${TUNN_VERBOSE:=false}" + +declare -A TUNN_INVERT +TUNN_INVERT[true]=false +TUNN_INVERT[false]=true + +usage() { + ECHO="echo -e" + $ECHO "tunn [operation] [options] [arguments]" + $ECHO + $ECHO "Default settings are obtained from the config file at ${TUNN_CONFIG}." + $ECHO "To override the config file location, put this in your .bashrc or other login file:" + $ECHO "\texport TUNN_CONFIG=/my/preferred/file" + $ECHO "The available config variables are: TUNN_PREFIX, TUNN_PORT, TUNN_VERBOSE." + $ECHO "Their values should be specified in the config file using bash syntax, e.g.:" + $ECHO "\tTUNN_PORT=8XXX" + $ECHO "(If TUNN_PORT is not specified in the config file or via the command line option," + $ECHO "the default value is taken from the last three digits of your UID.)" + $ECHO + $ECHO "Operations:" + $ECHO + $ECHO "make \t make new tunnel" + $ECHO "\t-n [name] \t tunnel socket name prefix (default: ${TUNN_PREFIX})" + $ECHO "\t-p [port] \t tunnel port (default: $TUNN_PORT)" + $ECHO "\t[destination] \t ssh destination for tunnel (required)" + $ECHO + $ECHO "list \t list open tunnels" + $ECHO + $ECHO "kill \t kill specified tunnel" + $ECHO "\t[index] \t index of tunnel (required)" + $ECHO + $ECHO "Common options:" + $ECHO "-u \t (unclean) do not auto-remove closed tunnels from list" + $ECHO "-v \t toggle verbosity (default: $TUNN_VERBOSE)" + $ECHO "-h \t print this message and exit" + exit "$1" +} + +# get operation +TUNN_OP="$1" +shift 1 + +TUNN_OPFN="" +case "$TUNN_OP" in + make) TUNN_OPFN=tunn_make + ;; + list) TUNN_OPFN=tunn_list + ;; + kill) TUNN_OPFN=tunn_kill + ;; + *) usage 1 + ;; +esac + +while getopts "n:p:uvh" opt; do + case "$opt" in + n) tunn_check_op make; TUNN_PREFIX="$OPTARG" + ;; + p) tunn_check_op make; TUNN_PORT="$OPTARG" + ;; + u) TUNN_UNCLEAN=true + ;; + v) TUNN_VERBOSE="${TUNN_INVERT[$TUNN_VERBOSE]}" + ;; + h) usage 0 + ;; + *) usage 1 + ;; + esac +done + +# ensure list file exists +if [ ! -e "$TUNN_LISTFILE" ]; then + touch "$TUNN_LISTFILE" +fi + +# get args for operation (if any) +shift $((OPTIND - 1)) + +# execute operation +$TUNN_OPFN "$@"