From 125330b5a5be093ec722e359e4d240355591f3c5 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Sun, 18 Jun 2017 02:22:43 +0900 Subject: [PATCH 01/21] Implement a local ssh proxy to access servers in remote private networks --- .gitignore | 1 + geofrontcli/cli.py | 44 +++++------ geofrontcli/client.py | 18 ++++- geofrontcli/proxy.py | 179 ++++++++++++++++++++++++++++++++++++++++++ setup.py | 9 +++ 5 files changed, 226 insertions(+), 25 deletions(-) create mode 100644 geofrontcli/proxy.py diff --git a/.gitignore b/.gitignore index 13f8832..a08c5ba 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ .tox build dist +venv diff --git a/geofrontcli/cli.py b/geofrontcli/cli.py index 9bb6db9..8fd453f 100644 --- a/geofrontcli/cli.py +++ b/geofrontcli/cli.py @@ -261,9 +261,9 @@ def authorize(args, alias=None): ) -def get_ssh_options(remote): +def mangle_ssh_args(remote): """Translate the given ``remote`` to a corresponding :program:`ssh` - options. For example, it returns the following list for ``'user@host'``:: + arguments. For example, it returns the following list for ``'user@host'``:: ['-l', 'user', 'host'] @@ -273,18 +273,11 @@ def get_ssh_options(remote): ['-p', '22', 'host'] """ - remote_match = REMOTE_PATTERN.match(remote) - if not remote_match: - raise ValueError('invalid remote format: ' + str(remote)) - options = [] - user = remote_match.group('user') - if user: - options.extend(['-l', user]) - port = remote_match.group('port') - if port: - options.extend(['-p', port]) - options.append(remote_match.group('host')) - return options + return [ + '-l', remote['user'], + '-p', str(remote['port']), + remote['host'], + ] @subparser @@ -296,14 +289,10 @@ def colonize(args): """ client = get_client() remote = client.remotes.get(args.remote, args.remote) - try: - options = get_ssh_options(remote) - except ValueError as e: - colonize.error(str(e)) cmd = [args.ssh] if args.identity_file: cmd.extend(['-i', args.identity_file]) - cmd.extend(options) + cmd.extend(mangle_ssh_args(remote)) cmd.extend([ 'mkdir', '~/.ssh', '&>', '/dev/null', '||', 'true', ';', 'echo', repr(str(client.master_key)), @@ -324,15 +313,22 @@ def colonize(args): @subparser def ssh(args, alias=None): """SSH to the remote through Geofront's temporary authorization.""" + if args.proxy and sys.version_info < (3, 6): + logger.error('To use the SSH proxy, you need to run geofront-cli on ' + 'Python 3.6 or higher.', + extra={'user_waiting': False}) remote = authorize.call(args, alias=alias) - try: - options = get_ssh_options(remote) - except ValueError as e: - ssh.error(str(e)) - subprocess.call([args.ssh] + options) + if args.proxy: + client = get_client() + client.ssh_proxy(remote, args.ssh, alias or args.remote) + else: + subprocess.call([args.ssh] + mangle_ssh_args(remote)) ssh.add_argument('remote', help='the remote alias to ssh') +ssh.add_argument('-p', '--proxy', action='store_true', default=False, + help='use a proxy tunneled via HTTPS to ssh into servers ' + 'inside remote private networks') def parse_scp_path(path, args): diff --git a/geofrontcli/client.py b/geofrontcli/client.py index 95af5d2..1284266 100644 --- a/geofrontcli/client.py +++ b/geofrontcli/client.py @@ -20,6 +20,8 @@ from .key import PublicKey from .ssl import create_urllib_https_handler from .version import MIN_PROTOCOL_VERSION, MAX_PROTOCOL_VERSION, VERSION +if sys.version_info >= (3, 6): # pragma: no cover + from .proxy import start_ssh_proxy __all__ = ('REMOTE_PATTERN', 'BufferedResponse', 'Client', 'ExpiredTokenIdError', @@ -235,7 +237,21 @@ def authorize(self, alias): logger.info('Access to %s has authorized! The access will be ' 'available only for a time.', alias, extra={'user_waiting': False}) - return '{0[user]}@{0[host]}:{0[port]}'.format(result['remote']) + #return '{0[user]}@{0[host]}:{0[port]}'.format(result['remote']) + return result['remote'] + + if sys.version_info >= (3, 6): # pragma: no cover + def ssh_proxy(self, remote, ssh_executable, alias): + logger = self.logger.getChild('ssh_proxy') + try: + path = ('ws', 'tokens', self.token_id, 'remotes', alias, 'ssh') + url = './{0}/'.format('/'.join(path)) + url = urljoin(self.server_url, url) + except TokenIdError: + logger.info('Authentication is required.', + extra={'user_waiting': False}) + raise + start_ssh_proxy(url, remote, ssh_executable) def __repr__(self): return '{0.__module__}.{0.__name__}({1!r})'.format( diff --git a/geofrontcli/proxy.py b/geofrontcli/proxy.py new file mode 100644 index 0000000..1ac1a2c --- /dev/null +++ b/geofrontcli/proxy.py @@ -0,0 +1,179 @@ +""":mod:`geofrontcli.proxy` --- Local SSH proxy over HTTPS/WebSocket +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +""" +import asyncio +from contextlib import closing +import csv +import logging +from pathlib import Path +import socket +import sys +import traceback + +import aiohttp +import aiotools +from dirspec.basedir import load_config_paths, save_config_path + +from .version import VERSION + +__all__ = ('start_ssh_proxy', ) + +CONFIG_RESOURCE = 'geofront-cli' +PROXY_PORT_MAP_FILENAME = 'proxyports.csv' + +logger = logging.getLogger(__name__) + + +def get_unused_port(): + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as temp_sock: + temp_sock.bind(('localhost', 0)) + return temp_sock.getsockname()[1] + + +def load_proxy_port_map(): + data = dict() + for path in load_config_paths(CONFIG_RESOURCE): + path = Path(path.decode()) / PROXY_PORT_MAP_FILENAME + if path.is_file(): + with open(path) as f: + for row in csv.reader(f): + data[row[0]] = int(row[1]) + return data + + +def save_proxy_port_map(data): + config_path = Path(save_config_path(CONFIG_RESOURCE).decode()) \ + / PROXY_PORT_MAP_FILENAME + with open(config_path, 'w') as f: + writer = csv.writer(f) + for key, val in data.items(): + writer.writerow((key, val)) + logger.info(f'To change/delete port-host mapping, check out {config_path}.', + extra={'user_waiting': False}) + + +def get_port_for_remote(host): + data = load_proxy_port_map() + if host in data: + return data[host] + else: + port = get_unused_port() + data[host] = port + logger.info(f'Mapped port {port} with host {host}.', + extra={'user_waiting': False}) + save_proxy_port_map(data) + return port + + +async def pipe(url, remote, ssh_executable): + loop = asyncio.get_event_loop() + headers = { + 'User-Agent': 'geofront-cli/{0} (Python-asyncio/{1})'.format( + VERSION, sys.version[:3] + ), + } + + async def handle_ssh_sock(ws, ssh_sock): + while True: + try: + data = await loop.sock_recv(ssh_sock, 4096) + except asyncio.CancelledError: + break + if not data: + break + ws.send_bytes(data) + + async def handle_subproc(cmd, pipe_task): + try: + proc = await asyncio.create_subprocess_exec( + *cmd, + stdin=None, stdout=None, stderr=None, # inherit + close_fds=True, + ) + await proc.wait() + pipe_task.cancel() # signal to terminate + except: + logger.error('Unexpected error!', extra={'user_waiting': False}) + traceback.print_exc() + + local_sock = None + ssh_sock = None + ssh_reader_task = None + subproc_task = None + + logger.info(f"Making a local SSH proxy to {remote['host']}...", + extra={'user_waiting': True}) + + # TODO: response header version check? + session = aiohttp.ClientSession() + try: + sock_type = socket.SOCK_STREAM + if hasattr(socket, 'SOCK_NONBLOCK'): # only for Linux + sock_type |= socket.SOCK_NONBLOCK + local_sock = socket.socket(socket.AF_INET, sock_type) + local_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + bind_port = get_port_for_remote(f"{remote['host']}:{remote['port']}") + try: + local_sock.bind(('localhost', bind_port)) + except OSError: + logger.error(f'Cannot bind to port {bind_port}!', extra={'user_waiting': False}) + return + local_sock.listen(1) + logger.info(f'Connecting to local SSH proxy at port {bind_port}...', + extra={'user_waiting': False}) + async with session.ws_connect(url, headers=headers) as ws: + cmd = [ + ssh_executable, + '-l', remote['user'], + '-p', str(bind_port), + 'localhost', + ] + subproc_task = loop.create_task(handle_subproc(cmd, asyncio.Task.current_task())) + await asyncio.sleep(0) # required! + ssh_sock, _ = await loop.sock_accept(local_sock) + ssh_sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + local_sock.close() # used only once + ssh_reader_task = loop.create_task(handle_ssh_sock(ws, ssh_sock)) + async for msg in ws: + if msg.type == aiohttp.WSMsgType.BINARY: + await loop.sock_sendall(ssh_sock, msg.data) + elif msg.type == aiohttp.WSMsgType.CLOSED: + break + elif msg.type == aiohttp.WSMsgType.ERROR: + logger.error('Server disconnected unexpectedly.', + extra={'user_waiting': False}) + break + except aiohttp.ClientError: + logger.error('Connection error!', extra={'user_waiting': False}) + raise + except asyncio.CancelledError: + pass + except: + logger.error('Unexpected error!', extra={'user_waiting': False}) + traceback.print_exc() + finally: + if subproc_task and not subproc_task.done(): + subproc_task.cancel() + await subproc_task + if ssh_reader_task and not ssh_reader_task.done(): + ssh_reader_task.cancel() + await ssh_reader_task + if ssh_sock: + ssh_sock.close() + session.close() + loop.stop() + +@aiotools.actxmgr +async def serve_proxy(loop, pidx, args): + pipe_task = None + try: + pipe_task = loop.create_task(pipe(*args)) + yield + finally: + if pipe_task and not pipe_task.done(): + pipe_task.cancel() + await pipe_task + +def start_ssh_proxy(url, remote, ssh_executable): + aiotools.start_server(serve_proxy, args=(url, remote, ssh_executable), num_proc=1) diff --git a/setup.py b/setup.py index 30c3e37..b9e4a9c 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,11 @@ def readme(): 'enum34', } +py36_or_higher_requires = { + 'aiohttp ~= 2.1.0', + 'aiotools >= 0.3', +} + win32_requires = { 'pypiwin32', } @@ -39,6 +44,9 @@ def readme(): if sys.version_info < (3, 4): install_requires.update(below_py34_requires) +if sys.version_info >= (3, 6): + install_requires.update(py36_or_higher_requires) + if sys.platform == 'win32': install_requires.update(win32_requires) @@ -63,6 +71,7 @@ def readme(): install_requires=list(install_requires), extras_require={ ":python_version<'3.4'": list(below_py34_requires), + ":python_version>='3.6'": list(py36_or_higher_requires), ":sys_platform=='win32'": list(win32_requires), }, classifiers=[ From bedda491aa36138db5eea7698f77818571736850 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Sun, 18 Jun 2017 02:28:06 +0900 Subject: [PATCH 02/21] Comments and clean up. --- geofrontcli/proxy.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/geofrontcli/proxy.py b/geofrontcli/proxy.py index 1ac1a2c..335bcad 100644 --- a/geofrontcli/proxy.py +++ b/geofrontcli/proxy.py @@ -19,6 +19,7 @@ __all__ = ('start_ssh_proxy', ) + CONFIG_RESOURCE = 'geofront-cli' PROXY_PORT_MAP_FILENAME = 'proxyports.csv' @@ -34,7 +35,7 @@ def get_unused_port(): def load_proxy_port_map(): data = dict() for path in load_config_paths(CONFIG_RESOURCE): - path = Path(path.decode()) / PROXY_PORT_MAP_FILENAME + path = Path(path.decode()) / PROXY_PORT_MAP_FILENAME if path.is_file(): with open(path) as f: for row in csv.reader(f): @@ -67,6 +68,7 @@ def get_port_for_remote(host): async def pipe(url, remote, ssh_executable): + """The main duplex pipe task that proxies the incoming SSH traffic via WebSockets.""" loop = asyncio.get_event_loop() headers = { 'User-Agent': 'geofront-cli/{0} (Python-asyncio/{1})'.format( @@ -75,6 +77,7 @@ async def pipe(url, remote, ssh_executable): } async def handle_ssh_sock(ws, ssh_sock): + """A sub-task that proxies the outgoing SSH traffic via WebSocket.""" while True: try: data = await loop.sock_recv(ssh_sock, 4096) @@ -85,6 +88,7 @@ async def handle_ssh_sock(ws, ssh_sock): ws.send_bytes(data) async def handle_subproc(cmd, pipe_task): + """Launch the local SSH agent and wait until it terminates.""" try: proc = await asyncio.create_subprocess_exec( *cmd, @@ -164,8 +168,10 @@ async def handle_subproc(cmd, pipe_task): session.close() loop.stop() + @aiotools.actxmgr async def serve_proxy(loop, pidx, args): + """The initialize and shtudown routines for the local SSH proxy.""" pipe_task = None try: pipe_task = loop.create_task(pipe(*args)) @@ -175,5 +181,6 @@ async def serve_proxy(loop, pidx, args): pipe_task.cancel() await pipe_task + def start_ssh_proxy(url, remote, ssh_executable): aiotools.start_server(serve_proxy, args=(url, remote, ssh_executable), num_proc=1) From c23742b873c95f224e225e309c651eb535935bd8 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Sun, 18 Jun 2017 02:42:06 +0900 Subject: [PATCH 03/21] Fix flake8 errors --- geofrontcli/cli.py | 12 ++---------- geofrontcli/client.py | 1 - geofrontcli/proxy.py | 20 ++++++++++++-------- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/geofrontcli/cli.py b/geofrontcli/cli.py index 8fd453f..3090401 100644 --- a/geofrontcli/cli.py +++ b/geofrontcli/cli.py @@ -263,16 +263,7 @@ def authorize(args, alias=None): def mangle_ssh_args(remote): """Translate the given ``remote`` to a corresponding :program:`ssh` - arguments. For example, it returns the following list for ``'user@host'``:: - - ['-l', 'user', 'host'] - - The remote can contain the port number or omit the user login as well - e.g. ``'host:22'``:: - - ['-p', '22', 'host'] - - """ + arguments including the login name and the port number explicitly.""" return [ '-l', remote['user'], '-p', str(remote['port']), @@ -313,6 +304,7 @@ def colonize(args): @subparser def ssh(args, alias=None): """SSH to the remote through Geofront's temporary authorization.""" + logger = logging.getLogger('geofrontcli') if args.proxy and sys.version_info < (3, 6): logger.error('To use the SSH proxy, you need to run geofront-cli on ' 'Python 3.6 or higher.', diff --git a/geofrontcli/client.py b/geofrontcli/client.py index 1284266..4373e29 100644 --- a/geofrontcli/client.py +++ b/geofrontcli/client.py @@ -237,7 +237,6 @@ def authorize(self, alias): logger.info('Access to %s has authorized! The access will be ' 'available only for a time.', alias, extra={'user_waiting': False}) - #return '{0[user]}@{0[host]}:{0[port]}'.format(result['remote']) return result['remote'] if sys.version_info >= (3, 6): # pragma: no cover diff --git a/geofrontcli/proxy.py b/geofrontcli/proxy.py index 335bcad..d0b73fd 100644 --- a/geofrontcli/proxy.py +++ b/geofrontcli/proxy.py @@ -27,9 +27,9 @@ def get_unused_port(): - with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as temp_sock: - temp_sock.bind(('localhost', 0)) - return temp_sock.getsockname()[1] + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + s.bind(('localhost', 0)) + return s.getsockname()[1] def load_proxy_port_map(): @@ -50,7 +50,7 @@ def save_proxy_port_map(data): writer = csv.writer(f) for key, val in data.items(): writer.writerow((key, val)) - logger.info(f'To change/delete port-host mapping, check out {config_path}.', + logger.info(f'To modify port-host mapping, check out {config_path}.', extra={'user_waiting': False}) @@ -68,7 +68,7 @@ def get_port_for_remote(host): async def pipe(url, remote, ssh_executable): - """The main duplex pipe task that proxies the incoming SSH traffic via WebSockets.""" + """The main task that proxies the incoming SSH traffic via WebSockets.""" loop = asyncio.get_event_loop() headers = { 'User-Agent': 'geofront-cli/{0} (Python-asyncio/{1})'.format( @@ -121,7 +121,8 @@ async def handle_subproc(cmd, pipe_task): try: local_sock.bind(('localhost', bind_port)) except OSError: - logger.error(f'Cannot bind to port {bind_port}!', extra={'user_waiting': False}) + logger.error(f'Cannot bind to port {bind_port}!', + extra={'user_waiting': False}) return local_sock.listen(1) logger.info(f'Connecting to local SSH proxy at port {bind_port}...', @@ -133,7 +134,8 @@ async def handle_subproc(cmd, pipe_task): '-p', str(bind_port), 'localhost', ] - subproc_task = loop.create_task(handle_subproc(cmd, asyncio.Task.current_task())) + subproc_task = loop.create_task( + handle_subproc(cmd, asyncio.Task.current_task())) await asyncio.sleep(0) # required! ssh_sock, _ = await loop.sock_accept(local_sock) ssh_sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) @@ -183,4 +185,6 @@ async def serve_proxy(loop, pidx, args): def start_ssh_proxy(url, remote, ssh_executable): - aiotools.start_server(serve_proxy, args=(url, remote, ssh_executable), num_proc=1) + aiotools.start_server(serve_proxy, + args=(url, remote, ssh_executable), + num_proc=1) From 917f2e4bf556ad212b3ee66b94c2273354ace53b Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Sun, 18 Jun 2017 02:52:11 +0900 Subject: [PATCH 04/21] Conform with Spoqa's import style rules --- geofrontcli/proxy.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/geofrontcli/proxy.py b/geofrontcli/proxy.py index d0b73fd..3b91d3b 100644 --- a/geofrontcli/proxy.py +++ b/geofrontcli/proxy.py @@ -3,16 +3,16 @@ """ import asyncio -from contextlib import closing +import contextlib import csv import logging -from pathlib import Path +import pathlib import socket import sys import traceback -import aiohttp -import aiotools +from aiohttp import ClientError, ClientSession, WSMsgType +from aiotools import actxmgr, start_server from dirspec.basedir import load_config_paths, save_config_path from .version import VERSION @@ -27,7 +27,8 @@ def get_unused_port(): - with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + with contextlib.closing(s): s.bind(('localhost', 0)) return s.getsockname()[1] @@ -35,7 +36,7 @@ def get_unused_port(): def load_proxy_port_map(): data = dict() for path in load_config_paths(CONFIG_RESOURCE): - path = Path(path.decode()) / PROXY_PORT_MAP_FILENAME + path = pathlib.Path(path.decode()) / PROXY_PORT_MAP_FILENAME if path.is_file(): with open(path) as f: for row in csv.reader(f): @@ -44,7 +45,7 @@ def load_proxy_port_map(): def save_proxy_port_map(data): - config_path = Path(save_config_path(CONFIG_RESOURCE).decode()) \ + config_path = pathlib.Path(save_config_path(CONFIG_RESOURCE).decode()) \ / PROXY_PORT_MAP_FILENAME with open(config_path, 'w') as f: writer = csv.writer(f) @@ -110,7 +111,7 @@ async def handle_subproc(cmd, pipe_task): extra={'user_waiting': True}) # TODO: response header version check? - session = aiohttp.ClientSession() + session = ClientSession() try: sock_type = socket.SOCK_STREAM if hasattr(socket, 'SOCK_NONBLOCK'): # only for Linux @@ -142,15 +143,15 @@ async def handle_subproc(cmd, pipe_task): local_sock.close() # used only once ssh_reader_task = loop.create_task(handle_ssh_sock(ws, ssh_sock)) async for msg in ws: - if msg.type == aiohttp.WSMsgType.BINARY: + if msg.type == WSMsgType.BINARY: await loop.sock_sendall(ssh_sock, msg.data) - elif msg.type == aiohttp.WSMsgType.CLOSED: + elif msg.type == WSMsgType.CLOSED: break - elif msg.type == aiohttp.WSMsgType.ERROR: + elif msg.type == WSMsgType.ERROR: logger.error('Server disconnected unexpectedly.', extra={'user_waiting': False}) break - except aiohttp.ClientError: + except ClientError: logger.error('Connection error!', extra={'user_waiting': False}) raise except asyncio.CancelledError: @@ -171,7 +172,7 @@ async def handle_subproc(cmd, pipe_task): loop.stop() -@aiotools.actxmgr +@actxmgr async def serve_proxy(loop, pidx, args): """The initialize and shtudown routines for the local SSH proxy.""" pipe_task = None @@ -185,6 +186,6 @@ async def serve_proxy(loop, pidx, args): def start_ssh_proxy(url, remote, ssh_executable): - aiotools.start_server(serve_proxy, - args=(url, remote, ssh_executable), - num_proc=1) + start_server(serve_proxy, + args=(url, remote, ssh_executable), + num_proc=1) From 45e02a97daa185067e6b57178e7580db4b7cb79b Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Sun, 18 Jun 2017 03:21:10 +0900 Subject: [PATCH 05/21] Fix Client.remotes to work with new mangle_ssh_args() function --- geofrontcli/client.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/geofrontcli/client.py b/geofrontcli/client.py index 4373e29..fec418f 100644 --- a/geofrontcli/client.py +++ b/geofrontcli/client.py @@ -194,11 +194,9 @@ def remotes(self): mimetype, _ = parse_mimetype(r.headers['Content-Type']) assert mimetype == 'application/json' result = json.loads(r.read().decode('utf-8')) - fmt = '{0[user]}@{0[host]}:{0[port]}'.format logger.info('Total %d remotes.', len(result), extra={'user_waiting': False}) - return dict((alias, fmt(remote)) - for alias, remote in result.items()) + return result except: logger.info('Failed to fetch the list of remotes.', extra={'user_waiting': False}) From cc3649ea1afb63b2590598b8fb403f439d7d67f7 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Sun, 18 Jun 2017 03:22:09 +0900 Subject: [PATCH 06/21] Fix flake8 line continuation. --- geofrontcli/proxy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/geofrontcli/proxy.py b/geofrontcli/proxy.py index 3b91d3b..30a7284 100644 --- a/geofrontcli/proxy.py +++ b/geofrontcli/proxy.py @@ -45,8 +45,8 @@ def load_proxy_port_map(): def save_proxy_port_map(data): - config_path = pathlib.Path(save_config_path(CONFIG_RESOURCE).decode()) \ - / PROXY_PORT_MAP_FILENAME + config_path = (pathlib.Path(save_config_path(CONFIG_RESOURCE).decode()) + / PROXY_PORT_MAP_FILENAME) with open(config_path, 'w') as f: writer = csv.writer(f) for key, val in data.items(): From a4f5fa5a6c981227fe23b1e1e1d8f6890b108c1e Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Thu, 22 Jun 2017 13:49:18 +0900 Subject: [PATCH 07/21] Improve tunneling support a lot * Now it uses a commad-arg template for generalized support of tunneling in different CLI commands (both ssh and scp) * If user part is specified in CLI remote aliases used as host names, it SKIPS authorization -- assuming that individual users would have configured their own authorization (e.g., personal keypairs to personal accounts in shared development server) - If the username is same to the remote's configured username, then it performs authorization as it has been doing. * The server should provide a GET version of /tokens/.../remotes/.../ API which returns a single "remote" dictionary containing the information of the given remote. --- geofrontcli/cli.py | 142 +++++++++++++++++++++++++++++------------- geofrontcli/client.py | 28 ++++++++- geofrontcli/proxy.py | 20 +++--- geofrontcli/utils.py | 28 +++++++++ 4 files changed, 162 insertions(+), 56 deletions(-) create mode 100644 geofrontcli/utils.py diff --git a/geofrontcli/cli.py b/geofrontcli/cli.py index 3090401..c66fb09 100644 --- a/geofrontcli/cli.py +++ b/geofrontcli/cli.py @@ -8,8 +8,10 @@ import logging import os import os.path +import pprint import subprocess import sys +import time import webbrowser from dirspec.basedir import load_config_paths, save_config_path @@ -21,6 +23,7 @@ NoTokenIdError, ProtocolVersionError, RemoteError, TokenIdError, UnfinishedAuthenticationError) from .key import PublicKey +from .utils import resolve_cmdarg_template from .version import VERSION @@ -205,11 +208,9 @@ def masterkey(args): def align_remote_list(remotes): - maxlength = max(map(len, remotes)) if remotes else 0 + maxlength = max(map(len, remotes)) if remotes else 1 for alias, remote in sorted(remotes.items()): - if remote.endswith(':22'): - remote = remote[:-3] - yield '{0:{1}} {2}'.format(alias, maxlength, remote) + yield '{0:{1}} {2}'.format(alias, maxlength, remote['host']) @subparser @@ -217,6 +218,7 @@ def remotes(args): """List available remotes.""" client = get_client() remotes = client.remotes + time.sleep(0.11) if args.alias: for alias in sorted(remotes): print(alias) @@ -233,6 +235,21 @@ def remotes(args): ) +@subparser +def remote(args): + """Get the information of a specific remote.""" + client = get_client() + remote = client.remote(args.remote) + time.sleep(0.11) + pprint.pprint(remote) + + +remote.add_argument( + 'remote', + help='the remote alias that you want to get information about' +) + + @subparser def authorize(args, alias=None): """Temporarily authorize you to access the given remote. @@ -302,70 +319,104 @@ def colonize(args): @subparser -def ssh(args, alias=None): +def ssh(args): """SSH to the remote through Geofront's temporary authorization.""" - logger = logging.getLogger('geofrontcli') - if args.proxy and sys.version_info < (3, 6): + if args.tunnel and sys.version_info < (3, 6): + logger = logging.getLogger('geofrontcli') logger.error('To use the SSH proxy, you need to run geofront-cli on ' 'Python 3.6 or higher.', extra={'user_waiting': False}) - remote = authorize.call(args, alias=alias) - if args.proxy: - client = get_client() - client.ssh_proxy(remote, args.ssh, alias or args.remote) + remote_match = REMOTE_PATTERN.match(args.remote) + if not remote_match: + raise ValueError('invalid remote format: ' + str(args.remote)) + alias = remote_match.group('host') + user = remote_match.group('user') + # port from remote_match is ignored + client = get_client() + remote = client.remote(alias, quiet=True) + if user and user != remote['user']: + print('---------- user override ---------') + remote['user'] = user # override username else: - subprocess.call([args.ssh] + mangle_ssh_args(remote)) + print('---------- normal auth ---------') + remote = authorize.call(args, alias=alias) + template = [ + args.ssh, + '-l', '$user', + '-p', '$port', + '$host', + ] + if args.tunnel: + client.ssh_proxy(template, remote, alias or args.remote) + else: + cmdargs = resolve_cmdarg_template(template, remote) + subprocess.call(cmdargs) ssh.add_argument('remote', help='the remote alias to ssh') -ssh.add_argument('-p', '--proxy', action='store_true', default=False, - help='use a proxy tunneled via HTTPS to ssh into servers ' - 'inside remote private networks') +ssh.add_argument('-t', '--tunnel', action='store_true', default=False, + help='use SSH tunneling via HTTPS WebSockets to access' + 'servers inside remote private networks') def parse_scp_path(path, args): """Parse remote:path format.""" if ':' not in path: return None, path - alias, path = path.split(':', 1) - remote = authorize.call(args, alias=alias) - return remote, path + host, path = path.split(':', 1) + return host, path @subparser def scp(args): - options = [] - src_remote, src_path = parse_scp_path(args.source, args) - dst_remote, dst_path = parse_scp_path(args.destination, args) - if src_remote and dst_remote: + """SCP from/to the remote through Geofront's temporary authorization.""" + if args.tunnel and sys.version_info < (3, 6): + logger = logging.getLogger('geofrontcli') + logger.error('To use the SSH proxy, you need to run geofront-cli on ' + 'Python 3.6 or higher.', + extra={'user_waiting': False}) + template = [args.scp] + src_host, src_path = parse_scp_path(args.source, args) + dst_host, dst_path = parse_scp_path(args.destination, args) + if src_host and dst_remote: scp.error('source and destination cannot be both ' 'remote paths at a time') - elif not (src_remote or dst_remote): + elif not (src_host or dst_host): scp.error('one of source and destination has to be a remote path') if args.ssh: - options.extend(['-S', args.ssh]) + template.extend(['-S', args.ssh]) if args.recursive: - options.append('-r') - remote = src_remote or dst_remote - remote_match = REMOTE_PATTERN.match(remote) - if not remote_match: - raise ValueError('invalid remote format: ' + str(remote)) - port = remote_match.group('port') - if port: - options.extend(['-P', port]) - host = remote_match.group('host') - user = remote_match.group('user') - if user: - host = user + '@' + host - if src_remote: - options.append(host + ':' + src_path) + template.append('-r') + host = src_host or dst_host + host_match = REMOTE_PATTERN.match(host) + if not host_match: + raise ValueError('invalid remote format: ' + str(host)) + alias = host_match.group('host') + user = host_match.group('user') + # port from host_match is ignored + template.extend(['-P', '$port']) + if src_host: + template.append('$user@$host:' + src_path) + else: + template.append(src_path) + if dst_host: + template.append('$user@$host:' + dst_path) + else: + template.append(dst_path) + client = get_client() + remote = client.remote(alias, quiet=True) + if user and user != remote_info['user']: + remote['user'] = user # override username else: - options.append(src_path) - if dst_remote: - options.append(host + ':' + dst_path) + remote = authorize.call(args, alias=alias) + if args.tunnel: + client.ssh_proxy(template, remote, alias) else: - options.append(dst_path) - subprocess.call([args.scp] + options) + subprocess.call(resolve_cmdarg_template(template, { + 'host': remote['host'], + 'user': remote['user'], + 'port': remote['port'], + })) scp.add_argument( @@ -376,9 +427,12 @@ def scp(args): ) scp.add_argument( '-r', '-R', '--recursive', - action='store_true', + action='store_true', default=False, help='recursively copy entire directories' ) +scp.add_argument('-t', '--tunnel', action='store_true', default=False, + help='use SSH tunneling via HTTPS WebSockets to access' + 'servers inside remote private networks') scp.add_argument('source', help='the source path to copy') scp.add_argument('destination', help='the destination path') diff --git a/geofrontcli/client.py b/geofrontcli/client.py index fec418f..6508aca 100644 --- a/geofrontcli/client.py +++ b/geofrontcli/client.py @@ -202,6 +202,30 @@ def remotes(self): extra={'user_waiting': False}) raise + def remote(self, alias, quiet=False): + """(:class:`dict`) The remote information including user, host, and + port. + + """ + logger = self.logger.getChild('remote') + if not quiet: + logger.info('Loading the remote information from the Geofront ' + 'server...', extra={'user_waiting': True}) + try: + path = ('tokens', self.token_id, 'remotes', alias) + with self.request('GET', path) as r: + assert r.code == 200 + mimetype, _ = parse_mimetype(r.headers['Content-Type']) + assert mimetype == 'application/json' + result = json.loads(r.read().decode('utf-8')) + if not quiet: + logger.info('Done.', extra={'user_waiting': False}) + except: + logger.info('Failed to fetch the remote information.', + extra={'user_waiting': False}) + raise + return result['remote'] + def authorize(self, alias): """Temporarily authorize you to access the given remote ``alias``. A made authorization keeps alive in a minute, and then will be expired. @@ -238,7 +262,7 @@ def authorize(self, alias): return result['remote'] if sys.version_info >= (3, 6): # pragma: no cover - def ssh_proxy(self, remote, ssh_executable, alias): + def ssh_proxy(self, cmd_template, remote, alias): logger = self.logger.getChild('ssh_proxy') try: path = ('ws', 'tokens', self.token_id, 'remotes', alias, 'ssh') @@ -248,7 +272,7 @@ def ssh_proxy(self, remote, ssh_executable, alias): logger.info('Authentication is required.', extra={'user_waiting': False}) raise - start_ssh_proxy(url, remote, ssh_executable) + start_ssh_proxy(cmd_template, url, remote) def __repr__(self): return '{0.__module__}.{0.__name__}({1!r})'.format( diff --git a/geofrontcli/proxy.py b/geofrontcli/proxy.py index 30a7284..3d5df4f 100644 --- a/geofrontcli/proxy.py +++ b/geofrontcli/proxy.py @@ -15,6 +15,7 @@ from aiotools import actxmgr, start_server from dirspec.basedir import load_config_paths, save_config_path +from .utils import resolve_cmdarg_template from .version import VERSION __all__ = ('start_ssh_proxy', ) @@ -68,7 +69,7 @@ def get_port_for_remote(host): return port -async def pipe(url, remote, ssh_executable): +async def pipe(cmd_tpl, url, remote): """The main task that proxies the incoming SSH traffic via WebSockets.""" loop = asyncio.get_event_loop() headers = { @@ -129,14 +130,13 @@ async def handle_subproc(cmd, pipe_task): logger.info(f'Connecting to local SSH proxy at port {bind_port}...', extra={'user_waiting': False}) async with session.ws_connect(url, headers=headers) as ws: - cmd = [ - ssh_executable, - '-l', remote['user'], - '-p', str(bind_port), - 'localhost', - ] + cmdargs = resolve_cmdarg_template(cmd_tpl, { + 'host': 'localhost', + 'user': remote['user'], + 'port': str(bind_port), + }) subproc_task = loop.create_task( - handle_subproc(cmd, asyncio.Task.current_task())) + handle_subproc(cmdargs, asyncio.Task.current_task())) await asyncio.sleep(0) # required! ssh_sock, _ = await loop.sock_accept(local_sock) ssh_sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) @@ -185,7 +185,7 @@ async def serve_proxy(loop, pidx, args): await pipe_task -def start_ssh_proxy(url, remote, ssh_executable): +def start_ssh_proxy(cmd_tpl, url, remote): start_server(serve_proxy, - args=(url, remote, ssh_executable), + args=(cmd_tpl, url, remote), num_proc=1) diff --git a/geofrontcli/utils.py b/geofrontcli/utils.py new file mode 100644 index 0000000..35c887b --- /dev/null +++ b/geofrontcli/utils.py @@ -0,0 +1,28 @@ +""":mod:`geofrontcli.utils` --- Utility functions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +""" +import re + +__all__ = ('resolve_cmdarg_template', ) + + +CMDARG_VAR_PATTERN = re.compile(r'\$(?P[A-Za-z]\w*)') + + +def resolve_cmdarg_template(template, vars): + resolved = template[:] + + def resolve_var(matchobj): + name = matchobj.group('name') + return str(vars[name]) + + for idx, piece in enumerate(resolved): + if isinstance(piece, bytes): + continue + new_piece, num_replaced = CMDARG_VAR_PATTERN.subn(resolve_var, piece) + if num_replaced: + resolved[idx] = new_piece + + print(f'--- resolved template ---\n{resolved}') + return resolved From 8827f5b40a54cd0d6c4121dc46709bd01edffde0 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Thu, 22 Jun 2017 13:52:33 +0900 Subject: [PATCH 08/21] Remove debug prints. --- geofrontcli/cli.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/geofrontcli/cli.py b/geofrontcli/cli.py index c66fb09..344f45f 100644 --- a/geofrontcli/cli.py +++ b/geofrontcli/cli.py @@ -335,10 +335,8 @@ def ssh(args): client = get_client() remote = client.remote(alias, quiet=True) if user and user != remote['user']: - print('---------- user override ---------') remote['user'] = user # override username else: - print('---------- normal auth ---------') remote = authorize.call(args, alias=alias) template = [ args.ssh, From b98042a5835a858201884eb672e6cc2821ed302c Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Thu, 22 Jun 2017 13:59:37 +0900 Subject: [PATCH 09/21] Remove debug print --- geofrontcli/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/geofrontcli/utils.py b/geofrontcli/utils.py index 35c887b..74a15ec 100644 --- a/geofrontcli/utils.py +++ b/geofrontcli/utils.py @@ -24,5 +24,4 @@ def resolve_var(matchobj): if num_replaced: resolved[idx] = new_piece - print(f'--- resolved template ---\n{resolved}') return resolved From 0be0ee7041ff4c115c7a7fb0e77a717611353640 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Thu, 22 Jun 2017 14:07:19 +0900 Subject: [PATCH 10/21] Bump version to 0.5.0 * Also upgrade min/max protocol version, as we require the new remote GET API even when not using tunneling. --- geofrontcli/version.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/geofrontcli/version.py b/geofrontcli/version.py index eb0e158..b8912a0 100644 --- a/geofrontcli/version.py +++ b/geofrontcli/version.py @@ -6,16 +6,16 @@ #: (:class:`tuple`) The triple of version numbers e.g. ``(1, 2, 3)``. -VERSION_INFO = (0, 4, 1) +VERSION_INFO = (0, 5, 0) #: (:class:`str`) The version string e.g. ``'1.2.3'``. VERSION = '{0}.{1}.{2}'.format(*VERSION_INFO) #: (:class:`tuple`) The minimum compatible version of server protocol. -MIN_PROTOCOL_VERSION = (0, 2, 0) +MIN_PROTOCOL_VERSION = (0, 5, 0) #: (:class:`tuple`) The maximum compatible version of server protocol. -MAX_PROTOCOL_VERSION = (0, 4, 999) +MAX_PROTOCOL_VERSION = (0, 6, 999) if __name__ == '__main__': From 0e91c4d21c8a3add588fa1686993ba46b96e745d Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Thu, 22 Jun 2017 15:03:20 +0900 Subject: [PATCH 11/21] Fix typo, oooops... --- geofrontcli/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geofrontcli/cli.py b/geofrontcli/cli.py index 344f45f..265c336 100644 --- a/geofrontcli/cli.py +++ b/geofrontcli/cli.py @@ -376,7 +376,7 @@ def scp(args): template = [args.scp] src_host, src_path = parse_scp_path(args.source, args) dst_host, dst_path = parse_scp_path(args.destination, args) - if src_host and dst_remote: + if src_host and dst_host: scp.error('source and destination cannot be both ' 'remote paths at a time') elif not (src_host or dst_host): From cf39e08d1930cbe972094d5b033d7a47cbb7a5de Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Fri, 23 Jun 2017 01:23:14 +0900 Subject: [PATCH 12/21] Support more SSH options (-i and -D) It is useful when using alternative logins without authorization (e.g., shared development servers) --- geofrontcli/cli.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/geofrontcli/cli.py b/geofrontcli/cli.py index 265c336..b0a3e88 100644 --- a/geofrontcli/cli.py +++ b/geofrontcli/cli.py @@ -342,8 +342,12 @@ def ssh(args): args.ssh, '-l', '$user', '-p', '$port', - '$host', ] + if args.identity: + template.extend(['-i', args.identity]) + if args.dynamic_port: + template.extend(['-D', args.dynamic_port]) + template.append('$host') if args.tunnel: client.ssh_proxy(template, remote, alias or args.remote) else: @@ -352,6 +356,10 @@ def ssh(args): ssh.add_argument('remote', help='the remote alias to ssh') +ssh.add_argument('-i', '--identity', + help='alternative SSH identity (private key)') +ssh.add_argument('-D', '--dynamic-port', + help='port number to use for dynamic TCP forwarding') ssh.add_argument('-t', '--tunnel', action='store_true', default=False, help='use SSH tunneling via HTTPS WebSockets to access' 'servers inside remote private networks') @@ -385,6 +393,8 @@ def scp(args): template.extend(['-S', args.ssh]) if args.recursive: template.append('-r') + if args.identity: + template.extend(['-i', args.identity]) host = src_host or dst_host host_match = REMOTE_PATTERN.match(host) if not host_match: @@ -428,6 +438,8 @@ def scp(args): action='store_true', default=False, help='recursively copy entire directories' ) +scp.add_argument('-i', '--identity', + help='alternative SSH identity (private key)') scp.add_argument('-t', '--tunnel', action='store_true', default=False, help='use SSH tunneling via HTTPS WebSockets to access' 'servers inside remote private networks') From 142b5d1706ca4b148a14fbc4f36edf092f40415f Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Fri, 23 Jun 2017 13:45:35 +0900 Subject: [PATCH 13/21] Print more details for "remotes -v" command --- geofrontcli/cli.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/geofrontcli/cli.py b/geofrontcli/cli.py index b0a3e88..4535d59 100644 --- a/geofrontcli/cli.py +++ b/geofrontcli/cli.py @@ -208,9 +208,20 @@ def masterkey(args): def align_remote_list(remotes): - maxlength = max(map(len, remotes)) if remotes else 1 + if remotes: + maxlen_alias = max(map(len, remotes.keys())) + maxlen_user = max(map(lambda v: len(v['user']), remotes.values())) + maxlen_host = max(map(lambda v: len(v['host']), remotes.values())) + else: + maxlen_alias = 1 + maxlen_user = 1 + maxlen_host = 1 for alias, remote in sorted(remotes.items()): - yield '{0:{1}} {2}'.format(alias, maxlength, remote['host']) + yield '{0:{1}} {2:{3}} @ {4:{5}} : {6}'.format( + alias, maxlen_alias, + remote['user'], maxlen_user, + remote['host'], maxlen_host, + remote['port']) @subparser From 5a32a34fe1926df418e7e8687839cc19f68be734 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Fri, 23 Jun 2017 17:04:47 +0900 Subject: [PATCH 14/21] Minor fixes * Use revised masterkey URL path * Prevent Ctrl+C from producing too much unnecessary exception tracebacks --- geofrontcli/cli.py | 2 ++ geofrontcli/client.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/geofrontcli/cli.py b/geofrontcli/cli.py index 4535d59..ac2b2b6 100644 --- a/geofrontcli/cli.py +++ b/geofrontcli/cli.py @@ -548,6 +548,8 @@ def main(args=None): parser.exit('geofront-cli seems incompatible with the server.\n' 'Try `pip install --upgrade geofront-cli` command.\n' 'The server version is {0}.'.format(e.server_version)) + except KeyboardInterrupt: + parser.exit('Aborted.') else: parser.print_usage() diff --git a/geofrontcli/client.py b/geofrontcli/client.py index 6508aca..dffc4fb 100644 --- a/geofrontcli/client.py +++ b/geofrontcli/client.py @@ -169,7 +169,7 @@ def identity(self): @property def master_key(self): """(:class:`~.key.PublicKey`) The current master key.""" - path = ('tokens', self.token_id, 'masterkey') + path = ('masterkey',) headers = {'Accept': 'text/plain'} with self.request('GET', path, headers=headers) as r: if r.code == 200: From b994ab30241bc4e51e692433d7a8d82f78f96fa7 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Wed, 2 Aug 2017 03:30:52 +0900 Subject: [PATCH 15/21] Upgrade aiotools version and bump version to 0.5.1 --- .gitignore | 1 + geofrontcli/proxy.py | 3 ++- geofrontcli/version.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index a08c5ba..e03df32 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ .cache .coverage .tox +.envrc build dist venv diff --git a/geofrontcli/proxy.py b/geofrontcli/proxy.py index 3d5df4f..b8027f0 100644 --- a/geofrontcli/proxy.py +++ b/geofrontcli/proxy.py @@ -188,4 +188,5 @@ async def serve_proxy(loop, pidx, args): def start_ssh_proxy(cmd_tpl, url, remote): start_server(serve_proxy, args=(cmd_tpl, url, remote), - num_proc=1) + use_threading=True, + num_workers=1) diff --git a/geofrontcli/version.py b/geofrontcli/version.py index b8912a0..dc80e44 100644 --- a/geofrontcli/version.py +++ b/geofrontcli/version.py @@ -6,7 +6,7 @@ #: (:class:`tuple`) The triple of version numbers e.g. ``(1, 2, 3)``. -VERSION_INFO = (0, 5, 0) +VERSION_INFO = (0, 5, 1) #: (:class:`str`) The version string e.g. ``'1.2.3'``. VERSION = '{0}.{1}.{2}'.format(*VERSION_INFO) diff --git a/setup.py b/setup.py index b9e4a9c..60b421e 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ def readme(): py36_or_higher_requires = { 'aiohttp ~= 2.1.0', - 'aiotools >= 0.3', + 'aiotools ~= 0.4.0', } win32_requires = { From 21dfdf97ffa4eaa579f7c1e008c989111efd03d4 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Wed, 2 Aug 2017 03:34:15 +0900 Subject: [PATCH 16/21] Fix termination hang for latest aiotools --- geofrontcli/proxy.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/geofrontcli/proxy.py b/geofrontcli/proxy.py index b8027f0..de24d11 100644 --- a/geofrontcli/proxy.py +++ b/geofrontcli/proxy.py @@ -6,7 +6,9 @@ import contextlib import csv import logging +import os import pathlib +import signal import socket import sys import traceback @@ -169,7 +171,8 @@ async def handle_subproc(cmd, pipe_task): if ssh_sock: ssh_sock.close() session.close() - loop.stop() + # inform the main that we finished + os.kill(0, signal.SIGINT) @actxmgr From bd98eff338a4d9cf587d86378293af36eb2ab575 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Mon, 11 Dec 2017 14:02:37 +0900 Subject: [PATCH 17/21] Upgrade aiotools and aiohttp versions to latest ones. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 60b421e..f9d4f1c 100644 --- a/setup.py +++ b/setup.py @@ -33,8 +33,8 @@ def readme(): } py36_or_higher_requires = { - 'aiohttp ~= 2.1.0', - 'aiotools ~= 0.4.0', + 'aiohttp ~= 2.3.0', + 'aiotools ~= 0.5.0', } win32_requires = { From 4cc580c09e9f2e0c91382e2ec4887aa0df0461f8 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Mon, 11 Dec 2017 16:26:54 +0900 Subject: [PATCH 18/21] Modernize console entry points config in setup.py --- setup.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index f9d4f1c..4c92772 100644 --- a/setup.py +++ b/setup.py @@ -63,11 +63,12 @@ def readme(): maintainer_email='dev' '@' 'spoqa.com', license='GPLv3 or later', packages=find_packages(exclude=['tests']), - entry_points=''' - [console_scripts] - geofront-cli = geofrontcli.cli:main - gfg = geofrontcli.cli:main_go - ''', + entry_points={ + 'console_scripts': [ + 'geofront-cli = geofrontcli.cli:main', + 'gfg = geofrontcli.cli:main_go', + ], + }, install_requires=list(install_requires), extras_require={ ":python_version<'3.4'": list(below_py34_requires), From 7a1855a0249f61df0587f9c653859acc5eb5e207 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Mon, 11 Dec 2017 16:27:29 +0900 Subject: [PATCH 19/21] Reduce too-verbose HTTP exception logs * It is enough to see the response code and status message, not the whole stack trace. (The stack always indicates the line 85 of client.py) --- geofrontcli/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geofrontcli/client.py b/geofrontcli/client.py index dffc4fb..10feace 100644 --- a/geofrontcli/client.py +++ b/geofrontcli/client.py @@ -84,7 +84,7 @@ def request(self, method, url, data=None, headers={}): try: response = self.opener.open(request) except HTTPError as e: - logger.exception(e) + logger.error('{0}: returned {1} {2}'.format(url, e.code, e.reason)) response = e server_version = response.headers.get('X-Geofront-Version') if server_version: From aa30a48cee5031a5c620f199467ce033bee42570 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Mon, 11 Dec 2017 16:28:46 +0900 Subject: [PATCH 20/21] Do not proceed futher when the user is using unsupported Python version. --- geofrontcli/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/geofrontcli/cli.py b/geofrontcli/cli.py index ac2b2b6..3a65634 100644 --- a/geofrontcli/cli.py +++ b/geofrontcli/cli.py @@ -337,6 +337,7 @@ def ssh(args): logger.error('To use the SSH proxy, you need to run geofront-cli on ' 'Python 3.6 or higher.', extra={'user_waiting': False}) + return remote_match = REMOTE_PATTERN.match(args.remote) if not remote_match: raise ValueError('invalid remote format: ' + str(args.remote)) From bd0b07446403537dff86997632ead0c3a58543b9 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Tue, 12 Dec 2017 00:50:25 +0900 Subject: [PATCH 21/21] Let's use reduced but more helpful error display --- geofrontcli/cli.py | 46 +++++++++++++++++++++++++++++++------------ geofrontcli/client.py | 12 +++++++---- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/geofrontcli/cli.py b/geofrontcli/cli.py index 3a65634..6d70672 100644 --- a/geofrontcli/cli.py +++ b/geofrontcli/cli.py @@ -57,6 +57,8 @@ version='%(prog)s ' + VERSION) subparsers = parser.add_subparsers() +logger = logging.getLogger('geofrontcli') + def get_server_url(): for path in load_config_paths(CONFIG_RESOURCE): @@ -123,13 +125,17 @@ def authenticate(args): """Authenticate to Geofront server.""" client = get_client() while True: - with client.authenticate() as url: - if args.open_browser: - print('Continue to authenticate in your web browser...') - webbrowser.open(url) - else: - print('Continue to authenticate in your web browser:') - print(url) + try: + with client.authenticate() as url: + if args.open_browser: + print('Continue to authenticate in your web browser...') + webbrowser.open(url) + else: + print('Continue to authenticate in your web browser:') + print(url) + except Exception as e: + # exception info is already provided in client.Client.authenticate() + return input('Press return to continue') try: client.identity @@ -228,7 +234,11 @@ def align_remote_list(remotes): def remotes(args): """List available remotes.""" client = get_client() - remotes = client.remotes + try: + remotes = client.remotes + except Exception: + # exception info is already provided in client.Client.remotes() + return time.sleep(0.11) if args.alias: for alias in sorted(remotes): @@ -307,7 +317,11 @@ def colonize(args): """ client = get_client() - remote = client.remotes.get(args.remote, args.remote) + try: + remote = client.remotes.get(args.remote, args.remote) + except: + # exception info is already provided in client.Client.remote() + return cmd = [args.ssh] if args.identity_file: cmd.extend(['-i', args.identity_file]) @@ -333,7 +347,6 @@ def colonize(args): def ssh(args): """SSH to the remote through Geofront's temporary authorization.""" if args.tunnel and sys.version_info < (3, 6): - logger = logging.getLogger('geofrontcli') logger.error('To use the SSH proxy, you need to run geofront-cli on ' 'Python 3.6 or higher.', extra={'user_waiting': False}) @@ -345,7 +358,11 @@ def ssh(args): user = remote_match.group('user') # port from remote_match is ignored client = get_client() - remote = client.remote(alias, quiet=True) + try: + remote = client.remote(alias, quiet=True) + except Exception: + # exception info is already provided in client.Client.remote() + return if user and user != remote['user']: remote['user'] = user # override username else: @@ -389,7 +406,6 @@ def parse_scp_path(path, args): def scp(args): """SCP from/to the remote through Geofront's temporary authorization.""" if args.tunnel and sys.version_info < (3, 6): - logger = logging.getLogger('geofrontcli') logger.error('To use the SSH proxy, you need to run geofront-cli on ' 'Python 3.6 or higher.', extra={'user_waiting': False}) @@ -463,7 +479,11 @@ def scp(args): def go(args): """Select a remote and SSH to it at once (in interactive way).""" client = get_client() - remotes = client.remotes + try: + remotes = client.remotes + except Exception: + # exception info is already provided in client.Client.remotes() + return chosen = iterfzf(align_remote_list(remotes)) if chosen is None: return diff --git a/geofrontcli/client.py b/geofrontcli/client.py index 10feace..e7fa50b 100644 --- a/geofrontcli/client.py +++ b/geofrontcli/client.py @@ -13,7 +13,7 @@ from keyring import get_password, set_password from six import string_types -from six.moves.urllib.error import HTTPError +from six.moves.urllib.error import HTTPError, URLError from six.moves.urllib.parse import urljoin from six.moves.urllib.request import OpenerDirector, Request, build_opener @@ -85,7 +85,11 @@ def request(self, method, url, data=None, headers={}): response = self.opener.open(request) except HTTPError as e: logger.error('{0}: returned {1} {2}'.format(url, e.code, e.reason)) - response = e + raise + except URLError as e: + logger.error('{0}: errored {1}'.format(url, e.reason)) + logger.error('Maybe you are not connected to the Internet!') + raise server_version = response.headers.get('X-Geofront-Version') if server_version: try: @@ -197,7 +201,7 @@ def remotes(self): logger.info('Total %d remotes.', len(result), extra={'user_waiting': False}) return result - except: + except Exception: logger.info('Failed to fetch the list of remotes.', extra={'user_waiting': False}) raise @@ -220,7 +224,7 @@ def remote(self, alias, quiet=False): result = json.loads(r.read().decode('utf-8')) if not quiet: logger.info('Done.', extra={'user_waiting': False}) - except: + except Exception: logger.info('Failed to fetch the remote information.', extra={'user_waiting': False}) raise