From 1868487c1145554d1f4d9df2925a0ce7e4792e00 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 30 May 2014 13:07:50 +0800 Subject: [PATCH 001/344] Update README.md --- README.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0ce6560..a03c2c8 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,8 @@ Install Shadowsocks. #### Debian / Ubuntu: - apt-get install python-pip python-gevent python-m2crypto - pip install shadowsocks + apt-get install build-essential python-pip python-m2crypto + pip install gevent shadowsocks #### CentOS: @@ -73,7 +73,8 @@ Explanation of the fields: | fast_open | use [TCP_FASTOPEN][2], true / false | | workers | number of workers, available on Unix/Linux | -Run `ssserver -c /etc/shadowsocks.json` on your server. To run it in the background, [use supervisor][8]. +Run `ssserver -c /etc/shadowsocks.json` on your server. To run it in the background, +[use supervisor][8]. On your client machine, run `sslocal -c /etc/shadowsocks.json`. @@ -83,7 +84,8 @@ Change the proxy settings in your browser to hostname: 127.0.0.1 port: your local_port -**Notice: If you want to use encryption methods other than "table", please install M2Crypto (See Encryption Section).** +**Notice: If you want to use encryption methods other than "table", please +install M2Crypto (See Encryption Section).** It's recommended to use shadowsocks with AutoProxy or Proxy SwitchySharp. @@ -96,6 +98,15 @@ You can use args to override settings from `config.json`. ssserver -p server_port -k password -m bf-cfb --workers 2 ssserver -c /etc/shadowsocks/config.json +gevent +------ + +If you suffer from any strange problem when you have installed gevent 0.9.x, +install a new version. + + pip install gevent --upgrade + + Salsa20 ------- From 5ad8cd6f3d0b3280f44ed3c6d9aef0d7e640225a Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 30 May 2014 13:10:37 +0800 Subject: [PATCH 002/344] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a03c2c8..442de0f 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Install Shadowsocks. #### Debian / Ubuntu: - apt-get install build-essential python-pip python-m2crypto + apt-get install build-essential python-pip python-m2crypto python-dev pip install gevent shadowsocks #### CentOS: From 4f464001450ea991d8f0221a4ea3161c83857cab Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 30 May 2014 16:28:44 +0800 Subject: [PATCH 003/344] add ssloop branch --- shadowsocks/eventloop.py | 9 +++ shadowsocks/server.py | 10 --- shadowsocks/tcprelay.py | 154 +++++++++++++++++++++++++++++++++++++++ shadowsocks/udprelay.py | 24 +++--- 4 files changed, 177 insertions(+), 20 deletions(-) create mode 100644 shadowsocks/tcprelay.py diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index 89ffbcb..1c48718 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -25,6 +25,7 @@ # https://github.com/clowwindy/ssloop +import os import select from collections import defaultdict @@ -187,3 +188,11 @@ def errno_from_exception(e): return e.args[0] else: return None + + +# from tornado +def get_sock_error(sock): + errno = sock.getsockopt(socket.SOL_SOCKET, + socket.SO_ERROR) + return socket.error(errno, os.strerror(errno)) + diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 29cd99e..1926c51 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -29,16 +29,6 @@ else: import json -# TODO remove gevent -try: - import gevent - import gevent.monkey - gevent.monkey.patch_all(dns=gevent.version_info[0] >= 1) -except ImportError: - gevent = None - print >>sys.stderr, 'warning: gevent not found, using threading instead' - - import socket import select import threading diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py new file mode 100644 index 0000000..4a922ed --- /dev/null +++ b/shadowsocks/tcprelay.py @@ -0,0 +1,154 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import time +import threading +import socket +import logging +import struct +import encrypt +import eventloop +import errno + + +class TCPRelayHandler(object): + def __init__(self, fd_to_handlers, loop, conn, config, is_local): + self._fd_to_handlers = fd_to_handlers + self._loop = loop + self._local_conn = conn + self._remote_conn = None + self._remains_data_for_local = None + self._remains_data_for_remote = None + self._config = config + self._is_local = is_local + self._stage = 0 + fd_to_handlers[conn.fileno()] = self + conn.setblocking(False) + loop.add(conn, eventloop.POLL_IN) + + def on_local_read(self): + pass + + def on_remote_read(self): + pass + + def on_local_write(self): + pass + + def on_remote_write(self): + pass + + def on_local_error(self): + self.destroy() + + def on_remote_error(self): + self.destroy() + + def handle_event(self, sock, event): + # order is important + if sock == self._local_conn: + if event & eventloop.POLL_IN: + self.on_local_read() + if event & eventloop.POLL_OUT: + self.on_local_write() + if event & eventloop.POLL_ERR: + self.on_local_error() + elif sock == self._remote_conn: + if event & eventloop.POLL_IN: + self.on_remote_read() + if event & eventloop.POLL_OUT: + self.on_remote_write() + if event & eventloop.POLL_ERR: + self.on_remote_error() + else: + logging.warn('unknown socket') + + def destroy(self): + if self._local_conn: + self._local_conn.close() + eventloop.remove(self._local_conn) + # TODO maybe better to delete the key + self._fd_to_handlers[self._local_conn.fileno()] = None + if self._remote_conn: + self._remote_conn.close() + eventloop.remove(self._remote_conn) + self._fd_to_handlers[self._local_conn.fileno()] = None + + +class TCPRelay(object): + def __init__(self, config, is_local): + self._config = config + self._is_local = is_local + self._closed = False + self._fd_to_handlers = {} + + addrs = socket.getaddrinfo(self._listen_addr, self._listen_port, 0, + socket.SOCK_STREAM, socket.SOL_TCP) + if len(addrs) == 0: + raise Exception("can't get addrinfo for %s:%d" % + (self._listen_addr, self._listen_port)) + af, socktype, proto, canonname, sa = addrs[0] + server_socket = socket.socket(af, socktype, proto) + server_socket.bind((self._listen_addr, self._listen_port)) + server_socket.setblocking(False) + self._server_socket = server_socket + + def _run(self): + server_socket = self._server_socket + self._eventloop = eventloop.EventLoop() + self._eventloop.add(server_socket, eventloop.POLL_IN) + last_time = time.time() + while not self._closed: + try: + events = self._eventloop.poll(1) + except (OSError, IOError) as e: + if eventloop.errno_from_exception(e) == errno.EPIPE: + # Happens when the client closes the connection + continue + else: + logging.error(e) + continue + for sock, event in events: + if sock == self._server_socket: + try: + conn = self._server_socket.accept() + TCPRelayHandler(loop, conn, remote_addr, remote_port, + password, method, timeout, is_local) + except (OSError, IOError) as e: + error_no = eventloop.errno_from_exception(e) + if error_no in [errno.EAGAIN, errno.EINPROGRESS]: + continue + else: + handler = self._fd_to_handlers.get(sock.fileno(), None) + if handler: + handler.handle_event(sock, event) + else: + logging.warn('can not find handler for fd %d', + sock.fileno()) + now = time.time() + if now - last_time > 5: + # TODO sweep timeouts + last_time = now + + + diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 6012661..dff9125 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -123,16 +123,20 @@ def client_key(a, b, c, d): class UDPRelay(object): - def __init__(self, listen_addr='127.0.0.1', listen_port=1080, - remote_addr='127.0.0.1', remote_port=8387, password=None, - method='table', timeout=300, is_local=True): - self._listen_addr = listen_addr - self._listen_port = listen_port - self._remote_addr = remote_addr - self._remote_port = remote_port - self._password = password - self._method = method - self._timeout = timeout + def __init__(self, config, is_local=True): + if is_local: + self._listen_addr = config['local_address'] + self._listen_port = config['local_port'] + self._remote_addr = config['server'] + self._remote_port = config['server_port'] + else: + self._listen_addr = config['server'] + self._listen_port = config['server_port'] + self._remote_addr = None + self._remote_port = None + self._password = config['password'] + self._method = config['method'] + self._timeout = config['timeout'] self._is_local = is_local self._cache = lru_cache.LRUCache(timeout=timeout, close_callback=self._close_client) From 2cdddd451561afb90849b10e1d1398273bc456fd Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 30 May 2014 18:17:40 +0800 Subject: [PATCH 004/344] update config --- shadowsocks/server.py | 45 ++++++++++++++++--------------------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 1926c51..43fac82 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -39,6 +39,7 @@ import getopt import encrypt import os import utils +import tcprelay import udprelay @@ -216,55 +217,43 @@ def main(): utils.print_server_help() sys.exit(2) - config_server = config['server'] - config_server_port = config['server_port'] - config_key = config['password'] - config_method = config.get('method', None) - config_port_password = config.get('port_password', None) - config_timeout = int(config.get('timeout', 300)) - config_fast_open = config.get('fast_open', False) - config_workers = config.get('workers', 1) - - if not config_key and not config_path: - sys.exit('config not specified, please read ' - 'https://github.com/clowwindy/shadowsocks') + config['password'] = config.get('password', None) + config['method'] = config.get('method', None) + config['port_password'] = config.get('port_password', None) + config['timeout'] = int(config.get('timeout', 300)) + config['fast_open'] = config.get('fast_open', False) + config['workers'] = config.get('workers', 1) utils.check_config(config) - if config_port_password: - if config_server_port or config_key: + if config['port_password']: + if config['server_port'] or config['password']: logging.warn('warning: port_password should not be used with ' 'server_port and password. server_port and password ' 'will be ignored') else: - config_port_password = {} - config_port_password[str(config_server_port)] = config_key + config['port_password'] = {} + config['port_password'][str(config['server_port'])] = config['password'] - encrypt.init_table(config_key, config_method) - addrs = socket.getaddrinfo(config_server, int(8387)) + encrypt.init_table(config['password'], config['method']) + addrs = socket.getaddrinfo(config['server'], int(8387)) if not addrs: logging.error('cant resolve listen address') sys.exit(1) ThreadingTCPServer.address_family = addrs[0][0] tcp_servers = [] udp_servers = [] - for port, key in config_port_password.items(): - tcp_server = ThreadingTCPServer((config_server, int(port)), - Socks5Server) - tcp_server.key = key - tcp_server.method = config_method - tcp_server.timeout = int(config_timeout) + for port, key in config['port_password'].items(): logging.info("starting server at %s:%d" % tuple(tcp_server.server_address[:2])) + tcp_server = tcprelay.TCPRelay(config, False) tcp_servers.append(tcp_server) - udp_server = udprelay.UDPRelay(config_server, int(port), None, None, - key, config_method, int(config_timeout), - False) + udp_server = udprelay.UDPRelay(config, False) udp_servers.append(udp_server) def run_server(): for tcp_server in tcp_servers: - threading.Thread(target=tcp_server.serve_forever).start() + tcp_server.start() for udp_server in udp_servers: udp_server.start() From 0d6c39900b737b994577fe7f98a324b7ba203452 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 30 May 2014 23:55:33 +0800 Subject: [PATCH 005/344] more work --- shadowsocks/common.py | 64 ++++++++++++ shadowsocks/tcprelay.py | 215 ++++++++++++++++++++++++++++++++-------- shadowsocks/udprelay.py | 41 +------- 3 files changed, 241 insertions(+), 79 deletions(-) create mode 100644 shadowsocks/common.py diff --git a/shadowsocks/common.py b/shadowsocks/common.py new file mode 100644 index 0000000..256fbef --- /dev/null +++ b/shadowsocks/common.py @@ -0,0 +1,64 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import socket +import struct +import logging + + +def parse_header(data): + addrtype = ord(data[0]) + dest_addr = None + dest_port = None + header_length = 0 + if addrtype == 1: + if len(data) >= 7: + dest_addr = socket.inet_ntoa(data[1:5]) + dest_port = struct.unpack('>H', data[5:7])[0] + header_length = 7 + else: + logging.warn('header is too short') + elif addrtype == 3: + if len(data) > 2: + addrlen = ord(data[1]) + if len(data) >= 2 + addrlen: + dest_addr = data[2:2 + addrlen] + dest_port = struct.unpack('>H', data[2 + addrlen:4 + + addrlen])[0] + header_length = 4 + addrlen + else: + logging.warn('header is too short') + else: + logging.warn('header is too short') + elif addrtype == 4: + if len(data) >= 19: + dest_addr = socket.inet_ntop(socket.AF_INET6, data[1:17]) + dest_port = struct.unpack('>H', data[17:19])[0] + header_length = 19 + else: + logging.warn('header is too short') + else: + logging.warn('unsupported addrtype %d, maybe wrong password' % addrtype) + if dest_addr is None: + return None + return (addrtype, dest_addr, dest_port, header_length) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 4a922ed..3ce8b54 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -22,41 +22,170 @@ # SOFTWARE. import time -import threading import socket import logging -import struct import encrypt -import eventloop import errno +import eventloop +from common import parse_header + + +# local: +# stage 0 init +# stage 1 hello received, hello sent +# stage 4 addr received, reply sent +# stage 5 remote connected + +# remote: +# stage 0 init +# stage 4 addr received, reply sent +# stage 5 remote connected + + +BUF_SIZE = 8 * 1024 class TCPRelayHandler(object): - def __init__(self, fd_to_handlers, loop, conn, config, is_local): + def __init__(self, fd_to_handlers, loop, local_sock, config, is_local): self._fd_to_handlers = fd_to_handlers self._loop = loop - self._local_conn = conn - self._remote_conn = None - self._remains_data_for_local = None - self._remains_data_for_remote = None + self._local_sock = local_sock + self._remote_sock = None self._config = config self._is_local = is_local self._stage = 0 - fd_to_handlers[conn.fileno()] = self - conn.setblocking(False) - loop.add(conn, eventloop.POLL_IN) + self._encryptor = encrypt.Encryptor(config['password'], + config['method']) + self._data_to_write_to_local = [] + self._data_to_write_to_remote = [] + fd_to_handlers[local_sock.fileno()] = self + local_sock.setblocking(False) + loop.add(local_sock, eventloop.POLL_IN) + + def resume_reading(self, sock): + pass + + def pause_reading(self, sock): + pass + + def resume_writing(self, sock): + pass + + def pause_writing(self, sock): + pass + + def write_all_to_sock(self, data, sock): + # write to sock + # put remaining bytes into buffer + # return true if all written + # return false if some bytes left in buffer + # raise if encounter error + return True def on_local_read(self): - pass + if not self._local_sock: + return + is_local = self._is_local + data = self._local_sock.recv(BUF_SIZE) + if not is_local: + data = self._encryptor.decrypt(data) + if self._stage == 5: + if self._is_local: + data = self._encryptor.encrypt(data) + if not self.write_all_to_sock(data, self._remote_sock): + self.pause_reading(self._local_sock) + return + if is_local and self._stage == 0: + # TODO check auth method + self.write_all_to_sock('\x05\00', self._local_sock) + self._stage = 1 + return + if self._stage == 4: + self._data_to_write_to_remote.append(data) + if (is_local and self._stage == 0) or \ + (not is_local and self._stage == 1): + try: + if is_local: + cmd = ord(data[1]) + # TODO check cmd == 1 + assert cmd == 1 + # just trim VER CMD RSV + data = data[3:] + header_result = parse_header(data) + if header_result is None: + raise Exception('can not parse header') + addrtype, remote_addr, remote_port, header_length =\ + header_result + logging.info('connecting %s:%d' % (remote_addr, remote_port)) + if is_local: + # forward address to remote + self._data_to_write_to_remote.append(data[:header_length]) + self.write_all_to_sock('\x05\x00\x00\x01' + + '\x00\x00\x00\x00\x10\x10', + self._local_sock) + else: + remote_addr = self._config['server'] + remote_port = self._config['server_port'] + + # TODO async DNS + addrs = socket.getaddrinfo(remote_addr, remote_port, 0, + socket.SOCK_STREAM, socket.SOL_TCP) + if len(addrs) == 0: + raise Exception("can't get addrinfo for %s:%d" % + (remote_addr, remote_port)) + af, socktype, proto, canonname, sa = addrs[0] + self._remote_sock = socket.socket(af, socktype, proto) + self._remote_sock.setblocking(False) + # TODO support TCP fast open + self._remote_sock.connect(sa) + self._loop.add(self._remote_sock, eventloop.POLL_OUT) + + if len(data) > header_length: + self._data_to_write_to_remote.append(data[header_length:]) + + self._stage = 4 + self.pause_reading(self._local_sock) + return + except Exception: + import traceback + traceback.print_exc() + # TODO use logging when debug completed + self.destroy() + + if self._stage == 4: + self._data_to_write_to_remote.append(data) def on_remote_read(self): - pass + data = self._remote_sock.recv(BUF_SIZE) + if self._is_local: + data = self._encryptor.decrypt(data) + try: + if not self.write_all_to_sock(data, self._local_sock): + self.pause_reading(self._remote_sock) + self.resume_writing(self._local_sock) + except Exception: + import traceback + traceback.print_exc() + # TODO use logging when debug completed + self.destroy() def on_local_write(self): - pass + if self._data_to_write_to_local: + written = self.write_all_to_sock( + ''.join(self._data_to_write_to_local), self._local_sock) + if written: + self.pause_writing(self._local_sock) + else: + self.pause_writing(self._local_sock) def on_remote_write(self): - pass + if self._data_to_write_to_remote: + written = self.write_all_to_sock( + ''.join(self._data_to_write_to_remote), self._remote_sock) + if written: + self.pause_writing(self._remote_sock) + else: + self.pause_writing(self._remote_sock) def on_local_error(self): self.destroy() @@ -66,33 +195,34 @@ class TCPRelayHandler(object): def handle_event(self, sock, event): # order is important - if sock == self._local_conn: - if event & eventloop.POLL_IN: - self.on_local_read() - if event & eventloop.POLL_OUT: - self.on_local_write() - if event & eventloop.POLL_ERR: - self.on_local_error() - elif sock == self._remote_conn: + if sock == self._remote_sock: if event & eventloop.POLL_IN: self.on_remote_read() if event & eventloop.POLL_OUT: self.on_remote_write() if event & eventloop.POLL_ERR: self.on_remote_error() + elif sock == self._local_sock: + if event & eventloop.POLL_IN: + self.on_local_read() + if event & eventloop.POLL_OUT: + self.on_local_write() + if event & eventloop.POLL_ERR: + self.on_local_error() else: logging.warn('unknown socket') def destroy(self): - if self._local_conn: - self._local_conn.close() - eventloop.remove(self._local_conn) - # TODO maybe better to delete the key - self._fd_to_handlers[self._local_conn.fileno()] = None - if self._remote_conn: - self._remote_conn.close() - eventloop.remove(self._remote_conn) - self._fd_to_handlers[self._local_conn.fileno()] = None + if self._remote_sock: + self._remote_sock.close() + self._loop.remove(self._remote_sock) + del self._fd_to_handlers[self._remote_sock.fileno()] + self._remote_sock = None + if self._local_sock: + self._local_sock.close() + self._loop.remove(self._local_sock) + del self._fd_to_handlers[self._local_sock.fileno()] + self._local_sock = None class TCPRelay(object): @@ -102,15 +232,23 @@ class TCPRelay(object): self._closed = False self._fd_to_handlers = {} - addrs = socket.getaddrinfo(self._listen_addr, self._listen_port, 0, + if is_local: + listen_addr = config['local_address'] + listen_port = config['local_port'] + else: + listen_addr = config['server'] + listen_port = config['server_port'] + + addrs = socket.getaddrinfo(listen_addr, listen_port, 0, socket.SOCK_STREAM, socket.SOL_TCP) if len(addrs) == 0: raise Exception("can't get addrinfo for %s:%d" % - (self._listen_addr, self._listen_port)) + (listen_addr, listen_port)) af, socktype, proto, canonname, sa = addrs[0] server_socket = socket.socket(af, socktype, proto) - server_socket.bind((self._listen_addr, self._listen_port)) + server_socket.bind(sa) server_socket.setblocking(False) + server_socket.listen(1024) self._server_socket = server_socket def _run(self): @@ -132,8 +270,8 @@ class TCPRelay(object): if sock == self._server_socket: try: conn = self._server_socket.accept() - TCPRelayHandler(loop, conn, remote_addr, remote_port, - password, method, timeout, is_local) + TCPRelayHandler(self._eventloop, conn, self._config, + self._is_local) except (OSError, IOError) as e: error_no = eventloop.errno_from_exception(e) if error_no in [errno.EAGAIN, errno.EINPROGRESS]: @@ -149,6 +287,3 @@ class TCPRelay(object): if now - last_time > 5: # TODO sweep timeouts last_time = now - - - diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index dff9125..68f6474 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -71,53 +71,16 @@ import threading import socket import logging import struct +import errno import encrypt import eventloop import lru_cache -import errno +from common import parse_header BUF_SIZE = 65536 -def parse_header(data): - addrtype = ord(data[0]) - dest_addr = None - dest_port = None - header_length = 0 - if addrtype == 1: - if len(data) >= 7: - dest_addr = socket.inet_ntoa(data[1:5]) - dest_port = struct.unpack('>H', data[5:7])[0] - header_length = 7 - else: - logging.warn('[udp] header is too short') - elif addrtype == 3: - if len(data) > 2: - addrlen = ord(data[1]) - if len(data) >= 2 + addrlen: - dest_addr = data[2:2 + addrlen] - dest_port = struct.unpack('>H', data[2 + addrlen:4 + - addrlen])[0] - header_length = 4 + addrlen - else: - logging.warn('[udp] header is too short') - else: - logging.warn('[udp] header is too short') - elif addrtype == 4: - if len(data) >= 19: - dest_addr = socket.inet_ntop(socket.AF_INET6, data[1:17]) - dest_port = struct.unpack('>H', data[17:19])[0] - header_length = 19 - else: - logging.warn('[udp] header is too short') - else: - logging.warn('unsupported addrtype %d' % addrtype) - if dest_addr is None: - return None - return (addrtype, dest_addr, dest_port, header_length) - - def client_key(a, b, c, d): return '%s:%s:%s:%s' % (a, b, c, d) From d1aec8bc2847ff4e1cf7652ea1e1844cf80f1ffb Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 31 May 2014 18:50:42 +0800 Subject: [PATCH 006/344] Update README.md --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 442de0f..08bd427 100644 --- a/README.md +++ b/README.md @@ -84,9 +84,6 @@ Change the proxy settings in your browser to hostname: 127.0.0.1 port: your local_port -**Notice: If you want to use encryption methods other than "table", please -install M2Crypto (See Encryption Section).** - It's recommended to use shadowsocks with AutoProxy or Proxy SwitchySharp. Command line args From c2f6bc369cc3642c5810539857da2116a9f0d40c Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 11:01:11 +0800 Subject: [PATCH 007/344] more work --- shadowsocks/common.py | 46 ++++++++++++++++++++++++++++++++++++++++- shadowsocks/server.py | 11 ++++++---- shadowsocks/tcprelay.py | 26 +++++++++++++++++++---- shadowsocks/udprelay.py | 8 ++++--- shadowsocks/utils.py | 44 --------------------------------------- 5 files changed, 79 insertions(+), 56 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 256fbef..ca5fb21 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -26,6 +26,50 @@ import struct import logging +def inet_ntop(family, ipstr): + if family == socket.AF_INET: + return socket.inet_ntoa(ipstr) + elif family == socket.AF_INET6: + v6addr = ':'.join(('%02X%02X' % (ord(i), ord(j))) + for i, j in zip(ipstr[::2], ipstr[1::2])) + return v6addr + + +def inet_pton(family, addr): + if family == socket.AF_INET: + return socket.inet_aton(addr) + elif family == socket.AF_INET6: + if '.' in addr: # a v4 addr + v4addr = addr[addr.rindex(':') + 1:] + v4addr = socket.inet_aton(v4addr) + v4addr = map(lambda x: ('%02X' % ord(x)), v4addr) + v4addr.insert(2, ':') + newaddr = addr[:addr.rindex(':') + 1] + ''.join(v4addr) + return inet_pton(family, newaddr) + dbyts = [0] * 8 # 8 groups + grps = addr.split(':') + for i, v in enumerate(grps): + if v: + dbyts[i] = int(v, 16) + else: + for j, w in enumerate(grps[::-1]): + if w: + dbyts[7 - j] = int(w, 16) + else: + break + break + return ''.join((chr(i // 256) + chr(i % 256)) for i in dbyts) + else: + raise RuntimeError("What family?") + + +if not hasattr(socket, 'inet_pton'): + socket.inet_pton = inet_pton + +if not hasattr(socket, 'inet_ntop'): + socket.inet_ntop = inet_ntop + + def parse_header(data): addrtype = ord(data[0]) dest_addr = None @@ -61,4 +105,4 @@ def parse_header(data): logging.warn('unsupported addrtype %d, maybe wrong password' % addrtype) if dest_addr is None: return None - return (addrtype, dest_addr, dest_port, header_length) + return addrtype, dest_addr, dest_port, header_length diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 43fac82..d7fd215 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -243,9 +243,12 @@ def main(): ThreadingTCPServer.address_family = addrs[0][0] tcp_servers = [] udp_servers = [] - for port, key in config['port_password'].items(): + for port, password in config['port_password'].items(): + a_config = config.copy() + a_config['server_port'] = port + a_config['password'] = password logging.info("starting server at %s:%d" % - tuple(tcp_server.server_address[:2])) + (a_config['server'], port)) tcp_server = tcprelay.TCPRelay(config, False) tcp_servers.append(tcp_server) udp_server = udprelay.UDPRelay(config, False) @@ -257,11 +260,11 @@ def main(): for udp_server in udp_servers: udp_server.start() - if int(config_workers) > 1: + if int(config['workers']) > 1: if os.name == 'posix': children = [] is_child = False - for i in xrange(0, int(config_workers)): + for i in xrange(0, int(config['workers'])): r = os.fork() if r == 0: logging.info('worker started') diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 3ce8b54..039eed9 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -26,6 +26,7 @@ import socket import logging import encrypt import errno +import threading import eventloop from common import parse_header @@ -121,8 +122,8 @@ class TCPRelayHandler(object): # forward address to remote self._data_to_write_to_remote.append(data[:header_length]) self.write_all_to_sock('\x05\x00\x00\x01' + - '\x00\x00\x00\x00\x10\x10', - self._local_sock) + '\x00\x00\x00\x00\x10\x10', + self._local_sock) else: remote_addr = self._config['server'] remote_port = self._config['server_port'] @@ -230,6 +231,7 @@ class TCPRelay(object): self._config = config self._is_local = is_local self._closed = False + self._thread = None self._fd_to_handlers = {} if is_local: @@ -270,8 +272,8 @@ class TCPRelay(object): if sock == self._server_socket: try: conn = self._server_socket.accept() - TCPRelayHandler(self._eventloop, conn, self._config, - self._is_local) + TCPRelayHandler(self._fd_to_handlers, self._eventloop, + conn, self._config, self._is_local) except (OSError, IOError) as e: error_no = eventloop.errno_from_exception(e) if error_no in [errno.EAGAIN, errno.EINPROGRESS]: @@ -287,3 +289,19 @@ class TCPRelay(object): if now - last_time > 5: # TODO sweep timeouts last_time = now + + def start(self): + if self._closed: + raise Exception('closed') + t = threading.Thread(target=self._run) + t.setName('UDPThread') + t.setDaemon(False) + t.start() + self._thread = t + + def close(self): + self._closed = True + self._server_socket.close() + + def thread(self): + return self._thread \ No newline at end of file diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 68f6474..8950de4 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -101,10 +101,12 @@ class UDPRelay(object): self._method = config['method'] self._timeout = config['timeout'] self._is_local = is_local - self._cache = lru_cache.LRUCache(timeout=timeout, + self._cache = lru_cache.LRUCache(timeout=config['timeout'], close_callback=self._close_client) - self._client_fd_to_server_addr = lru_cache.LRUCache(timeout=timeout) + self._client_fd_to_server_addr = \ + lru_cache.LRUCache(timeout=config['timeout']) self._closed = False + self._thread = None addrs = socket.getaddrinfo(self._listen_addr, self._listen_port, 0, socket.SOCK_DGRAM, socket.SOL_UDP) @@ -206,7 +208,7 @@ class UDPRelay(object): return # addrtype, dest_addr, dest_port, header_length = header_result response = '\x00\x00\x00' + data - client_addr = self._client_fd_to_server_addr.get(sock.fileno(), None) + client_addr = self._client_fd_to_server_addr.get(sock.fileno()) if client_addr: self._server_socket.sendto(response, client_addr) else: diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index f37eed9..05e45b4 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -22,53 +22,9 @@ # SOFTWARE. import os -import socket import logging -def inet_ntop(family, ipstr): - if family == socket.AF_INET: - return socket.inet_ntoa(ipstr) - elif family == socket.AF_INET6: - v6addr = ':'.join(('%02X%02X' % (ord(i), ord(j))) - for i, j in zip(ipstr[::2], ipstr[1::2])) - return v6addr - - -def inet_pton(family, addr): - if family == socket.AF_INET: - return socket.inet_aton(addr) - elif family == socket.AF_INET6: - if '.' in addr: # a v4 addr - v4addr = addr[addr.rindex(':') + 1:] - v4addr = socket.inet_aton(v4addr) - v4addr = map(lambda x: ('%02X' % ord(x)), v4addr) - v4addr.insert(2, ':') - newaddr = addr[:addr.rindex(':') + 1] + ''.join(v4addr) - return inet_pton(family, newaddr) - dbyts = [0] * 8 # 8 groups - grps = addr.split(':') - for i, v in enumerate(grps): - if v: - dbyts[i] = int(v, 16) - else: - for j, w in enumerate(grps[::-1]): - if w: - dbyts[7 - j] = int(w, 16) - else: - break - break - return ''.join((chr(i // 256) + chr(i % 256)) for i in dbyts) - else: - raise RuntimeError("What family?") - - -if not hasattr(socket, 'inet_pton'): - socket.inet_pton = inet_pton - -if not hasattr(socket, 'inet_ntop'): - socket.inet_ntop = inet_ntop - def find_config(): config_path = 'config.json' if os.path.exists(config_path): From 9b2a45583edf0c557fffe5a7eb82ec9f99f5e9bf Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 11:22:57 +0800 Subject: [PATCH 008/344] more work --- README.md | 66 +++++++++++++++++------------------------------------- README.rst | 38 +++++++++---------------------- setup.py | 4 ++-- 3 files changed, 34 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index 442de0f..0547ba7 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,15 @@ shadowsocks =========== -Current version: 1.4.5 [![Build Status][1]][0] +Current version: 2.0 [![Build Status][]][Shadowsocks] -shadowsocks is a lightweight tunnel proxy which can help you get through firewalls. +shadowsocks is a lightweight tunnel proxy that help you get through firewalls. + +2.0 is currently under development. Please use 1.4.x. Both TCP CONNECT and UDP ASSOCIATE are implemented. -[中文说明][3] +[中文说明] Install ------- @@ -22,7 +24,7 @@ Install Shadowsocks. #### Debian / Ubuntu: apt-get install build-essential python-pip python-m2crypto python-dev - pip install gevent shadowsocks + pip install shadowsocks #### CentOS: @@ -39,7 +41,7 @@ Install Shadowsocks. #### Windows: -Choose a [GUI client][7] +Choose a [GUI client] Usage ----- @@ -70,11 +72,11 @@ Explanation of the fields: | password | password used for encryption | | timeout | in seconds | | method | encryption method, "aes-256-cfb" is recommended | -| fast_open | use [TCP_FASTOPEN][2], true / false | +| fast_open | use [TCP_FASTOPEN], true / false | | workers | number of workers, available on Unix/Linux | -Run `ssserver -c /etc/shadowsocks.json` on your server. To run it in the background, -[use supervisor][8]. +Run `ssserver -c /etc/shadowsocks.json` on your server. To run it in the +background, use [Supervisor]. On your client machine, run `sslocal -c /etc/shadowsocks.json`. @@ -84,9 +86,6 @@ Change the proxy settings in your browser to hostname: 127.0.0.1 port: your local_port -**Notice: If you want to use encryption methods other than "table", please -install M2Crypto (See Encryption Section).** - It's recommended to use shadowsocks with AutoProxy or Proxy SwitchySharp. Command line args @@ -98,29 +97,6 @@ You can use args to override settings from `config.json`. ssserver -p server_port -k password -m bf-cfb --workers 2 ssserver -c /etc/shadowsocks/config.json -gevent ------- - -If you suffer from any strange problem when you have installed gevent 0.9.x, -install a new version. - - pip install gevent --upgrade - - -Salsa20 -------- - -Salsa20 is a fast stream cipher. - -Use "salsa20-ctr" in shadowsocks.json. - -And install these packages: - -#### Debian / Ubuntu: - - apt-get install python-numpy - pip install salsa20 - Wiki ---- @@ -132,18 +108,18 @@ MIT Bugs and Issues ---------------- -Please visit [issue tracker][5] +Please visit [Issue Tracker] Mailing list: http://groups.google.com/group/shadowsocks -Also see [troubleshooting][6] +Also see [Troubleshooting] -[0]: https://travis-ci.org/clowwindy/shadowsocks -[1]: https://travis-ci.org/clowwindy/shadowsocks.png?branch=master -[2]: https://github.com/clowwindy/shadowsocks/wiki/TCP-Fast-Open -[3]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E -[4]: http://chandlerproject.org/Projects/MeTooCrypto -[5]: https://github.com/clowwindy/shadowsocks/issues?state=open -[6]: https://github.com/clowwindy/shadowsocks/wiki/Troubleshooting -[7]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients -[8]: https://github.com/clowwindy/shadowsocks/wiki/Configure-Shadowsocks-with-Supervisor + +[Shadowsocks]: https://travis-ci.org/clowwindy/shadowsocks +[Build Status]: https://travis-ci.org/clowwindy/shadowsocks.png?branch=2.0 +[TCP_FASTOPEN]: https://github.com/clowwindy/shadowsocks/wiki/TCP-Fast-Open +[Issue Tracker]: https://github.com/clowwindy/shadowsocks/issues?state=open +[GUI client]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients +[Supervisor]: https://github.com/clowwindy/shadowsocks/wiki/Configure-Shadowsocks-with-Supervisor +[Troubleshooting]: https://github.com/clowwindy/shadowsocks/wiki/Troubleshooting +[中文说明]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E diff --git a/README.rst b/README.rst index a2bc336..953e114 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,13 @@ shadowsocks =========== -shadowsocks is a lightweight tunnel proxy which can help you get through +|Build Status| + +shadowsocks is a lightweight tunnel proxy that help you get through firewalls. +2.0 is currently under development. Please use 1.4.x. + Both TCP CONNECT and UDP ASSOCIATE are implemented. `中文说明 `__ @@ -25,7 +29,7 @@ Debian / Ubuntu: :: - apt-get install python-pip python-gevent python-m2crypto + apt-get install build-essential python-pip python-m2crypto python-dev pip install shadowsocks CentOS: @@ -98,8 +102,8 @@ Explanation of the fields: +------------------+-----------------------------------------------------------------------------------------------------+ Run ``ssserver -c /etc/shadowsocks.json`` on your server. To run it in -the background, `use -supervisor `__. +the background, use +`Supervisor `__. On your client machine, run ``sslocal -c /etc/shadowsocks.json``. @@ -111,9 +115,6 @@ Change the proxy settings in your browser to hostname: 127.0.0.1 port: your local_port -**Notice: If you want to use encryption methods other than "table", -please install M2Crypto (See Encryption Section).** - It's recommended to use shadowsocks with AutoProxy or Proxy SwitchySharp. @@ -128,23 +129,6 @@ You can use args to override settings from ``config.json``. ssserver -p server_port -k password -m bf-cfb --workers 2 ssserver -c /etc/shadowsocks/config.json -Salsa20 -------- - -Salsa20 is a fast stream cipher. - -Use "salsa20-ctr" in shadowsocks.json. - -And install these packages: - -Debian / Ubuntu: -^^^^^^^^^^^^^^^^ - -:: - - apt-get install python-numpy - pip install salsa20 - Wiki ---- @@ -158,13 +142,13 @@ MIT Bugs and Issues --------------- -Please visit `issue -tracker `__ +Please visit `Issue +Tracker `__ Mailing list: http://groups.google.com/group/shadowsocks Also see -`troubleshooting `__ +`Troubleshooting `__ .. |Build Status| image:: https://travis-ci.org/clowwindy/shadowsocks.png?branch=master :target: https://travis-ci.org/clowwindy/shadowsocks diff --git a/setup.py b/setup.py index 259c629..9db7faa 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="1.4.5", + version="2.0", license='MIT', description="a lightweight tunnel proxy", author='clowwindy', @@ -16,7 +16,7 @@ setup( package_data={ 'shadowsocks': ['README.rst', 'LICENSE', 'config.json'] }, - install_requires=['setuptools'], + install_requires=[], entry_points=""" [console_scripts] sslocal = shadowsocks.local:main From 4ccf4081b3a30f5886b2379303f8747733076faf Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 11:26:19 +0800 Subject: [PATCH 009/344] more work --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0547ba7..2144bef 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ shadowsocks =========== -Current version: 2.0 [![Build Status][]][Shadowsocks] +Current version: 2.0 [![Build Status]][Travis CI] shadowsocks is a lightweight tunnel proxy that help you get through firewalls. @@ -115,8 +115,8 @@ Mailing list: http://groups.google.com/group/shadowsocks Also see [Troubleshooting] -[Shadowsocks]: https://travis-ci.org/clowwindy/shadowsocks [Build Status]: https://travis-ci.org/clowwindy/shadowsocks.png?branch=2.0 +[Travis CI]: https://travis-ci.org/clowwindy/shadowsocks [TCP_FASTOPEN]: https://github.com/clowwindy/shadowsocks/wiki/TCP-Fast-Open [Issue Tracker]: https://github.com/clowwindy/shadowsocks/issues?state=open [GUI client]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients From bacf9cb44398531fff65cfab3fd3030d9f3e5e41 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 13:42:48 +0800 Subject: [PATCH 010/344] more work --- shadowsocks/common.py | 11 +++- shadowsocks/eventloop.py | 5 +- shadowsocks/tcprelay.py | 132 ++++++++++++++++++++++++++------------- 3 files changed, 99 insertions(+), 49 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index ca5fb21..4f7aac6 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -70,19 +70,24 @@ if not hasattr(socket, 'inet_ntop'): socket.inet_ntop = inet_ntop +ADDRTYPE_IPV4 = 1 +ADDRTYPE_IPV6 = 4 +ADDRTYPE_HOST = 3 + + def parse_header(data): addrtype = ord(data[0]) dest_addr = None dest_port = None header_length = 0 - if addrtype == 1: + if addrtype == ADDRTYPE_IPV4: if len(data) >= 7: dest_addr = socket.inet_ntoa(data[1:5]) dest_port = struct.unpack('>H', data[5:7])[0] header_length = 7 else: logging.warn('header is too short') - elif addrtype == 3: + elif addrtype == ADDRTYPE_HOST: if len(data) > 2: addrlen = ord(data[1]) if len(data) >= 2 + addrlen: @@ -94,7 +99,7 @@ def parse_header(data): logging.warn('header is too short') else: logging.warn('header is too short') - elif addrtype == 4: + elif addrtype == ADDRTYPE_IPV6: if len(data) >= 19: dest_addr = socket.inet_ntop(socket.AF_INET6, data[1:17]) dest_port = struct.unpack('>H', data[17:19])[0] diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index 1c48718..b432252 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -26,6 +26,7 @@ import os +import socket import select from collections import defaultdict @@ -192,7 +193,5 @@ def errno_from_exception(e): # from tornado def get_sock_error(sock): - errno = sock.getsockopt(socket.SOL_SOCKET, - socket.SO_ERROR) + errno = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) return socket.error(errno, os.strerror(errno)) - diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 039eed9..88abf30 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -30,6 +30,7 @@ import threading import eventloop from common import parse_header +CMD_CONNECT = 1 # local: # stage 0 init @@ -42,6 +43,18 @@ from common import parse_header # stage 4 addr received, reply sent # stage 5 remote connected +STAGE_INIT = 0 +STAGE_HELLO = 1 +STAGE_REPLY = 4 +STAGE_STREAM = 5 + +# stream direction +STREAM_UP = 0 +STREAM_DOWN = 1 + +# stream status +STATUS_WAIT_READING = 0 +STATUS_WAIT_WRITING = 1 BUF_SIZE = 8 * 1024 @@ -54,34 +67,69 @@ class TCPRelayHandler(object): self._remote_sock = None self._config = config self._is_local = is_local - self._stage = 0 + self._stage = STAGE_INIT self._encryptor = encrypt.Encryptor(config['password'], config['method']) self._data_to_write_to_local = [] self._data_to_write_to_remote = [] + self._upstream_status = STATUS_WAIT_READING + self._downstream_status = STATUS_WAIT_READING fd_to_handlers[local_sock.fileno()] = self local_sock.setblocking(False) loop.add(local_sock, eventloop.POLL_IN) - def resume_reading(self, sock): - pass - - def pause_reading(self, sock): - pass - - def resume_writing(self, sock): - pass - - def pause_writing(self, sock): - pass + def update_stream(self, stream, status): + dirty = False + if stream == STREAM_DOWN: + if self._downstream_status != status: + self._downstream_status = status + dirty = True + elif stream == STREAM_UP: + if self._upstream_status != status: + self._upstream_status = status + dirty = True + if dirty: + if self._local_sock: + event = eventloop.POLL_ERR + if self._downstream_status == STATUS_WAIT_WRITING: + event |= eventloop.POLL_OUT + if self._upstream_status == STATUS_WAIT_READING: + event |= eventloop.POLL_IN + self._loop.modify(self._local_sock, event) + if self._remote_sock: + event = eventloop.POLL_ERR + if self._downstream_status == STATUS_WAIT_READING: + event |= eventloop.POLL_IN + if self._upstream_status == STATUS_WAIT_WRITING: + event |= eventloop.POLL_OUT + self._loop.modify(self._remote_sock, event) def write_all_to_sock(self, data, sock): - # write to sock - # put remaining bytes into buffer - # return true if all written - # return false if some bytes left in buffer - # raise if encounter error - return True + if not data or not sock: + return + uncomplete = False + try: + l = len(data) + s = sock.send(data) + if s < l: + data = data[s:] + uncomplete = True + except (OSError, IOError) as e: + error_no = eventloop.errno_from_exception(e) + if error_no in (errno.EAGAIN, errno.EINPROGRESS): + uncomplete = True + else: + logging.error(e) + self.destroy() + if uncomplete: + if sock == self._local_sock: + self._data_to_write_to_local.append(data) + self.update_stream(STREAM_DOWN, STATUS_WAIT_WRITING) + elif sock == self._remote_sock: + self._data_to_write_to_remote.append(data) + self.update_stream(STREAM_UP, STATUS_WAIT_WRITING) + else: + logging.error('write_all_to_sock:unknown socket') def on_local_read(self): if not self._local_sock: @@ -90,26 +138,25 @@ class TCPRelayHandler(object): data = self._local_sock.recv(BUF_SIZE) if not is_local: data = self._encryptor.decrypt(data) - if self._stage == 5: + if self._stage == STAGE_STREAM: if self._is_local: data = self._encryptor.encrypt(data) - if not self.write_all_to_sock(data, self._remote_sock): - self.pause_reading(self._local_sock) + self.write_all_to_sock(data, self._remote_sock) return - if is_local and self._stage == 0: + if is_local and self._stage == STAGE_INIT: # TODO check auth method self.write_all_to_sock('\x05\00', self._local_sock) - self._stage = 1 + self._stage = STAGE_HELLO return - if self._stage == 4: + if self._stage == STAGE_REPLY: self._data_to_write_to_remote.append(data) - if (is_local and self._stage == 0) or \ - (not is_local and self._stage == 1): + if (is_local and self._stage == STAGE_HELLO) or \ + (not is_local and self._stage == STAGE_INIT): try: if is_local: cmd = ord(data[1]) # TODO check cmd == 1 - assert cmd == 1 + assert cmd == CMD_CONNECT # just trim VER CMD RSV data = data[3:] header_result = parse_header(data) @@ -145,7 +192,7 @@ class TCPRelayHandler(object): self._data_to_write_to_remote.append(data[header_length:]) self._stage = 4 - self.pause_reading(self._local_sock) + self.update_stream(STREAM_UP, STATUS_WAIT_WRITING) return except Exception: import traceback @@ -153,7 +200,7 @@ class TCPRelayHandler(object): # TODO use logging when debug completed self.destroy() - if self._stage == 4: + elif self._stage == STAGE_REPLY: self._data_to_write_to_remote.append(data) def on_remote_read(self): @@ -161,9 +208,7 @@ class TCPRelayHandler(object): if self._is_local: data = self._encryptor.decrypt(data) try: - if not self.write_all_to_sock(data, self._local_sock): - self.pause_reading(self._remote_sock) - self.resume_writing(self._local_sock) + self.write_all_to_sock(data, self._local_sock) except Exception: import traceback traceback.print_exc() @@ -172,26 +217,26 @@ class TCPRelayHandler(object): def on_local_write(self): if self._data_to_write_to_local: - written = self.write_all_to_sock( - ''.join(self._data_to_write_to_local), self._local_sock) - if written: - self.pause_writing(self._local_sock) + data = ''.join(self._data_to_write_to_local) + self._data_to_write_to_local = [] + self.write_all_to_sock(data, self._local_sock) else: - self.pause_writing(self._local_sock) + self.update_stream(STREAM_DOWN, STATUS_WAIT_READING) def on_remote_write(self): if self._data_to_write_to_remote: - written = self.write_all_to_sock( - ''.join(self._data_to_write_to_remote), self._remote_sock) - if written: - self.pause_writing(self._remote_sock) + data = ''.join(self._data_to_write_to_remote) + self._data_to_write_to_remote = [] + self.write_all_to_sock(data, self._remote_sock) else: - self.pause_writing(self._remote_sock) + self.update_stream(STREAM_DOWN, STATUS_WAIT_READING) def on_local_error(self): + logging.error(eventloop.get_sock_error(self._local_sock)) self.destroy() def on_remote_error(self): + logging.error(eventloop.get_sock_error(self._remote_sock)) self.destroy() def handle_event(self, sock, event): @@ -276,7 +321,7 @@ class TCPRelay(object): conn, self._config, self._is_local) except (OSError, IOError) as e: error_no = eventloop.errno_from_exception(e) - if error_no in [errno.EAGAIN, errno.EINPROGRESS]: + if error_no in (errno.EAGAIN, errno.EINPROGRESS): continue else: handler = self._fd_to_handlers.get(sock.fileno(), None) @@ -291,6 +336,7 @@ class TCPRelay(object): last_time = now def start(self): + # TODO combine loops on multiple ports into one single loop if self._closed: raise Exception('closed') t = threading.Thread(target=self._run) From 47fd479d6e62a65212ce3ae37d17c1215ee5049d Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 14:10:18 +0800 Subject: [PATCH 011/344] almost done --- shadowsocks/local.py | 343 ++-------------------------------------- shadowsocks/server.py | 199 +---------------------- shadowsocks/tcprelay.py | 11 +- shadowsocks/udprelay.py | 2 +- shadowsocks/utils.py | 95 +++++++++++ 5 files changed, 122 insertions(+), 528 deletions(-) diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 049c8ea..66070aa 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -21,353 +21,40 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from __future__ import with_statement import sys -if sys.version_info < (2, 6): - import simplejson as json -else: - import json - -try: - import gevent - import gevent.monkey - gevent.monkey.patch_all(dns=gevent.version_info[0] >= 1) -except ImportError: - gevent = None - print >>sys.stderr, 'warning: gevent not found, using threading instead' - import socket -import eventloop -import errno -import select -import SocketServer -import struct import os -import random -import re import logging -import getopt import encrypt import utils +import tcprelay import udprelay -MSG_FASTOPEN = 0x20000000 - - -def send_all(sock, data): - bytes_sent = 0 - while True: - r = sock.send(data[bytes_sent:]) - if r < 0: - return r - bytes_sent += r - if bytes_sent == len(data): - return bytes_sent - - -class ThreadingTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): - allow_reuse_address = True - - def get_request(self): - connection = self.socket.accept() - connection[0].settimeout(config_timeout) - return connection - - -class Socks5Server(SocketServer.StreamRequestHandler): - @staticmethod - def get_server(): - a_port = config_server_port - a_server = config_server - if isinstance(config_server_port, list): - # support config like "server_port": [8081, 8082] - a_port = random.choice(config_server_port) - if isinstance(config_server, list): - # support config like "server": ["123.123.123.1", "123.123.123.2"] - a_server = random.choice(config_server) - - r = re.match(r'^(.*):(\d+)$', a_server) - if r: - # support config like "server": "123.123.123.1:8381" - # or "server": ["123.123.123.1:8381", "123.123.123.2:8381"] - a_server = r.group(1) - a_port = int(r.group(2)) - return a_server, a_port - - @staticmethod - def handle_tcp(sock, remote, encryptor, pending_data=None, - server=None, port=None): - connected = False - try: - if config_fast_open: - fdset = [sock] - else: - fdset = [sock, remote] - while True: - should_break = False - r, w, e = select.select(fdset, [], [], config_timeout) - if not r: - logging.warn('read time out') - break - if sock in r: - if not connected and config_fast_open: - data = sock.recv(4096) - data = encryptor.encrypt(pending_data + data) - pending_data = None - logging.info('fast open %s:%d' % (server, port)) - try: - remote.sendto(data, MSG_FASTOPEN, (server, port)) - except (OSError, IOError) as e: - if eventloop.errno_from_exception(e) == errno.EINPROGRESS: - pass - else: - raise e - connected = True - fdset = [sock, remote] - else: - data = sock.recv(4096) - if pending_data: - data = pending_data + data - pending_data = None - data = encryptor.encrypt(data) - if len(data) <= 0: - should_break = True - else: - result = send_all(remote, data) - if result < len(data): - raise Exception('failed to send all data') - - if remote in r: - data = encryptor.decrypt(remote.recv(4096)) - if len(data) <= 0: - should_break = True - else: - result = send_all(sock, data) - if result < len(data): - raise Exception('failed to send all data') - if should_break: - # make sure all data are read before we close the sockets - # TODO: we haven't read ALL the data, actually - # http://cs.ecs.baylor.edu/~donahoo/practical/CSockets/TCPRST.pdf - break - finally: - sock.close() - remote.close() - - def handle(self): - try: - encryptor = encrypt.Encryptor(config_password, config_method) - sock = self.connection - sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - data = sock.recv(262) - if not data: - sock.close() - return - if len(data) < 3: - return - method = ord(data[2]) - if method == 2: - logging.warn('client tries to use username/password auth, prete' - 'nding the password is OK') - sock.send('\x05\x02') - try: - ver_ulen = sock.recv(2) - ulen = ord(ver_ulen[1]) - if ulen: - username = sock.recv(ulen) - assert(ulen == len(username)) - plen = ord(sock.recv(1)) - if plen: - _password = sock.recv(plen) - assert(plen == len(_password)) - sock.send('\x01\x00') - except Exception as e: - logging.error(e) - return - elif method == 0: - sock.send("\x05\x00") - else: - logging.error('unsupported method %d' % method) - return - data = self.rfile.read(4) or '\x00' * 4 - mode = ord(data[1]) - if mode == 1: - pass - elif mode == 3: - # UDP - logging.debug('UDP assc request') - if sock.family == socket.AF_INET6: - header = '\x05\x00\x00\x04' - else: - header = '\x05\x00\x00\x01' - addr, port = sock.getsockname() - addr_to_send = socket.inet_pton(sock.family, addr) - port_to_send = struct.pack('>H', port) - sock.send(header + addr_to_send + port_to_send) - while True: - data = sock.recv(4096) - if not data: - break - return - else: - logging.warn('unknown mode %d' % mode) - return - addrtype = ord(data[3]) - addr_to_send = data[3] - if addrtype == 1: - addr_ip = self.rfile.read(4) - addr = socket.inet_ntoa(addr_ip) - addr_to_send += addr_ip - elif addrtype == 3: - addr_len = self.rfile.read(1) - addr = self.rfile.read(ord(addr_len)) - addr_to_send += addr_len + addr - elif addrtype == 4: - addr_ip = self.rfile.read(16) - addr = socket.inet_ntop(socket.AF_INET6, addr_ip) - addr_to_send += addr_ip - else: - logging.warn('addr_type not supported') - # not supported - return - addr_port = self.rfile.read(2) - addr_to_send += addr_port - port = struct.unpack('>H', addr_port) - try: - reply = "\x05\x00\x00\x01" - reply += socket.inet_aton('0.0.0.0') + struct.pack(">H", 2222) - self.wfile.write(reply) - # reply immediately - a_server, a_port = Socks5Server.get_server() - addrs = socket.getaddrinfo(a_server, a_port) - if addrs: - af, socktype, proto, canonname, sa = addrs[0] - if config_fast_open: - remote = socket.socket(af, socktype, proto) - remote.setsockopt(socket.IPPROTO_TCP, - socket.TCP_NODELAY, 1) - Socks5Server.handle_tcp(sock, remote, encryptor, - addr_to_send, a_server, a_port) - else: - logging.info('connecting %s:%d' % (addr, port[0])) - remote = socket.create_connection((a_server, a_port), - timeout=config_timeout) - remote.settimeout(config_timeout) - remote.setsockopt(socket.IPPROTO_TCP, - socket.TCP_NODELAY, 1) - Socks5Server.handle_tcp(sock, remote, encryptor, - addr_to_send) - except (OSError, IOError) as e: - logging.warn(e) - return - except (OSError, IOError) as e: - raise e - logging.warn(e) - - def main(): - global config_server, config_server_port, config_password, config_method,\ - config_fast_open, config_timeout - - logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s %(levelname)-8s %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', filemode='a+') + utils.check_python() # fix py2exe if hasattr(sys, "frozen") and sys.frozen in \ ("windows_exe", "console_exe"): p = os.path.dirname(os.path.abspath(sys.executable)) os.chdir(p) - version = '' + + utils.print_shadowsocks() + + config = utils.get_config(True) + + encrypt.init_table(config['password'], config['method']) + try: - import pkg_resources - version = pkg_resources.get_distribution('shadowsocks').version - except: - pass - print 'shadowsocks %s' % version - - config_password = None - config_method = None - - config_path = utils.find_config() - try: - optlist, args = getopt.getopt(sys.argv[1:], 's:b:p:k:l:m:c:t:', - ['fast-open']) - for key, value in optlist: - if key == '-c': - config_path = value - - if config_path: - logging.info('loading config from %s' % config_path) - with open(config_path, 'rb') as f: - try: - config = json.load(f) - except ValueError as e: - logging.error('found an error in config.json: %s', - e.message) - sys.exit(1) - else: - config = {} - - optlist, args = getopt.getopt(sys.argv[1:], 's:b:p:k:l:m:c:t:', - ['fast-open']) - for key, value in optlist: - if key == '-p': - config['server_port'] = int(value) - elif key == '-k': - config['password'] = value - elif key == '-l': - config['local_port'] = int(value) - elif key == '-s': - config['server'] = value - elif key == '-m': - config['method'] = value - elif key == '-b': - config['local_address'] = value - elif key == '--fast-open': - config['fast_open'] = True - except getopt.GetoptError as e: - logging.error(e) - utils.print_local_help() - sys.exit(2) - - config_server = config['server'] - config_server_port = config['server_port'] - config_local_port = config['local_port'] - config_password = config['password'] - config_method = config.get('method', None) - config_local_address = config.get('local_address', '127.0.0.1') - config_timeout = int(config.get('timeout', 300)) - config_fast_open = config.get('fast_open', False) - - if not config_password and not config_path: - sys.exit('config not specified, please read ' - 'https://github.com/clowwindy/shadowsocks') - - utils.check_config(config) - - encrypt.init_table(config_password, config_method) - - addrs = socket.getaddrinfo(config_local_address, config_local_port) - if not addrs: - logging.error('cant resolve listen address') - sys.exit(1) - ThreadingTCPServer.address_family = addrs[0][0] - try: - udprelay.UDPRelay(config_local_address, int(config_local_port), - config_server, config_server_port, config_password, - config_method, int(config_timeout), True).start() - server = ThreadingTCPServer((config_local_address, config_local_port), - Socks5Server) - server.timeout = int(config_timeout) logging.info("starting local at %s:%d" % - tuple(server.server_address[:2])) - server.serve_forever() - except socket.error, e: - logging.error(e) + (config['local_address'], config['local_port'])) + + udprelay.UDPRelay(config, True).start() + tcprelay.TCPRelay(config, True).start() + while sys.stdin.read(): + pass except KeyboardInterrupt: - server.shutdown() sys.exit(0) if __name__ == '__main__': diff --git a/shadowsocks/server.py b/shadowsocks/server.py index d7fd215..bb64bcc 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -21,21 +21,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from __future__ import with_statement import sys -if sys.version_info < (2, 6): - import simplejson as json -else: - import json - - import socket -import select -import threading -import SocketServer -import struct import logging -import getopt import encrypt import os import utils @@ -43,188 +31,12 @@ import tcprelay import udprelay -def send_all(sock, data): - bytes_sent = 0 - while True: - r = sock.send(data[bytes_sent:]) - if r < 0: - return r - bytes_sent += r - if bytes_sent == len(data): - return bytes_sent - - -class ThreadingTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): - allow_reuse_address = True - - def server_activate(self): - if config_fast_open: - try: - self.socket.setsockopt(socket.SOL_TCP, 23, 5) - except socket.error: - logging.error('warning: fast open is not available') - self.socket.listen(self.request_queue_size) - - def get_request(self): - connection = self.socket.accept() - connection[0].settimeout(config_timeout) - return connection - - -class Socks5Server(SocketServer.StreamRequestHandler): - def handle_tcp(self, sock, remote): - try: - fdset = [sock, remote] - while True: - should_break = False - r, w, e = select.select(fdset, [], [], config_timeout) - if not r: - logging.warn('read time out') - break - if sock in r: - data = self.decrypt(sock.recv(4096)) - if len(data) <= 0: - should_break = True - else: - result = send_all(remote, data) - if result < len(data): - raise Exception('failed to send all data') - if remote in r: - data = self.encrypt(remote.recv(4096)) - if len(data) <= 0: - should_break = True - else: - result = send_all(sock, data) - if result < len(data): - raise Exception('failed to send all data') - if should_break: - # make sure all data are read before we close the sockets - # TODO: we haven't read ALL the data, actually - # http://cs.ecs.baylor.edu/~donahoo/practical/CSockets/TCPRST.pdf - break - - finally: - sock.close() - remote.close() - - def encrypt(self, data): - return self.encryptor.encrypt(data) - - def decrypt(self, data): - return self.encryptor.decrypt(data) - - def handle(self): - try: - self.encryptor = encrypt.Encryptor(self.server.key, - self.server.method) - sock = self.connection - sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - iv_len = self.encryptor.iv_len() - data = sock.recv(iv_len) - if iv_len > 0 and not data: - sock.close() - return - if iv_len: - self.decrypt(data) - data = sock.recv(1) - if not data: - sock.close() - return - addrtype = ord(self.decrypt(data)) - if addrtype == 1: - addr = socket.inet_ntoa(self.decrypt(self.rfile.read(4))) - elif addrtype == 3: - addr = self.decrypt( - self.rfile.read(ord(self.decrypt(sock.recv(1))))) - elif addrtype == 4: - addr = socket.inet_ntop(socket.AF_INET6, - self.decrypt(self.rfile.read(16))) - else: - # not supported - logging.warn('addr_type not supported, maybe wrong password') - return - port = struct.unpack('>H', self.decrypt(self.rfile.read(2))) - try: - logging.info('connecting %s:%d' % (addr, port[0])) - remote = socket.create_connection((addr, port[0]), - timeout=config_timeout) - remote.settimeout(config_timeout) - remote.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - except socket.error, e: - # Connection refused - logging.warn(e) - return - self.handle_tcp(sock, remote) - except socket.error, e: - logging.warn(e) - - def main(): - global config_server, config_server_port, config_method, config_fast_open, \ - config_timeout + utils.check_python() - logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s %(levelname)-8s %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', filemode='a+') + utils.print_shadowsocks() - - version = '' - try: - import pkg_resources - version = pkg_resources.get_distribution('shadowsocks').version - except: - pass - print 'shadowsocks %s' % version - - config_path = utils.find_config() - try: - optlist, args = getopt.getopt(sys.argv[1:], 's:p:k:m:c:t:', - ['fast-open', 'workers:']) - for key, value in optlist: - if key == '-c': - config_path = value - - if config_path: - logging.info('loading config from %s' % config_path) - with open(config_path, 'rb') as f: - try: - config = json.load(f) - except ValueError as e: - logging.error('found an error in config.json: %s', - e.message) - sys.exit(1) - else: - config = {} - - optlist, args = getopt.getopt(sys.argv[1:], 's:p:k:m:c:t:', - ['fast-open', 'workers=']) - for key, value in optlist: - if key == '-p': - config['server_port'] = int(value) - elif key == '-k': - config['password'] = value - elif key == '-s': - config['server'] = value - elif key == '-m': - config['method'] = value - elif key == '-t': - config['timeout'] = value - elif key == '--fast-open': - config['fast_open'] = True - elif key == '--workers': - config['workers'] = value - except getopt.GetoptError: - utils.print_server_help() - sys.exit(2) - - config['password'] = config.get('password', None) - config['method'] = config.get('method', None) - config['port_password'] = config.get('port_password', None) - config['timeout'] = int(config.get('timeout', 300)) - config['fast_open'] = config.get('fast_open', False) - config['workers'] = config.get('workers', 1) - - utils.check_config(config) + config = utils.get_config(True) if config['port_password']: if config['server_port'] or config['password']: @@ -236,11 +48,6 @@ def main(): config['port_password'][str(config['server_port'])] = config['password'] encrypt.init_table(config['password'], config['method']) - addrs = socket.getaddrinfo(config['server'], int(8387)) - if not addrs: - logging.error('cant resolve listen address') - sys.exit(1) - ThreadingTCPServer.address_family = addrs[0][0] tcp_servers = [] udp_servers = [] for port, password in config['port_password'].items(): diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 88abf30..335849b 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -76,7 +76,7 @@ class TCPRelayHandler(object): self._downstream_status = STATUS_WAIT_READING fd_to_handlers[local_sock.fileno()] = self local_sock.setblocking(False) - loop.add(local_sock, eventloop.POLL_IN) + loop.add(local_sock, eventloop.POLL_IN | eventloop.POLL_ERR) def update_stream(self, stream, status): dirty = False @@ -186,7 +186,8 @@ class TCPRelayHandler(object): self._remote_sock.setblocking(False) # TODO support TCP fast open self._remote_sock.connect(sa) - self._loop.add(self._remote_sock, eventloop.POLL_OUT) + self._loop.add(self._remote_sock, + eventloop.POLL_ERR | eventloop.POLL_OUT) if len(data) > header_length: self._data_to_write_to_remote.append(data[header_length:]) @@ -301,7 +302,8 @@ class TCPRelay(object): def _run(self): server_socket = self._server_socket self._eventloop = eventloop.EventLoop() - self._eventloop.add(server_socket, eventloop.POLL_IN) + self._eventloop.add(server_socket, + eventloop.POLL_IN | eventloop.POLL_ERR) last_time = time.time() while not self._closed: try: @@ -315,6 +317,9 @@ class TCPRelay(object): continue for sock, event in events: if sock == self._server_socket: + if event & eventloop.POLL_ERR: + # TODO + raise Exception('server_socket error') try: conn = self._server_socket.accept() TCPRelayHandler(self._fd_to_handlers, self._eventloop, diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 8950de4..68414e8 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -86,7 +86,7 @@ def client_key(a, b, c, d): class UDPRelay(object): - def __init__(self, config, is_local=True): + def __init__(self, config, is_local): if is_local: self._listen_addr = config['local_address'] self._listen_port = config['local_port'] diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 05e45b4..db19bec 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -22,9 +22,29 @@ # SOFTWARE. import os +import json +import sys +import getopt import logging +def check_python(): + info = sys.version_info + if not (info.major == 2 and info.minor >= 6): + print 'Python 2.6 or 2.7 required' + sys.exit(1) + + +def print_shadowsocks(): + version = '' + try: + import pkg_resources + version = pkg_resources.get_distribution('shadowsocks').version + except Exception: + pass + print 'shadowsocks %s' % version + + def find_config(): config_path = 'config.json' if os.path.exists(config_path): @@ -36,6 +56,12 @@ def find_config(): def check_config(config): + config['password'] = config.get('password', None) + config['method'] = config.get('method', None) + config['port_password'] = config.get('port_password', None) + config['timeout'] = int(config.get('timeout', 300)) + config['fast_open'] = config.get('fast_open', False) + config['workers'] = config.get('workers', 1) if config.get('local_address', '') in ['0.0.0.0']: logging.warn('warning: local set to listen 0.0.0.0, which is not safe') if config.get('server', '') in ['127.0.0.1', 'localhost']: @@ -52,6 +78,75 @@ def check_config(config): int(config.get('timeout'))) +def get_config(is_local): + if is_local: + shortopts = 's:b:p:k:l:m:c:t:v' + longopts = ['fast-open'] + else: + shortopts = 's:p:k:m:c:t:' + longopts = ['fast-open', 'workers:'] + try: + config_path = find_config() + optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) + for key, value in optlist: + if key == '-c': + config_path = value + + if config_path: + logging.info('loading config from %s' % config_path) + with open(config_path, 'rb') as f: + try: + config = json.load(f) + except ValueError as e: + logging.error('found an error in config.json: %s', + e.message) + sys.exit(1) + else: + config = {} + + optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) + for key, value in optlist: + if key == '-p': + config['server_port'] = int(value) + elif key == '-k': + config['password'] = value + elif key == '-l': + config['local_port'] = int(value) + elif key == '-s': + config['server'] = value + elif key == '-m': + config['method'] = value + elif key == '-b': + config['local_address'] = value + elif key == '-v': + config['verbose'] = True + elif key == '--fast-open': + config['fast_open'] = True + elif key == '--workers': + config['workers'] = value + except getopt.GetoptError as e: + logging.error(e) + if is_local: + print_local_help() + else: + print_server_help() + sys.exit(2) + + if not config['password'] and not config_path: + sys.exit('config not specified, please read ' + 'https://github.com/clowwindy/shadowsocks') + + check_config(config) + + if config['verbose']: + level = logging.DEBUG + else: + level = logging.WARNING + logging.basicConfig(level=level, + format='%(asctime)s %(levelname)-8s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', filemode='a+') + + def print_local_help(): print '''usage: sslocal [-h] -s SERVER_ADDR -p SERVER_PORT [-b LOCAL_ADDR] -l LOCAL_PORT -k PASSWORD -m METHOD [-t TIMEOUT] [-c CONFIG] From cd22d474fe6744a8b1ef41902f4f6090cee9742b Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 14:11:38 +0800 Subject: [PATCH 012/344] almost done --- shadowsocks/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index db19bec..84a514b 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -145,6 +145,7 @@ def get_config(is_local): logging.basicConfig(level=level, format='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S', filemode='a+') + return config def print_local_help(): From 5e9a9c9beb7b7734a5ca6b457938b87296646fd9 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 14:20:40 +0800 Subject: [PATCH 013/344] almost done --- shadowsocks/local.py | 4 ++-- shadowsocks/server.py | 4 ++-- shadowsocks/utils.py | 12 +++++++++--- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 66070aa..7219c08 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -40,10 +40,10 @@ def main(): p = os.path.dirname(os.path.abspath(sys.executable)) os.chdir(p) - utils.print_shadowsocks() - config = utils.get_config(True) + utils.print_shadowsocks() + encrypt.init_table(config['password'], config['method']) try: diff --git a/shadowsocks/server.py b/shadowsocks/server.py index bb64bcc..bfb2f8b 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -34,9 +34,9 @@ import udprelay def main(): utils.check_python() - utils.print_shadowsocks() + config = utils.get_config(False) - config = utils.get_config(True) + utils.print_shadowsocks() if config['port_password']: if config['server_port'] or config['password']: diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 84a514b..0daa507 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -125,7 +125,7 @@ def get_config(is_local): elif key == '--workers': config['workers'] = value except getopt.GetoptError as e: - logging.error(e) + print >>sys.stderr, e if is_local: print_local_help() else: @@ -151,7 +151,7 @@ def get_config(is_local): def print_local_help(): print '''usage: sslocal [-h] -s SERVER_ADDR -p SERVER_PORT [-b LOCAL_ADDR] -l LOCAL_PORT -k PASSWORD -m METHOD [-t TIMEOUT] [-c CONFIG] - [--fast-open] + [--fast-open] [-v] optional arguments: -h, --help show this help message and exit @@ -164,12 +164,15 @@ optional arguments: -t TIMEOUT timeout in seconds -c CONFIG path to config file --fast-open use TCP_FASTOPEN, requires Linux 3.7+ + -v verbose mode + +Online help: ''' def print_server_help(): print '''usage: ssserver [-h] -s SERVER_ADDR -p SERVER_PORT -k PASSWORD - -m METHOD [-t TIMEOUT] [-c CONFIG] [--fast-open] + -m METHOD [-t TIMEOUT] [-c CONFIG] [--fast-open] [-v] optional arguments: -h, --help show this help message and exit @@ -181,4 +184,7 @@ optional arguments: -c CONFIG path to config file --fast-open use TCP_FASTOPEN, requires Linux 3.7+ --workers WORKERS number of workers, available on Unix/Linux + -v verbose mode + +Online help: ''' \ No newline at end of file From 3cefe374c2b48fa7dda2fb152365a40d88086f20 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 14:53:06 +0800 Subject: [PATCH 014/344] socket.SO_REUSEADDR --- shadowsocks/encrypt.py | 6 +++--- shadowsocks/local.py | 2 +- shadowsocks/tcprelay.py | 1 + shadowsocks/utils.py | 15 +++++++++------ 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 267565c..9d94dd1 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -157,9 +157,9 @@ class Encryptor(object): self.cipher_iv = iv[:m[1]] # this iv is for cipher not decipher if method != 'salsa20-ctr': import M2Crypto.EVP - return M2Crypto.EVP.Cipher(method.replace('-', '_'), key, iv, op, - key_as_bytes=0, d='md5', salt=None, i=1, - padding=1) + return M2Crypto.EVP.Cipher(method.replace('-', '_'), key, iv, + op, key_as_bytes=0, d='md5', + salt=None, i=1, padding=1) else: return encrypt_salsa20.Salsa20Cipher(method, key, iv, op) diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 7219c08..0ccc813 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -55,7 +55,7 @@ def main(): while sys.stdin.read(): pass except KeyboardInterrupt: - sys.exit(0) + os._exit(0) if __name__ == '__main__': main() diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 335849b..b7902d9 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -294,6 +294,7 @@ class TCPRelay(object): (listen_addr, listen_port)) af, socktype, proto, canonname, sa = addrs[0] server_socket = socket.socket(af, socktype, proto) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server_socket.bind(sa) server_socket.setblocking(False) server_socket.listen(1024) diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 0daa507..180cdcb 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -56,12 +56,6 @@ def find_config(): def check_config(config): - config['password'] = config.get('password', None) - config['method'] = config.get('method', None) - config['port_password'] = config.get('port_password', None) - config['timeout'] = int(config.get('timeout', 300)) - config['fast_open'] = config.get('fast_open', False) - config['workers'] = config.get('workers', 1) if config.get('local_address', '') in ['0.0.0.0']: logging.warn('warning: local set to listen 0.0.0.0, which is not safe') if config.get('server', '') in ['127.0.0.1', 'localhost']: @@ -136,6 +130,15 @@ def get_config(is_local): sys.exit('config not specified, please read ' 'https://github.com/clowwindy/shadowsocks') + config['password'] = config.get('password', None) + config['method'] = config.get('method', None) + config['port_password'] = config.get('port_password', None) + config['timeout'] = int(config.get('timeout', 300)) + config['fast_open'] = config.get('fast_open', False) + config['workers'] = config.get('workers', 1) + config['verbose'] = config.get('verbose', False) + config['local_address'] = config.get('local_address', '127.0.0.1') + check_config(config) if config['verbose']: From c721a1c02f8a8278ccd7d12e619c84667111ae67 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 15:58:37 +0800 Subject: [PATCH 015/344] local works --- shadowsocks/tcprelay.py | 69 +++++++++++++++++++++++++++++++---------- shadowsocks/utils.py | 6 ++-- 2 files changed, 56 insertions(+), 19 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index b7902d9..2a9d552 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -53,8 +53,9 @@ STREAM_UP = 0 STREAM_DOWN = 1 # stream status -STATUS_WAIT_READING = 0 -STATUS_WAIT_WRITING = 1 +STATUS_WAIT_INIT = 0 +STATUS_WAIT_READING = 1 +STATUS_WAIT_WRITING = 2 BUF_SIZE = 8 * 1024 @@ -72,8 +73,8 @@ class TCPRelayHandler(object): config['method']) self._data_to_write_to_local = [] self._data_to_write_to_remote = [] - self._upstream_status = STATUS_WAIT_READING - self._downstream_status = STATUS_WAIT_READING + self._upstream_status = STATUS_WAIT_INIT + self._downstream_status = STATUS_WAIT_INIT fd_to_handlers[local_sock.fileno()] = self local_sock.setblocking(False) loop.add(local_sock, eventloop.POLL_IN | eventloop.POLL_ERR) @@ -132,12 +133,24 @@ class TCPRelayHandler(object): logging.error('write_all_to_sock:unknown socket') def on_local_read(self): + # TODO update timeout if not self._local_sock: return is_local = self._is_local - data = self._local_sock.recv(BUF_SIZE) + data = None + try: + data = self._local_sock.recv(BUF_SIZE) + except (OSError, IOError) as e: + if eventloop.errno_from_exception(e) in \ + (errno.ETIMEDOUT, errno.EAGAIN): + return + if not data: + self.destroy() + return if not is_local: data = self._encryptor.decrypt(data) + if not data: + return if self._stage == STAGE_STREAM: if self._is_local: data = self._encryptor.encrypt(data) @@ -167,13 +180,17 @@ class TCPRelayHandler(object): logging.info('connecting %s:%d' % (remote_addr, remote_port)) if is_local: # forward address to remote - self._data_to_write_to_remote.append(data[:header_length]) self.write_all_to_sock('\x05\x00\x00\x01' + '\x00\x00\x00\x00\x10\x10', self._local_sock) - else: + data_to_send = self._encryptor.encrypt(data) + self._data_to_write_to_remote.append(data_to_send) remote_addr = self._config['server'] remote_port = self._config['server_port'] + else: + if len(data) > header_length: + self._data_to_write_to_remote.append( + data[header_length:]) # TODO async DNS addrs = socket.getaddrinfo(remote_addr, remote_port, 0, @@ -183,17 +200,20 @@ class TCPRelayHandler(object): (remote_addr, remote_port)) af, socktype, proto, canonname, sa = addrs[0] self._remote_sock = socket.socket(af, socktype, proto) + self._fd_to_handlers[self._remote_sock.fileno()] = self self._remote_sock.setblocking(False) # TODO support TCP fast open - self._remote_sock.connect(sa) + try: + self._remote_sock.connect(sa) + except (OSError, IOError) as e: + if eventloop.errno_from_exception(e) == errno.EINPROGRESS: + pass self._loop.add(self._remote_sock, eventloop.POLL_ERR | eventloop.POLL_OUT) - if len(data) > header_length: - self._data_to_write_to_remote.append(data[header_length:]) - self._stage = 4 self.update_stream(STREAM_UP, STATUS_WAIT_WRITING) + self.update_stream(STREAM_DOWN, STATUS_WAIT_READING) return except Exception: import traceback @@ -205,7 +225,17 @@ class TCPRelayHandler(object): self._data_to_write_to_remote.append(data) def on_remote_read(self): - data = self._remote_sock.recv(BUF_SIZE) + # TODO update timeout + data = None + try: + data = self._remote_sock.recv(BUF_SIZE) + except (OSError, IOError) as e: + if eventloop.errno_from_exception(e) in \ + (errno.ETIMEDOUT, errno.EAGAIN): + return + if not data: + self.destroy() + return if self._is_local: data = self._encryptor.decrypt(data) try: @@ -225,12 +255,13 @@ class TCPRelayHandler(object): self.update_stream(STREAM_DOWN, STATUS_WAIT_READING) def on_remote_write(self): + self._stage = STAGE_STREAM if self._data_to_write_to_remote: data = ''.join(self._data_to_write_to_remote) self._data_to_write_to_remote = [] self.write_all_to_sock(data, self._remote_sock) else: - self.update_stream(STREAM_DOWN, STATUS_WAIT_READING) + self.update_stream(STREAM_UP, STATUS_WAIT_READING) def on_local_error(self): logging.error(eventloop.get_sock_error(self._local_sock)) @@ -261,14 +292,14 @@ class TCPRelayHandler(object): def destroy(self): if self._remote_sock: - self._remote_sock.close() self._loop.remove(self._remote_sock) del self._fd_to_handlers[self._remote_sock.fileno()] + self._remote_sock.close() self._remote_sock = None if self._local_sock: - self._local_sock.close() self._loop.remove(self._local_sock) del self._fd_to_handlers[self._local_sock.fileno()] + self._local_sock.close() self._local_sock = None @@ -317,6 +348,7 @@ class TCPRelay(object): logging.error(e) continue for sock, event in events: + logging.debug('%s %d', sock, event) if sock == self._server_socket: if event & eventloop.POLL_ERR: # TODO @@ -324,11 +356,13 @@ class TCPRelay(object): try: conn = self._server_socket.accept() TCPRelayHandler(self._fd_to_handlers, self._eventloop, - conn, self._config, self._is_local) + conn[0], self._config, self._is_local) except (OSError, IOError) as e: error_no = eventloop.errno_from_exception(e) if error_no in (errno.EAGAIN, errno.EINPROGRESS): continue + else: + logging.error(e) else: handler = self._fd_to_handlers.get(sock.fileno(), None) if handler: @@ -336,6 +370,7 @@ class TCPRelay(object): else: logging.warn('can not find handler for fd %d', sock.fileno()) + self._eventloop.remove(sock) now = time.time() if now - last_time > 5: # TODO sweep timeouts @@ -346,7 +381,7 @@ class TCPRelay(object): if self._closed: raise Exception('closed') t = threading.Thread(target=self._run) - t.setName('UDPThread') + t.setName('TCPThread') t.setDaemon(False) t.start() self._thread = t diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 180cdcb..595068a 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -139,15 +139,17 @@ def get_config(is_local): config['verbose'] = config.get('verbose', False) config['local_address'] = config.get('local_address', '127.0.0.1') - check_config(config) - if config['verbose']: level = logging.DEBUG else: level = logging.WARNING + logging.getLogger('').handlers = [] logging.basicConfig(level=level, format='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S', filemode='a+') + + check_config(config) + return config From f49d086d6ad037bbb6de14506f5adef8c0d3bc68 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 16:21:33 +0800 Subject: [PATCH 016/344] add event names --- shadowsocks/eventloop.py | 19 +++++++++++- shadowsocks/local.py | 2 +- shadowsocks/tcprelay.py | 66 +++++++++++++++++++++++++--------------- 3 files changed, 60 insertions(+), 27 deletions(-) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index b432252..732951f 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -32,7 +32,7 @@ from collections import defaultdict __all__ = ['EventLoop', 'POLL_NULL', 'POLL_IN', 'POLL_OUT', 'POLL_ERR', - 'POLL_HUP', 'POLL_NVAL'] + 'POLL_HUP', 'POLL_NVAL', 'EVENT_NAMES'] POLL_NULL = 0x00 POLL_IN = 0x01 @@ -42,6 +42,16 @@ POLL_HUP = 0x10 POLL_NVAL = 0x20 +EVENT_NAMES = { + POLL_NULL: 'POLL_NULL', + POLL_IN: 'POLL_IN', + POLL_OUT: 'POLL_OUT', + POLL_ERR: 'POLL_ERR', + POLL_HUP: 'POLL_HUP', + POLL_NVAL: 'POLL_NVAL', +} + + class EpollLoop(object): def __init__(self): @@ -144,15 +154,22 @@ class EventLoop(object): def __init__(self): if hasattr(select, 'epoll'): self._impl = EpollLoop() + self._model = 'epoll' elif hasattr(select, 'kqueue'): self._impl = KqueueLoop() + self._model = 'kqueue' elif hasattr(select, 'select'): self._impl = SelectLoop() + self._model = 'select' else: raise Exception('can not find any available functions in select ' 'package') self._fd_to_f = {} + @property + def model(self): + return self._model + def poll(self, timeout=None): events = self._impl.poll(timeout) return ((self._fd_to_f[fd], event) for fd, event in events) diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 0ccc813..6d70a12 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -22,7 +22,6 @@ # SOFTWARE. import sys -import socket import os import logging import encrypt @@ -50,6 +49,7 @@ def main(): logging.info("starting local at %s:%d" % (config['local_address'], config['local_port'])) + # TODO combine the two threads into one loop on a single thread udprelay.UDPRelay(config, True).start() tcprelay.TCPRelay(config, True).start() while sys.stdin.read(): diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 2a9d552..6184ee7 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -56,6 +56,7 @@ STREAM_DOWN = 1 STATUS_WAIT_INIT = 0 STATUS_WAIT_READING = 1 STATUS_WAIT_WRITING = 2 +STATUS_WAIT_READWRITING = STATUS_WAIT_READING | STATUS_WAIT_WRITING BUF_SIZE = 8 * 1024 @@ -73,10 +74,11 @@ class TCPRelayHandler(object): config['method']) self._data_to_write_to_local = [] self._data_to_write_to_remote = [] - self._upstream_status = STATUS_WAIT_INIT + self._upstream_status = STATUS_WAIT_READING self._downstream_status = STATUS_WAIT_INIT fd_to_handlers[local_sock.fileno()] = self local_sock.setblocking(False) + local_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) loop.add(local_sock, eventloop.POLL_IN | eventloop.POLL_ERR) def update_stream(self, stream, status): @@ -92,16 +94,16 @@ class TCPRelayHandler(object): if dirty: if self._local_sock: event = eventloop.POLL_ERR - if self._downstream_status == STATUS_WAIT_WRITING: + if self._downstream_status & STATUS_WAIT_WRITING: event |= eventloop.POLL_OUT - if self._upstream_status == STATUS_WAIT_READING: + if self._upstream_status & STATUS_WAIT_READING: event |= eventloop.POLL_IN self._loop.modify(self._local_sock, event) if self._remote_sock: event = eventloop.POLL_ERR - if self._downstream_status == STATUS_WAIT_READING: + if self._downstream_status & STATUS_WAIT_READING: event |= eventloop.POLL_IN - if self._upstream_status == STATUS_WAIT_WRITING: + if self._upstream_status & STATUS_WAIT_WRITING: event |= eventloop.POLL_OUT self._loop.modify(self._remote_sock, event) @@ -131,6 +133,13 @@ class TCPRelayHandler(object): self.update_stream(STREAM_UP, STATUS_WAIT_WRITING) else: logging.error('write_all_to_sock:unknown socket') + else: + if sock == self._local_sock: + self.update_stream(STREAM_DOWN, STATUS_WAIT_READING) + elif sock == self._remote_sock: + self.update_stream(STREAM_UP, STATUS_WAIT_READING) + else: + logging.error('write_all_to_sock:unknown socket') def on_local_read(self): # TODO update timeout @@ -156,14 +165,16 @@ class TCPRelayHandler(object): data = self._encryptor.encrypt(data) self.write_all_to_sock(data, self._remote_sock) return - if is_local and self._stage == STAGE_INIT: + elif is_local and self._stage == STAGE_INIT: # TODO check auth method self.write_all_to_sock('\x05\00', self._local_sock) self._stage = STAGE_HELLO return - if self._stage == STAGE_REPLY: + elif self._stage == STAGE_REPLY: + if is_local: + data = self._encryptor.encrypt(data) self._data_to_write_to_remote.append(data) - if (is_local and self._stage == STAGE_HELLO) or \ + elif (is_local and self._stage == STAGE_HELLO) or \ (not is_local and self._stage == STAGE_INIT): try: if is_local: @@ -199,20 +210,22 @@ class TCPRelayHandler(object): raise Exception("can't get addrinfo for %s:%d" % (remote_addr, remote_port)) af, socktype, proto, canonname, sa = addrs[0] - self._remote_sock = socket.socket(af, socktype, proto) - self._fd_to_handlers[self._remote_sock.fileno()] = self - self._remote_sock.setblocking(False) + remote_sock = socket.socket(af, socktype, proto) + self._remote_sock = remote_sock + self._fd_to_handlers[remote_sock.fileno()] = self + remote_sock.setblocking(False) + remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) # TODO support TCP fast open try: - self._remote_sock.connect(sa) + remote_sock.connect(sa) except (OSError, IOError) as e: if eventloop.errno_from_exception(e) == errno.EINPROGRESS: pass - self._loop.add(self._remote_sock, + self._loop.add(remote_sock, eventloop.POLL_ERR | eventloop.POLL_OUT) - self._stage = 4 - self.update_stream(STREAM_UP, STATUS_WAIT_WRITING) + self._stage = STAGE_REPLY + self.update_stream(STREAM_UP, STATUS_WAIT_READWRITING) self.update_stream(STREAM_DOWN, STATUS_WAIT_READING) return except Exception: @@ -221,9 +234,6 @@ class TCPRelayHandler(object): # TODO use logging when debug completed self.destroy() - elif self._stage == STAGE_REPLY: - self._data_to_write_to_remote.append(data) - def on_remote_read(self): # TODO update timeout data = None @@ -334,6 +344,7 @@ class TCPRelay(object): def _run(self): server_socket = self._server_socket self._eventloop = eventloop.EventLoop() + logging.debug('using event model: %s', self._eventloop.model) self._eventloop.add(server_socket, eventloop.POLL_IN | eventloop.POLL_ERR) last_time = time.time() @@ -348,7 +359,9 @@ class TCPRelay(object): logging.error(e) continue for sock, event in events: - logging.debug('%s %d', sock, event) + if sock: + logging.debug('fd %d %s', sock.fileno(), + eventloop.EVENT_NAMES[event]) if sock == self._server_socket: if event & eventloop.POLL_ERR: # TODO @@ -364,13 +377,16 @@ class TCPRelay(object): else: logging.error(e) else: - handler = self._fd_to_handlers.get(sock.fileno(), None) - if handler: - handler.handle_event(sock, event) + if sock: + handler = self._fd_to_handlers.get(sock.fileno(), None) + if handler: + handler.handle_event(sock, event) + else: + logging.warn('can not find handler for fd %d', + sock.fileno()) + self._eventloop.remove(sock) else: - logging.warn('can not find handler for fd %d', - sock.fileno()) - self._eventloop.remove(sock) + logging.warn('poll removed fd') now = time.time() if now - last_time > 5: # TODO sweep timeouts From fd35f1d0673caff8ef0c1c62173feff4aa9aaff5 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 17:16:58 +0800 Subject: [PATCH 017/344] fix server --- shadowsocks/server.py | 4 ++-- shadowsocks/tcprelay.py | 2 ++ shadowsocks/utils.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index bfb2f8b..d3e384f 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -52,10 +52,10 @@ def main(): udp_servers = [] for port, password in config['port_password'].items(): a_config = config.copy() - a_config['server_port'] = port + a_config['server_port'] = int(port) a_config['password'] = password logging.info("starting server at %s:%d" % - (a_config['server'], port)) + (a_config['server'], int(port))) tcp_server = tcprelay.TCPRelay(config, False) tcp_servers.append(tcp_server) udp_server = udprelay.UDPRelay(config, False) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 6184ee7..3f72ed2 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -248,6 +248,8 @@ class TCPRelayHandler(object): return if self._is_local: data = self._encryptor.decrypt(data) + else: + data = self._encryptor.encrypt(data) try: self.write_all_to_sock(data, self._local_sock) except Exception: diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 595068a..bc9d92a 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -77,7 +77,7 @@ def get_config(is_local): shortopts = 's:b:p:k:l:m:c:t:v' longopts = ['fast-open'] else: - shortopts = 's:p:k:m:c:t:' + shortopts = 's:p:k:m:c:t:v' longopts = ['fast-open', 'workers:'] try: config_path = find_config() From 2672a6378e659707720206aa3f64ebc7023ff3c0 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 17:20:51 +0800 Subject: [PATCH 018/344] fix ctrl C --- shadowsocks/server.py | 5 +++++ test/aes.json | 2 +- test/salsa20.json | 2 +- test/server-multi-passwd.json | 2 +- test/table.json | 2 +- test/workers.json | 2 +- 6 files changed, 10 insertions(+), 5 deletions(-) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index d3e384f..b9b897e 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -66,6 +66,11 @@ def main(): tcp_server.start() for udp_server in udp_servers: udp_server.start() + try: + while sys.stdin.read(): + pass + except KeyboardInterrupt: + os._exit(0) if int(config['workers']) > 1: if os.name == 'posix': diff --git a/test/aes.json b/test/aes.json index 42ba376..cc9ffeb 100644 --- a/test/aes.json +++ b/test/aes.json @@ -1,7 +1,7 @@ { "server":"127.0.0.1", "server_port":8388, - "local_port":1080, + "local_port":1081, "password":"barfoo!", "timeout":300, "method":"aes-256-cfb", diff --git a/test/salsa20.json b/test/salsa20.json index 8035f06..182589e 100644 --- a/test/salsa20.json +++ b/test/salsa20.json @@ -1,7 +1,7 @@ { "server":"127.0.0.1", "server_port":8388, - "local_port":1080, + "local_port":1081, "password":"barfoo!", "timeout":300, "method":"salsa20-ctr", diff --git a/test/server-multi-passwd.json b/test/server-multi-passwd.json index 2c0609e..497c60a 100644 --- a/test/server-multi-passwd.json +++ b/test/server-multi-passwd.json @@ -1,7 +1,7 @@ { "server": "127.0.0.1", "server_port": 8384, - "local_port": 1080, + "local_port": 1081, "password": "foobar4", "port_password": { "8381": "foobar1", diff --git a/test/table.json b/test/table.json index 4b1c984..402a4b8 100644 --- a/test/table.json +++ b/test/table.json @@ -1,7 +1,7 @@ { "server":"127.0.0.1", "server_port":8388, - "local_port":1080, + "local_port":1081, "password":"barfoo!", "timeout":300, "method":"table", diff --git a/test/workers.json b/test/workers.json index c3d699b..1df28ce 100644 --- a/test/workers.json +++ b/test/workers.json @@ -1,7 +1,7 @@ { "server":"127.0.0.1", "server_port":8388, - "local_port":1080, + "local_port":1081, "password":"barfoo!", "timeout":300, "method":"aes-256-cfb", From 20ba04c4ce2de34d67165c6ffaf18ce7d3122ec9 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 17:24:10 +0800 Subject: [PATCH 019/344] fix tests --- .travis.yml | 5 +++-- shadowsocks/tcprelay.py | 2 +- shadowsocks/utils.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 20710fc..6e8a5d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,10 +4,11 @@ python: - 2.7 before_install: - sudo apt-get update -qq - - sudo apt-get install -qq build-essential libssl-dev swig libevent-dev python-gevent python-m2crypto python-numpy - - pip install m2crypto gevent salsa20 + - sudo apt-get install -qq build-essential libssl-dev swig python-m2crypto python-numpy + - pip install m2crypto salsa20 script: - python test.py -c test/table.json - python test.py -c test/aes.json - python test.py -c test/salsa20.json - python test.py -c test/server-multi-passwd.json + - python test.py -c test/workers.json diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 3f72ed2..e969d3e 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -188,7 +188,7 @@ class TCPRelayHandler(object): raise Exception('can not parse header') addrtype, remote_addr, remote_port, header_length =\ header_result - logging.info('connecting %s:%d' % (remote_addr, remote_port)) + logging.debug('connecting %s:%d' % (remote_addr, remote_port)) if is_local: # forward address to remote self.write_all_to_sock('\x05\x00\x00\x01' + diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index bc9d92a..9ee6b2f 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -142,7 +142,7 @@ def get_config(is_local): if config['verbose']: level = logging.DEBUG else: - level = logging.WARNING + level = logging.INFO logging.getLogger('').handlers = [] logging.basicConfig(level=level, format='%(asctime)s %(levelname)-8s %(message)s', From 21ab52c62bd87b3b9e87fb2d26123b0489504779 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 17:27:36 +0800 Subject: [PATCH 020/344] fix tests --- test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.py b/test.py index 0b46b33..4da0a79 100755 --- a/test.py +++ b/test.py @@ -111,7 +111,7 @@ try: if local_ready and server_ready and p3 is None: time.sleep(1) p3 = Popen(['curl', 'http://www.example.com/', '-v', '-L', - '--socks5-hostname', '127.0.0.1:1080'], shell=False, + '--socks5-hostname', '127.0.0.1:1081'], shell=False, bufsize=0, close_fds=True) break From 77f74a06688de5ce0d19412de68f99bcf94193db Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 17:29:18 +0800 Subject: [PATCH 021/344] fix python version check --- shadowsocks/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 9ee6b2f..c087ee3 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -30,7 +30,7 @@ import logging def check_python(): info = sys.version_info - if not (info.major == 2 and info.minor >= 6): + if not (info[0] == 2 and info[1] >= 6): print 'Python 2.6 or 2.7 required' sys.exit(1) From 7a983316550eb989183606b67e0773af7cef5a77 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 17:35:32 +0800 Subject: [PATCH 022/344] add salsa20 test --- shadowsocks/encrypt_salsa20.py | 4 ++-- test.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/shadowsocks/encrypt_salsa20.py b/shadowsocks/encrypt_salsa20.py index d69010f..62bca65 100644 --- a/shadowsocks/encrypt_salsa20.py +++ b/shadowsocks/encrypt_salsa20.py @@ -109,7 +109,7 @@ def test(): decipher = Salsa20Cipher('salsa20-ctr', 'k' * 32, 'i' * 8, 1) results = [] pos = 0 - print 'start' + print 'salsa20 test start' start = time.time() while pos < len(plain): l = random.randint(100, 32768) @@ -124,7 +124,7 @@ def test(): results.append(decipher.update(c[pos:pos + l])) pos += l end = time.time() - print BLOCK_SIZE * rounds / (end - start) + print 'speed: %d bytes/s' % (BLOCK_SIZE * rounds / (end - start)) assert ''.join(results) == plain diff --git a/test.py b/test.py index 4da0a79..d1a5a0c 100755 --- a/test.py +++ b/test.py @@ -1,15 +1,17 @@ #!/usr/bin/python # -*- coding: utf-8 -*- +import sys +sys.path.insert(0, 'shadowsocks') import os import signal -import sys import select import struct import hashlib import string import time from subprocess import Popen, PIPE +import encrypt_salsa20 target1 = [ [60, 53, 84, 138, 217, 94, 88, 23, 39, 242, 219, 35, 12, 157, 165, 181, 255, 143, 83, 247, 162, 16, 31, 209, 190, @@ -89,6 +91,8 @@ p2 = Popen(['python', 'shadowsocks/local.py', '-c', sys.argv[-1]], shell=False, stdout=PIPE, stderr=PIPE, close_fds=True) p3 = None +encrypt_salsa20.test() + print 'encryption test passed' try: @@ -120,7 +124,7 @@ try: if r == 0: print 'test passed' sys.exit(r) - + finally: for p in [p1, p2]: try: From 7a1647278fc028bcb077fab2841d0d534f1c0a5a Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 17:50:57 +0800 Subject: [PATCH 023/344] fix multiple servers --- shadowsocks/server.py | 6 +++--- test/server-multi-passwd.json | 5 +---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index b9b897e..a0b6dd7 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -56,9 +56,9 @@ def main(): a_config['password'] = password logging.info("starting server at %s:%d" % (a_config['server'], int(port))) - tcp_server = tcprelay.TCPRelay(config, False) + tcp_server = tcprelay.TCPRelay(a_config, False) tcp_servers.append(tcp_server) - udp_server = udprelay.UDPRelay(config, False) + udp_server = udprelay.UDPRelay(a_config, False) udp_servers.append(udp_server) def run_server(): @@ -96,7 +96,7 @@ def main(): # master for tcp_server in tcp_servers: - tcp_server.server_close() + tcp_server.close() for udp_server in udp_servers: udp_server.close() diff --git a/test/server-multi-passwd.json b/test/server-multi-passwd.json index 497c60a..eff5ed8 100644 --- a/test/server-multi-passwd.json +++ b/test/server-multi-passwd.json @@ -12,10 +12,7 @@ "8386": "foobar6", "8387": "foobar7", "8388": "foobar8", - "8389": "foobar9", - "8390": "foobar10", - "8391": "foobar11", - "8392": "foobar12" + "8389": "foobar9" }, "timeout": 60, "method": "aes-256-cfb" From 5e19fdc66b1f2a164abd6113d00e1b8dd205c0b2 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 17:56:41 +0800 Subject: [PATCH 024/344] refine exiting --- shadowsocks/local.py | 3 ++- shadowsocks/server.py | 13 +++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 6d70a12..24d3645 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -54,7 +54,8 @@ def main(): tcprelay.TCPRelay(config, True).start() while sys.stdin.read(): pass - except KeyboardInterrupt: + except (KeyboardInterrupt, IOError, OSError) as e: + logging.error(e) os._exit(0) if __name__ == '__main__': diff --git a/shadowsocks/server.py b/shadowsocks/server.py index a0b6dd7..7991dd8 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -62,14 +62,15 @@ def main(): udp_servers.append(udp_server) def run_server(): - for tcp_server in tcp_servers: - tcp_server.start() - for udp_server in udp_servers: - udp_server.start() try: + for tcp_server in tcp_servers: + tcp_server.start() + for udp_server in udp_servers: + udp_server.start() while sys.stdin.read(): pass - except KeyboardInterrupt: + except (KeyboardInterrupt, IOError, OSError) as e: + logging.error(e) os._exit(0) if int(config['workers']) > 1: @@ -86,7 +87,7 @@ def main(): else: children.append(r) if not is_child: - def handler(signum, frame): + def handler(signum, _): for pid in children: os.kill(pid, signum) os.waitpid(pid, 0) From 0c8a8ef23fc6c5992a0f9735cefaf3eb6089be53 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 19:09:52 +0800 Subject: [PATCH 025/344] refine loop --- shadowsocks/eventloop.py | 37 +++++++++++--- shadowsocks/local.py | 14 ++--- shadowsocks/server.py | 33 +++++------- shadowsocks/tcprelay.py | 107 ++++++++++++++++----------------------- shadowsocks/udprelay.py | 65 ++++++++++-------------- 5 files changed, 120 insertions(+), 136 deletions(-) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index 732951f..f068532 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -28,6 +28,8 @@ import os import socket import select +import errno +import logging from collections import defaultdict @@ -154,25 +156,24 @@ class EventLoop(object): def __init__(self): if hasattr(select, 'epoll'): self._impl = EpollLoop() - self._model = 'epoll' + model = 'epoll' elif hasattr(select, 'kqueue'): self._impl = KqueueLoop() - self._model = 'kqueue' + model = 'kqueue' elif hasattr(select, 'select'): self._impl = SelectLoop() - self._model = 'select' + model = 'select' else: raise Exception('can not find any available functions in select ' 'package') self._fd_to_f = {} - - @property - def model(self): - return self._model + self._handlers = [] + self.stopping = False + logging.debug('using event model: %s', model) def poll(self, timeout=None): events = self._impl.poll(timeout) - return ((self._fd_to_f[fd], event) for fd, event in events) + return [(self._fd_to_f[fd], fd, event) for fd, event in events] def add(self, f, mode): fd = f.fileno() @@ -188,6 +189,26 @@ class EventLoop(object): fd = f.fileno() self._impl.modify_fd(fd, mode) + def add_handler(self, handler): + self._handlers.append(handler) + + def run(self): + while not self.stopping: + events = None + try: + events = self.poll(1) + except (OSError, IOError) as e: + if errno_from_exception(e) == errno.EPIPE: + # Happens when the client closes the connection + continue + else: + logging.error(e) + continue + for handler in self._handlers: + # no exceptions should be raised by users + # TODO when there are a lot of handlers + handler(events) + # from tornado def errno_from_exception(e): diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 24d3645..0b4b17d 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -24,8 +24,9 @@ import sys import os import logging -import encrypt import utils +import encrypt +import eventloop import tcprelay import udprelay @@ -49,11 +50,12 @@ def main(): logging.info("starting local at %s:%d" % (config['local_address'], config['local_port'])) - # TODO combine the two threads into one loop on a single thread - udprelay.UDPRelay(config, True).start() - tcprelay.TCPRelay(config, True).start() - while sys.stdin.read(): - pass + tcp_server = tcprelay.TCPRelay(config, True) + udp_server = udprelay.UDPRelay(config, True) + loop = eventloop.EventLoop() + tcp_server.add_to_loop(loop) + udp_server.add_to_loop(loop) + loop.run() except (KeyboardInterrupt, IOError, OSError) as e: logging.error(e) os._exit(0) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 7991dd8..2e35698 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -22,11 +22,11 @@ # SOFTWARE. import sys -import socket -import logging -import encrypt import os +import logging import utils +import encrypt +import eventloop import tcprelay import udprelay @@ -56,19 +56,17 @@ def main(): a_config['password'] = password logging.info("starting server at %s:%d" % (a_config['server'], int(port))) - tcp_server = tcprelay.TCPRelay(a_config, False) - tcp_servers.append(tcp_server) - udp_server = udprelay.UDPRelay(a_config, False) - udp_servers.append(udp_server) + tcp_servers.append(tcprelay.TCPRelay(a_config, False)) + udp_servers.append(udprelay.UDPRelay(a_config, False)) def run_server(): try: + loop = eventloop.EventLoop() for tcp_server in tcp_servers: - tcp_server.start() + tcp_server.add_to_loop(loop) for udp_server in udp_servers: - udp_server.start() - while sys.stdin.read(): - pass + udp_server.add_to_loop(loop) + loop.run() except (KeyboardInterrupt, IOError, OSError) as e: logging.error(e) os._exit(0) @@ -96,10 +94,10 @@ def main(): signal.signal(signal.SIGTERM, handler) # master - for tcp_server in tcp_servers: - tcp_server.close() - for udp_server in udp_servers: - udp_server.close() + for a_tcp_server in tcp_servers: + a_tcp_server.close() + for a_udp_server in udp_servers: + a_udp_server.close() for child in children: os.waitpid(child, 0) @@ -111,7 +109,4 @@ def main(): if __name__ == '__main__': - try: - main() - except socket.error, e: - logging.error(e) + main() diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index e969d3e..6239629 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -26,7 +26,6 @@ import socket import logging import encrypt import errno -import threading import eventloop from common import parse_header @@ -303,6 +302,7 @@ class TCPRelayHandler(object): logging.warn('unknown socket') def destroy(self): + logging.debug('destroy') if self._remote_sock: self._loop.remove(self._remote_sock) del self._fd_to_handlers[self._remote_sock.fileno()] @@ -320,8 +320,9 @@ class TCPRelay(object): self._config = config self._is_local = is_local self._closed = False - self._thread = None + self._eventloop = None self._fd_to_handlers = {} + self._last_time = time.time() if is_local: listen_addr = config['local_address'] @@ -343,70 +344,48 @@ class TCPRelay(object): server_socket.listen(1024) self._server_socket = server_socket - def _run(self): - server_socket = self._server_socket - self._eventloop = eventloop.EventLoop() - logging.debug('using event model: %s', self._eventloop.model) - self._eventloop.add(server_socket, - eventloop.POLL_IN | eventloop.POLL_ERR) - last_time = time.time() - while not self._closed: - try: - events = self._eventloop.poll(1) - except (OSError, IOError) as e: - if eventloop.errno_from_exception(e) == errno.EPIPE: - # Happens when the client closes the connection - continue - else: - logging.error(e) - continue - for sock, event in events: - if sock: - logging.debug('fd %d %s', sock.fileno(), - eventloop.EVENT_NAMES[event]) - if sock == self._server_socket: - if event & eventloop.POLL_ERR: - # TODO - raise Exception('server_socket error') - try: - conn = self._server_socket.accept() - TCPRelayHandler(self._fd_to_handlers, self._eventloop, - conn[0], self._config, self._is_local) - except (OSError, IOError) as e: - error_no = eventloop.errno_from_exception(e) - if error_no in (errno.EAGAIN, errno.EINPROGRESS): - continue - else: - logging.error(e) - else: - if sock: - handler = self._fd_to_handlers.get(sock.fileno(), None) - if handler: - handler.handle_event(sock, event) - else: - logging.warn('can not find handler for fd %d', - sock.fileno()) - self._eventloop.remove(sock) - else: - logging.warn('poll removed fd') - now = time.time() - if now - last_time > 5: - # TODO sweep timeouts - last_time = now - - def start(self): - # TODO combine loops on multiple ports into one single loop + def add_to_loop(self, loop): if self._closed: - raise Exception('closed') - t = threading.Thread(target=self._run) - t.setName('TCPThread') - t.setDaemon(False) - t.start() - self._thread = t + raise Exception('already closed') + self._eventloop = loop + loop.add_handler(self._handle_events) + + self._eventloop.add(self._server_socket, + eventloop.POLL_IN | eventloop.POLL_ERR) + + def _handle_events(self, events): + for sock, fd, event in events: + if sock: + logging.debug('fd %d %s', fd, + eventloop.EVENT_NAMES.get(event, event)) + if sock == self._server_socket: + if event & eventloop.POLL_ERR: + # TODO + raise Exception('server_socket error') + try: + logging.debug('accept') + conn = self._server_socket.accept() + TCPRelayHandler(self._fd_to_handlers, self._eventloop, + conn[0], self._config, self._is_local) + except (OSError, IOError) as e: + error_no = eventloop.errno_from_exception(e) + if error_no in (errno.EAGAIN, errno.EINPROGRESS): + continue + else: + logging.error(e) + else: + if sock: + handler = self._fd_to_handlers.get(fd, None) + if handler: + handler.handle_event(sock, event) + else: + logging.warn('poll removed fd') + + now = time.time() + if now - self._last_time > 5: + # TODO sweep timeouts + self._last_time = now def close(self): self._closed = True self._server_socket.close() - - def thread(self): - return self._thread \ No newline at end of file diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 68414e8..2aeb069 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -67,7 +67,6 @@ import time -import threading import socket import logging import struct @@ -105,8 +104,10 @@ class UDPRelay(object): close_callback=self._close_client) self._client_fd_to_server_addr = \ lru_cache.LRUCache(timeout=config['timeout']) + self._eventloop = None self._closed = False - self._thread = None + self._last_time = time.time() + self._sockets = set() addrs = socket.getaddrinfo(self._listen_addr, self._listen_port, 0, socket.SOCK_DGRAM, socket.SOL_UDP) @@ -121,6 +122,7 @@ class UDPRelay(object): def _close_client(self, client): if hasattr(client, 'close'): + self._sockets.remove(client.fileno()) self._eventloop.remove(client) client.close() else: @@ -167,6 +169,7 @@ class UDPRelay(object): else: # drop return + self._sockets.add(client.fileno()) self._eventloop.add(client, eventloop.POLL_IN) data = data[header_length:] @@ -216,45 +219,29 @@ class UDPRelay(object): # simply drop that packet pass - def _run(self): - server_socket = self._server_socket - self._eventloop = eventloop.EventLoop() - self._eventloop.add(server_socket, eventloop.POLL_IN) - last_time = time.time() - while not self._closed: - try: - events = self._eventloop.poll(10) - except (OSError, IOError) as e: - if eventloop.errno_from_exception(e) == errno.EPIPE: - # Happens when the client closes the connection - continue - else: - logging.error(e) - continue - for sock, event in events: - if sock == self._server_socket: - self._handle_server() - else: - self._handle_client(sock) - now = time.time() - if now - last_time > 3.5: - self._cache.sweep() - if now - last_time > 7: - self._client_fd_to_server_addr.sweep() - last_time = now - - def start(self): + def add_to_loop(self, loop): if self._closed: - raise Exception('closed') - t = threading.Thread(target=self._run) - t.setName('UDPThread') - t.setDaemon(False) - t.start() - self._thread = t + raise Exception('already closed') + self._eventloop = loop + loop.add_handler(self._handle_events) + + server_socket = self._server_socket + self._eventloop.add(server_socket, + eventloop.POLL_IN | eventloop.POLL_ERR) + + def _handle_events(self, events): + for sock, fd, event in events: + if sock == self._server_socket: + self._handle_server() + elif sock and (fd in self._sockets): + self._handle_client(sock) + now = time.time() + if now - self._last_time > 3.5: + self._cache.sweep() + if now - self._last_time > 7: + self._client_fd_to_server_addr.sweep() + self._last_time = now def close(self): self._closed = True self._server_socket.close() - - def thread(self): - return self._thread From fe01025ba77909f5888e88184ba9d990bec965a6 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 19:12:15 +0800 Subject: [PATCH 026/344] rename tests --- .travis.yml | 10 +++++----- {test => tests}/aes.json | 0 {test => tests}/salsa20.json | 0 {test => tests}/server-multi-passwd.json | 0 {test => tests}/table.json | 0 {test => tests}/workers.json | 0 6 files changed, 5 insertions(+), 5 deletions(-) rename {test => tests}/aes.json (100%) rename {test => tests}/salsa20.json (100%) rename {test => tests}/server-multi-passwd.json (100%) rename {test => tests}/table.json (100%) rename {test => tests}/workers.json (100%) diff --git a/.travis.yml b/.travis.yml index 6e8a5d8..267427c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,8 @@ before_install: - sudo apt-get install -qq build-essential libssl-dev swig python-m2crypto python-numpy - pip install m2crypto salsa20 script: - - python test.py -c test/table.json - - python test.py -c test/aes.json - - python test.py -c test/salsa20.json - - python test.py -c test/server-multi-passwd.json - - python test.py -c test/workers.json + - python test.py -c tests/table.json + - python test.py -c tests/aes.json + - python test.py -c tests/salsa20.json + - python test.py -c tests/server-multi-passwd.json + - python test.py -c tests/workers.json diff --git a/test/aes.json b/tests/aes.json similarity index 100% rename from test/aes.json rename to tests/aes.json diff --git a/test/salsa20.json b/tests/salsa20.json similarity index 100% rename from test/salsa20.json rename to tests/salsa20.json diff --git a/test/server-multi-passwd.json b/tests/server-multi-passwd.json similarity index 100% rename from test/server-multi-passwd.json rename to tests/server-multi-passwd.json diff --git a/test/table.json b/tests/table.json similarity index 100% rename from test/table.json rename to tests/table.json diff --git a/test/workers.json b/tests/workers.json similarity index 100% rename from test/workers.json rename to tests/workers.json From d69272767f902db23560db0244bd0fd8bcc41be1 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 19:13:55 +0800 Subject: [PATCH 027/344] update readme python version now checked by utils --- README.md | 7 ------- README.rst | 9 --------- 2 files changed, 16 deletions(-) diff --git a/README.md b/README.md index 2144bef..e900f98 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,6 @@ Both TCP CONNECT and UDP ASSOCIATE are implemented. Install ------- -First, make sure you have Python 2.6 or 2.7. - - $ python --version - Python 2.6.8 - -Install Shadowsocks. - #### Debian / Ubuntu: apt-get install build-essential python-pip python-m2crypto python-dev diff --git a/README.rst b/README.rst index 953e114..139a1f1 100644 --- a/README.rst +++ b/README.rst @@ -15,15 +15,6 @@ Both TCP CONNECT and UDP ASSOCIATE are implemented. Install ------- -First, make sure you have Python 2.6 or 2.7. - -:: - - $ python --version - Python 2.6.8 - -Install Shadowsocks. - Debian / Ubuntu: ^^^^^^^^^^^^^^^^ From bbad408ace3506748e49ebdf6ab2cd1b5fe1ee6f Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 1 Jun 2014 19:34:35 +0800 Subject: [PATCH 028/344] rename things --- shadowsocks/eventloop.py | 1 - shadowsocks/tcprelay.py | 52 ++++++++++++++++++++-------------------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index f068532..b6a9ca5 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -194,7 +194,6 @@ class EventLoop(object): def run(self): while not self.stopping: - events = None try: events = self.poll(1) except (OSError, IOError) as e: diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 6239629..de11e86 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -51,11 +51,11 @@ STAGE_STREAM = 5 STREAM_UP = 0 STREAM_DOWN = 1 -# stream status -STATUS_WAIT_INIT = 0 -STATUS_WAIT_READING = 1 -STATUS_WAIT_WRITING = 2 -STATUS_WAIT_READWRITING = STATUS_WAIT_READING | STATUS_WAIT_WRITING +# stream wait status +WAIT_STATUS_INIT = 0 +WAIT_STATUS_READING = 1 +WAIT_STATUS_WRITING = 2 +WAIT_STATUS_READWRITING = WAIT_STATUS_READING | WAIT_STATUS_WRITING BUF_SIZE = 8 * 1024 @@ -73,8 +73,8 @@ class TCPRelayHandler(object): config['method']) self._data_to_write_to_local = [] self._data_to_write_to_remote = [] - self._upstream_status = STATUS_WAIT_READING - self._downstream_status = STATUS_WAIT_INIT + self._upstream_status = WAIT_STATUS_READING + self._downstream_status = WAIT_STATUS_INIT fd_to_handlers[local_sock.fileno()] = self local_sock.setblocking(False) local_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) @@ -93,20 +93,20 @@ class TCPRelayHandler(object): if dirty: if self._local_sock: event = eventloop.POLL_ERR - if self._downstream_status & STATUS_WAIT_WRITING: + if self._downstream_status & WAIT_STATUS_WRITING: event |= eventloop.POLL_OUT - if self._upstream_status & STATUS_WAIT_READING: + if self._upstream_status & WAIT_STATUS_READING: event |= eventloop.POLL_IN self._loop.modify(self._local_sock, event) if self._remote_sock: event = eventloop.POLL_ERR - if self._downstream_status & STATUS_WAIT_READING: + if self._downstream_status & WAIT_STATUS_READING: event |= eventloop.POLL_IN - if self._upstream_status & STATUS_WAIT_WRITING: + if self._upstream_status & WAIT_STATUS_WRITING: event |= eventloop.POLL_OUT self._loop.modify(self._remote_sock, event) - def write_all_to_sock(self, data, sock): + def write_to_sock(self, data, sock): if not data or not sock: return uncomplete = False @@ -126,17 +126,17 @@ class TCPRelayHandler(object): if uncomplete: if sock == self._local_sock: self._data_to_write_to_local.append(data) - self.update_stream(STREAM_DOWN, STATUS_WAIT_WRITING) + self.update_stream(STREAM_DOWN, WAIT_STATUS_WRITING) elif sock == self._remote_sock: self._data_to_write_to_remote.append(data) - self.update_stream(STREAM_UP, STATUS_WAIT_WRITING) + self.update_stream(STREAM_UP, WAIT_STATUS_WRITING) else: logging.error('write_all_to_sock:unknown socket') else: if sock == self._local_sock: - self.update_stream(STREAM_DOWN, STATUS_WAIT_READING) + self.update_stream(STREAM_DOWN, WAIT_STATUS_READING) elif sock == self._remote_sock: - self.update_stream(STREAM_UP, STATUS_WAIT_READING) + self.update_stream(STREAM_UP, WAIT_STATUS_READING) else: logging.error('write_all_to_sock:unknown socket') @@ -162,11 +162,11 @@ class TCPRelayHandler(object): if self._stage == STAGE_STREAM: if self._is_local: data = self._encryptor.encrypt(data) - self.write_all_to_sock(data, self._remote_sock) + self.write_to_sock(data, self._remote_sock) return elif is_local and self._stage == STAGE_INIT: # TODO check auth method - self.write_all_to_sock('\x05\00', self._local_sock) + self.write_to_sock('\x05\00', self._local_sock) self._stage = STAGE_HELLO return elif self._stage == STAGE_REPLY: @@ -190,7 +190,7 @@ class TCPRelayHandler(object): logging.debug('connecting %s:%d' % (remote_addr, remote_port)) if is_local: # forward address to remote - self.write_all_to_sock('\x05\x00\x00\x01' + + self.write_to_sock('\x05\x00\x00\x01' + '\x00\x00\x00\x00\x10\x10', self._local_sock) data_to_send = self._encryptor.encrypt(data) @@ -224,8 +224,8 @@ class TCPRelayHandler(object): eventloop.POLL_ERR | eventloop.POLL_OUT) self._stage = STAGE_REPLY - self.update_stream(STREAM_UP, STATUS_WAIT_READWRITING) - self.update_stream(STREAM_DOWN, STATUS_WAIT_READING) + self.update_stream(STREAM_UP, WAIT_STATUS_READWRITING) + self.update_stream(STREAM_DOWN, WAIT_STATUS_READING) return except Exception: import traceback @@ -250,7 +250,7 @@ class TCPRelayHandler(object): else: data = self._encryptor.encrypt(data) try: - self.write_all_to_sock(data, self._local_sock) + self.write_to_sock(data, self._local_sock) except Exception: import traceback traceback.print_exc() @@ -261,18 +261,18 @@ class TCPRelayHandler(object): if self._data_to_write_to_local: data = ''.join(self._data_to_write_to_local) self._data_to_write_to_local = [] - self.write_all_to_sock(data, self._local_sock) + self.write_to_sock(data, self._local_sock) else: - self.update_stream(STREAM_DOWN, STATUS_WAIT_READING) + self.update_stream(STREAM_DOWN, WAIT_STATUS_READING) def on_remote_write(self): self._stage = STAGE_STREAM if self._data_to_write_to_remote: data = ''.join(self._data_to_write_to_remote) self._data_to_write_to_remote = [] - self.write_all_to_sock(data, self._remote_sock) + self.write_to_sock(data, self._remote_sock) else: - self.update_stream(STREAM_UP, STATUS_WAIT_READING) + self.update_stream(STREAM_UP, WAIT_STATUS_READING) def on_local_error(self): logging.error(eventloop.get_sock_error(self._local_sock)) From e66636867ad811a6e98a44b47ca8508945a0cd4a Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 2 Jun 2014 12:27:59 +0800 Subject: [PATCH 029/344] align --- shadowsocks/tcprelay.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index de11e86..66fa09e 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -191,8 +191,8 @@ class TCPRelayHandler(object): if is_local: # forward address to remote self.write_to_sock('\x05\x00\x00\x01' + - '\x00\x00\x00\x00\x10\x10', - self._local_sock) + '\x00\x00\x00\x00\x10\x10', + self._local_sock) data_to_send = self._encryptor.encrypt(data) self._data_to_write_to_remote.append(data_to_send) remote_addr = self._config['server'] From c225b808d6fd82d2e9853b274d9add392a9ece4a Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 2 Jun 2014 12:53:22 +0800 Subject: [PATCH 030/344] UDP assoc --- shadowsocks/tcprelay.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 66fa09e..85a5d90 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -23,17 +23,21 @@ import time import socket +import errno +import struct import logging import encrypt -import errno import eventloop from common import parse_header CMD_CONNECT = 1 +CMD_BIND = 2 +CMD_UDP_ASSOCIATE = 3 # local: # stage 0 init # stage 1 hello received, hello sent +# stage 2 UDP assoc # stage 4 addr received, reply sent # stage 5 remote connected @@ -44,6 +48,7 @@ CMD_CONNECT = 1 STAGE_INIT = 0 STAGE_HELLO = 1 +STAGE_UDP_ASSOC = 2 STAGE_REPLY = 4 STAGE_STREAM = 5 @@ -178,7 +183,28 @@ class TCPRelayHandler(object): try: if is_local: cmd = ord(data[1]) - # TODO check cmd == 1 + if cmd == CMD_UDP_ASSOCIATE: + logging.debug('UDP associate') + if self._local_sock.family == socket.AF_INET6: + header = '\x05\x00\x00\x04' + else: + header = '\x05\x00\x00\x01' + addr, port = self._local_sock.getsockname() + addr_to_send = socket.inet_pton(self._local_sock.family, + addr) + port_to_send = struct.pack('>H', port) + self.write_to_sock(header + addr_to_send + port_to_send, + self._local_sock) + self._stage = STAGE_UDP_ASSOC + # just wait for the client to disconnect + return + + elif cmd == CMD_CONNECT: + data = data[3:] + else: + logging.error('unknown command %d', cmd) + self.destroy() + return assert cmd == CMD_CONNECT # just trim VER CMD RSV data = data[3:] From cadf0e11bb11d8263089b5f9d0c66abc6f611cf2 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 2 Jun 2014 13:11:20 +0800 Subject: [PATCH 031/344] fix header parsing --- shadowsocks/tcprelay.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 85a5d90..8ada73d 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -200,14 +200,12 @@ class TCPRelayHandler(object): return elif cmd == CMD_CONNECT: + # just trim VER CMD RSV data = data[3:] else: logging.error('unknown command %d', cmd) self.destroy() return - assert cmd == CMD_CONNECT - # just trim VER CMD RSV - data = data[3:] header_result = parse_header(data) if header_result is None: raise Exception('can not parse header') From 0e662e04b614e47f40aae3178f764377c5efba6c Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 2 Jun 2014 13:12:15 +0800 Subject: [PATCH 032/344] lint --- shadowsocks/tcprelay.py | 1 - 1 file changed, 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 8ada73d..51e186a 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -198,7 +198,6 @@ class TCPRelayHandler(object): self._stage = STAGE_UDP_ASSOC # just wait for the client to disconnect return - elif cmd == CMD_CONNECT: # just trim VER CMD RSV data = data[3:] From c5bcb9a050a395c19b82b4340866a627dcd23c18 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 2 Jun 2014 17:01:35 +0800 Subject: [PATCH 033/344] add timeout support --- shadowsocks/tcprelay.py | 114 +++++++++++++++++++++++++++++++++++----- shadowsocks/utils.py | 6 ++- 2 files changed, 105 insertions(+), 15 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 51e186a..e1b8cf9 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -30,6 +30,10 @@ import encrypt import eventloop from common import parse_header + +TIMEOUTS_CLEAN_SIZE = 512 +TIMEOUT_PRECISION = 4 + CMD_CONNECT = 1 CMD_BIND = 2 CMD_UDP_ASSOCIATE = 3 @@ -66,7 +70,9 @@ BUF_SIZE = 8 * 1024 class TCPRelayHandler(object): - def __init__(self, fd_to_handlers, loop, local_sock, config, is_local): + def __init__(self, server, fd_to_handlers, loop, local_sock, config, + is_local): + self._server = server self._fd_to_handlers = fd_to_handlers self._loop = loop self._local_sock = local_sock @@ -80,10 +86,25 @@ class TCPRelayHandler(object): self._data_to_write_to_remote = [] self._upstream_status = WAIT_STATUS_READING self._downstream_status = WAIT_STATUS_INIT + self._remote_address = None fd_to_handlers[local_sock.fileno()] = self local_sock.setblocking(False) local_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) loop.add(local_sock, eventloop.POLL_IN | eventloop.POLL_ERR) + self.last_activity = 0 + self.update_activity() + + def __hash__(self): + # default __hash__ is id / 16 + # we want to eliminate collisions + return id(self) + + @property + def remote_address(self): + return self._remote_address + + def update_activity(self): + self._server.update_activity(self) def update_stream(self, stream, status): dirty = False @@ -146,7 +167,7 @@ class TCPRelayHandler(object): logging.error('write_all_to_sock:unknown socket') def on_local_read(self): - # TODO update timeout + self.update_activity() if not self._local_sock: return is_local = self._is_local @@ -211,6 +232,7 @@ class TCPRelayHandler(object): addrtype, remote_addr, remote_port, header_length =\ header_result logging.debug('connecting %s:%d' % (remote_addr, remote_port)) + self._remote_address = (remote_addr, remote_port) if is_local: # forward address to remote self.write_to_sock('\x05\x00\x00\x01' + @@ -257,7 +279,7 @@ class TCPRelayHandler(object): self.destroy() def on_remote_read(self): - # TODO update timeout + self.update_activity() data = None try: data = self._remote_sock.recv(BUF_SIZE) @@ -325,7 +347,11 @@ class TCPRelayHandler(object): logging.warn('unknown socket') def destroy(self): - logging.debug('destroy') + if self._remote_address: + logging.debug('destroy: %s:%d' % + self._remote_address) + else: + logging.debug('destroy') if self._remote_sock: self._loop.remove(self._remote_sock) del self._fd_to_handlers[self._remote_sock.fileno()] @@ -336,6 +362,7 @@ class TCPRelayHandler(object): del self._fd_to_handlers[self._local_sock.fileno()] self._local_sock.close() self._local_sock = None + self._server.remove_handler(self) class TCPRelay(object): @@ -347,6 +374,12 @@ class TCPRelay(object): self._fd_to_handlers = {} self._last_time = time.time() + self._timeout = config['timeout'] + self._timeouts = [] # a list for all the handlers + self._timeout_offset = 0 # last checked position for timeout + # we trim the timeouts once a while + self._handler_to_timeouts = {} # key: handler value: index in timeouts + if is_local: listen_addr = config['local_address'] listen_port = config['local_port'] @@ -376,19 +409,74 @@ class TCPRelay(object): self._eventloop.add(self._server_socket, eventloop.POLL_IN | eventloop.POLL_ERR) + def remove_handler(self, handler): + index = self._handler_to_timeouts.get(hash(handler), -1) + if index >= 0: + # delete is O(n), so we just set it to None + self._timeouts[index] = None + del self._handler_to_timeouts[hash(handler)] + + def update_activity(self, handler): + """ set handler to active """ + now = int(time.time()) + if now - handler.last_activity < TIMEOUT_PRECISION: + # thus we can lower timeout modification frequency + return + handler.last_activity = now + index = self._handler_to_timeouts.get(hash(handler), -1) + if index >= 0: + # delete is O(n), so we just set it to None + self._timeouts[index] = None + length = len(self._timeouts) + self._timeouts.append(handler) + self._handler_to_timeouts[hash(handler)] = length + + def _sweep_timeout(self): + # tornado's timeout memory management is more flexible that we need + # we just need a sorted last_activity queue and it's faster that heapq + # in fact we can do O(1) insertion/remove so we invent our own + if self._timeouts: + now = time.time() + length = len(self._timeouts) + pos = self._timeout_offset + while pos < length: + handler = self._timeouts[pos] + if handler: + if now - handler.last_activity < self._timeout: + break + else: + if handler.remote_address: + logging.warn('timed out: %s:%d' % + handler.remote_address) + else: + logging.warn('timed out') + handler.destroy() + self._timeouts[pos] = None # free memory + pos += 1 + else: + pos += 1 + if pos > TIMEOUTS_CLEAN_SIZE and pos > length >> 1: + # clean up the timeout queue when it gets larger than half + # of the queue + self._timeouts = self._timeouts[pos:] + for key in self._handler_to_timeouts: + self._handler_to_timeouts[key] -= pos + pos = 0 + self._timeout_offset = pos + def _handle_events(self, events): for sock, fd, event in events: - if sock: - logging.debug('fd %d %s', fd, - eventloop.EVENT_NAMES.get(event, event)) + # if sock: + # logging.debug('fd %d %s', fd, + # eventloop.EVENT_NAMES.get(event, event)) if sock == self._server_socket: if event & eventloop.POLL_ERR: # TODO raise Exception('server_socket error') try: - logging.debug('accept') + # logging.debug('accept') conn = self._server_socket.accept() - TCPRelayHandler(self._fd_to_handlers, self._eventloop, + TCPRelayHandler(self, self._fd_to_handlers, self._eventloop, conn[0], self._config, self._is_local) except (OSError, IOError) as e: error_no = eventloop.errno_from_exception(e) @@ -404,10 +492,10 @@ class TCPRelay(object): else: logging.warn('poll removed fd') - now = time.time() - if now - self._last_time > 5: - # TODO sweep timeouts - self._last_time = now + now = time.time() + if now - self._last_time > TIMEOUT_PRECISION: + self._sweep_timeout() + self._last_time = now def close(self): self._closed = True diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index c087ee3..d1941f9 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -64,10 +64,10 @@ def check_config(config): if (config.get('method', '') or '').lower() == 'rc4': logging.warn('warning: RC4 is not safe; please use a safer cipher, ' 'like AES-256-CFB') - if (int(config.get('timeout', 300)) or 300) < 100: + if config.get('timeout', 300) < 100: logging.warn('warning: your timeout %d seems too short' % int(config.get('timeout'))) - if (int(config.get('timeout', 300)) or 300) > 600: + if config.get('timeout', 300) > 600: logging.warn('warning: your timeout %d seems too long' % int(config.get('timeout'))) @@ -114,6 +114,8 @@ def get_config(is_local): config['local_address'] = value elif key == '-v': config['verbose'] = True + elif key == '-t': + config['timeout'] = int(value) elif key == '--fast-open': config['fast_open'] = True elif key == '--workers': From 66bbe50719b53ec1142649e92e26dfd096d235dd Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 2 Jun 2014 17:03:08 +0800 Subject: [PATCH 034/344] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e900f98..76fea19 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Current version: 2.0 [![Build Status]][Travis CI] shadowsocks is a lightweight tunnel proxy that help you get through firewalls. -2.0 is currently under development. Please use 1.4.x. +2.0 is currently experimental. Please use 1.4.x. Both TCP CONNECT and UDP ASSOCIATE are implemented. From 74566308d1dd9075b8b8e9a3104d0dd1561e820b Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 2 Jun 2014 17:17:36 +0800 Subject: [PATCH 035/344] update readme --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 76fea19..4ad1396 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,12 @@ shadowsocks =========== -Current version: 2.0 [![Build Status]][Travis CI] +Current version: 2.0-pre [![Build Status]][Travis CI] shadowsocks is a lightweight tunnel proxy that help you get through firewalls. 2.0 is currently experimental. Please use 1.4.x. -Both TCP CONNECT and UDP ASSOCIATE are implemented. - [中文说明] Install From 1966b643fcea57036c401bb7654ae38a02de1390 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 2 Jun 2014 18:08:22 +0800 Subject: [PATCH 036/344] update log --- CHANGES | 4 ++++ shadowsocks/tcprelay.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 4f19880..13a7058 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +2.0 2014-06-02 +- Use a new event model +- Remove gevent + 1.4.5 2014-05-24 - Add timeout in TCP server - Close sockets in master process diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index e1b8cf9..979d68f 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -231,7 +231,7 @@ class TCPRelayHandler(object): raise Exception('can not parse header') addrtype, remote_addr, remote_port, header_length =\ header_result - logging.debug('connecting %s:%d' % (remote_addr, remote_port)) + logging.info('connecting %s:%d' % (remote_addr, remote_port)) self._remote_address = (remote_addr, remote_port) if is_local: # forward address to remote From 5d022e282864f1848b5e643f3cf880471939cd2a Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 2 Jun 2014 18:11:38 +0800 Subject: [PATCH 037/344] fix None --- shadowsocks/tcprelay.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 979d68f..318dcec 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -320,11 +320,13 @@ class TCPRelayHandler(object): self.update_stream(STREAM_UP, WAIT_STATUS_READING) def on_local_error(self): - logging.error(eventloop.get_sock_error(self._local_sock)) + if self._local_sock: + logging.error(eventloop.get_sock_error(self._local_sock)) self.destroy() def on_remote_error(self): - logging.error(eventloop.get_sock_error(self._remote_sock)) + if self._remote_sock: + logging.error(eventloop.get_sock_error(self._remote_sock)) self.destroy() def handle_event(self, sock, event): From 476738d2732956356e1c00b6b0abc414a9dde7de Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 2 Jun 2014 18:16:24 +0800 Subject: [PATCH 038/344] fix errno --- shadowsocks/eventloop.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index b6a9ca5..d75905d 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -230,5 +230,5 @@ def errno_from_exception(e): # from tornado def get_sock_error(sock): - errno = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) - return socket.error(errno, os.strerror(errno)) + error_number = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) + return socket.error(error_number, os.strerror(error_number)) From 904498fdc7dcdedbc513470fe7cfe96e9468a4c9 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 2 Jun 2014 18:18:57 +0800 Subject: [PATCH 039/344] update readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4ad1396..98f30f2 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Current version: 2.0-pre [![Build Status]][Travis CI] shadowsocks is a lightweight tunnel proxy that help you get through firewalls. -2.0 is currently experimental. Please use 1.4.x. +2.0-pre is currently experimental. Please use [1.4.x]. [中文说明] @@ -106,6 +106,7 @@ Mailing list: http://groups.google.com/group/shadowsocks Also see [Troubleshooting] +[1.4.x]: https://github.com/clowwindy/shadowsocks/tree/1.4 [Build Status]: https://travis-ci.org/clowwindy/shadowsocks.png?branch=2.0 [Travis CI]: https://travis-ci.org/clowwindy/shadowsocks [TCP_FASTOPEN]: https://github.com/clowwindy/shadowsocks/wiki/TCP-Fast-Open From 6c35dfd8327c645029f18ccd82f468ac7764f0f6 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 2 Jun 2014 18:24:26 +0800 Subject: [PATCH 040/344] fix typo --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 318dcec..c447ab8 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -435,7 +435,7 @@ class TCPRelay(object): def _sweep_timeout(self): # tornado's timeout memory management is more flexible that we need - # we just need a sorted last_activity queue and it's faster that heapq + # we just need a sorted last_activity queue and it's faster than heapq # in fact we can do O(1) insertion/remove so we invent our own if self._timeouts: now = time.time() From 30e8d5d1a13b71a92fbdf9352e7b715eb32ea78c Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 2 Jun 2014 18:28:57 +0800 Subject: [PATCH 041/344] add pypy --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 267427c..81cfb2b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python python: - 2.6 - 2.7 + - pypy before_install: - sudo apt-get update -qq - sudo apt-get install -qq build-essential libssl-dev swig python-m2crypto python-numpy From fd6a0a8af0c33c7ba721bae3e17ff09dd1146534 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 2 Jun 2014 18:40:03 +0800 Subject: [PATCH 042/344] update travis --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 98f30f2..f7575b7 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ Also see [Troubleshooting] [1.4.x]: https://github.com/clowwindy/shadowsocks/tree/1.4 -[Build Status]: https://travis-ci.org/clowwindy/shadowsocks.png?branch=2.0 +[Build Status]: https://travis-ci.org/clowwindy/shadowsocks.png?branch=master [Travis CI]: https://travis-ci.org/clowwindy/shadowsocks [TCP_FASTOPEN]: https://github.com/clowwindy/shadowsocks/wiki/TCP-Fast-Open [Issue Tracker]: https://github.com/clowwindy/shadowsocks/issues?state=open From f6a9cad684000a092b115e7adc5eae861426b7fb Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 3 Jun 2014 21:22:53 +0800 Subject: [PATCH 043/344] add fast open support in server --- shadowsocks/tcprelay.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index c447ab8..a6e3e3e 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -400,6 +400,11 @@ class TCPRelay(object): server_socket.bind(sa) server_socket.setblocking(False) server_socket.listen(1024) + if config['fast_open']: + try: + server_socket.setsockopt(socket.SOL_TCP, 23, 5) + except socket.error: + logging.error('warning: fast open is not available') self._server_socket = server_socket def add_to_loop(self, loop): From 0c12ee04cbe08570e1916618f5cb4b375b965207 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 3 Jun 2014 21:59:34 +0800 Subject: [PATCH 044/344] implement fast open in local --- .travis.yml | 1 + shadowsocks/tcprelay.py | 57 +++++++++++++++++++++++++++++++++-------- tests/fastopen.json | 10 ++++++++ 3 files changed, 57 insertions(+), 11 deletions(-) create mode 100644 tests/fastopen.json diff --git a/.travis.yml b/.travis.yml index 81cfb2b..90be114 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,3 +13,4 @@ script: - python test.py -c tests/salsa20.json - python test.py -c tests/server-multi-passwd.json - python test.py -c tests/workers.json + - python test.py -c tests/fastopen.json diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index a6e3e3e..dcfa8ed 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -34,6 +34,8 @@ from common import parse_header TIMEOUTS_CLEAN_SIZE = 512 TIMEOUT_PRECISION = 4 +MSG_FASTOPEN = 0x20000000 + CMD_CONNECT = 1 CMD_BIND = 2 CMD_UDP_ASSOCIATE = 3 @@ -199,6 +201,34 @@ class TCPRelayHandler(object): if is_local: data = self._encryptor.encrypt(data) self._data_to_write_to_remote.append(data) + if is_local and self._upstream_status == WAIT_STATUS_INIT and \ + self._config['fast_open']: + try: + data = ''.join(self._data_to_write_to_local) + l = len(data) + s = self._remote_sock.sendto(data, MSG_FASTOPEN, + self.remote_address) + if s < l: + data = data[s:] + self._data_to_write_to_local = [data] + self.update_stream(STREAM_UP, WAIT_STATUS_READWRITING) + self.update_stream(STREAM_DOWN, WAIT_STATUS_READING) + else: + self._data_to_write_to_local = [] + self.update_stream(STREAM_UP, WAIT_STATUS_READING) + self.update_stream(STREAM_DOWN, WAIT_STATUS_READING) + self._stage = STAGE_STREAM + except (OSError, IOError) as e: + if eventloop.errno_from_exception(e) == errno.EINPROGRESS: + self.update_stream(STREAM_UP, WAIT_STATUS_READWRITING) + self.update_stream(STREAM_DOWN, WAIT_STATUS_READING) + elif eventloop.errno_from_exception(e) == errno.ENOTCONN: + logging.error('fast open not supported on this OS') + self._config['fast_open'] = False + self.destroy() + else: + logging.error(e) + self.destroy() elif (is_local and self._stage == STAGE_HELLO) or \ (not is_local and self._stage == STAGE_INIT): try: @@ -259,18 +289,23 @@ class TCPRelayHandler(object): self._fd_to_handlers[remote_sock.fileno()] = self remote_sock.setblocking(False) remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) - # TODO support TCP fast open - try: - remote_sock.connect(sa) - except (OSError, IOError) as e: - if eventloop.errno_from_exception(e) == errno.EINPROGRESS: - pass - self._loop.add(remote_sock, - eventloop.POLL_ERR | eventloop.POLL_OUT) - self._stage = STAGE_REPLY - self.update_stream(STREAM_UP, WAIT_STATUS_READWRITING) - self.update_stream(STREAM_DOWN, WAIT_STATUS_READING) + if self._is_local and self._config['fast_open']: + # wait for more data to arrive and send them in one SYN + self._stage = STAGE_REPLY + # TODO when there is already data in this packet + else: + try: + remote_sock.connect(sa) + except (OSError, IOError) as e: + if eventloop.errno_from_exception(e) == \ + errno.EINPROGRESS: + pass + self._loop.add(remote_sock, + eventloop.POLL_ERR | eventloop.POLL_OUT) + self._stage = STAGE_REPLY + self.update_stream(STREAM_UP, WAIT_STATUS_READWRITING) + self.update_stream(STREAM_DOWN, WAIT_STATUS_READING) return except Exception: import traceback diff --git a/tests/fastopen.json b/tests/fastopen.json new file mode 100644 index 0000000..4f3da05 --- /dev/null +++ b/tests/fastopen.json @@ -0,0 +1,10 @@ +{ + "server":"127.0.0.1", + "server_port":8388, + "local_port":1081, + "password":"barfoo!", + "timeout":300, + "method":"aes-256-cfb", + "local_address":"127.0.0.1", + "fast_open":true +} From adda6d6e290a1ba02e9b7b3bbbf08d50c6fb37f4 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 3 Jun 2014 22:15:59 +0800 Subject: [PATCH 045/344] lint code --- shadowsocks/tcprelay.py | 283 ++++++++++++++++++++-------------------- 1 file changed, 142 insertions(+), 141 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index dcfa8ed..298b9dd 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -94,7 +94,7 @@ class TCPRelayHandler(object): local_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) loop.add(local_sock, eventloop.POLL_IN | eventloop.POLL_ERR) self.last_activity = 0 - self.update_activity() + self._update_activity() def __hash__(self): # default __hash__ is id / 16 @@ -105,10 +105,10 @@ class TCPRelayHandler(object): def remote_address(self): return self._remote_address - def update_activity(self): + def _update_activity(self): self._server.update_activity(self) - def update_stream(self, stream, status): + def _update_stream(self, stream, status): dirty = False if stream == STREAM_DOWN: if self._downstream_status != status: @@ -134,7 +134,7 @@ class TCPRelayHandler(object): event |= eventloop.POLL_OUT self._loop.modify(self._remote_sock, event) - def write_to_sock(self, data, sock): + def _write_to_sock(self, data, sock): if not data or not sock: return uncomplete = False @@ -154,22 +154,133 @@ class TCPRelayHandler(object): if uncomplete: if sock == self._local_sock: self._data_to_write_to_local.append(data) - self.update_stream(STREAM_DOWN, WAIT_STATUS_WRITING) + self._update_stream(STREAM_DOWN, WAIT_STATUS_WRITING) elif sock == self._remote_sock: self._data_to_write_to_remote.append(data) - self.update_stream(STREAM_UP, WAIT_STATUS_WRITING) + self._update_stream(STREAM_UP, WAIT_STATUS_WRITING) else: logging.error('write_all_to_sock:unknown socket') else: if sock == self._local_sock: - self.update_stream(STREAM_DOWN, WAIT_STATUS_READING) + self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) elif sock == self._remote_sock: - self.update_stream(STREAM_UP, WAIT_STATUS_READING) + self._update_stream(STREAM_UP, WAIT_STATUS_READING) else: logging.error('write_all_to_sock:unknown socket') - def on_local_read(self): - self.update_activity() + def _handle_stage_reply(self, data): + if self._is_local: + data = self._encryptor.encrypt(data) + self._data_to_write_to_remote.append(data) + if self._is_local and self._upstream_status == WAIT_STATUS_INIT and \ + self._config['fast_open']: + try: + data = ''.join(self._data_to_write_to_local) + l = len(data) + s = self._remote_sock.sendto(data, MSG_FASTOPEN, + self.remote_address) + if s < l: + data = data[s:] + self._data_to_write_to_local = [data] + self._update_stream(STREAM_UP, WAIT_STATUS_READWRITING) + self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) + else: + self._data_to_write_to_local = [] + self._update_stream(STREAM_UP, WAIT_STATUS_READING) + self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) + self._stage = STAGE_STREAM + except (OSError, IOError) as e: + if eventloop.errno_from_exception(e) == errno.EINPROGRESS: + self._update_stream(STREAM_UP, WAIT_STATUS_READWRITING) + self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) + elif eventloop.errno_from_exception(e) == errno.ENOTCONN: + logging.error('fast open not supported on this OS') + self._config['fast_open'] = False + self.destroy() + else: + logging.error(e) + self.destroy() + + def _handle_stage_hello(self, data): + try: + if self._is_local: + cmd = ord(data[1]) + if cmd == CMD_UDP_ASSOCIATE: + logging.debug('UDP associate') + if self._local_sock.family == socket.AF_INET6: + header = '\x05\x00\x00\x04' + else: + header = '\x05\x00\x00\x01' + addr, port = self._local_sock.getsockname() + addr_to_send = socket.inet_pton(self._local_sock.family, + addr) + port_to_send = struct.pack('>H', port) + self._write_to_sock(header + addr_to_send + port_to_send, + self._local_sock) + self._stage = STAGE_UDP_ASSOC + # just wait for the client to disconnect + return + elif cmd == CMD_CONNECT: + # just trim VER CMD RSV + data = data[3:] + else: + logging.error('unknown command %d', cmd) + self.destroy() + return + header_result = parse_header(data) + if header_result is None: + raise Exception('can not parse header') + addrtype, remote_addr, remote_port, header_length = header_result + logging.info('connecting %s:%d' % (remote_addr, remote_port)) + self._remote_address = (remote_addr, remote_port) + if self._is_local: + # forward address to remote + self._write_to_sock('\x05\x00\x00\x01\x00\x00\x00\x00\x10\x10', + self._local_sock) + data_to_send = self._encryptor.encrypt(data) + self._data_to_write_to_remote.append(data_to_send) + remote_addr = self._config['server'] + remote_port = self._config['server_port'] + else: + if len(data) > header_length: + self._data_to_write_to_remote.append(data[header_length:]) + + # TODO async DNS + addrs = socket.getaddrinfo(remote_addr, remote_port, 0, + socket.SOCK_STREAM, socket.SOL_TCP) + if len(addrs) == 0: + raise Exception("can't get addrinfo for %s:%d" % + (remote_addr, remote_port)) + af, socktype, proto, canonname, sa = addrs[0] + remote_sock = socket.socket(af, socktype, proto) + self._remote_sock = remote_sock + self._fd_to_handlers[remote_sock.fileno()] = self + remote_sock.setblocking(False) + remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) + + if self._is_local and self._config['fast_open']: + # wait for more data to arrive and send them in one SYN + self._stage = STAGE_REPLY + # TODO when there is already data in this packet + else: + try: + remote_sock.connect(sa) + except (OSError, IOError) as e: + if eventloop.errno_from_exception(e) == errno.EINPROGRESS: + pass + self._loop.add(remote_sock, + eventloop.POLL_ERR | eventloop.POLL_OUT) + self._stage = STAGE_REPLY + self._update_stream(STREAM_UP, WAIT_STATUS_READWRITING) + self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) + except Exception: + import traceback + traceback.print_exc() + # TODO use logging when debug completed + self.destroy() + + def _on_local_read(self): + self._update_activity() if not self._local_sock: return is_local = self._is_local @@ -190,131 +301,21 @@ class TCPRelayHandler(object): if self._stage == STAGE_STREAM: if self._is_local: data = self._encryptor.encrypt(data) - self.write_to_sock(data, self._remote_sock) + self._write_to_sock(data, self._remote_sock) return elif is_local and self._stage == STAGE_INIT: # TODO check auth method - self.write_to_sock('\x05\00', self._local_sock) + self._write_to_sock('\x05\00', self._local_sock) self._stage = STAGE_HELLO return elif self._stage == STAGE_REPLY: - if is_local: - data = self._encryptor.encrypt(data) - self._data_to_write_to_remote.append(data) - if is_local and self._upstream_status == WAIT_STATUS_INIT and \ - self._config['fast_open']: - try: - data = ''.join(self._data_to_write_to_local) - l = len(data) - s = self._remote_sock.sendto(data, MSG_FASTOPEN, - self.remote_address) - if s < l: - data = data[s:] - self._data_to_write_to_local = [data] - self.update_stream(STREAM_UP, WAIT_STATUS_READWRITING) - self.update_stream(STREAM_DOWN, WAIT_STATUS_READING) - else: - self._data_to_write_to_local = [] - self.update_stream(STREAM_UP, WAIT_STATUS_READING) - self.update_stream(STREAM_DOWN, WAIT_STATUS_READING) - self._stage = STAGE_STREAM - except (OSError, IOError) as e: - if eventloop.errno_from_exception(e) == errno.EINPROGRESS: - self.update_stream(STREAM_UP, WAIT_STATUS_READWRITING) - self.update_stream(STREAM_DOWN, WAIT_STATUS_READING) - elif eventloop.errno_from_exception(e) == errno.ENOTCONN: - logging.error('fast open not supported on this OS') - self._config['fast_open'] = False - self.destroy() - else: - logging.error(e) - self.destroy() + self._handle_stage_reply(data) elif (is_local and self._stage == STAGE_HELLO) or \ (not is_local and self._stage == STAGE_INIT): - try: - if is_local: - cmd = ord(data[1]) - if cmd == CMD_UDP_ASSOCIATE: - logging.debug('UDP associate') - if self._local_sock.family == socket.AF_INET6: - header = '\x05\x00\x00\x04' - else: - header = '\x05\x00\x00\x01' - addr, port = self._local_sock.getsockname() - addr_to_send = socket.inet_pton(self._local_sock.family, - addr) - port_to_send = struct.pack('>H', port) - self.write_to_sock(header + addr_to_send + port_to_send, - self._local_sock) - self._stage = STAGE_UDP_ASSOC - # just wait for the client to disconnect - return - elif cmd == CMD_CONNECT: - # just trim VER CMD RSV - data = data[3:] - else: - logging.error('unknown command %d', cmd) - self.destroy() - return - header_result = parse_header(data) - if header_result is None: - raise Exception('can not parse header') - addrtype, remote_addr, remote_port, header_length =\ - header_result - logging.info('connecting %s:%d' % (remote_addr, remote_port)) - self._remote_address = (remote_addr, remote_port) - if is_local: - # forward address to remote - self.write_to_sock('\x05\x00\x00\x01' + - '\x00\x00\x00\x00\x10\x10', - self._local_sock) - data_to_send = self._encryptor.encrypt(data) - self._data_to_write_to_remote.append(data_to_send) - remote_addr = self._config['server'] - remote_port = self._config['server_port'] - else: - if len(data) > header_length: - self._data_to_write_to_remote.append( - data[header_length:]) + self._handle_stage_hello(data) - # TODO async DNS - addrs = socket.getaddrinfo(remote_addr, remote_port, 0, - socket.SOCK_STREAM, socket.SOL_TCP) - if len(addrs) == 0: - raise Exception("can't get addrinfo for %s:%d" % - (remote_addr, remote_port)) - af, socktype, proto, canonname, sa = addrs[0] - remote_sock = socket.socket(af, socktype, proto) - self._remote_sock = remote_sock - self._fd_to_handlers[remote_sock.fileno()] = self - remote_sock.setblocking(False) - remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) - - if self._is_local and self._config['fast_open']: - # wait for more data to arrive and send them in one SYN - self._stage = STAGE_REPLY - # TODO when there is already data in this packet - else: - try: - remote_sock.connect(sa) - except (OSError, IOError) as e: - if eventloop.errno_from_exception(e) == \ - errno.EINPROGRESS: - pass - self._loop.add(remote_sock, - eventloop.POLL_ERR | eventloop.POLL_OUT) - self._stage = STAGE_REPLY - self.update_stream(STREAM_UP, WAIT_STATUS_READWRITING) - self.update_stream(STREAM_DOWN, WAIT_STATUS_READING) - return - except Exception: - import traceback - traceback.print_exc() - # TODO use logging when debug completed - self.destroy() - - def on_remote_read(self): - self.update_activity() + def _on_remote_read(self): + self._update_activity() data = None try: data = self._remote_sock.recv(BUF_SIZE) @@ -330,36 +331,36 @@ class TCPRelayHandler(object): else: data = self._encryptor.encrypt(data) try: - self.write_to_sock(data, self._local_sock) + self._write_to_sock(data, self._local_sock) except Exception: import traceback traceback.print_exc() # TODO use logging when debug completed self.destroy() - def on_local_write(self): + def _on_local_write(self): if self._data_to_write_to_local: data = ''.join(self._data_to_write_to_local) self._data_to_write_to_local = [] - self.write_to_sock(data, self._local_sock) + self._write_to_sock(data, self._local_sock) else: - self.update_stream(STREAM_DOWN, WAIT_STATUS_READING) + self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) - def on_remote_write(self): + def _on_remote_write(self): self._stage = STAGE_STREAM if self._data_to_write_to_remote: data = ''.join(self._data_to_write_to_remote) self._data_to_write_to_remote = [] - self.write_to_sock(data, self._remote_sock) + self._write_to_sock(data, self._remote_sock) else: - self.update_stream(STREAM_UP, WAIT_STATUS_READING) + self._update_stream(STREAM_UP, WAIT_STATUS_READING) - def on_local_error(self): + def _on_local_error(self): if self._local_sock: logging.error(eventloop.get_sock_error(self._local_sock)) self.destroy() - def on_remote_error(self): + def _on_remote_error(self): if self._remote_sock: logging.error(eventloop.get_sock_error(self._remote_sock)) self.destroy() @@ -368,18 +369,18 @@ class TCPRelayHandler(object): # order is important if sock == self._remote_sock: if event & eventloop.POLL_IN: - self.on_remote_read() + self._on_remote_read() if event & eventloop.POLL_OUT: - self.on_remote_write() + self._on_remote_write() if event & eventloop.POLL_ERR: - self.on_remote_error() + self._on_remote_error() elif sock == self._local_sock: if event & eventloop.POLL_IN: - self.on_local_read() + self._on_local_read() if event & eventloop.POLL_OUT: - self.on_local_write() + self._on_local_write() if event & eventloop.POLL_ERR: - self.on_local_error() + self._on_local_error() else: logging.warn('unknown socket') From 8d25047d56b60e256d69d57ce137655976142363 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 3 Jun 2014 22:26:22 +0800 Subject: [PATCH 046/344] fix fastopen local --- shadowsocks/tcprelay.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 298b9dd..f0c2228 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -84,6 +84,7 @@ class TCPRelayHandler(object): self._stage = STAGE_INIT self._encryptor = encrypt.Encryptor(config['password'], config['method']) + self._fastopen_connected = False self._data_to_write_to_local = [] self._data_to_write_to_remote = [] self._upstream_status = WAIT_STATUS_READING @@ -172,9 +173,10 @@ class TCPRelayHandler(object): if self._is_local: data = self._encryptor.encrypt(data) self._data_to_write_to_remote.append(data) - if self._is_local and self._upstream_status == WAIT_STATUS_INIT and \ + if self._is_local and not self._fastopen_connected and \ self._config['fast_open']: try: + self._fastopen_connected = True data = ''.join(self._data_to_write_to_local) l = len(data) s = self._remote_sock.sendto(data, MSG_FASTOPEN, From a08edd20078a1dfafce9a92d8585245082db373a Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 3 Jun 2014 22:29:54 +0800 Subject: [PATCH 047/344] fix fastopen local --- shadowsocks/tcprelay.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index f0c2228..381e378 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -181,15 +181,16 @@ class TCPRelayHandler(object): l = len(data) s = self._remote_sock.sendto(data, MSG_FASTOPEN, self.remote_address) + self._loop.add(self._remote_sock, + eventloop.POLL_ERR | eventloop.POLL_OUT) + self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) if s < l: data = data[s:] self._data_to_write_to_local = [data] self._update_stream(STREAM_UP, WAIT_STATUS_READWRITING) - self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) else: self._data_to_write_to_local = [] self._update_stream(STREAM_UP, WAIT_STATUS_READING) - self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) self._stage = STAGE_STREAM except (OSError, IOError) as e: if eventloop.errno_from_exception(e) == errno.EINPROGRESS: From 944c422768086314a193d11ee24398283ff0e367 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 3 Jun 2014 23:37:36 +0800 Subject: [PATCH 048/344] fix fast open --- .travis.yml | 1 - shadowsocks/tcprelay.py | 8 +++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 90be114..81cfb2b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,4 +13,3 @@ script: - python test.py -c tests/salsa20.json - python test.py -c tests/server-multi-passwd.json - python test.py -c tests/workers.json - - python test.py -c tests/fastopen.json diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 381e378..5da00ee 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -180,10 +180,8 @@ class TCPRelayHandler(object): data = ''.join(self._data_to_write_to_local) l = len(data) s = self._remote_sock.sendto(data, MSG_FASTOPEN, - self.remote_address) - self._loop.add(self._remote_sock, - eventloop.POLL_ERR | eventloop.POLL_OUT) - self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) + (self._config['server'], + self._config['server_port'])) if s < l: data = data[s:] self._data_to_write_to_local = [data] @@ -195,7 +193,6 @@ class TCPRelayHandler(object): except (OSError, IOError) as e: if eventloop.errno_from_exception(e) == errno.EINPROGRESS: self._update_stream(STREAM_UP, WAIT_STATUS_READWRITING) - self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) elif eventloop.errno_from_exception(e) == errno.ENOTCONN: logging.error('fast open not supported on this OS') self._config['fast_open'] = False @@ -264,6 +261,7 @@ class TCPRelayHandler(object): if self._is_local and self._config['fast_open']: # wait for more data to arrive and send them in one SYN self._stage = STAGE_REPLY + self._loop.add(remote_sock, eventloop.POLL_ERR) # TODO when there is already data in this packet else: try: From 8aae1f07ca751a4604ca36ccbafb27d1d1dbd006 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 3 Jun 2014 23:44:49 +0800 Subject: [PATCH 049/344] warn table encryption --- shadowsocks/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index d1941f9..66e0dd6 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -61,6 +61,9 @@ def check_config(config): if config.get('server', '') in ['127.0.0.1', 'localhost']: logging.warn('warning: server set to listen %s:%s, are you sure?' % (config['server'], config['server_port'])) + if (config.get('method', '') or '').lower() == '': + logging.warn('warning: table is not safe; please use a safer cipher, ' + 'like AES-256-CFB') if (config.get('method', '') or '').lower() == 'rc4': logging.warn('warning: RC4 is not safe; please use a safer cipher, ' 'like AES-256-CFB') From 816eee8dc1ac78c7dda1138e732cdc6881f8b65c Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 4 Jun 2014 18:04:41 +0800 Subject: [PATCH 050/344] Update README.md --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f7575b7..eea41f3 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,10 @@ shadowsocks =========== -Current version: 2.0-pre [![Build Status]][Travis CI] +[![Build Status]][Travis CI] +![Downloads Stat] + +Current version: 2.0-pre shadowsocks is a lightweight tunnel proxy that help you get through firewalls. @@ -107,11 +110,12 @@ Also see [Troubleshooting] [1.4.x]: https://github.com/clowwindy/shadowsocks/tree/1.4 -[Build Status]: https://travis-ci.org/clowwindy/shadowsocks.png?branch=master -[Travis CI]: https://travis-ci.org/clowwindy/shadowsocks [TCP_FASTOPEN]: https://github.com/clowwindy/shadowsocks/wiki/TCP-Fast-Open [Issue Tracker]: https://github.com/clowwindy/shadowsocks/issues?state=open [GUI client]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients [Supervisor]: https://github.com/clowwindy/shadowsocks/wiki/Configure-Shadowsocks-with-Supervisor [Troubleshooting]: https://github.com/clowwindy/shadowsocks/wiki/Troubleshooting [中文说明]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E +[Build Status]: http://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat +[Travis CI]: https://travis-ci.org/clowwindy/shadowsocks +[Downloads Stat]: http://img.shields.io/pypi/dm/shadowsocks.svg?style=flat From 345696cf296f3a81f082dfe2e01bbff1a1ddc28e Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 4 Jun 2014 18:12:04 +0800 Subject: [PATCH 051/344] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index eea41f3..6ba6dea 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ shadowsocks =========== -[![Build Status]][Travis CI] -![Downloads Stat] +[![Build Status]][Travis CI] ![PyPI version] ![Downloads Stat] Current version: 2.0-pre @@ -119,3 +118,4 @@ Also see [Troubleshooting] [Build Status]: http://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat [Travis CI]: https://travis-ci.org/clowwindy/shadowsocks [Downloads Stat]: http://img.shields.io/pypi/dm/shadowsocks.svg?style=flat +[PyPI version]: http://img.shields.io/pypi/v/shadowsocks.svg?style=flat From f8326289681cb4ac2c50bf08994bd07261c60ff3 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 4 Jun 2014 18:22:09 +0800 Subject: [PATCH 052/344] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6ba6dea..8d30991 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ shadowsocks =========== -[![Build Status]][Travis CI] ![PyPI version] ![Downloads Stat] +[![PyPI version]][PyPI] [![Build Status]][Travis CI] Current version: 2.0-pre @@ -117,5 +117,5 @@ Also see [Troubleshooting] [中文说明]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E [Build Status]: http://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat [Travis CI]: https://travis-ci.org/clowwindy/shadowsocks -[Downloads Stat]: http://img.shields.io/pypi/dm/shadowsocks.svg?style=flat +[PyPI]: https://pypi.python.org/pypi/shadowsocks [PyPI version]: http://img.shields.io/pypi/v/shadowsocks.svg?style=flat From 5f29cd6fb5f4e5a20345bf9b3f58fca875c854f9 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 4 Jun 2014 18:32:58 +0800 Subject: [PATCH 053/344] Update README.md --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 8d30991..9fb27f5 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ shadowsocks is a lightweight tunnel proxy that help you get through firewalls. 2.0-pre is currently experimental. Please use [1.4.x]. -[中文说明] +[中文说明][Chinese Readme] Install ------- @@ -109,13 +109,13 @@ Also see [Troubleshooting] [1.4.x]: https://github.com/clowwindy/shadowsocks/tree/1.4 -[TCP_FASTOPEN]: https://github.com/clowwindy/shadowsocks/wiki/TCP-Fast-Open -[Issue Tracker]: https://github.com/clowwindy/shadowsocks/issues?state=open -[GUI client]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients -[Supervisor]: https://github.com/clowwindy/shadowsocks/wiki/Configure-Shadowsocks-with-Supervisor -[Troubleshooting]: https://github.com/clowwindy/shadowsocks/wiki/Troubleshooting -[中文说明]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E [Build Status]: http://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat -[Travis CI]: https://travis-ci.org/clowwindy/shadowsocks +[Chinese Readme]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E +[GUI client]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients +[Issue Tracker]: https://github.com/clowwindy/shadowsocks/issues?state=open [PyPI]: https://pypi.python.org/pypi/shadowsocks [PyPI version]: http://img.shields.io/pypi/v/shadowsocks.svg?style=flat +[Supervisor]: https://github.com/clowwindy/shadowsocks/wiki/Configure-Shadowsocks-with-Supervisor +[TCP_FASTOPEN]: https://github.com/clowwindy/shadowsocks/wiki/TCP-Fast-Open +[Travis CI]: https://travis-ci.org/clowwindy/shadowsocks +[Troubleshooting]: https://github.com/clowwindy/shadowsocks/wiki/Troubleshooting From b50a89b1f531e897d9b893bbccac4e22fd1db0a8 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 5 Jun 2014 19:33:45 +0800 Subject: [PATCH 054/344] add server-multi-passwd-table test --- .travis.yml | 1 + tests/aes.json | 2 +- tests/fastopen.json | 2 +- tests/salsa20.json | 2 +- tests/server-multi-passwd-table.json | 19 +++++++++++++++++++ tests/table.json | 2 +- tests/workers.json | 2 +- 7 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 tests/server-multi-passwd-table.json diff --git a/.travis.yml b/.travis.yml index 81cfb2b..f9f7d7a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,4 +12,5 @@ script: - python test.py -c tests/aes.json - python test.py -c tests/salsa20.json - python test.py -c tests/server-multi-passwd.json + - python test.py -c tests/server-multi-passwd-table.json - python test.py -c tests/workers.json diff --git a/tests/aes.json b/tests/aes.json index cc9ffeb..7c0f8f2 100644 --- a/tests/aes.json +++ b/tests/aes.json @@ -3,7 +3,7 @@ "server_port":8388, "local_port":1081, "password":"barfoo!", - "timeout":300, + "timeout":60, "method":"aes-256-cfb", "local_address":"127.0.0.1", "fast_open":false diff --git a/tests/fastopen.json b/tests/fastopen.json index 4f3da05..82a9cb2 100644 --- a/tests/fastopen.json +++ b/tests/fastopen.json @@ -3,7 +3,7 @@ "server_port":8388, "local_port":1081, "password":"barfoo!", - "timeout":300, + "timeout":60, "method":"aes-256-cfb", "local_address":"127.0.0.1", "fast_open":true diff --git a/tests/salsa20.json b/tests/salsa20.json index 182589e..af1d266 100644 --- a/tests/salsa20.json +++ b/tests/salsa20.json @@ -3,7 +3,7 @@ "server_port":8388, "local_port":1081, "password":"barfoo!", - "timeout":300, + "timeout":60, "method":"salsa20-ctr", "local_address":"127.0.0.1", "fast_open":false diff --git a/tests/server-multi-passwd-table.json b/tests/server-multi-passwd-table.json new file mode 100644 index 0000000..a2c0a80 --- /dev/null +++ b/tests/server-multi-passwd-table.json @@ -0,0 +1,19 @@ +{ + "server": "127.0.0.1", + "server_port": 8384, + "local_port": 1081, + "password": "foobar4", + "port_password": { + "8381": "foobar1", + "8382": "foobar2", + "8383": "foobar3", + "8384": "foobar4", + "8385": "foobar5", + "8386": "foobar6", + "8387": "foobar7", + "8388": "foobar8", + "8389": "foobar9" + }, + "timeout": 60, + "method": "table" +} diff --git a/tests/table.json b/tests/table.json index 402a4b8..21cff86 100644 --- a/tests/table.json +++ b/tests/table.json @@ -3,7 +3,7 @@ "server_port":8388, "local_port":1081, "password":"barfoo!", - "timeout":300, + "timeout":60, "method":"table", "local_address":"127.0.0.1", "fast_open":false diff --git a/tests/workers.json b/tests/workers.json index 1df28ce..35f3962 100644 --- a/tests/workers.json +++ b/tests/workers.json @@ -3,7 +3,7 @@ "server_port":8388, "local_port":1081, "password":"barfoo!", - "timeout":300, + "timeout":60, "method":"aes-256-cfb", "local_address":"127.0.0.1", "workers": 4 From d3f294217c45198706b42ebd05683bd012d7ec60 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 5 Jun 2014 19:47:03 +0800 Subject: [PATCH 055/344] close #121 --- shadowsocks/utils.py | 4 ++++ tests/aes.json | 2 +- tests/fastopen.json | 2 +- tests/salsa20.json | 2 +- tests/table.json | 2 +- tests/workers.json | 2 +- 6 files changed, 9 insertions(+), 5 deletions(-) diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 66e0dd6..ed95443 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -73,6 +73,10 @@ def check_config(config): if config.get('timeout', 300) > 600: logging.warn('warning: your timeout %d seems too long' % int(config.get('timeout'))) + if config.get('password') in ['mypassword', 'barfoo!']: + logging.error('DON\'T USE DEFAULT PASSWORD! Please change it in your ' + 'config.json!') + exit(1) def get_config(is_local): diff --git a/tests/aes.json b/tests/aes.json index 7c0f8f2..a3d95b9 100644 --- a/tests/aes.json +++ b/tests/aes.json @@ -2,7 +2,7 @@ "server":"127.0.0.1", "server_port":8388, "local_port":1081, - "password":"barfoo!", + "password":"aes_password", "timeout":60, "method":"aes-256-cfb", "local_address":"127.0.0.1", diff --git a/tests/fastopen.json b/tests/fastopen.json index 82a9cb2..f3980b6 100644 --- a/tests/fastopen.json +++ b/tests/fastopen.json @@ -2,7 +2,7 @@ "server":"127.0.0.1", "server_port":8388, "local_port":1081, - "password":"barfoo!", + "password":"fastopen_password", "timeout":60, "method":"aes-256-cfb", "local_address":"127.0.0.1", diff --git a/tests/salsa20.json b/tests/salsa20.json index af1d266..5ca6c45 100644 --- a/tests/salsa20.json +++ b/tests/salsa20.json @@ -2,7 +2,7 @@ "server":"127.0.0.1", "server_port":8388, "local_port":1081, - "password":"barfoo!", + "password":"salsa20_password", "timeout":60, "method":"salsa20-ctr", "local_address":"127.0.0.1", diff --git a/tests/table.json b/tests/table.json index 21cff86..cca6ac2 100644 --- a/tests/table.json +++ b/tests/table.json @@ -2,7 +2,7 @@ "server":"127.0.0.1", "server_port":8388, "local_port":1081, - "password":"barfoo!", + "password":"table_password", "timeout":60, "method":"table", "local_address":"127.0.0.1", diff --git a/tests/workers.json b/tests/workers.json index 35f3962..2015ff6 100644 --- a/tests/workers.json +++ b/tests/workers.json @@ -2,7 +2,7 @@ "server":"127.0.0.1", "server_port":8388, "local_port":1081, - "password":"barfoo!", + "password":"workers_password", "timeout":60, "method":"aes-256-cfb", "local_address":"127.0.0.1", From e98e9232e86969215f6b4b8f668329a16a2b6793 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 5 Jun 2014 20:52:16 +0800 Subject: [PATCH 056/344] update readme --- README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/README.md b/README.md index 9fb27f5..2f6010a 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,7 @@ shadowsocks [![PyPI version]][PyPI] [![Build Status]][Travis CI] -Current version: 2.0-pre - -shadowsocks is a lightweight tunnel proxy that help you get through firewalls. - -2.0-pre is currently experimental. Please use [1.4.x]. +A fast tunnel proxy that help you get through firewalls. [中文说明][Chinese Readme] From dbaf83eb7b2e4a2419570a5cc7a93b4654fa6790 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 5 Jun 2014 20:52:29 +0800 Subject: [PATCH 057/344] fix a typo --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 5da00ee..8f12218 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -476,7 +476,7 @@ class TCPRelay(object): self._handler_to_timeouts[hash(handler)] = length def _sweep_timeout(self): - # tornado's timeout memory management is more flexible that we need + # tornado's timeout memory management is more flexible than we need # we just need a sorted last_activity queue and it's faster than heapq # in fact we can do O(1) insertion/remove so we invent our own if self._timeouts: From f3defc8b92ad63a9c085a36f1cc7a20c6268c890 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 5 Jun 2014 20:55:20 +0800 Subject: [PATCH 058/344] update CHANGES --- CHANGES | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 13a7058..5935325 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,8 @@ -2.0 2014-06-02 +2.0 2014-06-05 - Use a new event model - Remove gevent +- Refuse to use default password +- Fix a problem when using multiple passwords with table encryption 1.4.5 2014-05-24 - Add timeout in TCP server From a7bdac35789f0b50246f1b8a01888386860e012c Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 5 Jun 2014 21:02:25 +0800 Subject: [PATCH 059/344] update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9db7faa..0263846 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="shadowsocks", version="2.0", license='MIT', - description="a lightweight tunnel proxy", + description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', author_email='clowwindy42@gmail.com', url='https://github.com/clowwindy/shadowsocks', From aab163f067ba4f848c64a93f19d2e44c1a53a8cc Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 5 Jun 2014 21:02:36 +0800 Subject: [PATCH 060/344] update README.rst --- README.rst | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 139a1f1..9a263fa 100644 --- a/README.rst +++ b/README.rst @@ -1,14 +1,9 @@ shadowsocks =========== -|Build Status| +|PyPI version| |Build Status| -shadowsocks is a lightweight tunnel proxy that help you get through -firewalls. - -2.0 is currently under development. Please use 1.4.x. - -Both TCP CONNECT and UDP ASSOCIATE are implemented. +A fast tunnel proxy that help you get through firewalls. `中文说明 `__ @@ -141,5 +136,7 @@ Mailing list: http://groups.google.com/group/shadowsocks Also see `Troubleshooting `__ -.. |Build Status| image:: https://travis-ci.org/clowwindy/shadowsocks.png?branch=master +.. |PyPI version| image:: http://img.shields.io/pypi/v/shadowsocks.svg?style=flat + :target: https://pypi.python.org/pypi/shadowsocks +.. |Build Status| image:: http://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat :target: https://travis-ci.org/clowwindy/shadowsocks From d88d2802b0d1d8724ff23e2f430a4f0685b84b7e Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 5 Jun 2014 21:29:05 +0800 Subject: [PATCH 061/344] change status image to https --- README.md | 4 ++-- README.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2f6010a..1a2f7da 100644 --- a/README.md +++ b/README.md @@ -105,12 +105,12 @@ Also see [Troubleshooting] [1.4.x]: https://github.com/clowwindy/shadowsocks/tree/1.4 -[Build Status]: http://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat +[Build Status]: https://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat [Chinese Readme]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E [GUI client]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients [Issue Tracker]: https://github.com/clowwindy/shadowsocks/issues?state=open [PyPI]: https://pypi.python.org/pypi/shadowsocks -[PyPI version]: http://img.shields.io/pypi/v/shadowsocks.svg?style=flat +[PyPI version]: https://img.shields.io/pypi/v/shadowsocks.svg?style=flat [Supervisor]: https://github.com/clowwindy/shadowsocks/wiki/Configure-Shadowsocks-with-Supervisor [TCP_FASTOPEN]: https://github.com/clowwindy/shadowsocks/wiki/TCP-Fast-Open [Travis CI]: https://travis-ci.org/clowwindy/shadowsocks diff --git a/README.rst b/README.rst index 9a263fa..77edba2 100644 --- a/README.rst +++ b/README.rst @@ -136,7 +136,7 @@ Mailing list: http://groups.google.com/group/shadowsocks Also see `Troubleshooting `__ -.. |PyPI version| image:: http://img.shields.io/pypi/v/shadowsocks.svg?style=flat +.. |PyPI version| image:: https://img.shields.io/pypi/v/shadowsocks.svg?style=flat :target: https://pypi.python.org/pypi/shadowsocks -.. |Build Status| image:: http://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat +.. |Build Status| image:: https://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat :target: https://travis-ci.org/clowwindy/shadowsocks From c354d8a8afc0f5e242e76a25d93d5179d071c2c3 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 6 Jun 2014 21:33:10 +0800 Subject: [PATCH 062/344] add asyncdns --- shadowsocks/asyncdns.py | 73 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 shadowsocks/asyncdns.py diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py new file mode 100644 index 0000000..a75be90 --- /dev/null +++ b/shadowsocks/asyncdns.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import socket +import struct +import common + + +_request_count = 1 + +# rfc1035 +# 1 1 1 1 1 1 +# 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 +# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +# | ID | +# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +# |QR| Opcode |AA|TC|RD|RA| Z | RCODE | +# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +# | QDCOUNT | +# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +# | ANCOUNT | +# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +# | NSCOUNT | +# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +# | ARCOUNT | +# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + + +def pack_request(address): + global _request_count + _request_count += 1 + header = struct.pack('!HBBHHHH', _request_count, 1, 0, 1, 0, 0, 0) + pass + + +def unpack_response(data): + # TODO detect address type + pass + + +def resolve(address, callback): + callback(address) + + +def test(): + def _callback(address): + print address + + resolve('www.baidu.com', _callback) + + +if __name__ == '__main__': + test() From a185e1671b52c12148ab7989e4faf756ab650876 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 6 Jun 2014 22:52:02 +0800 Subject: [PATCH 063/344] add some status protection --- setup.py | 2 +- shadowsocks/eventloop.py | 6 ++++-- shadowsocks/tcprelay.py | 14 +++++++++++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 0263846..efdb366 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.0", + version="2.0.1", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index d75905d..c9de6ef 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -204,9 +204,11 @@ class EventLoop(object): logging.error(e) continue for handler in self._handlers: - # no exceptions should be raised by users # TODO when there are a lot of handlers - handler(events) + try: + handler(events) + except (OSError, IOError) as e: + logging.error(e) # from tornado diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 8f12218..5f66962 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -57,6 +57,7 @@ STAGE_HELLO = 1 STAGE_UDP_ASSOC = 2 STAGE_REPLY = 4 STAGE_STREAM = 5 +STAGE_DESTROYED = -1 # stream direction STREAM_UP = 0 @@ -137,7 +138,7 @@ class TCPRelayHandler(object): def _write_to_sock(self, data, sock): if not data or not sock: - return + return False uncomplete = False try: l = len(data) @@ -152,6 +153,7 @@ class TCPRelayHandler(object): else: logging.error(e) self.destroy() + return False if uncomplete: if sock == self._local_sock: self._data_to_write_to_local.append(data) @@ -168,6 +170,7 @@ class TCPRelayHandler(object): self._update_stream(STREAM_UP, WAIT_STATUS_READING) else: logging.error('write_all_to_sock:unknown socket') + return True def _handle_stage_reply(self, data): if self._is_local: @@ -367,10 +370,14 @@ class TCPRelayHandler(object): self.destroy() def handle_event(self, sock, event): + if self._stage == STAGE_DESTROYED: + return # order is important if sock == self._remote_sock: if event & eventloop.POLL_IN: self._on_remote_read() + if self._stage == STAGE_DESTROYED: + return if event & eventloop.POLL_OUT: self._on_remote_write() if event & eventloop.POLL_ERR: @@ -378,6 +385,8 @@ class TCPRelayHandler(object): elif sock == self._local_sock: if event & eventloop.POLL_IN: self._on_local_read() + if self._stage == STAGE_DESTROYED: + return if event & eventloop.POLL_OUT: self._on_local_write() if event & eventloop.POLL_ERR: @@ -386,6 +395,9 @@ class TCPRelayHandler(object): logging.warn('unknown socket') def destroy(self): + if self._stage == STAGE_DESTROYED: + return + self._stage = STAGE_DESTROYED if self._remote_address: logging.debug('destroy: %s:%d' % self._remote_address) From 816592b67467fa435903f76b6dcd9c23a0b4783e Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 6 Jun 2014 22:57:57 +0800 Subject: [PATCH 064/344] add more log --- shadowsocks/eventloop.py | 4 ++++ shadowsocks/server.py | 2 ++ shadowsocks/tcprelay.py | 12 ++++++++---- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index c9de6ef..e8af800 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -202,6 +202,8 @@ class EventLoop(object): continue else: logging.error(e) + import traceback + traceback.print_exc() continue for handler in self._handlers: # TODO when there are a lot of handlers @@ -209,6 +211,8 @@ class EventLoop(object): handler(events) except (OSError, IOError) as e: logging.error(e) + import traceback + traceback.print_exc() # from tornado diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 2e35698..13c2452 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -69,6 +69,8 @@ def main(): loop.run() except (KeyboardInterrupt, IOError, OSError) as e: logging.error(e) + import traceback + traceback.print_exc() os._exit(0) if int(config['workers']) > 1: diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 5f66962..64b65d2 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -26,6 +26,7 @@ import socket import errno import struct import logging +import traceback import encrypt import eventloop from common import parse_header @@ -152,6 +153,7 @@ class TCPRelayHandler(object): uncomplete = True else: logging.error(e) + traceback.print_exc() self.destroy() return False if uncomplete: @@ -202,6 +204,7 @@ class TCPRelayHandler(object): self.destroy() else: logging.error(e) + traceback.print_exc() self.destroy() def _handle_stage_hello(self, data): @@ -277,8 +280,8 @@ class TCPRelayHandler(object): self._stage = STAGE_REPLY self._update_stream(STREAM_UP, WAIT_STATUS_READWRITING) self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) - except Exception: - import traceback + except Exception as e: + logging.error(e) traceback.print_exc() # TODO use logging when debug completed self.destroy() @@ -336,8 +339,8 @@ class TCPRelayHandler(object): data = self._encryptor.encrypt(data) try: self._write_to_sock(data, self._local_sock) - except Exception: - import traceback + except Exception as e: + logging.error(e) traceback.print_exc() # TODO use logging when debug completed self.destroy() @@ -540,6 +543,7 @@ class TCPRelay(object): continue else: logging.error(e) + traceback.print_exc() else: if sock: handler = self._fd_to_handlers.get(fd, None) From d7d125082fec771f0cbd222b75f6ea9df49942ec Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 6 Jun 2014 23:10:00 +0800 Subject: [PATCH 065/344] handle POLL_ERR first --- shadowsocks/tcprelay.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 64b65d2..c93b6fc 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -364,11 +364,13 @@ class TCPRelayHandler(object): def _on_local_error(self): if self._local_sock: + logging.debug('got local error') logging.error(eventloop.get_sock_error(self._local_sock)) self.destroy() def _on_remote_error(self): if self._remote_sock: + logging.debug('got remote error') logging.error(eventloop.get_sock_error(self._remote_sock)) self.destroy() @@ -377,23 +379,27 @@ class TCPRelayHandler(object): return # order is important if sock == self._remote_sock: + if event & eventloop.POLL_ERR: + self._on_remote_error() + if self._stage == STAGE_DESTROYED: + return if event & eventloop.POLL_IN: self._on_remote_read() if self._stage == STAGE_DESTROYED: return if event & eventloop.POLL_OUT: self._on_remote_write() - if event & eventloop.POLL_ERR: - self._on_remote_error() elif sock == self._local_sock: + if event & eventloop.POLL_ERR: + self._on_local_error() + if self._stage == STAGE_DESTROYED: + return if event & eventloop.POLL_IN: self._on_local_read() if self._stage == STAGE_DESTROYED: return if event & eventloop.POLL_OUT: self._on_local_write() - if event & eventloop.POLL_ERR: - self._on_local_error() else: logging.warn('unknown socket') @@ -525,9 +531,9 @@ class TCPRelay(object): def _handle_events(self, events): for sock, fd, event in events: - # if sock: - # logging.debug('fd %d %s', fd, - # eventloop.EVENT_NAMES.get(event, event)) + if sock: + logging.debug('fd %d %s', fd, + eventloop.EVENT_NAMES.get(event, event)) if sock == self._server_socket: if event & eventloop.POLL_ERR: # TODO From 4b630d08142a0a727ed34221b0f17a56697211a1 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 6 Jun 2014 23:16:19 +0800 Subject: [PATCH 066/344] update CHANGES --- CHANGES | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES b/CHANGES index 5935325..3c46d24 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +2.0.1 2014-06-05 +- Better logging +- Maybe fix bad file descriptor + 2.0 2014-06-05 - Use a new event model - Remove gevent From 795e0f4b8e565a3b63bc05317fb48daceb0b8794 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 7 Jun 2014 00:11:14 +0800 Subject: [PATCH 067/344] update ns --- shadowsocks/asyncdns.py | 53 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index a75be90..6a3f80a 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -45,28 +45,71 @@ _request_count = 1 # | ARCOUNT | # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +QTYPE_ANY = 255 +QTYPE_A = 1 +QTYPE_AAAA = 28 +QCLASS_IN = 1 + + +def convert_address(address): + address = address.strip('.') + labels = address.split('.') + results = [] + for label in labels: + l = len(label) + if l > 63: + return None + results.append(chr(l)) + results.append(label) + results.append('\0') + return ''.join(results) + def pack_request(address): global _request_count - _request_count += 1 header = struct.pack('!HBBHHHH', _request_count, 1, 0, 1, 0, 0, 0) - pass + addr = convert_address(address) + qtype_qclass = struct.pack('!HH', QTYPE_A, QCLASS_IN) + _request_count += 1 + return header + addr + qtype_qclass def unpack_response(data): + if len(data) > 12: + header = struct.unpack('!HBBHHHH', data[:12]) + res_id = header[0] + res_qr = header[1] & 128 + res_tc = header[1] & 2 + res_ra = header[2] & 128 + res_rcode = header[2] & 15 + res_qdcount = header[3] + res_ancount = header[4] + res_nscount = header[5] + res_arcount = header[6] + print header # TODO detect address type - pass + print data + print data.encode('hex') + return data def resolve(address, callback): - callback(address) + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.SOL_UDP) + req = pack_request(address) + if req is None: + # TODO + return + sock.sendto(req, ('8.8.8.8', 53)) + res, addr = sock.recvfrom(1024) + parsed_res = unpack_response(res) + callback(parsed_res) def test(): def _callback(address): print address - resolve('www.baidu.com', _callback) + resolve('www.google.com', _callback) if __name__ == '__main__': From 7a772055301f0c5cebf5bbd5724411881b9b73a9 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 7 Jun 2014 11:50:30 +0800 Subject: [PATCH 068/344] update ns --- shadowsocks/asyncdns.py | 175 +++++++++++++++++++++++++++++++++++----- 1 file changed, 156 insertions(+), 19 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 6a3f80a..b80c5dd 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -29,6 +29,20 @@ import common _request_count = 1 # rfc1035 +# format +# +---------------------+ +# | Header | +# +---------------------+ +# | Question | the question for the name server +# +---------------------+ +# | Answer | RRs answering the question +# +---------------------+ +# | Authority | RRs pointing toward an authority +# +---------------------+ +# | Additional | RRs holding additional information +# +---------------------+ +# +# header # 1 1 1 1 1 1 # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ @@ -48,10 +62,22 @@ _request_count = 1 QTYPE_ANY = 255 QTYPE_A = 1 QTYPE_AAAA = 28 +QTYPE_CNAME = 5 QCLASS_IN = 1 -def convert_address(address): +def parse_ip(addrtype, data, length, offset): + if addrtype == QTYPE_A: + return socket.inet_ntop(socket.AF_INET, data[offset:offset + length]) + elif addrtype == QTYPE_AAAA: + return socket.inet_ntop(socket.AF_INET6, data[offset:offset + length]) + elif addrtype == QTYPE_CNAME: + return parse_name(data, offset, length)[1] + else: + return data + + +def pack_address(address): address = address.strip('.') labels = address.split('.') results = [] @@ -68,32 +94,137 @@ def convert_address(address): def pack_request(address): global _request_count header = struct.pack('!HBBHHHH', _request_count, 1, 0, 1, 0, 0, 0) - addr = convert_address(address) - qtype_qclass = struct.pack('!HH', QTYPE_A, QCLASS_IN) + addr = pack_address(address) + qtype_qclass = struct.pack('!HH', QTYPE_ANY, QCLASS_IN) _request_count += 1 + if _request_count > 65535: + _request_count = 1 return header + addr + qtype_qclass +def parse_name(data, offset, length=512): + p = offset + if (ord(data[offset]) & (128 + 64)) == (128 + 64): + # pointer + pointer = struct.unpack('!H', data[offset:offset + 2])[0] + pointer = pointer & 0x3FFF + if pointer == offset: + return (0, None) + return (2, parse_name(data, pointer)) + else: + labels = [] + l = ord(data[p]) + while l > 0 and p < offset + length: + if (l & (128 + 64)) == (128 + 64): + # pointer + pointer = struct.unpack('!H', data[p:p + 2])[0] + pointer = pointer & 0x3FFF + # if pointer == offset: + # return (0, None) + r = parse_name(data, pointer) + labels.append(r[1]) + p += 2 + # pointer is the end + return (p - offset + 1, '.'.join(labels)) + else: + labels.append(data[p + 1:p + 1 + l]) + p += 1 + l + l = ord(data[p]) + return (p - offset + 1, '.'.join(labels)) + + +# rfc1035 +# record +# 1 1 1 1 1 1 +# 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 +# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +# | | +# / / +# / NAME / +# | | +# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +# | TYPE | +# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +# | CLASS | +# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +# | TTL | +# | | +# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +# | RDLENGTH | +# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--| +# / RDATA / +# / / +# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +def parse_record(data, offset, question=False): + len, name = parse_name(data, offset) + # TODO + assert len + if not question: + record_type, record_class, record_ttl, record_rdlength = struct.unpack( + '!HHiH', data[offset + len:offset + len + 10] + ) + ip = parse_ip(record_type, data, record_rdlength, offset + len + 10) + return len + 10 + record_rdlength, \ + (name, ip, record_type, record_class, record_ttl) + else: + record_type, record_class = struct.unpack( + '!HH', data[offset + len:offset + len + 4] + ) + return len + 4, (name, None, record_type, record_class, None, None) + + def unpack_response(data): - if len(data) > 12: - header = struct.unpack('!HBBHHHH', data[:12]) - res_id = header[0] - res_qr = header[1] & 128 - res_tc = header[1] & 2 - res_ra = header[2] & 128 - res_rcode = header[2] & 15 - res_qdcount = header[3] - res_ancount = header[4] - res_nscount = header[5] - res_arcount = header[6] - print header - # TODO detect address type - print data - print data.encode('hex') - return data + try: + if len(data) >= 12: + header = struct.unpack('!HBBHHHH', data[:12]) + res_id = header[0] + res_qr = header[1] & 128 + res_tc = header[1] & 2 + res_ra = header[2] & 128 + res_rcode = header[2] & 15 + # TODO check tc and rcode + assert res_tc == 0 + assert res_rcode == 0 + res_qdcount = header[3] + res_ancount = header[4] + res_nscount = header[5] + res_arcount = header[6] + qds = [] + ans = [] + nss = [] + ars = [] + offset = 12 + for i in xrange(0, res_qdcount): + l, r = parse_record(data, offset, True) + offset += l + if r: + qds.append(r) + for i in xrange(0, res_ancount): + l, r = parse_record(data, offset) + offset += l + if r: + ans.append(r) + for i in xrange(0, res_nscount): + l, r = parse_record(data, offset) + offset += l + if r: + nss.append(r) + for i in xrange(0, res_arcount): + l, r = parse_record(data, offset) + offset += l + if r: + ars.append(r) + + return ans + + except Exception as e: + import traceback + traceback.print_exc() + return None def resolve(address, callback): + # TODO async sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.SOL_UDP) req = pack_request(address) if req is None: @@ -109,7 +240,13 @@ def test(): def _callback(address): print address + resolve('www.twitter.com', _callback) resolve('www.google.com', _callback) + resolve('ipv6.google.com', _callback) + resolve('ipv6.l.google.com', _callback) + resolve('www.baidu.com', _callback) + resolve('www.a.shifen.com', _callback) + resolve('m.baidu.jp', _callback) if __name__ == '__main__': From 6b763194954681753f079180a1d9c42bbb6929ae Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 8 Jun 2014 13:55:19 +0800 Subject: [PATCH 069/344] more work --- shadowsocks/asyncdns.py | 228 +++++++++++++++++++++++++++++++++------- 1 file changed, 188 insertions(+), 40 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index b80c5dd..5b5d6b2 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -23,7 +23,9 @@ import socket import struct +import logging import common +import eventloop _request_count = 1 @@ -66,18 +68,7 @@ QTYPE_CNAME = 5 QCLASS_IN = 1 -def parse_ip(addrtype, data, length, offset): - if addrtype == QTYPE_A: - return socket.inet_ntop(socket.AF_INET, data[offset:offset + length]) - elif addrtype == QTYPE_AAAA: - return socket.inet_ntop(socket.AF_INET6, data[offset:offset + length]) - elif addrtype == QTYPE_CNAME: - return parse_name(data, offset, length)[1] - else: - return data - - -def pack_address(address): +def build_address(address): address = address.strip('.') labels = address.split('.') results = [] @@ -91,17 +82,28 @@ def pack_address(address): return ''.join(results) -def pack_request(address): +def build_request(address, qtype): global _request_count header = struct.pack('!HBBHHHH', _request_count, 1, 0, 1, 0, 0, 0) - addr = pack_address(address) - qtype_qclass = struct.pack('!HH', QTYPE_ANY, QCLASS_IN) + addr = build_address(address) + qtype_qclass = struct.pack('!HH', qtype, QCLASS_IN) _request_count += 1 if _request_count > 65535: _request_count = 1 return header + addr + qtype_qclass +def parse_ip(addrtype, data, length, offset): + if addrtype == QTYPE_A: + return socket.inet_ntop(socket.AF_INET, data[offset:offset + length]) + elif addrtype == QTYPE_AAAA: + return socket.inet_ntop(socket.AF_INET6, data[offset:offset + length]) + elif addrtype == QTYPE_CNAME: + return parse_name(data, offset, length)[1] + else: + return data + + def parse_name(data, offset, length=512): p = offset if (ord(data[offset]) & (128 + 64)) == (128 + 64): @@ -110,7 +112,7 @@ def parse_name(data, offset, length=512): pointer = pointer & 0x3FFF if pointer == offset: return (0, None) - return (2, parse_name(data, pointer)) + return (2, parse_name(data, pointer)[1]) else: labels = [] l = ord(data[p]) @@ -173,7 +175,7 @@ def parse_record(data, offset, question=False): return len + 4, (name, None, record_type, record_class, None, None) -def unpack_response(data): +def parse_response(data): try: if len(data) >= 12: header = struct.unpack('!HBBHHHH', data[:12]) @@ -214,39 +216,185 @@ def unpack_response(data): offset += l if r: ars.append(r) - - return ans - + response = DNSResponse() + if qds: + response.hostname = qds[0][0] + for an in ans: + response.answers.append((an[1], an[2], an[3])) + return response except Exception as e: import traceback traceback.print_exc() return None -def resolve(address, callback): - # TODO async - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.SOL_UDP) - req = pack_request(address) - if req is None: - # TODO - return - sock.sendto(req, ('8.8.8.8', 53)) - res, addr = sock.recvfrom(1024) - parsed_res = unpack_response(res) - callback(parsed_res) +def is_ip(address): + for family in (socket.AF_INET, socket.AF_INET6): + try: + socket.inet_pton(family, address) + return True + except (OSError, IOError): + pass + return False + + +class DNSResponse(object): + def __init__(self): + self.hostname = None + self.answers = [] # each: (addr, type, class) + + def __str__(self): + return '%s: %s' % (self.hostname, str(self.answers)) + + +STATUS_IPV4 = 0 +STATUS_IPV6 = 1 + + +class DNSResolver(object): + + def __init__(self): + self._loop = None + self._hostname_status = {} + self._hostname_to_cb = {} + self._cb_to_hostname = {} + # TODO add caching + # TODO try ipv4 and ipv6 sequencely + self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, + socket.SOL_UDP) + self._sock.setblocking(False) + self._parse_config() + + def _parse_config(self): + try: + with open('/etc/resolv.conf', 'rb') as f: + servers = [] + content = f.readlines() + for line in content: + line = line.strip() + if line: + if line.startswith('nameserver'): + parts = line.split(' ') + if len(parts) >= 2: + server = parts[1] + if is_ip(server): + servers.append(server) + # TODO support more servers + if servers: + self._dns_server = (servers[0], 53) + return + except IOError: + pass + self._dns_server = ('8.8.8.8', 53) + + def add_to_loop(self, loop): + self._loop = loop + loop.add(self._sock, eventloop.POLL_IN) + loop.add_handler(self.handle_events) + + def _handle_data(self, data): + response = parse_response(data) + if response and response.hostname: + hostname = response.hostname + callbacks = self._hostname_to_cb.get(hostname, []) + ip = None + for answer in response.answers: + if answer[1] in (QTYPE_A, QTYPE_AAAA) and \ + answer[2] == QCLASS_IN: + ip = answer[0] + break + if not ip and self._hostname_status.get(hostname, STATUS_IPV6) \ + == STATUS_IPV4: + self._hostname_status[hostname] = STATUS_IPV6 + self._send_req(hostname, QTYPE_AAAA) + return + for callback in callbacks: + if self._cb_to_hostname.__contains__(callback): + del self._cb_to_hostname[callback] + callback((hostname, ip), None) + if self._hostname_to_cb.__contains__(hostname): + del self._hostname_to_cb[hostname] + + def handle_events(self, events): + for sock, fd, event in events: + if sock != self._sock: + continue + if event & eventloop.POLL_ERR: + logging.error('dns socket err') + self._loop.remove(self._sock) + self._sock.close() + self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, + socket.SOL_UDP) + self._sock.setblocking(False) + self._loop.add(self._sock, eventloop.POLL_IN) + else: + data, addr = sock.recvfrom(1024) + if addr != self._dns_server: + logging.warn('received a packet other than our dns') + break + self._handle_data(data) + break + + def remove_callback(self, callback): + hostname = self._cb_to_hostname.get(callback) + if hostname: + del self._cb_to_hostname[callback] + arr = self._hostname_to_cb.get(hostname, None) + if arr: + arr.remove(callback) + if not arr: + del self._hostname_to_cb[hostname] + + def _send_req(self, hostname, qtype): + logging.debug('resolving %s with type %d using server %s', hostname, + qtype, self._dns_server) + req = build_request(hostname, qtype) + self._sock.sendto(req, self._dns_server) + + def resolve(self, hostname, callback): + if not hostname: + callback(None, Exception('empty hostname')) + elif is_ip(hostname): + callback(hostname, None) + else: + arr = self._hostname_to_cb.get(hostname, None) + if not arr: + self._hostname_status[hostname] = STATUS_IPV4 + self._send_req(hostname, QTYPE_A) + self._hostname_to_cb[hostname] = [callback] + self._cb_to_hostname[callback] = hostname + else: + arr.append(callback) def test(): - def _callback(address): - print address + logging.getLogger('').handlers = [] + logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s %(levelname)-8s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', filemode='a+') - resolve('www.twitter.com', _callback) - resolve('www.google.com', _callback) - resolve('ipv6.google.com', _callback) - resolve('ipv6.l.google.com', _callback) - resolve('www.baidu.com', _callback) - resolve('www.a.shifen.com', _callback) - resolve('m.baidu.jp', _callback) + def _callback(address, error): + print error, address + + loop = eventloop.EventLoop() + resolver = DNSResolver() + resolver.add_to_loop(loop) + + resolver.resolve('8.8.8.8', _callback) + resolver.resolve('www.twitter.com', _callback) + resolver.resolve('www.google.com', _callback) + resolver.resolve('ipv6.google.com', _callback) + resolver.resolve('ipv6.l.google.com', _callback) + resolver.resolve('www.gmail.com', _callback) + resolver.resolve('r4---sn-3qqp-ioql.googlevideo.com', _callback) + resolver.resolve('www.baidu.com', _callback) + resolver.resolve('www.a.shifen.com', _callback) + resolver.resolve('m.baidu.jp', _callback) + resolver.resolve('www.youku.com', _callback) + resolver.resolve('www.twitter.com', _callback) + resolver.resolve('ipv6.google.com', _callback) + + loop.run() if __name__ == '__main__': From 45f9998fa9f62d7cae1abb9ad80d34f7c427b839 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 8 Jun 2014 14:17:26 +0800 Subject: [PATCH 070/344] more work --- shadowsocks/asyncdns.py | 78 ++++++++++++++++------------------------- shadowsocks/common.py | 12 ++++--- 2 files changed, 39 insertions(+), 51 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 5b5d6b2..426cbf5 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -28,6 +28,8 @@ import common import eventloop +common.patch_socket() + _request_count = 1 # rfc1035 @@ -99,40 +101,30 @@ def parse_ip(addrtype, data, length, offset): elif addrtype == QTYPE_AAAA: return socket.inet_ntop(socket.AF_INET6, data[offset:offset + length]) elif addrtype == QTYPE_CNAME: - return parse_name(data, offset, length)[1] + return parse_name(data, offset)[1] else: return data -def parse_name(data, offset, length=512): +def parse_name(data, offset): p = offset - if (ord(data[offset]) & (128 + 64)) == (128 + 64): - # pointer - pointer = struct.unpack('!H', data[offset:offset + 2])[0] - pointer = pointer & 0x3FFF - if pointer == offset: - return (0, None) - return (2, parse_name(data, pointer)[1]) - else: - labels = [] + labels = [] + l = ord(data[p]) + while l > 0: + if (l & (128 + 64)) == (128 + 64): + # pointer + pointer = struct.unpack('!H', data[p:p + 2])[0] + pointer &= 0x3FFF + r = parse_name(data, pointer) + labels.append(r[1]) + p += 2 + # pointer is the end + return p - offset, '.'.join(labels) + else: + labels.append(data[p + 1:p + 1 + l]) + p += 1 + l l = ord(data[p]) - while l > 0 and p < offset + length: - if (l & (128 + 64)) == (128 + 64): - # pointer - pointer = struct.unpack('!H', data[p:p + 2])[0] - pointer = pointer & 0x3FFF - # if pointer == offset: - # return (0, None) - r = parse_name(data, pointer) - labels.append(r[1]) - p += 2 - # pointer is the end - return (p - offset + 1, '.'.join(labels)) - else: - labels.append(data[p + 1:p + 1 + l]) - p += 1 + l - l = ord(data[p]) - return (p - offset + 1, '.'.join(labels)) + return p - offset + 1, '.'.join(labels) # rfc1035 @@ -158,33 +150,30 @@ def parse_name(data, offset, length=512): # / / # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ def parse_record(data, offset, question=False): - len, name = parse_name(data, offset) - # TODO - assert len + nlen, name = parse_name(data, offset) if not question: record_type, record_class, record_ttl, record_rdlength = struct.unpack( - '!HHiH', data[offset + len:offset + len + 10] + '!HHiH', data[offset + nlen:offset + nlen + 10] ) - ip = parse_ip(record_type, data, record_rdlength, offset + len + 10) - return len + 10 + record_rdlength, \ + ip = parse_ip(record_type, data, record_rdlength, offset + nlen + 10) + return nlen + 10 + record_rdlength, \ (name, ip, record_type, record_class, record_ttl) else: record_type, record_class = struct.unpack( - '!HH', data[offset + len:offset + len + 4] + '!HH', data[offset + nlen:offset + nlen + 4] ) - return len + 4, (name, None, record_type, record_class, None, None) + return nlen + 4, (name, None, record_type, record_class, None, None) def parse_response(data): try: if len(data) >= 12: header = struct.unpack('!HBBHHHH', data[:12]) - res_id = header[0] - res_qr = header[1] & 128 + # res_id = header[0] + # res_qr = header[1] & 128 res_tc = header[1] & 2 - res_ra = header[2] & 128 + # res_ra = header[2] & 128 res_rcode = header[2] & 15 - # TODO check tc and rcode assert res_tc == 0 assert res_rcode == 0 res_qdcount = header[3] @@ -193,8 +182,6 @@ def parse_response(data): res_arcount = header[6] qds = [] ans = [] - nss = [] - ars = [] offset = 12 for i in xrange(0, res_qdcount): l, r = parse_record(data, offset, True) @@ -209,13 +196,9 @@ def parse_response(data): for i in xrange(0, res_nscount): l, r = parse_record(data, offset) offset += l - if r: - nss.append(r) for i in xrange(0, res_arcount): l, r = parse_record(data, offset) offset += l - if r: - ars.append(r) response = DNSResponse() if qds: response.hostname = qds[0][0] @@ -225,6 +208,7 @@ def parse_response(data): except Exception as e: import traceback traceback.print_exc() + logging.error(e) return None @@ -380,9 +364,9 @@ def test(): resolver = DNSResolver() resolver.add_to_loop(loop) + resolver.resolve('www.google.com', _callback) resolver.resolve('8.8.8.8', _callback) resolver.resolve('www.twitter.com', _callback) - resolver.resolve('www.google.com', _callback) resolver.resolve('ipv6.google.com', _callback) resolver.resolve('ipv6.l.google.com', _callback) resolver.resolve('www.gmail.com', _callback) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 4f7aac6..6104478 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -63,11 +63,15 @@ def inet_pton(family, addr): raise RuntimeError("What family?") -if not hasattr(socket, 'inet_pton'): - socket.inet_pton = inet_pton +def patch_socket(): + if not hasattr(socket, 'inet_pton'): + socket.inet_pton = inet_pton -if not hasattr(socket, 'inet_ntop'): - socket.inet_ntop = inet_ntop + if not hasattr(socket, 'inet_ntop'): + socket.inet_ntop = inet_ntop + + +patch_socket() ADDRTYPE_IPV4 = 1 From 9c0bab297e9b36c2950ab5393473819e39b18781 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 8 Jun 2014 14:38:09 +0800 Subject: [PATCH 071/344] more work --- shadowsocks/asyncdns.py | 4 ++++ shadowsocks/lru_cache.py | 48 ++++++++++++++++++++-------------------- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 426cbf5..259511b 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -298,6 +298,8 @@ class DNSResolver(object): callback((hostname, ip), None) if self._hostname_to_cb.__contains__(hostname): del self._hostname_to_cb[hostname] + if self._hostname_status.__contains__(hostname): + del self._hostname_status[hostname] def handle_events(self, events): for sock, fd, event in events: @@ -328,6 +330,8 @@ class DNSResolver(object): arr.remove(callback) if not arr: del self._hostname_to_cb[hostname] + if self._hostname_status.__contains__(hostname): + del self._hostname_status[hostname] def _send_req(self, hostname, qtype): logging.debug('resolving %s with type %d using server %s', hostname, diff --git a/shadowsocks/lru_cache.py b/shadowsocks/lru_cache.py index ce40d17..a26680c 100644 --- a/shadowsocks/lru_cache.py +++ b/shadowsocks/lru_cache.py @@ -13,52 +13,52 @@ class LRUCache(collections.MutableMapping): def __init__(self, timeout=60, close_callback=None, *args, **kwargs): self.timeout = timeout self.close_callback = close_callback - self.store = {} - self.time_to_keys = collections.defaultdict(list) - self.last_visits = [] + self._store = {} + self._time_to_keys = collections.defaultdict(list) + self._last_visits = [] self.update(dict(*args, **kwargs)) # use the free update to set keys def __getitem__(self, key): - "O(logm)" + # O(logm) t = time.time() - self.time_to_keys[t].append(key) - heapq.heappush(self.last_visits, t) - return self.store[key] + self._time_to_keys[t].append(key) + heapq.heappush(self._last_visits, t) + return self._store[key] def __setitem__(self, key, value): - "O(logm)" + # O(logm) t = time.time() - self.store[key] = value - self.time_to_keys[t].append(key) - heapq.heappush(self.last_visits, t) + self._store[key] = value + self._time_to_keys[t].append(key) + heapq.heappush(self._last_visits, t) def __delitem__(self, key): - "O(1)" - del self.store[key] + # O(1) + del self._store[key] def __iter__(self): - return iter(self.store) + return iter(self._store) def __len__(self): - return len(self.store) + return len(self._store) def sweep(self): - "O(m)" + # O(m) now = time.time() c = 0 - while len(self.last_visits) > 0: - least = self.last_visits[0] + while len(self._last_visits) > 0: + least = self._last_visits[0] if now - least <= self.timeout: break - for key in self.time_to_keys[least]: - heapq.heappop(self.last_visits) - if self.store.__contains__(key): - value = self.store[key] + for key in self._time_to_keys[least]: + heapq.heappop(self._last_visits) + if self._store.__contains__(key): + value = self._store[key] if self.close_callback is not None: self.close_callback(value) - del self.store[key] + del self._store[key] c += 1 - del self.time_to_keys[least] + del self._time_to_keys[least] if c: logging.debug('%d keys swept' % c) From fbc4906445860d9ee84adec8947303634ba06442 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 8 Jun 2014 15:41:39 +0800 Subject: [PATCH 072/344] add cache --- shadowsocks/asyncdns.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 259511b..7c52b16 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -25,6 +25,7 @@ import socket import struct import logging import common +import lru_cache import eventloop @@ -242,8 +243,7 @@ class DNSResolver(object): self._hostname_status = {} self._hostname_to_cb = {} self._cb_to_hostname = {} - # TODO add caching - # TODO try ipv4 and ipv6 sequencely + self._cache = lru_cache.LRUCache(timeout=300) self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.SOL_UDP) self._sock.setblocking(False) @@ -276,11 +276,21 @@ class DNSResolver(object): loop.add(self._sock, eventloop.POLL_IN) loop.add_handler(self.handle_events) + def _call_callback(self, hostname, ip): + callbacks = self._hostname_to_cb.get(hostname, []) + for callback in callbacks: + if self._cb_to_hostname.__contains__(callback): + del self._cb_to_hostname[callback] + callback((hostname, ip), None) + if self._hostname_to_cb.__contains__(hostname): + del self._hostname_to_cb[hostname] + if self._hostname_status.__contains__(hostname): + del self._hostname_status[hostname] + def _handle_data(self, data): response = parse_response(data) if response and response.hostname: hostname = response.hostname - callbacks = self._hostname_to_cb.get(hostname, []) ip = None for answer in response.answers: if answer[1] in (QTYPE_A, QTYPE_AAAA) and \ @@ -291,15 +301,9 @@ class DNSResolver(object): == STATUS_IPV4: self._hostname_status[hostname] = STATUS_IPV6 self._send_req(hostname, QTYPE_AAAA) - return - for callback in callbacks: - if self._cb_to_hostname.__contains__(callback): - del self._cb_to_hostname[callback] - callback((hostname, ip), None) - if self._hostname_to_cb.__contains__(hostname): - del self._hostname_to_cb[hostname] - if self._hostname_status.__contains__(hostname): - del self._hostname_status[hostname] + else: + self._cache[hostname] = ip + self._call_callback(hostname, ip) def handle_events(self, events): for sock, fd, event in events: @@ -344,6 +348,10 @@ class DNSResolver(object): callback(None, Exception('empty hostname')) elif is_ip(hostname): callback(hostname, None) + elif self._cache.__contains__(hostname): + logging.debug('hit cache: %s', hostname) + ip = self._cache[hostname] + callback(ip, None) else: arr = self._hostname_to_cb.get(hostname, None) if not arr: From 71c3759c126ebbc7cfc1ea749e08913f4c116474 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 8 Jun 2014 15:58:59 +0800 Subject: [PATCH 073/344] add async dns --- shadowsocks/asyncdns.py | 12 +++++++++--- shadowsocks/local.py | 9 +++++++-- shadowsocks/server.py | 8 ++++++-- shadowsocks/tcprelay.py | 5 ++++- shadowsocks/udprelay.py | 5 ++++- 5 files changed, 30 insertions(+), 9 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 7c52b16..2272ea2 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -244,9 +244,7 @@ class DNSResolver(object): self._hostname_to_cb = {} self._cb_to_hostname = {} self._cache = lru_cache.LRUCache(timeout=300) - self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, - socket.SOL_UDP) - self._sock.setblocking(False) + self._sock = None self._parse_config() def _parse_config(self): @@ -272,7 +270,12 @@ class DNSResolver(object): self._dns_server = ('8.8.8.8', 53) def add_to_loop(self, loop): + if self._loop: + raise Exception('already add to loop') self._loop = loop + self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, + socket.SOL_UDP) + self._sock.setblocking(False) loop.add(self._sock, eventloop.POLL_IN) loop.add_handler(self.handle_events) @@ -362,6 +365,9 @@ class DNSResolver(object): else: arr.append(callback) + def close(self): + self._sock.close() + def test(): logging.getLogger('').handlers = [] diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 0b4b17d..0a00136 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -29,6 +29,7 @@ import encrypt import eventloop import tcprelay import udprelay +import asyncdns def main(): @@ -50,14 +51,18 @@ def main(): logging.info("starting local at %s:%d" % (config['local_address'], config['local_port'])) - tcp_server = tcprelay.TCPRelay(config, True) - udp_server = udprelay.UDPRelay(config, True) + dns_resolver = asyncdns.DNSResolver() + tcp_server = tcprelay.TCPRelay(config, dns_resolver, True) + udp_server = udprelay.UDPRelay(config, dns_resolver, True) loop = eventloop.EventLoop() + dns_resolver.add_to_loop(loop) tcp_server.add_to_loop(loop) udp_server.add_to_loop(loop) loop.run() except (KeyboardInterrupt, IOError, OSError) as e: logging.error(e) + import traceback + traceback.print_exc() os._exit(0) if __name__ == '__main__': diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 13c2452..9145ce4 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -29,6 +29,7 @@ import encrypt import eventloop import tcprelay import udprelay +import asyncdns def main(): @@ -50,18 +51,20 @@ def main(): encrypt.init_table(config['password'], config['method']) tcp_servers = [] udp_servers = [] + dns_resolver = asyncdns.DNSResolver() for port, password in config['port_password'].items(): a_config = config.copy() a_config['server_port'] = int(port) a_config['password'] = password logging.info("starting server at %s:%d" % (a_config['server'], int(port))) - tcp_servers.append(tcprelay.TCPRelay(a_config, False)) - udp_servers.append(udprelay.UDPRelay(a_config, False)) + tcp_servers.append(tcprelay.TCPRelay(a_config, dns_resolver, False)) + udp_servers.append(udprelay.UDPRelay(a_config, dns_resolver, False)) def run_server(): try: loop = eventloop.EventLoop() + dns_resolver.add_to_loop(loop) for tcp_server in tcp_servers: tcp_server.add_to_loop(loop) for udp_server in udp_servers: @@ -100,6 +103,7 @@ def main(): a_tcp_server.close() for a_udp_server in udp_servers: a_udp_server.close() + dns_resolver.close() for child in children: os.waitpid(child, 0) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index c93b6fc..381a809 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -426,9 +426,10 @@ class TCPRelayHandler(object): class TCPRelay(object): - def __init__(self, config, is_local): + def __init__(self, config, dns_resolver, is_local): self._config = config self._is_local = is_local + self._dns_resolver = dns_resolver self._closed = False self._eventloop = None self._fd_to_handlers = {} @@ -466,6 +467,8 @@ class TCPRelay(object): self._server_socket = server_socket def add_to_loop(self, loop): + if self._eventloop: + raise Exception('already add to loop') if self._closed: raise Exception('already closed') self._eventloop = loop diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 2aeb069..5218feb 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -85,7 +85,7 @@ def client_key(a, b, c, d): class UDPRelay(object): - def __init__(self, config, is_local): + def __init__(self, config, dns_resolver, is_local): if is_local: self._listen_addr = config['local_address'] self._listen_port = config['local_port'] @@ -96,6 +96,7 @@ class UDPRelay(object): self._listen_port = config['server_port'] self._remote_addr = None self._remote_port = None + self._dns_resolver = dns_resolver self._password = config['password'] self._method = config['method'] self._timeout = config['timeout'] @@ -220,6 +221,8 @@ class UDPRelay(object): pass def add_to_loop(self, loop): + if self._eventloop: + raise Exception('already add to loop') if self._closed: raise Exception('already closed') self._eventloop = loop From de5461a17e5fbcfc8cb9533d81f8f7925a88784b Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 8 Jun 2014 16:08:00 +0800 Subject: [PATCH 074/344] refine test --- shadowsocks/asyncdns.py | 51 +++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 2272ea2..45676ad 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -21,6 +21,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import time import socket import struct import logging @@ -29,9 +30,9 @@ import lru_cache import eventloop -common.patch_socket() +CACHE_SWEEP_INTERVAL = 30 -_request_count = 1 +common.patch_socket() # rfc1035 # format @@ -85,14 +86,10 @@ def build_address(address): return ''.join(results) -def build_request(address, qtype): - global _request_count - header = struct.pack('!HBBHHHH', _request_count, 1, 0, 1, 0, 0, 0) +def build_request(address, qtype, request_id): + header = struct.pack('!HBBHHHH', request_id, 1, 0, 1, 0, 0, 0) addr = build_address(address) qtype_qclass = struct.pack('!HH', qtype, QCLASS_IN) - _request_count += 1 - if _request_count > 65535: - _request_count = 1 return header + addr + qtype_qclass @@ -240,10 +237,12 @@ class DNSResolver(object): def __init__(self): self._loop = None + self._request_id = 1 self._hostname_status = {} self._hostname_to_cb = {} self._cb_to_hostname = {} self._cache = lru_cache.LRUCache(timeout=300) + self._last_time = time.time() self._sock = None self._parse_config() @@ -327,6 +326,10 @@ class DNSResolver(object): break self._handle_data(data) break + now = time.time() + if now - self._last_time > CACHE_SWEEP_INTERVAL: + self._cache.sweep() + self._last_time = now def remove_callback(self, callback): hostname = self._cb_to_hostname.get(callback) @@ -343,7 +346,10 @@ class DNSResolver(object): def _send_req(self, hostname, qtype): logging.debug('resolving %s with type %d using server %s', hostname, qtype, self._dns_server) - req = build_request(hostname, qtype) + self._request_id += 1 + if self._request_id > 32768: + self._request_id = 1 + req = build_request(hostname, qtype, self._request_id) self._sock.sendto(req, self._dns_server) def resolve(self, hostname, callback): @@ -382,19 +388,20 @@ def test(): resolver = DNSResolver() resolver.add_to_loop(loop) - resolver.resolve('www.google.com', _callback) - resolver.resolve('8.8.8.8', _callback) - resolver.resolve('www.twitter.com', _callback) - resolver.resolve('ipv6.google.com', _callback) - resolver.resolve('ipv6.l.google.com', _callback) - resolver.resolve('www.gmail.com', _callback) - resolver.resolve('r4---sn-3qqp-ioql.googlevideo.com', _callback) - resolver.resolve('www.baidu.com', _callback) - resolver.resolve('www.a.shifen.com', _callback) - resolver.resolve('m.baidu.jp', _callback) - resolver.resolve('www.youku.com', _callback) - resolver.resolve('www.twitter.com', _callback) - resolver.resolve('ipv6.google.com', _callback) + for hostname in ['www.google.com', + '8.8.8.8', + 'www.twitter.com', + 'ipv6.google.com', + 'ipv6.l.google.com', + 'www.gmail.com', + 'r4---sn-3qqp-ioql.googlevideo.com', + 'www.baidu.com', + 'www.a.shifen.com', + 'm.baidu.jp', + 'www.youku.com', + 'www.twitter.com', + 'ipv6.google.com']: + resolver.resolve(hostname, _callback) loop.run() From cdd333ea973c47cb744f00200f6f0b50b9728947 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 8 Jun 2014 16:35:39 +0800 Subject: [PATCH 075/344] update dns --- shadowsocks/asyncdns.py | 42 +++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 45676ad..02311d2 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -22,6 +22,7 @@ # SOFTWARE. import time +import os import socket import struct import logging @@ -173,7 +174,7 @@ def parse_response(data): # res_ra = header[2] & 128 res_rcode = header[2] & 15 assert res_tc == 0 - assert res_rcode == 0 + assert res_rcode in [0, 3] res_qdcount = header[3] res_ancount = header[4] res_nscount = header[5] @@ -238,15 +239,19 @@ class DNSResolver(object): def __init__(self): self._loop = None self._request_id = 1 + self._hosts = {} self._hostname_status = {} self._hostname_to_cb = {} self._cb_to_hostname = {} self._cache = lru_cache.LRUCache(timeout=300) self._last_time = time.time() self._sock = None - self._parse_config() + self._parse_resolv() + self._parse_hosts() + # TODO monitor hosts change and reload hosts + # TODO parse /etc/gai.conf and follow its rules - def _parse_config(self): + def _parse_resolv(self): try: with open('/etc/resolv.conf', 'rb') as f: servers = [] @@ -255,7 +260,7 @@ class DNSResolver(object): line = line.strip() if line: if line.startswith('nameserver'): - parts = line.split(' ') + parts = line.split() if len(parts) >= 2: server = parts[1] if is_ip(server): @@ -268,6 +273,25 @@ class DNSResolver(object): pass self._dns_server = ('8.8.8.8', 53) + def _parse_hosts(self): + etc_path = '/etc/hosts' + if os.environ.__contains__('WINDIR'): + etc_path = os.environ['WINDIR'] + '/system32/drivers/etc/hosts' + try: + with open(etc_path, 'rb') as f: + for line in f.readlines(): + line = line.strip() + parts = line.split() + if len(parts) >= 2: + ip = parts[0] + if is_ip(ip): + for i in xrange(1, len(parts)): + hostname = parts[i] + if hostname: + self._hosts[hostname] = ip + except IOError: + self._hosts['localhost'] = '127.0.0.1' + def add_to_loop(self, loop): if self._loop: raise Exception('already add to loop') @@ -356,11 +380,15 @@ class DNSResolver(object): if not hostname: callback(None, Exception('empty hostname')) elif is_ip(hostname): - callback(hostname, None) + callback((hostname, hostname), None) + elif self._hosts.__contains__(hostname): + logging.debug('hit hosts: %s', hostname) + ip = self._hosts[hostname] + callback((hostname, ip), None) elif self._cache.__contains__(hostname): logging.debug('hit cache: %s', hostname) ip = self._cache[hostname] - callback(ip, None) + callback((hostname, ip), None) else: arr = self._hostname_to_cb.get(hostname, None) if not arr: @@ -390,6 +418,8 @@ def test(): for hostname in ['www.google.com', '8.8.8.8', + 'localhost', + 'activate.adobe.com', 'www.twitter.com', 'ipv6.google.com', 'ipv6.l.google.com', From 5c274a1bc79911f347bfe75e31ed1194f31ac8e0 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 8 Jun 2014 16:38:30 +0800 Subject: [PATCH 076/344] update dns --- shadowsocks/asyncdns.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 02311d2..5f9f91b 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -296,6 +296,7 @@ class DNSResolver(object): if self._loop: raise Exception('already add to loop') self._loop = loop + # TODO when dns server is IPv6 self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.SOL_UDP) self._sock.setblocking(False) @@ -339,6 +340,7 @@ class DNSResolver(object): logging.error('dns socket err') self._loop.remove(self._sock) self._sock.close() + # TODO when dns server is IPv6 self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.SOL_UDP) self._sock.setblocking(False) From bcdc1e96719efdfd9f6084b1f48da233e2188d3d Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 8 Jun 2014 17:12:00 +0800 Subject: [PATCH 077/344] add async dns to tcp relay --- shadowsocks/asyncdns.py | 13 +++-- shadowsocks/tcprelay.py | 102 ++++++++++++++++++++++++++-------------- 2 files changed, 77 insertions(+), 38 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 5f9f91b..20d6ff0 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -173,8 +173,8 @@ def parse_response(data): res_tc = header[1] & 2 # res_ra = header[2] & 128 res_rcode = header[2] & 15 - assert res_tc == 0 - assert res_rcode in [0, 3] + # assert res_tc == 0 + # assert res_rcode in [0, 3] res_qdcount = header[3] res_ancount = header[4] res_nscount = header[5] @@ -308,7 +308,11 @@ class DNSResolver(object): for callback in callbacks: if self._cb_to_hostname.__contains__(callback): del self._cb_to_hostname[callback] - callback((hostname, ip), None) + if ip: + callback((hostname, ip), None) + else: + callback((hostname, None), + Exception('unknown hostname %s' % hostname)) if self._hostname_to_cb.__contains__(hostname): del self._hostname_to_cb[hostname] if self._hostname_status.__contains__(hostname): @@ -329,7 +333,8 @@ class DNSResolver(object): self._hostname_status[hostname] = STATUS_IPV6 self._send_req(hostname, QTYPE_AAAA) else: - self._cache[hostname] = ip + if ip: + self._cache[hostname] = ip self._call_callback(hostname, ip) def handle_events(self, events): diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 381a809..08fb2f5 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -56,6 +56,7 @@ CMD_UDP_ASSOCIATE = 3 STAGE_INIT = 0 STAGE_HELLO = 1 STAGE_UDP_ASSOC = 2 +STAGE_DNS = 3 STAGE_REPLY = 4 STAGE_STREAM = 5 STAGE_DESTROYED = -1 @@ -75,13 +76,14 @@ BUF_SIZE = 8 * 1024 class TCPRelayHandler(object): def __init__(self, server, fd_to_handlers, loop, local_sock, config, - is_local): + dns_resolver, is_local): self._server = server self._fd_to_handlers = fd_to_handlers self._loop = loop self._local_sock = local_sock self._remote_sock = None self._config = config + self._dns_resolver = dns_resolver self._is_local = is_local self._stage = STAGE_INIT self._encryptor = encrypt.Encryptor(config['password'], @@ -239,53 +241,83 @@ class TCPRelayHandler(object): addrtype, remote_addr, remote_port, header_length = header_result logging.info('connecting %s:%d' % (remote_addr, remote_port)) self._remote_address = (remote_addr, remote_port) + # pause reading + self._update_stream(STREAM_UP, WAIT_STATUS_WRITING) + self._stage = STAGE_DNS if self._is_local: # forward address to remote self._write_to_sock('\x05\x00\x00\x01\x00\x00\x00\x00\x10\x10', self._local_sock) data_to_send = self._encryptor.encrypt(data) self._data_to_write_to_remote.append(data_to_send) - remote_addr = self._config['server'] - remote_port = self._config['server_port'] + # notice here may go into _handle_dns_resolved directly + self._dns_resolver.resolve(self._config['server'], + self._handle_dns_resolved) else: if len(data) > header_length: self._data_to_write_to_remote.append(data[header_length:]) - - # TODO async DNS - addrs = socket.getaddrinfo(remote_addr, remote_port, 0, - socket.SOCK_STREAM, socket.SOL_TCP) - if len(addrs) == 0: - raise Exception("can't get addrinfo for %s:%d" % - (remote_addr, remote_port)) - af, socktype, proto, canonname, sa = addrs[0] - remote_sock = socket.socket(af, socktype, proto) - self._remote_sock = remote_sock - self._fd_to_handlers[remote_sock.fileno()] = self - remote_sock.setblocking(False) - remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) - - if self._is_local and self._config['fast_open']: - # wait for more data to arrive and send them in one SYN - self._stage = STAGE_REPLY - self._loop.add(remote_sock, eventloop.POLL_ERR) - # TODO when there is already data in this packet - else: - try: - remote_sock.connect(sa) - except (OSError, IOError) as e: - if eventloop.errno_from_exception(e) == errno.EINPROGRESS: - pass - self._loop.add(remote_sock, - eventloop.POLL_ERR | eventloop.POLL_OUT) - self._stage = STAGE_REPLY - self._update_stream(STREAM_UP, WAIT_STATUS_READWRITING) - self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) + # notice here may go into _handle_dns_resolved directly + self._dns_resolver.resolve(remote_addr, + self._handle_dns_resolved) except Exception as e: logging.error(e) traceback.print_exc() # TODO use logging when debug completed self.destroy() + def _handle_dns_resolved(self, result, error): + if error: + logging.error(error) + self.destroy() + return + if result: + ip = result[1] + if ip: + try: + self._stage = STAGE_REPLY + remote_addr = self._remote_address[0] + remote_port = self._remote_address[1] + if self._is_local: + remote_addr = self._config['server'] + remote_port = self._config['server_port'] + addrs = socket.getaddrinfo(ip, remote_port, 0, + socket.SOCK_STREAM, + socket.SOL_TCP) + if len(addrs) == 0: + raise Exception("getaddrinfo failed for %s:%d" % + (remote_addr, remote_port)) + af, socktype, proto, canonname, sa = addrs[0] + remote_sock = socket.socket(af, socktype, proto) + self._remote_sock = remote_sock + self._fd_to_handlers[remote_sock.fileno()] = self + remote_sock.setblocking(False) + remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, + 1) + + if self._is_local and self._config['fast_open']: + # wait for more data to arrive and send them in one SYN + self._stage = STAGE_REPLY + self._loop.add(remote_sock, eventloop.POLL_ERR) + self._update_stream(STREAM_UP, WAIT_STATUS_READING) + # TODO when there is already data in this packet + else: + try: + remote_sock.connect(sa) + except (OSError, IOError) as e: + if eventloop.errno_from_exception(e) == \ + errno.EINPROGRESS: + pass + self._loop.add(remote_sock, + eventloop.POLL_ERR | eventloop.POLL_OUT) + self._stage = STAGE_REPLY + self._update_stream(STREAM_UP, WAIT_STATUS_READWRITING) + self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) + return + except (OSError, IOError) as e: + logging.error(e) + traceback.print_exc() + self.destroy() + def _on_local_read(self): self._update_activity() if not self._local_sock: @@ -422,6 +454,7 @@ class TCPRelayHandler(object): del self._fd_to_handlers[self._local_sock.fileno()] self._local_sock.close() self._local_sock = None + self._dns_resolver.remove_callback(self._handle_dns_resolved) self._server.remove_handler(self) @@ -545,7 +578,8 @@ class TCPRelay(object): # logging.debug('accept') conn = self._server_socket.accept() TCPRelayHandler(self, self._fd_to_handlers, self._eventloop, - conn[0], self._config, self._is_local) + conn[0], self._config, self._dns_resolver, + self._is_local) except (OSError, IOError) as e: error_no = eventloop.errno_from_exception(e) if error_no in (errno.EAGAIN, errno.EINPROGRESS): From 24eadfbb5fc9c40ec697c1be004ea01b148041d0 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 8 Jun 2014 17:30:50 +0800 Subject: [PATCH 078/344] send again even if in queue --- shadowsocks/asyncdns.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 20d6ff0..9382122 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -405,6 +405,8 @@ class DNSResolver(object): self._cb_to_hostname[callback] = hostname else: arr.append(callback) + # TODO send again only if waited too long + self._send_req(hostname, QTYPE_A) def close(self): self._sock.close() From 26fa1ab5557fd35b8f813ca4f5a837d730ac8ecd Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 8 Jun 2014 18:28:02 +0800 Subject: [PATCH 079/344] refine comments --- shadowsocks/tcprelay.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 08fb2f5..b378107 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -45,11 +45,13 @@ CMD_UDP_ASSOCIATE = 3 # stage 0 init # stage 1 hello received, hello sent # stage 2 UDP assoc +# stage 3 DNS # stage 4 addr received, reply sent # stage 5 remote connected # remote: # stage 0 init +# stage 3 DNS # stage 4 addr received, reply sent # stage 5 remote connected From 6e93bdd5a5d55ccb41976c12a6af1460c2e26982 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 8 Jun 2014 10:42:40 +0800 Subject: [PATCH 080/344] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1a2f7da..b5f8428 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Install #### Debian / Ubuntu: - apt-get install build-essential python-pip python-m2crypto python-dev + apt-get install python-pip python-m2crypto pip install shadowsocks #### CentOS: From 4faf5cf8aeb112ccfcf6eee539772edc4d66ddd1 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 11 Jun 2014 14:20:13 +0800 Subject: [PATCH 081/344] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b5f8428..e63b870 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Install #### OS X: + brew install swig git clone https://github.com/clowwindy/M2Crypto.git cd M2Crypto pip install . From 2306add1dfebc65c69bd26140742dd42df08bebe Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 11 Jun 2014 14:42:51 +0800 Subject: [PATCH 082/344] split parse_header() --- shadowsocks/asyncdns.py | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 9382122..58da4e1 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -164,21 +164,34 @@ def parse_record(data, offset, question=False): return nlen + 4, (name, None, record_type, record_class, None, None) +def parse_header(data): + if len(data) >= 12: + header = struct.unpack('!HBBHHHH', data[:12]) + res_id = header[0] + res_qr = header[1] & 128 + res_tc = header[1] & 2 + res_ra = header[2] & 128 + res_rcode = header[2] & 15 + # assert res_tc == 0 + # assert res_rcode in [0, 3] + res_qdcount = header[3] + res_ancount = header[4] + res_nscount = header[5] + res_arcount = header[6] + return (res_id, res_qr, res_tc, res_ra, res_rcode, res_qdcount, + res_ancount, res_nscount, res_arcount) + return None + + def parse_response(data): try: if len(data) >= 12: - header = struct.unpack('!HBBHHHH', data[:12]) - # res_id = header[0] - # res_qr = header[1] & 128 - res_tc = header[1] & 2 - # res_ra = header[2] & 128 - res_rcode = header[2] & 15 - # assert res_tc == 0 - # assert res_rcode in [0, 3] - res_qdcount = header[3] - res_ancount = header[4] - res_nscount = header[5] - res_arcount = header[6] + header = parse_header(data) + if not header: + return None + res_id, res_qr, res_tc, res_ra, res_rcode, res_qdcount, \ + res_ancount, res_nscount, res_arcount = header + qds = [] ans = [] offset = 12 From cb2e2ce0df724b5e03b8409ee791f2c5c7ec9de8 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 11 Jun 2014 15:01:31 +0800 Subject: [PATCH 083/344] bump --- CHANGES | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 3c46d24..dd4976b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +2.0.2 2014-06-11 +- Add asynchronous DNS in TCP relay + 2.0.1 2014-06-05 - Better logging - Maybe fix bad file descriptor diff --git a/setup.py b/setup.py index efdb366..c55a4e6 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.0.1", + version="2.0.2", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From 9422ef2fd3f965bfb2655cdc916410fa6e5fe70d Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 11 Jun 2014 19:21:17 +0800 Subject: [PATCH 084/344] Fix table encryption with UDP --- CHANGES | 3 +++ shadowsocks/encrypt.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index dd4976b..f569c16 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +2.0.3 2014-06-11 +- Fix table encryption with UDP + 2.0.2 2014-06-11 - Add asynchronous DNS in TCP relay diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 9d94dd1..fc6b5ce 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -201,9 +201,9 @@ def encrypt_all(password, method, op, data): if not method: [encrypt_table, decrypt_table] = init_table(password) if op: - return string.translate(encrypt_table, data) + return string.translate(data, encrypt_table) else: - return string.translate(decrypt_table, data) + return string.translate(data, decrypt_table) else: import M2Crypto.EVP result = [] From a7592b8625d34f0c6b2b6333c82b2385517e0f2f Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 11 Jun 2014 19:28:04 +0800 Subject: [PATCH 085/344] bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c55a4e6..576aa8a 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.0.2", + version="2.0.3", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From 4fb6b9df987f14485dc5ada10adb8ca2def82357 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 12 Jun 2014 00:06:40 +0800 Subject: [PATCH 086/344] add CONTRIBUTING.md --- CONTRIBUTING.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..cca3953 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,34 @@ +How to contribute +================= + +Before you submit issues, please take a few minutes to read this guide. + +在你提交问题前，请花两分钟阅读一下这份指南。 + +Issues +------ + +Please include the following information in your submission: + +1. How did you set up your environment? (OS, version of Shadowsocks) +2. Where did you see this error, was it on local or on server? +3. What happened in your browser? Just no response, or any error message? +4. 10 lines of log on the local side of shadowsocks when the error happened. +5. 10 lines of log on the server side of shadowsocks when the error happened. +6. Any other useful information. + +Skip any of them if you don't know its meaning. + +问题反馈 +------- + +请提交下面的信息： + +1. 你是如何搭建环境的？（操作系统，Shadowsocks 版本） +2. 错误是发生在哪里，本地还是服务器？ +3. 浏览器里的现象是什么？一直转菊花，还是有提示错误？ +4. 发生错误时，本地端的十行完整的日志。 +5. 发生错误时，服务器端的十行完整的日志。 +6. 其它你认为可能和问题有关的信息。 + +如果你不清楚其中某条的含义， 可以直接跳过那一条。 From df836bed7767da996cc602c2d2ba2bd5c106a78f Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 12 Jun 2014 16:21:08 +0800 Subject: [PATCH 087/344] fix DNS record parsing --- shadowsocks/asyncdns.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 58da4e1..cbdd6a5 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -70,6 +70,7 @@ QTYPE_ANY = 255 QTYPE_A = 1 QTYPE_AAAA = 28 QTYPE_CNAME = 5 +QTYPE_NS = 2 QCLASS_IN = 1 @@ -99,10 +100,10 @@ def parse_ip(addrtype, data, length, offset): return socket.inet_ntop(socket.AF_INET, data[offset:offset + length]) elif addrtype == QTYPE_AAAA: return socket.inet_ntop(socket.AF_INET6, data[offset:offset + length]) - elif addrtype == QTYPE_CNAME: + elif addrtype in [QTYPE_CNAME, QTYPE_NS]: return parse_name(data, offset)[1] else: - return data + return data[offset:offset + length] def parse_name(data, offset): From a109f571e37996612186f9195161dec98987738b Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 12 Jun 2014 17:11:05 +0800 Subject: [PATCH 088/344] fix worker master --- CHANGES | 3 +++ shadowsocks/asyncdns.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index f569c16..de63c86 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +2.0.4 2014-06-12 +- Fix worker master + 2.0.3 2014-06-11 - Fix table encryption with UDP diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index cbdd6a5..f7934cb 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -423,7 +423,9 @@ class DNSResolver(object): self._send_req(hostname, QTYPE_A) def close(self): - self._sock.close() + if self._sock: + self._sock.close() + self._sock = None def test(): From 06a87c385b17439cc7dab5bb0ff01a7be0bc55b0 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 12 Jun 2014 17:15:40 +0800 Subject: [PATCH 089/344] bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 576aa8a..bbe30df 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.0.3", + version="2.0.4", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From f5caa5b3d268a251d0395e6bbcd3fe7bb4a85632 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 13 Jun 2014 13:03:23 +0800 Subject: [PATCH 090/344] update README.rst --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 77edba2..b77e1ba 100644 --- a/README.rst +++ b/README.rst @@ -15,7 +15,7 @@ Debian / Ubuntu: :: - apt-get install build-essential python-pip python-m2crypto python-dev + apt-get install python-pip python-m2crypto pip install shadowsocks CentOS: @@ -32,6 +32,7 @@ OS X: :: + brew install swig git clone https://github.com/clowwindy/M2Crypto.git cd M2Crypto pip install . From ed5888387bb7fda10d6cd94c4da52eeefe255bf6 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 14 Jun 2014 11:11:07 +0800 Subject: [PATCH 091/344] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index e63b870..7fad7e6 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,6 @@ Mailing list: http://groups.google.com/group/shadowsocks Also see [Troubleshooting] -[1.4.x]: https://github.com/clowwindy/shadowsocks/tree/1.4 [Build Status]: https://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat [Chinese Readme]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E [GUI client]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients From a68b9a052057913e15384b05c50fd8c085aa7f40 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 14 Jun 2014 11:17:13 +0800 Subject: [PATCH 092/344] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7fad7e6..2224665 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Change the proxy settings in your browser to hostname: 127.0.0.1 port: your local_port -It's recommended to use shadowsocks with AutoProxy or Proxy SwitchySharp. +It's recommended to use shadowsocks with [SwitchySharp]. Command line args ------------------ @@ -115,3 +115,4 @@ Also see [Troubleshooting] [TCP_FASTOPEN]: https://github.com/clowwindy/shadowsocks/wiki/TCP-Fast-Open [Travis CI]: https://travis-ci.org/clowwindy/shadowsocks [Troubleshooting]: https://github.com/clowwindy/shadowsocks/wiki/Troubleshooting +[SwitchySharp]: https://chrome.google.com/webstore/detail/proxy-switchysharp/dpplabbmogkhghncfbfdeeokoefdjegm From 71b4d34ce4d0f76ab0076ce7dd7478b7fef70fa5 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 16 Jun 2014 00:52:29 +0800 Subject: [PATCH 093/344] Update README.md --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2224665..6b0b706 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,11 @@ A fast tunnel proxy that help you get through firewalls. Install ------- +You'll have a client on your local machine, and install a server on a +remote server. + +For client, find any [GUI client]. For server: + #### Debian / Ubuntu: apt-get install python-pip python-m2crypto @@ -31,7 +36,8 @@ Install #### Windows: -Choose a [GUI client] +Since installing M2Crypto on Windows is a bit frustrating, use +[node.js version] instead. Usage ----- @@ -109,6 +115,7 @@ Also see [Troubleshooting] [Chinese Readme]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E [GUI client]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients [Issue Tracker]: https://github.com/clowwindy/shadowsocks/issues?state=open +[node.js version]: https://github.com/clowwindy/shadowsocks-nodejs [PyPI]: https://pypi.python.org/pypi/shadowsocks [PyPI version]: https://img.shields.io/pypi/v/shadowsocks.svg?style=flat [Supervisor]: https://github.com/clowwindy/shadowsocks/wiki/Configure-Shadowsocks-with-Supervisor From c1a6dc4c9e96b1526895fd5625ba5525b2b8d84b Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 16 Jun 2014 01:13:54 +0800 Subject: [PATCH 094/344] Update README.md --- README.md | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 6b0b706..5666d1e 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,13 @@ Install You'll have a client on your local machine, and install a server on a remote server. -For client, find any [GUI client]. For server: +### Client + +* [Windows] / [OS X] +* [Android] / [iOS] +* [OpenWRT] + +### Server #### Debian / Ubuntu: @@ -26,23 +32,10 @@ For client, find any [GUI client]. For server: easy_install pip pip install shadowsocks -#### OS X: +Configuration +------------- - brew install swig - git clone https://github.com/clowwindy/M2Crypto.git - cd M2Crypto - pip install . - pip install shadowsocks - -#### Windows: - -Since installing M2Crypto on Windows is a bit frustrating, use -[node.js version] instead. - -Usage ------ - -Create a config file `/etc/shadowsocks.json` (or put it in other path). +On your server create a config file `/etc/shadowsocks.json` (or put it in other path). Example: { @@ -74,7 +67,8 @@ Explanation of the fields: Run `ssserver -c /etc/shadowsocks.json` on your server. To run it in the background, use [Supervisor]. -On your client machine, run `sslocal -c /etc/shadowsocks.json`. +On your client machine, use the same configuration as your server, and +start your client. Change the proxy settings in your browser to @@ -111,11 +105,14 @@ Mailing list: http://groups.google.com/group/shadowsocks Also see [Troubleshooting] +[Android]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#android [Build Status]: https://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat [Chinese Readme]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E [GUI client]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients +[iOS]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#ios [Issue Tracker]: https://github.com/clowwindy/shadowsocks/issues?state=open -[node.js version]: https://github.com/clowwindy/shadowsocks-nodejs +[OpenWRT]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#openwrt +[OS X]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#os-x [PyPI]: https://pypi.python.org/pypi/shadowsocks [PyPI version]: https://img.shields.io/pypi/v/shadowsocks.svg?style=flat [Supervisor]: https://github.com/clowwindy/shadowsocks/wiki/Configure-Shadowsocks-with-Supervisor @@ -123,3 +120,4 @@ Also see [Troubleshooting] [Travis CI]: https://travis-ci.org/clowwindy/shadowsocks [Troubleshooting]: https://github.com/clowwindy/shadowsocks/wiki/Troubleshooting [SwitchySharp]: https://chrome.google.com/webstore/detail/proxy-switchysharp/dpplabbmogkhghncfbfdeeokoefdjegm +[Windows]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#windows From 3c0024cef0d638f461b1ed9341a6b7042d34ad24 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 16 Jun 2014 01:26:57 +0800 Subject: [PATCH 095/344] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5666d1e..715a52e 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,8 @@ You can use args to override settings from `config.json`. ssserver -p server_port -k password -m bf-cfb --workers 2 ssserver -c /etc/shadowsocks/config.json +List all available args with `-h`. + Wiki ---- From eeda2c258d693660730c703f9b1956b1958cea44 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 16 Jun 2014 02:14:49 +0800 Subject: [PATCH 096/344] Update README.md --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 715a52e..0d5e4f5 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ remote server. Configuration ------------- -On your server create a config file `/etc/shadowsocks.json` (or put it in other path). +On your server create a config file `/etc/shadowsocks.json`. Example: { @@ -70,13 +70,17 @@ background, use [Supervisor]. On your client machine, use the same configuration as your server, and start your client. -Change the proxy settings in your browser to +If you use Chrome, it's recommended to use [SwitchySharp]. Change the proxy +settings to protocol: socks5 hostname: 127.0.0.1 port: your local_port -It's recommended to use shadowsocks with [SwitchySharp]. +If you can't install [SwitchySharp], you can launch Chrome with the following +arguments to force Chrome to use the proxy: + + Chrome.exe --proxy-server="socks5://127.0.0.1:1080" --host-resolver-rules="MAP * 0.0.0.0 , EXCLUDE localhost" Command line args ------------------ From 1c847a9a58d9fed0a6d1f18fcf281d937fc666f9 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 16 Jun 2014 02:29:06 +0800 Subject: [PATCH 097/344] Update README.md --- README.md | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 0d5e4f5..102b2b7 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,9 @@ arguments to force Chrome to use the proxy: Chrome.exe --proxy-server="socks5://127.0.0.1:1080" --host-resolver-rules="MAP * 0.0.0.0 , EXCLUDE localhost" +If you can't even download Chrome, find a friend to download a +[Chrome Standalone] installer for you. + Command line args ------------------ @@ -111,19 +114,20 @@ Mailing list: http://groups.google.com/group/shadowsocks Also see [Troubleshooting] -[Android]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#android -[Build Status]: https://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat -[Chinese Readme]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E -[GUI client]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients -[iOS]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#ios -[Issue Tracker]: https://github.com/clowwindy/shadowsocks/issues?state=open -[OpenWRT]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#openwrt -[OS X]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#os-x -[PyPI]: https://pypi.python.org/pypi/shadowsocks -[PyPI version]: https://img.shields.io/pypi/v/shadowsocks.svg?style=flat -[Supervisor]: https://github.com/clowwindy/shadowsocks/wiki/Configure-Shadowsocks-with-Supervisor -[TCP_FASTOPEN]: https://github.com/clowwindy/shadowsocks/wiki/TCP-Fast-Open -[Travis CI]: https://travis-ci.org/clowwindy/shadowsocks -[Troubleshooting]: https://github.com/clowwindy/shadowsocks/wiki/Troubleshooting -[SwitchySharp]: https://chrome.google.com/webstore/detail/proxy-switchysharp/dpplabbmogkhghncfbfdeeokoefdjegm -[Windows]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#windows +[Android]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#android +[Build Status]: https://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat +[Chinese Readme]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E +[Chrome Standalone]: https://support.google.com/installer/answer/126299 +[GUI client]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients +[iOS]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#ios +[Issue Tracker]: https://github.com/clowwindy/shadowsocks/issues?state=open +[OpenWRT]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#openwrt +[OS X]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#os-x +[PyPI]: https://pypi.python.org/pypi/shadowsocks +[PyPI version]: https://img.shields.io/pypi/v/shadowsocks.svg?style=flat +[Supervisor]: https://github.com/clowwindy/shadowsocks/wiki/Configure-Shadowsocks-with-Supervisor +[TCP_FASTOPEN]: https://github.com/clowwindy/shadowsocks/wiki/TCP-Fast-Open +[Travis CI]: https://travis-ci.org/clowwindy/shadowsocks +[Troubleshooting]: https://github.com/clowwindy/shadowsocks/wiki/Troubleshooting +[SwitchySharp]: https://chrome.google.com/webstore/detail/proxy-switchysharp/dpplabbmogkhghncfbfdeeokoefdjegm +[Windows]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#windows From 08e351a55db2562187b8be6c0c5ab512a47130b6 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 18 Jun 2014 12:54:16 +0800 Subject: [PATCH 098/344] add more logs --- shadowsocks/tcprelay.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index b378107..a5fe879 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -397,19 +397,20 @@ class TCPRelayHandler(object): self._update_stream(STREAM_UP, WAIT_STATUS_READING) def _on_local_error(self): + logging.debug('got local error') if self._local_sock: - logging.debug('got local error') logging.error(eventloop.get_sock_error(self._local_sock)) self.destroy() def _on_remote_error(self): + logging.debug('got remote error') if self._remote_sock: - logging.debug('got remote error') logging.error(eventloop.get_sock_error(self._remote_sock)) self.destroy() def handle_event(self, sock, event): if self._stage == STAGE_DESTROYED: + logging.debug('ignore handle_event: destroyed') return # order is important if sock == self._remote_sock: @@ -439,6 +440,7 @@ class TCPRelayHandler(object): def destroy(self): if self._stage == STAGE_DESTROYED: + logging.debug('already destroyed') return self._stage = STAGE_DESTROYED if self._remote_address: @@ -447,11 +449,13 @@ class TCPRelayHandler(object): else: logging.debug('destroy') if self._remote_sock: + logging.debug('destroying remote') self._loop.remove(self._remote_sock) del self._fd_to_handlers[self._remote_sock.fileno()] self._remote_sock.close() self._remote_sock = None if self._local_sock: + logging.debug('destroying local') self._loop.remove(self._local_sock) del self._fd_to_handlers[self._local_sock.fileno()] self._local_sock.close() @@ -539,6 +543,7 @@ class TCPRelay(object): # we just need a sorted last_activity queue and it's faster than heapq # in fact we can do O(1) insertion/remove so we invent our own if self._timeouts: + logging.debug('sweeping timeouts') now = time.time() length = len(self._timeouts) pos = self._timeout_offset @@ -569,15 +574,15 @@ class TCPRelay(object): def _handle_events(self, events): for sock, fd, event in events: - if sock: - logging.debug('fd %d %s', fd, - eventloop.EVENT_NAMES.get(event, event)) + # if sock: + # logging.debug('fd %d %s', fd, + # eventloop.EVENT_NAMES.get(event, event)) if sock == self._server_socket: if event & eventloop.POLL_ERR: # TODO raise Exception('server_socket error') try: - # logging.debug('accept') + logging.debug('accept') conn = self._server_socket.accept() TCPRelayHandler(self, self._fd_to_handlers, self._eventloop, conn[0], self._config, self._dns_resolver, From a0e1a9f1b02cb4b2c3cfc69d48c6e009769d4d13 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 18 Jun 2014 12:59:00 +0800 Subject: [PATCH 099/344] close #122 --- shadowsocks/tcprelay.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index a5fe879..c188a9d 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -157,7 +157,8 @@ class TCPRelayHandler(object): uncomplete = True else: logging.error(e) - traceback.print_exc() + if self._config['verbose']: + traceback.print_exc() self.destroy() return False if uncomplete: @@ -208,7 +209,8 @@ class TCPRelayHandler(object): self.destroy() else: logging.error(e) - traceback.print_exc() + if self._config['verbose']: + traceback.print_exc() self.destroy() def _handle_stage_hello(self, data): @@ -263,7 +265,8 @@ class TCPRelayHandler(object): self._handle_dns_resolved) except Exception as e: logging.error(e) - traceback.print_exc() + if self._config['verbose']: + traceback.print_exc() # TODO use logging when debug completed self.destroy() @@ -317,7 +320,8 @@ class TCPRelayHandler(object): return except (OSError, IOError) as e: logging.error(e) - traceback.print_exc() + if self._config['verbose']: + traceback.print_exc() self.destroy() def _on_local_read(self): @@ -375,7 +379,8 @@ class TCPRelayHandler(object): self._write_to_sock(data, self._local_sock) except Exception as e: logging.error(e) - traceback.print_exc() + if self._config['verbose']: + traceback.print_exc() # TODO use logging when debug completed self.destroy() @@ -593,7 +598,8 @@ class TCPRelay(object): continue else: logging.error(e) - traceback.print_exc() + if self._config['verbose']: + traceback.print_exc() else: if sock: handler = self._fd_to_handlers.get(fd, None) From 743d3cddb59798954c5dc5ee2097db0cb910fde1 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 18 Jun 2014 15:50:05 +0800 Subject: [PATCH 100/344] close #132 --- .travis.yml | 1 + CHANGES | 3 ++ shadowsocks/server.py | 7 +++- shadowsocks/tcprelay.py | 20 ++++++--- test.py | 92 ++++------------------------------------- 5 files changed, 33 insertions(+), 90 deletions(-) diff --git a/.travis.yml b/.travis.yml index f9f7d7a..dabbb80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ script: - python test.py -c tests/table.json - python test.py -c tests/aes.json - python test.py -c tests/salsa20.json + - python test.py -c tests/server-multi-ports.json - python test.py -c tests/server-multi-passwd.json - python test.py -c tests/server-multi-passwd-table.json - python test.py -c tests/workers.json diff --git a/CHANGES b/CHANGES index de63c86..bcae1a6 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +2.0.5 2014-06-18 +- Support a simple config format for multiple ports + 2.0.4 2014-06-12 - Fix worker master diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 9145ce4..d5648bc 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -46,7 +46,12 @@ def main(): 'will be ignored') else: config['port_password'] = {} - config['port_password'][str(config['server_port'])] = config['password'] + server_port = config['server_port'] + if type(server_port) == list: + for a_server_port in server_port: + config['port_password'][a_server_port] = config['password'] + else: + config['port_password'][str(server_port)] = config['password'] encrypt.init_table(config['password'], config['method']) tcp_servers = [] diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index c188a9d..81b9d92 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -27,6 +27,7 @@ import errno import struct import logging import traceback +import random import encrypt import eventloop from common import parse_header @@ -96,6 +97,8 @@ class TCPRelayHandler(object): self._upstream_status = WAIT_STATUS_READING self._downstream_status = WAIT_STATUS_INIT self._remote_address = None + if is_local: + self._chosen_server = self._get_a_server() fd_to_handlers[local_sock.fileno()] = self local_sock.setblocking(False) local_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) @@ -112,6 +115,15 @@ class TCPRelayHandler(object): def remote_address(self): return self._remote_address + def _get_a_server(self): + server = self._config['server'] + server_port = self._config['server_port'] + if type(server_port) == list: + server_port = random.choice(server_port) + logging.debug('chosen server: %s:%d', server, server_port) + # TODO support multiple server IP + return server, server_port + def _update_activity(self): self._server.update_activity(self) @@ -190,8 +202,7 @@ class TCPRelayHandler(object): data = ''.join(self._data_to_write_to_local) l = len(data) s = self._remote_sock.sendto(data, MSG_FASTOPEN, - (self._config['server'], - self._config['server_port'])) + self._chosen_server) if s < l: data = data[s:] self._data_to_write_to_local = [data] @@ -255,7 +266,7 @@ class TCPRelayHandler(object): data_to_send = self._encryptor.encrypt(data) self._data_to_write_to_remote.append(data_to_send) # notice here may go into _handle_dns_resolved directly - self._dns_resolver.resolve(self._config['server'], + self._dns_resolver.resolve(self._chosen_server[0], self._handle_dns_resolved) else: if len(data) > header_length: @@ -283,8 +294,7 @@ class TCPRelayHandler(object): remote_addr = self._remote_address[0] remote_port = self._remote_address[1] if self._is_local: - remote_addr = self._config['server'] - remote_port = self._config['server_port'] + remote_addr, remote_port = self._chosen_server addrs = socket.getaddrinfo(ip, remote_port, 0, socket.SOCK_STREAM, socket.SOL_TCP) diff --git a/test.py b/test.py index d1a5a0c..154f352 100755 --- a/test.py +++ b/test.py @@ -2,99 +2,23 @@ # -*- coding: utf-8 -*- import sys -sys.path.insert(0, 'shadowsocks') import os import signal import select -import struct -import hashlib -import string import time from subprocess import Popen, PIPE -import encrypt_salsa20 - -target1 = [ - [60, 53, 84, 138, 217, 94, 88, 23, 39, 242, 219, 35, 12, 157, 165, 181, 255, 143, 83, 247, 162, 16, 31, 209, 190, - 171, 115, 65, 38, 41, 21, 245, 236, 46, 121, 62, 166, 233, 44, 154, 153, 145, 230, 49, 128, 216, 173, 29, 241, 119, - 64, 229, 194, 103, 131, 110, 26, 197, 218, 59, 204, 56, 27, 34, 141, 221, 149, 239, 192, 195, 24, 155, 170, 183, 11 - , 254, 213, 37, 137, 226, 75, 203, 55, 19, 72, 248, 22, 129, 33, 175, 178, 10, 198, 71, 77, 36, 113, 167, 48, 2, - 117, 140, 142, 66, 199, 232, 243, 32, 123, 54, 51, 82, 57, 177, 87, 251, 150, 196, 133, 5, 253, 130, 8, 184, 14, - 152, 231, 3, 186, 159, 76, 89, 228, 205, 156, 96, 163, 146, 18, 91, 132, 85, 80, 109, 172, 176, 105, 13, 50, 235, - 127, 0, 189, 95, 98, 136, 250, 200, 108, 179, 211, 214, 106, 168, 78, 79, 74, 210, 30, 73, 201, 151, 208, 114, 101, - 174, 92, 52, 120, 240, 15, 169, 220, 182, 81, 224, 43, 185, 40, 99, 180, 17, 212, 158, 42, 90, 9, 191, 45, 6, 25, 4 - , 222, 67, 126, 1, 116, 124, 206, 69, 61, 7, 68, 97, 202, 63, 244, 20, 28, 58, 93, 134, 104, 144, 227, 147, 102, - 118, 135, 148, 47, 238, 86, 112, 122, 70, 107, 215, 100, 139, 223, 225, 164, 237, 111, 125, 207, 160, 187, 246, 234 - , 161, 188, 193, 249, 252], - [151, 205, 99, 127, 201, 119, 199, 211, 122, 196, 91, 74, 12, 147, 124, 180, 21, 191, 138, 83, 217, 30, 86, 7, 70, - 200, 56, 62, 218, 47, 168, 22, 107, 88, 63, 11, 95, 77, 28, 8, 188, 29, 194, 186, 38, 198, 33, 230, 98, 43, 148, - 110, 177, 1, 109, 82, 61, 112, 219, 59, 0, 210, 35, 215, 50, 27, 103, 203, 212, 209, 235, 93, 84, 169, 166, 80, 130 - , 94, 164, 165, 142, 184, 111, 18, 2, 141, 232, 114, 6, 131, 195, 139, 176, 220, 5, 153, 135, 213, 154, 189, 238 - , 174, 226, 53, 222, 146, 162, 236, 158, 143, 55, 244, 233, 96, 173, 26, 206, 100, 227, 49, 178, 34, 234, 108, - 207, 245, 204, 150, 44, 87, 121, 54, 140, 118, 221, 228, 155, 78, 3, 239, 101, 64, 102, 17, 223, 41, 137, 225, 229, - 66, 116, 171, 125, 40, 39, 71, 134, 13, 193, 129, 247, 251, 20, 136, 242, 14, 36, 97, 163, 181, 72, 25, 144, 46, - 175, 89, 145, 113, 90, 159, 190, 15, 183, 73, 123, 187, 128, 248, 252, 152, 24, 197, 68, 253, 52, 69, 117, 57, 92, - 104, 157, 170, 214, 81, 60, 133, 208, 246, 172, 23, 167, 160, 192, 76, 161, 237, 45, 4, 58, 10, 182, 65, 202, 240, - 185, 241, 79, 224, 132, 51, 42, 126, 105, 37, 250, 149, 32, 243, 231, 67, 179, 48, 9, 106, 216, 31, 249, 19, 85, - 254, 156, 115, 255, 120, 75, 16]] - -target2 = [ - [124, 30, 170, 247, 27, 127, 224, 59, 13, 22, 196, 76, 72, 154, 32, 209, 4, 2, 131, 62, 101, 51, 230, 9, 166, 11, 99 - , 80, 208, 112, 36, 248, 81, 102, 130, 88, 218, 38, 168, 15, 241, 228, 167, 117, 158, 41, 10, 180, 194, 50, 204, - 243, 246, 251, 29, 198, 219, 210, 195, 21, 54, 91, 203, 221, 70, 57, 183, 17, 147, 49, 133, 65, 77, 55, 202, 122, - 162, 169, 188, 200, 190, 125, 63, 244, 96, 31, 107, 106, 74, 143, 116, 148, 78, 46, 1, 137, 150, 110, 181, 56, 95, - 139, 58, 3, 231, 66, 165, 142, 242, 43, 192, 157, 89, 175, 109, 220, 128, 0, 178, 42, 255, 20, 214, 185, 83, 160, - 253, 7, 23, 92, 111, 153, 26, 226, 33, 176, 144, 18, 216, 212, 28, 151, 71, 206, 222, 182, 8, 174, 205, 201, 152, - 240, 155, 108, 223, 104, 239, 98, 164, 211, 184, 34, 193, 14, 114, 187, 40, 254, 12, 67, 93, 217, 6, 94, 16, 19, 82 - , 86, 245, 24, 197, 134, 132, 138, 229, 121, 5, 235, 238, 85, 47, 103, 113, 179, 69, 250, 45, 135, 156, 25, 61, - 75, 44, 146, 189, 84, 207, 172, 119, 53, 123, 186, 120, 171, 68, 227, 145, 136, 100, 90, 48, 79, 159, 149, 39, 213, - 236, 126, 52, 60, 225, 199, 105, 73, 233, 252, 118, 215, 35, 115, 64, 37, 97, 129, 161, 177, 87, 237, 141, 173, 191 - , 163, 140, 234, 232, 249], - [117, 94, 17, 103, 16, 186, 172, 127, 146, 23, 46, 25, 168, 8, 163, 39, 174, 67, 137, 175, 121, 59, 9, 128, 179, 199 - , 132, 4, 140, 54, 1, 85, 14, 134, 161, 238, 30, 241, 37, 224, 166, 45, 119, 109, 202, 196, 93, 190, 220, 69, 49 - , 21, 228, 209, 60, 73, 99, 65, 102, 7, 229, 200, 19, 82, 240, 71, 105, 169, 214, 194, 64, 142, 12, 233, 88, 201 - , 11, 72, 92, 221, 27, 32, 176, 124, 205, 189, 177, 246, 35, 112, 219, 61, 129, 170, 173, 100, 84, 242, 157, 26, - 218, 20, 33, 191, 155, 232, 87, 86, 153, 114, 97, 130, 29, 192, 164, 239, 90, 43, 236, 208, 212, 185, 75, 210, 0, - 81, 227, 5, 116, 243, 34, 18, 182, 70, 181, 197, 217, 95, 183, 101, 252, 248, 107, 89, 136, 216, 203, 68, 91, 223, - 96, 141, 150, 131, 13, 152, 198, 111, 44, 222, 125, 244, 76, 251, 158, 106, 24, 42, 38, 77, 2, 213, 207, 249, 147, - 113, 135, 245, 118, 193, 47, 98, 145, 66, 160, 123, 211, 165, 78, 204, 80, 250, 110, 162, 48, 58, 10, 180, 55, 231, - 79, 149, 74, 62, 50, 148, 143, 206, 28, 15, 57, 159, 139, 225, 122, 237, 138, 171, 36, 56, 115, 63, 144, 154, 6, - 230, 133, 215, 41, 184, 22, 104, 254, 234, 253, 187, 226, 247, 188, 156, 151, 40, 108, 51, 83, 178, 52, 3, 31, 255, - 195, 53, 235, 126, 167, 120]] - - -def get_table(key): - m = hashlib.md5() - m.update(key) - s = m.digest() - (a, b) = struct.unpack(' Date: Wed, 18 Jun 2014 15:50:17 +0800 Subject: [PATCH 101/344] close #132 --- tests/server-multi-ports.json | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 tests/server-multi-ports.json diff --git a/tests/server-multi-ports.json b/tests/server-multi-ports.json new file mode 100644 index 0000000..5bdbcab --- /dev/null +++ b/tests/server-multi-ports.json @@ -0,0 +1,8 @@ +{ + "server": "127.0.0.1", + "server_port": [8384, 8345, 8346, 8347], + "local_port": 1081, + "password": "foobar4", + "timeout": 60, + "method": "aes-256-cfb" +} From e811bdb89107df2d9ccb08cab0b6afe779692128 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 18 Jun 2014 15:59:30 +0800 Subject: [PATCH 102/344] bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bbe30df..b379d22 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.0.4", + version="2.0.5", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From 46d23bf0d4bfa1692d1400a68d147b425ccdcd6c Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 19 Jun 2014 09:20:04 +0800 Subject: [PATCH 103/344] more log --- shadowsocks/eventloop.py | 3 ++- shadowsocks/tcprelay.py | 6 +++--- shadowsocks/udprelay.py | 12 +++++++++++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index e8af800..0b628e0 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -199,9 +199,10 @@ class EventLoop(object): except (OSError, IOError) as e: if errno_from_exception(e) == errno.EPIPE: # Happens when the client closes the connection + logging.error('poll:%s', e) continue else: - logging.error(e) + logging.error('poll:%s', e) import traceback traceback.print_exc() continue diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 81b9d92..285c166 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -589,9 +589,9 @@ class TCPRelay(object): def _handle_events(self, events): for sock, fd, event in events: - # if sock: - # logging.debug('fd %d %s', fd, - # eventloop.EVENT_NAMES.get(event, event)) + if sock: + logging.debug('fd %d %s', fd, + eventloop.EVENT_NAMES.get(event, event)) if sock == self._server_socket: if event & eventloop.POLL_ERR: # TODO diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 5218feb..c7faeee 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -133,6 +133,8 @@ class UDPRelay(object): def _handle_server(self): server = self._server_socket data, r_addr = server.recvfrom(BUF_SIZE) + if not data: + logging.debug('UDP handle_server: data is empty') if self._is_local: frag = ord(data[2]) if frag != 0: @@ -141,9 +143,10 @@ class UDPRelay(object): else: data = data[3:] else: - # decrypt data data = encrypt.encrypt_all(self._password, self._method, 0, data) + # decrypt data if not data: + logging.debug('UDP handle_server: data is empty after decrypt') return header_result = parse_header(data) if header_result is None: @@ -191,6 +194,9 @@ class UDPRelay(object): def _handle_client(self, sock): data, r_addr = sock.recvfrom(BUF_SIZE) + if not data: + logging.debug('UDP handle_client: data is empty') + return if not self._is_local: addrlen = len(r_addr[0]) if addrlen > 255: @@ -235,8 +241,12 @@ class UDPRelay(object): def _handle_events(self, events): for sock, fd, event in events: if sock == self._server_socket: + if event & eventloop.POLL_ERR: + logging.error('UDP server_socket err') self._handle_server() elif sock and (fd in self._sockets): + if event & eventloop.POLL_ERR: + logging.error('UDP client_socket err') self._handle_client(sock) now = time.time() if now - self._last_time > 3.5: From 35bfa56eaf6d66a13fea23f4fc523680e5ab5976 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 19 Jun 2014 09:41:44 +0800 Subject: [PATCH 104/344] fix POLL_HUP --- shadowsocks/tcprelay.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 285c166..a789cd2 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -433,7 +433,7 @@ class TCPRelayHandler(object): self._on_remote_error() if self._stage == STAGE_DESTROYED: return - if event & eventloop.POLL_IN: + if event & (eventloop.POLL_IN | eventloop.POLL_HUP): self._on_remote_read() if self._stage == STAGE_DESTROYED: return @@ -444,7 +444,7 @@ class TCPRelayHandler(object): self._on_local_error() if self._stage == STAGE_DESTROYED: return - if event & eventloop.POLL_IN: + if event & (eventloop.POLL_IN | eventloop.POLL_HUP): self._on_local_read() if self._stage == STAGE_DESTROYED: return From d949d11b281d81c0c47111537a0be649acfa471f Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 19 Jun 2014 10:01:55 +0800 Subject: [PATCH 105/344] larger buffer --- CHANGES | 4 ++++ setup.py | 2 +- shadowsocks/tcprelay.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index bcae1a6..85b1666 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +2.0.6 2014-06-19 +- Fix CPU 100% on POLL_HUP +- More friendly logging + 2.0.5 2014-06-18 - Support a simple config format for multiple ports diff --git a/setup.py b/setup.py index b379d22..fa7e13e 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.0.5", + version="2.0.6", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index a789cd2..10f73d6 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -74,7 +74,7 @@ WAIT_STATUS_READING = 1 WAIT_STATUS_WRITING = 2 WAIT_STATUS_READWRITING = WAIT_STATUS_READING | WAIT_STATUS_WRITING -BUF_SIZE = 8 * 1024 +BUF_SIZE = 32 * 1024 class TCPRelayHandler(object): From d639a375bc2a2dff29105a9c934b3ddf4559f939 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 19 Jun 2014 10:11:48 +0800 Subject: [PATCH 106/344] remove fd logging --- shadowsocks/tcprelay.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 10f73d6..ddc79a6 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -589,9 +589,9 @@ class TCPRelay(object): def _handle_events(self, events): for sock, fd, event in events: - if sock: - logging.debug('fd %d %s', fd, - eventloop.EVENT_NAMES.get(event, event)) + # if sock: + # logging.debug('fd %d %s', fd, + # eventloop.EVENT_NAMES.get(event, event)) if sock == self._server_socket: if event & eventloop.POLL_ERR: # TODO From 6c6afde2a559423704d03c55234ddececc75247b Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 20 Jun 2014 20:35:33 +0800 Subject: [PATCH 107/344] auto fallback when fast open not available --- shadowsocks/tcprelay.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index ddc79a6..77ff8cd 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -512,12 +512,13 @@ class TCPRelay(object): server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server_socket.bind(sa) server_socket.setblocking(False) - server_socket.listen(1024) if config['fast_open']: try: server_socket.setsockopt(socket.SOL_TCP, 23, 5) except socket.error: logging.error('warning: fast open is not available') + self._config['fast_open'] = False + server_socket.listen(1024) self._server_socket = server_socket def add_to_loop(self, loop): From 41010d810efc40b9a6a89742842ed688c3e70fb0 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 20 Jun 2014 21:06:15 +0800 Subject: [PATCH 108/344] support -vv verbose logging --- shadowsocks/tcprelay.py | 9 +++++---- shadowsocks/utils.py | 16 +++++++++++++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 77ff8cd..cfab5f2 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -30,6 +30,7 @@ import traceback import random import encrypt import eventloop +import utils from common import parse_header @@ -559,7 +560,7 @@ class TCPRelay(object): # we just need a sorted last_activity queue and it's faster than heapq # in fact we can do O(1) insertion/remove so we invent our own if self._timeouts: - logging.debug('sweeping timeouts') + logging.log(utils.VERBOSE_LEVEL, 'sweeping timeouts') now = time.time() length = len(self._timeouts) pos = self._timeout_offset @@ -590,9 +591,9 @@ class TCPRelay(object): def _handle_events(self, events): for sock, fd, event in events: - # if sock: - # logging.debug('fd %d %s', fd, - # eventloop.EVENT_NAMES.get(event, event)) + if sock: + logging.log(utils.VERBOSE_LEVEL, 'fd %d %s', fd, + eventloop.EVENT_NAMES.get(event, event)) if sock == self._server_socket: if event & eventloop.POLL_ERR: # TODO diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index ed95443..c1d29e6 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -28,6 +28,9 @@ import getopt import logging +VERBOSE_LEVEL = 5 + + def check_python(): info = sys.version_info if not (info[0] == 2 and info[1] >= 6): @@ -106,6 +109,7 @@ def get_config(is_local): config = {} optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) + v_count = 0 for key, value in optlist: if key == '-p': config['server_port'] = int(value) @@ -120,7 +124,10 @@ def get_config(is_local): elif key == '-b': config['local_address'] = value elif key == '-v': - config['verbose'] = True + v_count += 1 + print v_count + # '-vv' turns on more verbose mode + config['verbose'] = v_count elif key == '-t': config['timeout'] = int(value) elif key == '--fast-open': @@ -148,11 +155,14 @@ def get_config(is_local): config['verbose'] = config.get('verbose', False) config['local_address'] = config.get('local_address', '127.0.0.1') - if config['verbose']: + logging.getLogger('').handlers = [] + logging.addLevelName(VERBOSE_LEVEL, 'VERBOSE') + if config['verbose'] == 2: + level = VERBOSE_LEVEL + elif config['verbose']: level = logging.DEBUG else: level = logging.INFO - logging.getLogger('').handlers = [] logging.basicConfig(level=level, format='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S', filemode='a+') From cb5481499ef80d5be2c35b89f4cb58dcd8c42add Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 20 Jun 2014 22:32:02 +0800 Subject: [PATCH 109/344] fix tcp fastopen --- shadowsocks/tcprelay.py | 60 ++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index cfab5f2..225fb28 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -200,10 +200,12 @@ class TCPRelayHandler(object): self._config['fast_open']: try: self._fastopen_connected = True + remote_sock = self._create_remote_socket(self._chosen_server[0], + self._chosen_server[1]) + self._loop.add(remote_sock, eventloop.POLL_ERR) data = ''.join(self._data_to_write_to_local) l = len(data) - s = self._remote_sock.sendto(data, MSG_FASTOPEN, - self._chosen_server) + s = remote_sock.sendto(data, MSG_FASTOPEN, self._chosen_server) if s < l: data = data[s:] self._data_to_write_to_local = [data] @@ -282,6 +284,19 @@ class TCPRelayHandler(object): # TODO use logging when debug completed self.destroy() + def _create_remote_socket(self, ip, port): + addrs = socket.getaddrinfo(ip, port, 0, socket.SOCK_STREAM, + socket.SOL_TCP) + if len(addrs) == 0: + raise Exception("getaddrinfo failed for %s:%d" % (ip, port)) + af, socktype, proto, canonname, sa = addrs[0] + remote_sock = socket.socket(af, socktype, proto) + self._remote_sock = remote_sock + self._fd_to_handlers[remote_sock.fileno()] = self + remote_sock.setblocking(False) + remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) + return remote_sock + def _handle_dns_resolved(self, result, error): if error: logging.error(error) @@ -292,33 +307,22 @@ class TCPRelayHandler(object): if ip: try: self._stage = STAGE_REPLY - remote_addr = self._remote_address[0] - remote_port = self._remote_address[1] + remote_addr = ip if self._is_local: - remote_addr, remote_port = self._chosen_server - addrs = socket.getaddrinfo(ip, remote_port, 0, - socket.SOCK_STREAM, - socket.SOL_TCP) - if len(addrs) == 0: - raise Exception("getaddrinfo failed for %s:%d" % - (remote_addr, remote_port)) - af, socktype, proto, canonname, sa = addrs[0] - remote_sock = socket.socket(af, socktype, proto) - self._remote_sock = remote_sock - self._fd_to_handlers[remote_sock.fileno()] = self - remote_sock.setblocking(False) - remote_sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, - 1) + remote_port = self._chosen_server[1] + else: + remote_port = self._remote_address[1] if self._is_local and self._config['fast_open']: # wait for more data to arrive and send them in one SYN self._stage = STAGE_REPLY - self._loop.add(remote_sock, eventloop.POLL_ERR) self._update_stream(STREAM_UP, WAIT_STATUS_READING) # TODO when there is already data in this packet else: + remote_sock = self._create_remote_socket(remote_addr, + remote_port) try: - remote_sock.connect(sa) + remote_sock.connect((remote_addr, remote_port)) except (OSError, IOError) as e: if eventloop.errno_from_exception(e) == \ errno.EINPROGRESS: @@ -432,23 +436,23 @@ class TCPRelayHandler(object): if sock == self._remote_sock: if event & eventloop.POLL_ERR: self._on_remote_error() - if self._stage == STAGE_DESTROYED: - return + if self._stage == STAGE_DESTROYED: + return if event & (eventloop.POLL_IN | eventloop.POLL_HUP): self._on_remote_read() - if self._stage == STAGE_DESTROYED: - return + if self._stage == STAGE_DESTROYED: + return if event & eventloop.POLL_OUT: self._on_remote_write() elif sock == self._local_sock: if event & eventloop.POLL_ERR: self._on_local_error() - if self._stage == STAGE_DESTROYED: - return + if self._stage == STAGE_DESTROYED: + return if event & (eventloop.POLL_IN | eventloop.POLL_HUP): self._on_local_read() - if self._stage == STAGE_DESTROYED: - return + if self._stage == STAGE_DESTROYED: + return if event & eventloop.POLL_OUT: self._on_local_write() else: From 4fd19a7714cfd2deb36b4acd7674c23c145bc62b Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 20 Jun 2014 22:32:10 +0800 Subject: [PATCH 110/344] refine test --- test.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test.py b/test.py index 154f352..68b120f 100755 --- a/test.py +++ b/test.py @@ -13,10 +13,10 @@ encrypt_salsa20.test() print 'encryption test passed' -p1 = Popen(['python', 'shadowsocks/server.py', '-c', sys.argv[-1]], shell=False, - bufsize=0, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) -p2 = Popen(['python', 'shadowsocks/local.py', '-c', sys.argv[-1]], shell=False, - bufsize=0, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) +p1 = Popen(['python', 'shadowsocks/server.py', '-c', sys.argv[-1]], stdin=PIPE, + stdout=PIPE, stderr=PIPE, close_fds=True) +p2 = Popen(['python', 'shadowsocks/local.py', '-c', sys.argv[-1]], stdin=PIPE, + stdout=PIPE, stderr=PIPE, close_fds=True) p3 = None try: @@ -39,10 +39,9 @@ try: if local_ready and server_ready and p3 is None: time.sleep(1) p3 = Popen(['curl', 'http://www.example.com/', '-v', '-L', - '--socks5-hostname', '127.0.0.1:1081'], shell=False, - bufsize=0, close_fds=True) + '--socks5-hostname', '127.0.0.1:1081'], close_fds=True) break - + if p3 is not None: r = p3.wait() if r == 0: From 0592fcefbdb89b4cbd73fa306f0bf22bbea0b2f2 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 20 Jun 2014 22:58:01 +0800 Subject: [PATCH 111/344] refine test --- test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test.py b/test.py index 68b120f..a05afb5 100755 --- a/test.py +++ b/test.py @@ -7,11 +7,11 @@ import signal import select import time from subprocess import Popen, PIPE -from shadowsocks import encrypt_salsa20 -encrypt_salsa20.test() - -print 'encryption test passed' +if 'salsa20' in sys.argv[-1]: + from shadowsocks import encrypt_salsa20 + encrypt_salsa20.test() + print 'encryption test passed' p1 = Popen(['python', 'shadowsocks/server.py', '-c', sys.argv[-1]], stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) From 4b3c018aa970d46474badd5624402996546f79ce Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 20 Jun 2014 22:58:14 +0800 Subject: [PATCH 112/344] verify hostname --- shadowsocks/asyncdns.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index f7934cb..f0ea239 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -25,6 +25,7 @@ import time import os import socket import struct +import re import logging import common import lru_cache @@ -33,6 +34,8 @@ import eventloop CACHE_SWEEP_INTERVAL = 30 +VALID_HOSTNAME = re.compile("(?!-)[A-Z\d-]{1,63}(? 255: + return False + if hostname[-1] == ".": + hostname = hostname[:-1] + return all(VALID_HOSTNAME.match(x) for x in hostname.split(".")) + + class DNSResponse(object): def __init__(self): self.hostname = None @@ -411,6 +422,9 @@ class DNSResolver(object): ip = self._cache[hostname] callback((hostname, ip), None) else: + if not is_valid_hostname(hostname): + callback(None, Exception('invalid hostname: %s' % hostname)) + return arr = self._hostname_to_cb.get(hostname, None) if not arr: self._hostname_status[hostname] = STATUS_IPV4 From c3ae04d07dd41efe372a456f1fcb80ba4eb4376b Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 20 Jun 2014 23:23:14 +0800 Subject: [PATCH 113/344] update readme --- README.md | 4 ++-- README.rst | 68 ++++++++++++++++++++++++++++++++---------------------- 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 102b2b7..54e0e6c 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Also see [Troubleshooting] [Android]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#android -[Build Status]: https://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat +[Build Status]: https://travis-ci.org/clowwindy/shadowsocks.svg?branch=master [Chinese Readme]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E [Chrome Standalone]: https://support.google.com/installer/answer/126299 [GUI client]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients @@ -124,7 +124,7 @@ Also see [Troubleshooting] [OpenWRT]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#openwrt [OS X]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#os-x [PyPI]: https://pypi.python.org/pypi/shadowsocks -[PyPI version]: https://img.shields.io/pypi/v/shadowsocks.svg?style=flat +[PyPI version]: https://badge.fury.io/py/shadowsocks.svg [Supervisor]: https://github.com/clowwindy/shadowsocks/wiki/Configure-Shadowsocks-with-Supervisor [TCP_FASTOPEN]: https://github.com/clowwindy/shadowsocks/wiki/TCP-Fast-Open [Travis CI]: https://travis-ci.org/clowwindy/shadowsocks diff --git a/README.rst b/README.rst index b77e1ba..a420cbd 100644 --- a/README.rst +++ b/README.rst @@ -10,6 +10,23 @@ A fast tunnel proxy that help you get through firewalls. Install ------- +You'll have a client on your local machine, and install a server on a +remote server. + +Client +~~~~~~ + +- `Windows `__ + / `OS + X `__ +- `Android `__ + / + `iOS `__ +- `OpenWRT `__ + +Server +~~~~~~ + Debian / Ubuntu: ^^^^^^^^^^^^^^^^ @@ -27,28 +44,10 @@ CentOS: easy_install pip pip install shadowsocks -OS X: -^^^^^ +Configuration +------------- -:: - - brew install swig - git clone https://github.com/clowwindy/M2Crypto.git - cd M2Crypto - pip install . - pip install shadowsocks - -Windows: -^^^^^^^^ - -Choose a `GUI -client `__ - -Usage ------ - -Create a config file ``/etc/shadowsocks.json`` (or put it in other -path). Example: +On your server create a config file ``/etc/shadowsocks.json``. Example: :: @@ -92,9 +91,12 @@ Run ``ssserver -c /etc/shadowsocks.json`` on your server. To run it in the background, use `Supervisor `__. -On your client machine, run ``sslocal -c /etc/shadowsocks.json``. +On your client machine, use the same configuration as your server, and +start your client. -Change the proxy settings in your browser to +If you use Chrome, it's recommended to use +`SwitchySharp `__. +Change the proxy settings to :: @@ -102,8 +104,18 @@ Change the proxy settings in your browser to hostname: 127.0.0.1 port: your local_port -It's recommended to use shadowsocks with AutoProxy or Proxy -SwitchySharp. +If you can't install +`SwitchySharp `__, +you can launch Chrome with the following arguments to force Chrome to +use the proxy: + +:: + + Chrome.exe --proxy-server="socks5://127.0.0.1:1080" --host-resolver-rules="MAP * 0.0.0.0 , EXCLUDE localhost" + +If you can't even download Chrome, find a friend to download a `Chrome +Standalone `__ +installer for you. Command line args ----------------- @@ -116,6 +128,8 @@ You can use args to override settings from ``config.json``. ssserver -p server_port -k password -m bf-cfb --workers 2 ssserver -c /etc/shadowsocks/config.json +List all available args with ``-h``. + Wiki ---- @@ -137,7 +151,7 @@ Mailing list: http://groups.google.com/group/shadowsocks Also see `Troubleshooting `__ -.. |PyPI version| image:: https://img.shields.io/pypi/v/shadowsocks.svg?style=flat +.. |PyPI version| image:: https://badge.fury.io/py/shadowsocks.svg :target: https://pypi.python.org/pypi/shadowsocks -.. |Build Status| image:: https://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat +.. |Build Status| image:: https://travis-ci.org/clowwindy/shadowsocks.svg?branch=master :target: https://travis-ci.org/clowwindy/shadowsocks From 9dd392218929ee7292b6e03377057ee68a2f7271 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 21 Jun 2014 11:18:33 +0800 Subject: [PATCH 114/344] update readme --- README.md | 4 ++-- README.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 54e0e6c..102b2b7 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Also see [Troubleshooting] [Android]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#android -[Build Status]: https://travis-ci.org/clowwindy/shadowsocks.svg?branch=master +[Build Status]: https://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat [Chinese Readme]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E [Chrome Standalone]: https://support.google.com/installer/answer/126299 [GUI client]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients @@ -124,7 +124,7 @@ Also see [Troubleshooting] [OpenWRT]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#openwrt [OS X]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#os-x [PyPI]: https://pypi.python.org/pypi/shadowsocks -[PyPI version]: https://badge.fury.io/py/shadowsocks.svg +[PyPI version]: https://img.shields.io/pypi/v/shadowsocks.svg?style=flat [Supervisor]: https://github.com/clowwindy/shadowsocks/wiki/Configure-Shadowsocks-with-Supervisor [TCP_FASTOPEN]: https://github.com/clowwindy/shadowsocks/wiki/TCP-Fast-Open [Travis CI]: https://travis-ci.org/clowwindy/shadowsocks diff --git a/README.rst b/README.rst index a420cbd..f46ea8b 100644 --- a/README.rst +++ b/README.rst @@ -151,7 +151,7 @@ Mailing list: http://groups.google.com/group/shadowsocks Also see `Troubleshooting `__ -.. |PyPI version| image:: https://badge.fury.io/py/shadowsocks.svg +.. |PyPI version| image:: https://img.shields.io/pypi/v/shadowsocks.svg?style=flat :target: https://pypi.python.org/pypi/shadowsocks -.. |Build Status| image:: https://travis-ci.org/clowwindy/shadowsocks.svg?branch=master +.. |Build Status| image:: https://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat :target: https://travis-ci.org/clowwindy/shadowsocks From 1dd722acc7aab30e9ea7c48d6a502e6826dfd472 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 21 Jun 2014 11:23:44 +0800 Subject: [PATCH 115/344] bump --- CHANGES | 6 ++++++ setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 85b1666..92572b1 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,9 @@ +2.0.7 2014-06-21 +- Fix fastopen on local +- Fallback when fastopen is not available +- Add verbose logging mode -vv +- Verify if hostname is valid + 2.0.6 2014-06-19 - Fix CPU 100% on POLL_HUP - More friendly logging diff --git a/setup.py b/setup.py index fa7e13e..2f1f225 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.0.6", + version="2.0.7", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From f9fef0e2295ca3783217fe0c15ada8b0b2ee9bef Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 22 Jun 2014 11:58:22 +0800 Subject: [PATCH 116/344] add TypeError --- shadowsocks/asyncdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index f0ea239..f188faa 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -233,7 +233,7 @@ def is_ip(address): try: socket.inet_pton(family, address) return True - except (TypeError, OSError, IOError): + except (TypeError, ValueError, OSError, IOError): pass return False From 61d9e17cae72e5e9e094ee0df76a034eb7086157 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 22 Jun 2014 19:50:34 +0800 Subject: [PATCH 117/344] remove print --- shadowsocks/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index c1d29e6..492f357 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -125,7 +125,6 @@ def get_config(is_local): config['local_address'] = value elif key == '-v': v_count += 1 - print v_count # '-vv' turns on more verbose mode config['verbose'] = v_count elif key == '-t': From bbd649b1c1b175a73c06d81fb1625ecf76e29ee3 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 23 Jun 2014 17:02:32 +0800 Subject: [PATCH 118/344] Update CONTRIBUTING.md --- CONTRIBUTING.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cca3953..4a73830 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,9 +1,10 @@ How to contribute ================= -Before you submit issues, please take a few minutes to read this guide. +Before you submit issues, please read [Troubleshooting] and take a few minutes +to read this guide. -在你提交问题前，请花两分钟阅读一下这份指南。 +在你提交问题前，请先[自行诊断]一下，然后再提交，附上诊断过程中的问题和下列结果。 Issues ------ @@ -11,7 +12,7 @@ Issues Please include the following information in your submission: 1. How did you set up your environment? (OS, version of Shadowsocks) -2. Where did you see this error, was it on local or on server? +2. Did you see any error? Where did you see this error, was it on local or on server? 3. What happened in your browser? Just no response, or any error message? 4. 10 lines of log on the local side of shadowsocks when the error happened. 5. 10 lines of log on the server side of shadowsocks when the error happened. @@ -25,10 +26,13 @@ Skip any of them if you don't know its meaning. 请提交下面的信息： 1. 你是如何搭建环境的？（操作系统，Shadowsocks 版本） -2. 错误是发生在哪里，本地还是服务器？ +2. 有无错误提示？错误是发生在哪里，客户端还是服务器？ 3. 浏览器里的现象是什么？一直转菊花，还是有提示错误？ -4. 发生错误时，本地端的十行完整的日志。 -5. 发生错误时，服务器端的十行完整的日志。 +4. 发生错误时，客户端最后一页完整的日志。 +5. 发生错误时，服务器端最后一页完整的日志。 6. 其它你认为可能和问题有关的信息。 如果你不清楚其中某条的含义， 可以直接跳过那一条。 + +[Troubleshooting]: https://github.com/clowwindy/shadowsocks/wiki/Troubleshooting +[自行诊断]: https://github.com/clowwindy/shadowsocks/wiki/Troubleshooting From 868c7c5b3800e4340f2f92e558d9b0de3da52b40 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 23 Jun 2014 17:12:16 +0800 Subject: [PATCH 119/344] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 102b2b7..543f1a6 100644 --- a/README.md +++ b/README.md @@ -107,11 +107,10 @@ MIT Bugs and Issues ---------------- -Please visit [Issue Tracker] -Mailing list: http://groups.google.com/group/shadowsocks - -Also see [Troubleshooting] +* [Troubleshooting] +* [Issue Tracker] +* [Mailing list] [Android]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#android @@ -121,6 +120,7 @@ Also see [Troubleshooting] [GUI client]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients [iOS]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#ios [Issue Tracker]: https://github.com/clowwindy/shadowsocks/issues?state=open +[Mailing list]: http://groups.google.com/group/shadowsocks [OpenWRT]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#openwrt [OS X]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#os-x [PyPI]: https://pypi.python.org/pypi/shadowsocks From d4458bceb766d35849bfeac54a3df4281d107b9a Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 23 Jun 2014 21:35:23 +0800 Subject: [PATCH 120/344] add timeout support in DNS --- shadowsocks/asyncdns.py | 33 +++++++++++++++++++++++---------- shadowsocks/lru_cache.py | 9 +++++---- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index f188faa..ce0076d 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -32,7 +32,7 @@ import lru_cache import eventloop -CACHE_SWEEP_INTERVAL = 30 +CACHE_SWEEP_INTERVAL = 10 VALID_HOSTNAME = re.compile("(?!-)[A-Z\d-]{1,63}(? CACHE_SWEEP_INTERVAL: self._cache.sweep() + self._cb_to_hostname.sweep() self._last_time = now def remove_callback(self, callback): diff --git a/shadowsocks/lru_cache.py b/shadowsocks/lru_cache.py index a26680c..1313af5 100644 --- a/shadowsocks/lru_cache.py +++ b/shadowsocks/lru_cache.py @@ -50,13 +50,14 @@ class LRUCache(collections.MutableMapping): least = self._last_visits[0] if now - least <= self.timeout: break + if self.close_callback is not None: + for key in self._time_to_keys[least]: + if self._store.__contains__(key): + value = self._store[key] + self.close_callback(value) for key in self._time_to_keys[least]: heapq.heappop(self._last_visits) if self._store.__contains__(key): - value = self._store[key] - if self.close_callback is not None: - self.close_callback(value) - del self._store[key] c += 1 del self._time_to_keys[least] From 012ab7050008f19d7be304b21f7eb69a7b30de77 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 23 Jun 2014 21:46:23 +0800 Subject: [PATCH 121/344] use multiple DNS instead of trying each one --- shadowsocks/asyncdns.py | 36 +++++++++++------------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index ce0076d..a3c276c 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -32,7 +32,7 @@ import lru_cache import eventloop -CACHE_SWEEP_INTERVAL = 10 +CACHE_SWEEP_INTERVAL = 30 VALID_HOSTNAME = re.compile("(?!-)[A-Z\d-]{1,63}(?= 2: server = parts[1] if is_ip(server): - servers.append(server) - # TODO support more servers - + self._servers.append(server) except IOError: pass - if not servers: - servers.append('8.8.8.8') - self._dns_server = (servers[0], 53) - - def _timedout(self, hostname): - error = Exception('timed out when resolving %s with DNS %s' % - (hostname, self._dns_server)) - self._server_index += 1 - self._server_index %= len(self._servers) - self._dns_server = (self._servers[self._server_index], 53) - self._call_callback(hostname, None, error=error) + if not self._servers: + self._servers = ['8.8.4.4', '8.8.8.8'] def _parse_hosts(self): etc_path = '/etc/hosts' @@ -389,7 +375,7 @@ class DNSResolver(object): self._loop.add(self._sock, eventloop.POLL_IN) else: data, addr = sock.recvfrom(1024) - if addr != self._dns_server: + if addr[0] not in self._servers: logging.warn('received a packet other than our dns') break self._handle_data(data) @@ -397,7 +383,6 @@ class DNSResolver(object): now = time.time() if now - self._last_time > CACHE_SWEEP_INTERVAL: self._cache.sweep() - self._cb_to_hostname.sweep() self._last_time = now def remove_callback(self, callback): @@ -413,13 +398,14 @@ class DNSResolver(object): del self._hostname_status[hostname] def _send_req(self, hostname, qtype): - logging.debug('resolving %s with type %d using server %s', hostname, - qtype, self._dns_server) self._request_id += 1 if self._request_id > 32768: self._request_id = 1 req = build_request(hostname, qtype, self._request_id) - self._sock.sendto(req, self._dns_server) + for server in self._servers: + logging.debug('resolving %s with type %d using server %s', hostname, + qtype, server) + self._sock.sendto(req, (server, 53)) def resolve(self, hostname, callback): if not hostname: From f4e68dc64e7a310da1ec964778f3e39a70a6f318 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 23 Jun 2014 22:26:46 +0800 Subject: [PATCH 122/344] bump --- CHANGES | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 92572b1..a50ec1d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +2.0.8 2014-06-23 +- Use multiple DNS to query hostnames + 2.0.7 2014-06-21 - Fix fastopen on local - Fallback when fastopen is not available diff --git a/setup.py b/setup.py index 2f1f225..2bb28c9 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.0.7", + version="2.0.8", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From 63ed7a1bc653f643c291e9e3071c553c0058f533 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 26 Jun 2014 14:34:33 +0800 Subject: [PATCH 123/344] Update CONTRIBUTING.md --- CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4a73830..9c0c767 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,8 @@ How to contribute Before you submit issues, please read [Troubleshooting] and take a few minutes to read this guide. -在你提交问题前，请先[自行诊断]一下，然后再提交，附上诊断过程中的问题和下列结果。 +在你提交问题前，请先[自行诊断]一下，然后再提交，附上诊断过程中的问题和下列结果， +否则我们无法重现你的问题，也就帮不了你。 Issues ------ From d82441e0b46fbadac8a1c366ef5c98313826dc9e Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 3 Jul 2014 23:59:33 +0300 Subject: [PATCH 124/344] Update CONTRIBUTING.md --- CONTRIBUTING.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9c0c767..11054e2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,26 +1,12 @@ How to contribute ================= +在你提交问题前，请先[自行诊断]一下。提交时附上诊断过程中的问题和下列结果， +否则如果我们无法重现你的问题，也就不能帮助你。 + Before you submit issues, please read [Troubleshooting] and take a few minutes to read this guide. -在你提交问题前，请先[自行诊断]一下，然后再提交，附上诊断过程中的问题和下列结果， -否则我们无法重现你的问题，也就帮不了你。 - -Issues ------- - -Please include the following information in your submission: - -1. How did you set up your environment? (OS, version of Shadowsocks) -2. Did you see any error? Where did you see this error, was it on local or on server? -3. What happened in your browser? Just no response, or any error message? -4. 10 lines of log on the local side of shadowsocks when the error happened. -5. 10 lines of log on the server side of shadowsocks when the error happened. -6. Any other useful information. - -Skip any of them if you don't know its meaning. - 问题反馈 ------- @@ -35,5 +21,19 @@ Skip any of them if you don't know its meaning. 如果你不清楚其中某条的含义， 可以直接跳过那一条。 +Issues +------ + +Please include the following information in your submission: + +1. How did you set up your environment? (OS, version of Shadowsocks) +2. Did you see any error? Where did you see this error, was it on local or on server? +3. What happened in your browser? Just no response, or any error message? +4. 10 lines of log on the local side of shadowsocks when the error happened. +5. 10 lines of log on the server side of shadowsocks when the error happened. +6. Any other useful information. + +Skip any of them if you don't know its meaning. + [Troubleshooting]: https://github.com/clowwindy/shadowsocks/wiki/Troubleshooting [自行诊断]: https://github.com/clowwindy/shadowsocks/wiki/Troubleshooting From b7bfdb9918aa0dad08e852394cffdb9d9b24ddf7 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 4 Jul 2014 21:57:11 +0300 Subject: [PATCH 125/344] add timeout support in DNS; close #138 --- shadowsocks/tcprelay.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 225fb28..fa10c67 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -166,7 +166,8 @@ class TCPRelayHandler(object): uncomplete = True except (OSError, IOError) as e: error_no = eventloop.errno_from_exception(e) - if error_no in (errno.EAGAIN, errno.EINPROGRESS): + if error_no in (errno.EAGAIN, errno.EINPROGRESS, + errno.EWOULDBLOCK): uncomplete = True else: logging.error(e) @@ -349,7 +350,7 @@ class TCPRelayHandler(object): data = self._local_sock.recv(BUF_SIZE) except (OSError, IOError) as e: if eventloop.errno_from_exception(e) in \ - (errno.ETIMEDOUT, errno.EAGAIN): + (errno.ETIMEDOUT, errno.EAGAIN, errno.EWOULDBLOCK): return if not data: self.destroy() @@ -381,7 +382,7 @@ class TCPRelayHandler(object): data = self._remote_sock.recv(BUF_SIZE) except (OSError, IOError) as e: if eventloop.errno_from_exception(e) in \ - (errno.ETIMEDOUT, errno.EAGAIN): + (errno.ETIMEDOUT, errno.EAGAIN, errno.EWOULDBLOCK): return if not data: self.destroy() @@ -610,7 +611,8 @@ class TCPRelay(object): self._is_local) except (OSError, IOError) as e: error_no = eventloop.errno_from_exception(e) - if error_no in (errno.EAGAIN, errno.EINPROGRESS): + if error_no in (errno.EAGAIN, errno.EINPROGRESS, + errno.EWOULDBLOCK): continue else: logging.error(e) From 13436b224ec7f2e8001f960f1d2e05b2275843db Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 4 Jul 2014 21:57:11 +0300 Subject: [PATCH 126/344] add EWOULDBLOCK to fix Windows; close #138 --- shadowsocks/tcprelay.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 225fb28..fa10c67 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -166,7 +166,8 @@ class TCPRelayHandler(object): uncomplete = True except (OSError, IOError) as e: error_no = eventloop.errno_from_exception(e) - if error_no in (errno.EAGAIN, errno.EINPROGRESS): + if error_no in (errno.EAGAIN, errno.EINPROGRESS, + errno.EWOULDBLOCK): uncomplete = True else: logging.error(e) @@ -349,7 +350,7 @@ class TCPRelayHandler(object): data = self._local_sock.recv(BUF_SIZE) except (OSError, IOError) as e: if eventloop.errno_from_exception(e) in \ - (errno.ETIMEDOUT, errno.EAGAIN): + (errno.ETIMEDOUT, errno.EAGAIN, errno.EWOULDBLOCK): return if not data: self.destroy() @@ -381,7 +382,7 @@ class TCPRelayHandler(object): data = self._remote_sock.recv(BUF_SIZE) except (OSError, IOError) as e: if eventloop.errno_from_exception(e) in \ - (errno.ETIMEDOUT, errno.EAGAIN): + (errno.ETIMEDOUT, errno.EAGAIN, errno.EWOULDBLOCK): return if not data: self.destroy() @@ -610,7 +611,8 @@ class TCPRelay(object): self._is_local) except (OSError, IOError) as e: error_no = eventloop.errno_from_exception(e) - if error_no in (errno.EAGAIN, errno.EINPROGRESS): + if error_no in (errno.EAGAIN, errno.EINPROGRESS, + errno.EWOULDBLOCK): continue else: logging.error(e) From b0c8e50492bf66e257a0cd902541c467be888ac2 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 4 Jul 2014 22:25:27 +0300 Subject: [PATCH 127/344] fix #145 --- shadowsocks/utils.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 492f357..0b7b068 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -100,7 +100,7 @@ def get_config(is_local): logging.info('loading config from %s' % config_path) with open(config_path, 'rb') as f: try: - config = json.load(f) + config = json.load(f, object_hook=_decode_dict) except ValueError as e: logging.error('found an error in config.json: %s', e.message) @@ -210,4 +210,32 @@ optional arguments: -v verbose mode Online help: -''' \ No newline at end of file +''' + + +def _decode_list(data): + rv = [] + for item in data: + if isinstance(item, unicode): + item = item.encode('utf-8') + elif isinstance(item, list): + item = _decode_list(item) + elif isinstance(item, dict): + item = _decode_dict(item) + rv.append(item) + return rv + + +def _decode_dict(data): + rv = {} + for key, value in data.iteritems(): + if isinstance(key, unicode): + key = key.encode('utf-8') + if isinstance(value, unicode): + value = value.encode('utf-8') + elif isinstance(value, list): + value = _decode_list(value) + elif isinstance(value, dict): + value = _decode_dict(value) + rv[key] = value + return rv \ No newline at end of file From a614dd6e5b2bdf813bb34d0c2ceae6a99b398708 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 6 Jul 2014 18:43:50 +0300 Subject: [PATCH 128/344] bump --- CHANGES | 4 ++++ README.rst | 11 ++++------- setup.py | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGES b/CHANGES index a50ec1d..5a27a49 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +2.0.9 2014-07-06 +- Fix EWOULDBLOCK on Windows +- Fix Unicode config problem on some platforms + 2.0.8 2014-06-23 - Use multiple DNS to query hostnames diff --git a/README.rst b/README.rst index f46ea8b..983a375 100644 --- a/README.rst +++ b/README.rst @@ -143,13 +143,10 @@ MIT Bugs and Issues --------------- -Please visit `Issue -Tracker `__ - -Mailing list: http://groups.google.com/group/shadowsocks - -Also see -`Troubleshooting `__ +- `Troubleshooting `__ +- `Issue + Tracker `__ +- `Mailing list `__ .. |PyPI version| image:: https://img.shields.io/pypi/v/shadowsocks.svg?style=flat :target: https://pypi.python.org/pypi/shadowsocks diff --git a/setup.py b/setup.py index 2bb28c9..9025d91 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.0.8", + version="2.0.9", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From 130cf8fe3594327e3487783ff0b88311f3ecd729 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 11 Jul 2014 17:35:47 +0800 Subject: [PATCH 129/344] add UDP tests --- .travis.yml | 4 ++++ test.py | 22 +++++++++++++++++----- tests/socksify/install.sh | 10 ++++++++++ tests/socksify/socks.conf | 5 +++++ 4 files changed, 36 insertions(+), 5 deletions(-) create mode 100755 tests/socksify/install.sh create mode 100644 tests/socksify/socks.conf diff --git a/.travis.yml b/.travis.yml index dabbb80..6543a4e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,10 +3,14 @@ python: - 2.6 - 2.7 - pypy +cache: + directories: + - dante-1.4.0 before_install: - sudo apt-get update -qq - sudo apt-get install -qq build-essential libssl-dev swig python-m2crypto python-numpy - pip install m2crypto salsa20 + - sudo tests/socksify/install.sh script: - python test.py -c tests/table.json - python test.py -c tests/aes.json diff --git a/test.py b/test.py index a05afb5..4ee04a0 100755 --- a/test.py +++ b/test.py @@ -38,15 +38,27 @@ try: if local_ready and server_ready and p3 is None: time.sleep(1) - p3 = Popen(['curl', 'http://www.example.com/', '-v', '-L', - '--socks5-hostname', '127.0.0.1:1081'], close_fds=True) + break + p3 = Popen(['curl', 'http://www.example.com/', '-v', '-L', + '--socks5-hostname', '127.0.0.1:1081'], close_fds=True) if p3 is not None: r = p3.wait() - if r == 0: - print 'test passed' - sys.exit(r) + if r != 0: + sys.exit(r) + else: + sys.exit(1) + + p4 = Popen(['socksify', 'dig', '@8.8.8.8', 'www.google.com'], + close_fds=True) + if p4 is not None: + r = p4.wait() + if r != 0: + sys.exit(r) + else: + sys.exit(1) + print 'test passed' finally: for p in [p1, p2]: diff --git a/tests/socksify/install.sh b/tests/socksify/install.sh new file mode 100755 index 0000000..1927d76 --- /dev/null +++ b/tests/socksify/install.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +if [ ! -d dante-1.4.0 ]; then + wget http://www.inet.no/dante/files/dante-1.4.0.tar.gz || exit 1 + tar xf dante-1.4.0.tar.gz || exit 1 +fi +pushd dante-1.4.0 +./configure && make && make install || exit 1 +popd +cp tests/socksify/socks.conf /etc/ || exit 1 diff --git a/tests/socksify/socks.conf b/tests/socksify/socks.conf new file mode 100644 index 0000000..13db772 --- /dev/null +++ b/tests/socksify/socks.conf @@ -0,0 +1,5 @@ +route { + from: 0.0.0.0/0 to: 0.0.0.0/0 via: 127.0.0.1 port = 1081 + proxyprotocol: socks_v5 + method: none +} \ No newline at end of file From 68cf94b10ed46355c69b5a94fd173d54dbad55e9 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 11 Jul 2014 17:41:22 +0800 Subject: [PATCH 130/344] add dnsutils for tests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6543a4e..24e1d8f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ cache: - dante-1.4.0 before_install: - sudo apt-get update -qq - - sudo apt-get install -qq build-essential libssl-dev swig python-m2crypto python-numpy + - sudo apt-get install -qq build-essential libssl-dev swig python-m2crypto python-numpy dnsutils - pip install m2crypto salsa20 - sudo tests/socksify/install.sh script: From 122f23314378e951e06c89f290f126069d96e69d Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 11 Jul 2014 18:00:58 +0800 Subject: [PATCH 131/344] fix udprelay for local --- shadowsocks/udprelay.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index c7faeee..5149ac4 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -176,13 +176,14 @@ class UDPRelay(object): self._sockets.add(client.fileno()) self._eventloop.add(client, eventloop.POLL_IN) - data = data[header_length:] - if not data: - return if self._is_local: data = encrypt.encrypt_all(self._password, self._method, 1, data) if not data: return + else: + data = data[header_length:] + if not data: + return try: client.sendto(data, (server_addr, server_port)) except IOError as e: From c788aa59b625f198dd26b556aeb9f32b369ec73e Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 11 Jul 2014 18:30:24 +0800 Subject: [PATCH 132/344] fix UDP tests --- test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test.py b/test.py index 4ee04a0..594cfdc 100755 --- a/test.py +++ b/test.py @@ -66,5 +66,3 @@ finally: os.kill(p.pid, signal.SIGTERM) except OSError: pass - -sys.exit(-1) From 040c9564722e0cd9d8f2de519b39495a305dbd68 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 11 Jul 2014 18:40:18 +0800 Subject: [PATCH 133/344] fix server-multi-ports --- shadowsocks/udprelay.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 5149ac4..7a72c11 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -71,6 +71,7 @@ import socket import logging import struct import errno +import random import encrypt import eventloop import lru_cache @@ -86,6 +87,7 @@ def client_key(a, b, c, d): class UDPRelay(object): def __init__(self, config, dns_resolver, is_local): + self._config = config if is_local: self._listen_addr = config['local_address'] self._listen_port = config['local_port'] @@ -121,6 +123,15 @@ class UDPRelay(object): server_socket.setblocking(False) self._server_socket = server_socket + def _get_a_server(self): + server = self._config['server'] + server_port = self._config['server_port'] + if type(server_port) == list: + server_port = random.choice(server_port) + logging.debug('chosen server: %s:%d', server, server_port) + # TODO support multiple server IP + return server, server_port + def _close_client(self, client): if hasattr(client, 'close'): self._sockets.remove(client.fileno()) @@ -154,7 +165,7 @@ class UDPRelay(object): addrtype, dest_addr, dest_port, header_length = header_result if self._is_local: - server_addr, server_port = self._remote_addr, self._remote_port + server_addr, server_port = self._get_a_server() else: server_addr, server_port = dest_addr, dest_port From 2950f99da6c42dd08e8ddabd5d1b62061c18efc5 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 11 Jul 2014 18:46:26 +0800 Subject: [PATCH 134/344] fix salsa20 for UDP --- shadowsocks/encrypt.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index fc6b5ce..a590645 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -216,9 +216,12 @@ def encrypt_all(password, method, op, data): else: iv = data[:iv_len] data = data[iv_len:] - cipher = M2Crypto.EVP.Cipher(method.replace('-', '_'), key, iv, op, - key_as_bytes=0, d='md5', salt=None, i=1, - padding=1) + if method != 'salsa20-ctr': + cipher = M2Crypto.EVP.Cipher(method.replace('-', '_'), key, iv, + op, key_as_bytes=0, d='md5', + salt=None, i=1, padding=1) + else: + cipher = encrypt_salsa20.Salsa20Cipher(method, key, iv, op) result.append(cipher.update(data)) f = cipher.final() if f: From a1bc23c1612227aad2c12e1a554804ffcf87f307 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 11 Jul 2014 18:48:57 +0800 Subject: [PATCH 135/344] fix salsa20 --- shadowsocks/encrypt.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index a590645..857bdfe 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -223,7 +223,4 @@ def encrypt_all(password, method, op, data): else: cipher = encrypt_salsa20.Salsa20Cipher(method, key, iv, op) result.append(cipher.update(data)) - f = cipher.final() - if f: - result.append(f) return ''.join(result) From edeb66395a866f9647fa3a5b809a049a55834904 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 11 Jul 2014 19:00:24 +0800 Subject: [PATCH 136/344] bump --- CHANGES | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 5a27a49..8305637 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +2.0.10 2014-07-11 +- Fix UDP on local + 2.0.9 2014-07-06 - Fix EWOULDBLOCK on Windows - Fix Unicode config problem on some platforms diff --git a/setup.py b/setup.py index 9025d91..945ce87 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.0.9", + version="2.0.10", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From a323f878e1efcee0974cebe5489ded46f960740d Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 12 Jul 2014 10:31:35 +0800 Subject: [PATCH 137/344] prefer addrtype 1 and 4 over 3 in UDP relay --- shadowsocks/common.py | 15 +++++++++++++++ shadowsocks/udprelay.py | 5 ++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 6104478..7548ff3 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -79,6 +79,21 @@ ADDRTYPE_IPV6 = 4 ADDRTYPE_HOST = 3 +def pack_addr(address): + for family in (socket.AF_INET, socket.AF_INET6): + try: + r = socket.inet_pton(family, address) + if family == socket.AF_INET6: + return '\x04' + r + else: + return '\x01' + r + except (TypeError, ValueError, OSError, IOError): + pass + if len(address) > 255: + address = address[:255] # TODO + return '\x03' + chr(len(address)) + address + + def parse_header(data): addrtype = ord(data[0]) dest_addr = None diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 7a72c11..d7e3d4c 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -75,7 +75,7 @@ import random import encrypt import eventloop import lru_cache -from common import parse_header +from common import parse_header, pack_addr BUF_SIZE = 65536 @@ -214,8 +214,7 @@ class UDPRelay(object): if addrlen > 255: # drop return - data = '\x03' + chr(addrlen) + r_addr[0] + \ - struct.pack('>H', r_addr[1]) + data + data = pack_addr(r_addr[0]) + struct.pack('>H', r_addr[1]) + data response = encrypt.encrypt_all(self._password, self._method, 1, data) if not response: From fed480bb1727f616edb30aeded30528f6de868f1 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 12 Jul 2014 11:11:28 +0800 Subject: [PATCH 138/344] include LICENSE in package tar --- MANIFEST.in | 3 +++ setup.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..1dc4c8e --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +recursive-include *.py +include README.rst +include LICENSE diff --git a/setup.py b/setup.py index 945ce87..2bf1d1f 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.0.10", + version="2.0.11", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', @@ -14,7 +14,7 @@ setup( url='https://github.com/clowwindy/shadowsocks', packages=['shadowsocks'], package_data={ - 'shadowsocks': ['README.rst', 'LICENSE', 'config.json'] + 'shadowsocks': ['README.rst', 'LICENSE'] }, install_requires=[], entry_points=""" From e7e226768af30492fc47b8a3b951b737650ec5e5 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 12 Jul 2014 17:45:56 +0800 Subject: [PATCH 139/344] add instructions for installing m2crypto --- shadowsocks/encrypt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 857bdfe..63c7d8a 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -61,7 +61,8 @@ def init_table(key, method=None): __import__('M2Crypto') except ImportError: logging.error('M2Crypto is required to use encryption other than ' - 'default method') + 'default method, please run `apt-get install ' + 'python-m2crypto`') sys.exit(1) if not method: if key in cached_tables: From c66d9b2c202faf34b6fb5f29bf7c3511302ce361 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 12 Jul 2014 17:48:33 +0800 Subject: [PATCH 140/344] more details --- shadowsocks/encrypt.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 63c7d8a..86c5d6d 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -60,9 +60,8 @@ def init_table(key, method=None): try: __import__('M2Crypto') except ImportError: - logging.error('M2Crypto is required to use encryption other than ' - 'default method, please run `apt-get install ' - 'python-m2crypto`') + logging.error(('M2Crypto is required to use %s, please run' + ' `apt-get install python-m2crypto`') % method) sys.exit(1) if not method: if key in cached_tables: From 0df13b57153e5b0d5c084bc613247bd5b2354c43 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 12 Jul 2014 21:53:51 +0800 Subject: [PATCH 141/344] bump year --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 0d9e69a..98f608b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ Shadowsocks -Copyright (c) 2013 clowwindy +Copyright (c) 2014 clowwindy Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 68b9a063c5f73c4fd5efa5867ba767d4fdf4b0cf Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 12 Jul 2014 21:59:00 +0800 Subject: [PATCH 142/344] move tests and remove config.json --- .travis.yml | 14 +++++++------- config.json | 10 ---------- test.py => tests/test.py | 2 ++ test_latency.py => tests/test_latency.py | 0 4 files changed, 9 insertions(+), 17 deletions(-) delete mode 100644 config.json rename test.py => tests/test.py (98%) rename test_latency.py => tests/test_latency.py (100%) diff --git a/.travis.yml b/.travis.yml index 24e1d8f..85d7939 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,10 +12,10 @@ before_install: - pip install m2crypto salsa20 - sudo tests/socksify/install.sh script: - - python test.py -c tests/table.json - - python test.py -c tests/aes.json - - python test.py -c tests/salsa20.json - - python test.py -c tests/server-multi-ports.json - - python test.py -c tests/server-multi-passwd.json - - python test.py -c tests/server-multi-passwd-table.json - - python test.py -c tests/workers.json + - python tests/test.py -c tests/table.json + - python tests/test.py -c tests/aes.json + - python tests/test.py -c tests/salsa20.json + - python tests/test.py -c tests/server-multi-ports.json + - python tests/test.py -c tests/server-multi-passwd.json + - python tests/test.py -c tests/server-multi-passwd-table.json + - python tests/test.py -c tests/workers.json diff --git a/config.json b/config.json deleted file mode 100644 index 026ed38..0000000 --- a/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "server":"0.0.0.0", - "server_port":8388, - "local_port":1080, - "password":"barfoo!", - "timeout":300, - "method":"aes-256-cfb", - "local_address":"127.0.0.1", - "fast_open":false -} diff --git a/test.py b/tests/test.py similarity index 98% rename from test.py rename to tests/test.py index 594cfdc..da28e0b 100755 --- a/test.py +++ b/tests/test.py @@ -8,6 +8,8 @@ import select import time from subprocess import Popen, PIPE +sys.path.insert(0, '../') + if 'salsa20' in sys.argv[-1]: from shadowsocks import encrypt_salsa20 encrypt_salsa20.test() diff --git a/test_latency.py b/tests/test_latency.py similarity index 100% rename from test_latency.py rename to tests/test_latency.py From c1cf02cbea8ade829eb30eaa83e5b282bd31b32a Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 13 Jul 2014 01:29:57 +0800 Subject: [PATCH 143/344] fix tests --- tests/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test.py b/tests/test.py index da28e0b..33530cd 100755 --- a/tests/test.py +++ b/tests/test.py @@ -8,7 +8,7 @@ import select import time from subprocess import Popen, PIPE -sys.path.insert(0, '../') +sys.path.insert(0, './') if 'salsa20' in sys.argv[-1]: from shadowsocks import encrypt_salsa20 From da13e768b0ff1fd658ec5696ea31c1c0a63be62c Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 13 Jul 2014 11:24:59 +0800 Subject: [PATCH 144/344] bump --- CHANGES | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES b/CHANGES index 8305637..68e75bd 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +2.0.11 2014-07-12 +- Prefers IP addresses over hostnames, more friendly with socksify and openvpn + 2.0.10 2014-07-11 - Fix UDP on local From da842918bd60caeaf80a1c29ce51bd1e892902f1 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 23 Jul 2014 14:33:55 +0800 Subject: [PATCH 145/344] close #158 --- shadowsocks/utils.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 0b7b068..7724594 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -84,10 +84,10 @@ def check_config(config): def get_config(is_local): if is_local: - shortopts = 's:b:p:k:l:m:c:t:v' + shortopts = 'hs:b:p:k:l:m:c:t:v' longopts = ['fast-open'] else: - shortopts = 's:p:k:m:c:t:v' + shortopts = 'hs:p:k:m:c:t:v' longopts = ['fast-open', 'workers:'] try: config_path = find_config() @@ -133,6 +133,12 @@ def get_config(is_local): config['fast_open'] = True elif key == '--workers': config['workers'] = value + elif key == '-h': + if is_local: + print_local_help() + else: + print_server_help() + sys.exit(0) except getopt.GetoptError as e: print >>sys.stderr, e if is_local: From 72fe18212e8ba6c0d880bdb66e07270a489001d9 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 26 Jul 2014 16:14:48 +0800 Subject: [PATCH 146/344] add quiet mode; close #164 --- CHANGES | 4 ++++ setup.py | 2 +- shadowsocks/utils.py | 26 ++++++++++++++++++-------- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/CHANGES b/CHANGES index 68e75bd..0e6c2b4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +2.0.12 2014-07-26 +- Support -q quiet mode +- Exit 0 when showing help with -h + 2.0.11 2014-07-12 - Prefers IP addresses over hostnames, more friendly with socksify and openvpn diff --git a/setup.py b/setup.py index 2bf1d1f..b59b3df 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.0.11", + version="2.0.12", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 7724594..d570435 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -84,10 +84,10 @@ def check_config(config): def get_config(is_local): if is_local: - shortopts = 'hs:b:p:k:l:m:c:t:v' + shortopts = 'hs:b:p:k:l:m:c:t:vq' longopts = ['fast-open'] else: - shortopts = 'hs:p:k:m:c:t:v' + shortopts = 'hs:p:k:m:c:t:vq' longopts = ['fast-open', 'workers:'] try: config_path = find_config() @@ -139,6 +139,9 @@ def get_config(is_local): else: print_server_help() sys.exit(0) + elif key == '-q': + v_count -= 1 + config['verbose'] = v_count except getopt.GetoptError as e: print >>sys.stderr, e if is_local: @@ -162,10 +165,14 @@ def get_config(is_local): logging.getLogger('').handlers = [] logging.addLevelName(VERBOSE_LEVEL, 'VERBOSE') - if config['verbose'] == 2: + if config['verbose'] >= 2: level = VERBOSE_LEVEL - elif config['verbose']: + elif config['verbose'] == 1: level = logging.DEBUG + elif config['verbose'] == -1: + level = logging.WARN + elif config['verbose'] <= -2: + level = logging.ERROR else: level = logging.INFO logging.basicConfig(level=level, @@ -180,7 +187,7 @@ def get_config(is_local): def print_local_help(): print '''usage: sslocal [-h] -s SERVER_ADDR -p SERVER_PORT [-b LOCAL_ADDR] -l LOCAL_PORT -k PASSWORD -m METHOD [-t TIMEOUT] [-c CONFIG] - [--fast-open] [-v] + [--fast-open] [-v] [-q] optional arguments: -h, --help show this help message and exit @@ -193,7 +200,8 @@ optional arguments: -t TIMEOUT timeout in seconds -c CONFIG path to config file --fast-open use TCP_FASTOPEN, requires Linux 3.7+ - -v verbose mode + -v, -vv verbose mode + -q, -qq quiet mode, only show warnings/errors Online help: ''' @@ -201,7 +209,8 @@ Online help: def print_server_help(): print '''usage: ssserver [-h] -s SERVER_ADDR -p SERVER_PORT -k PASSWORD - -m METHOD [-t TIMEOUT] [-c CONFIG] [--fast-open] [-v] + -m METHOD [-t TIMEOUT] [-c CONFIG] [--fast-open] [--workders] + [-v] [-q] optional arguments: -h, --help show this help message and exit @@ -213,7 +222,8 @@ optional arguments: -c CONFIG path to config file --fast-open use TCP_FASTOPEN, requires Linux 3.7+ --workers WORKERS number of workers, available on Unix/Linux - -v verbose mode + -v, -vv verbose mode + -q, -qq quiet mode, only show warnings/errors Online help: ''' From fb7e83a7538582b3eeea6e8529e52d05cf7c0b1f Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 28 Jul 2014 00:26:30 +0800 Subject: [PATCH 147/344] less jump steps --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 543f1a6..fcf091c 100644 --- a/README.md +++ b/README.md @@ -117,12 +117,11 @@ Bugs and Issues [Build Status]: https://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat [Chinese Readme]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E [Chrome Standalone]: https://support.google.com/installer/answer/126299 -[GUI client]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients -[iOS]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#ios +[iOS]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Help [Issue Tracker]: https://github.com/clowwindy/shadowsocks/issues?state=open [Mailing list]: http://groups.google.com/group/shadowsocks [OpenWRT]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#openwrt -[OS X]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#os-x +[OS X]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Shadowsocks-for-OSX-Help [PyPI]: https://pypi.python.org/pypi/shadowsocks [PyPI version]: https://img.shields.io/pypi/v/shadowsocks.svg?style=flat [Supervisor]: https://github.com/clowwindy/shadowsocks/wiki/Configure-Shadowsocks-with-Supervisor From ca6f72e22b99c0b438021dc98dec32978d0093f4 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 1 Aug 2014 17:21:10 +0800 Subject: [PATCH 148/344] better error msg --- shadowsocks/utils.py | 67 +++++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index d570435..939ac3e 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -83,6 +83,8 @@ def check_config(config): def get_config(is_local): + logging.basicConfig(level=logging.INFO, + format='%(levelname)-s: %(message)s', filemode='a+') if is_local: shortopts = 'hs:b:p:k:l:m:c:t:vq' longopts = ['fast-open'] @@ -144,24 +146,36 @@ def get_config(is_local): config['verbose'] = v_count except getopt.GetoptError as e: print >>sys.stderr, e - if is_local: - print_local_help() - else: - print_server_help() + print_help(is_local) sys.exit(2) - if not config['password'] and not config_path: - sys.exit('config not specified, please read ' - 'https://github.com/clowwindy/shadowsocks') + if not config: + logging.error('config not specified') + print_help(is_local) + sys.exit(2) config['password'] = config.get('password', None) - config['method'] = config.get('method', None) + config['method'] = config.get('method', 'aes-256-cfb') config['port_password'] = config.get('port_password', None) config['timeout'] = int(config.get('timeout', 300)) config['fast_open'] = config.get('fast_open', False) config['workers'] = config.get('workers', 1) config['verbose'] = config.get('verbose', False) config['local_address'] = config.get('local_address', '127.0.0.1') + config['local_port'] = config.get('local_port', 1080) + if is_local: + if config.get('server', None) is None: + logging.error('server addr not specified') + print_local_help() + sys.exit(2) + else: + config['server'] = config.get('server', '0.0.0.0') + config['server_port'] = config.get('server_port', 8388) + + if not ('password' in config and config['password']): + logging.error('password not specified') + print_help(is_local) + sys.exit(2) logging.getLogger('').handlers = [] logging.addLevelName(VERBOSE_LEVEL, 'VERBOSE') @@ -184,20 +198,27 @@ def get_config(is_local): return config +def print_help(is_local): + if is_local: + print_local_help() + else: + print_server_help() + + def print_local_help(): - print '''usage: sslocal [-h] -s SERVER_ADDR -p SERVER_PORT [-b LOCAL_ADDR] - -l LOCAL_PORT -k PASSWORD -m METHOD [-t TIMEOUT] [-c CONFIG] - [--fast-open] [-v] [-q] + print '''usage: sslocal [-h] -s SERVER_ADDR [-p SERVER_PORT] + [-b LOCAL_ADDR] [-l LOCAL_PORT] -k PASSWORD [-m METHOD] + [-t TIMEOUT] [-c CONFIG] [--fast-open] [-v] [-q] optional arguments: -h, --help show this help message and exit -s SERVER_ADDR server address - -p SERVER_PORT server port - -b LOCAL_ADDR local binding address, default is 127.0.0.1 - -l LOCAL_PORT local port + -p SERVER_PORT server port, default: 8388 + -b LOCAL_ADDR local binding address, default: 127.0.0.1 + -l LOCAL_PORT local port, default: 1080 -k PASSWORD password - -m METHOD encryption method, for example, aes-256-cfb - -t TIMEOUT timeout in seconds + -m METHOD encryption method, default: aes-256-cfb + -t TIMEOUT timeout in seconds, default: 300 -c CONFIG path to config file --fast-open use TCP_FASTOPEN, requires Linux 3.7+ -v, -vv verbose mode @@ -208,17 +229,17 @@ Online help: def print_server_help(): - print '''usage: ssserver [-h] -s SERVER_ADDR -p SERVER_PORT -k PASSWORD - -m METHOD [-t TIMEOUT] [-c CONFIG] [--fast-open] [--workders] - [-v] [-q] + print '''usage: ssserver [-h] [-s SERVER_ADDR] [-p SERVER_PORT] -k PASSWORD + -m METHOD [-t TIMEOUT] [-c CONFIG] [--fast-open] + [--workers WORKERS] [-v] [-q] optional arguments: -h, --help show this help message and exit - -s SERVER_ADDR server address - -p SERVER_PORT server port + -s SERVER_ADDR server address, default: 0.0.0.0 + -p SERVER_PORT server port, default: 8388 -k PASSWORD password - -m METHOD encryption method, for example, aes-256-cfb - -t TIMEOUT timeout in seconds + -m METHOD encryption method, default: aes-256-cfb + -t TIMEOUT timeout in seconds, default: 300 -c CONFIG path to config file --fast-open use TCP_FASTOPEN, requires Linux 3.7+ --workers WORKERS number of workers, available on Unix/Linux From 4c29b54b7d6693ddf64a2cc4bb0a4ea270379e35 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 1 Aug 2014 17:59:17 +0800 Subject: [PATCH 149/344] bump --- CHANGES | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 0e6c2b4..a75653c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +2.1.0 2014-08-01 +- Does not ship config.json +- Better error message + 2.0.12 2014-07-26 - Support -q quiet mode - Exit 0 when showing help with -h diff --git a/setup.py b/setup.py index b59b3df..be07821 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.0.12", + version="2.1.0", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From 8165904eafa629361cc4cf5f714d85d621d0330b Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 9 Aug 2014 14:53:01 +0800 Subject: [PATCH 150/344] bump 3.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index be07821..d611351 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.1.0", + version="3.0", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From e51aa4f137ff1066bbff35d30c4094abf9fafe63 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 10 Aug 2014 12:23:44 +0800 Subject: [PATCH 151/344] use only ipv4 dns server --- CHANGES | 3 +++ setup.py | 2 +- shadowsocks/asyncdns.py | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index a75653c..55f4f4a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +2.1.0 2014-08-10 +- Use only IPv4 DNS server + 2.1.0 2014-08-01 - Does not ship config.json - Better error message diff --git a/setup.py b/setup.py index d611351..be07821 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="3.0", + version="2.1.0", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index a3c276c..f745be7 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -232,7 +232,7 @@ def is_ip(address): for family in (socket.AF_INET, socket.AF_INET6): try: socket.inet_pton(family, address) - return True + return family except (TypeError, ValueError, OSError, IOError): pass return False @@ -289,7 +289,7 @@ class DNSResolver(object): parts = line.split() if len(parts) >= 2: server = parts[1] - if is_ip(server): + if is_ip(server) == socket.AF_INET: self._servers.append(server) except IOError: pass From c7d2d76767d419e0b7e6c8e0b2357a4e91f28514 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 3 Aug 2014 09:40:12 +0800 Subject: [PATCH 152/344] fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fcf091c..4784b9b 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ shadowsocks [![PyPI version]][PyPI] [![Build Status]][Travis CI] -A fast tunnel proxy that help you get through firewalls. +A fast tunnel proxy that helps you bypass firewalls. [中文说明][Chinese Readme] From 3b242bee5eb191599a0d051497003127986ea290 Mon Sep 17 00:00:00 2001 From: "Shell.Xu" Date: Sat, 23 Aug 2014 01:19:44 +0800 Subject: [PATCH 153/344] * debian inital package files. --- debian/README.Debian | 6 + debian/README.source | 9 ++ debian/changelog | 5 + debian/compat | 1 + debian/control | 15 ++ debian/copyright | 45 ++++++ debian/docs | 2 + debian/init.d.ex | 154 ++++++++++++++++++ debian/manpage.1.ex | 56 +++++++ debian/manpage.sgml.ex | 154 ++++++++++++++++++ debian/manpage.xml.ex | 291 ++++++++++++++++++++++++++++++++++ debian/menu.ex | 2 + debian/postinst.ex | 39 +++++ debian/postrm.ex | 37 +++++ debian/preinst.ex | 35 ++++ debian/prerm.ex | 38 +++++ debian/rules | 13 ++ debian/shadowsocks.default.ex | 10 ++ debian/source/format | 1 + debian/watch.ex | 23 +++ 20 files changed, 936 insertions(+) create mode 100644 debian/README.Debian create mode 100644 debian/README.source create mode 100644 debian/changelog create mode 100644 debian/compat create mode 100644 debian/control create mode 100644 debian/copyright create mode 100644 debian/docs create mode 100644 debian/init.d.ex create mode 100644 debian/manpage.1.ex create mode 100644 debian/manpage.sgml.ex create mode 100644 debian/manpage.xml.ex create mode 100644 debian/menu.ex create mode 100644 debian/postinst.ex create mode 100644 debian/postrm.ex create mode 100644 debian/preinst.ex create mode 100644 debian/prerm.ex create mode 100755 debian/rules create mode 100644 debian/shadowsocks.default.ex create mode 100644 debian/source/format create mode 100644 debian/watch.ex diff --git a/debian/README.Debian b/debian/README.Debian new file mode 100644 index 0000000..c3a2cad --- /dev/null +++ b/debian/README.Debian @@ -0,0 +1,6 @@ +shadowsocks for Debian +---------------------- + + + + -- Shell.Xu Sat, 23 Aug 2014 00:56:04 +0800 diff --git a/debian/README.source b/debian/README.source new file mode 100644 index 0000000..50cd37d --- /dev/null +++ b/debian/README.source @@ -0,0 +1,9 @@ +shadowsocks for Debian +---------------------- + + + + + + diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..445c1b6 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +shadowsocks (2.1.0-1) unstable; urgency=low + + * Initial release (Closes: 758900) + + -- Shell.Xu Sat, 23 Aug 2014 00:56:04 +0800 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..45a4fb7 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +8 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..3e6cad6 --- /dev/null +++ b/debian/control @@ -0,0 +1,15 @@ +Source: python-shadowsocks +Section: python +Priority: extra +Maintainer: Shell.Xu +Build-Depends: debhelper (>= 8), python-setuptools +Standards-Version: 3.9.3 +Homepage: https://github.com/clowwindy/shadowsocks +#Vcs-Git: git://git.debian.org/collab-maint/shadowsocks.git +#Vcs-Browser: http://git.debian.org/?p=collab-maint/shadowsocks.git;a=summary + +Package: python-shadowsocks +Architecture: all +Depends: ${shlibs:Depends}, ${misc:Depends}, python-m2crypto +Description: A fast tunnel proxy that helps you bypass firewalls + A secure socks5 proxy, designed to protect your Internet traffic. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..7be1b17 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,45 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: shadowsocks +Source: https://github.com/clowwindy/shadowsocks + +Files: * +Copyright: 2014 clowwindy +License: MIT + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + . + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + . + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +Files: debian/* +Copyright: 2014 Shell.Xu +License: MIT + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + . + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + . + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. diff --git a/debian/docs b/debian/docs new file mode 100644 index 0000000..0208fc1 --- /dev/null +++ b/debian/docs @@ -0,0 +1,2 @@ +README.md +README.rst diff --git a/debian/init.d.ex b/debian/init.d.ex new file mode 100644 index 0000000..8a14b5b --- /dev/null +++ b/debian/init.d.ex @@ -0,0 +1,154 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: shadowsocks +# Required-Start: $network $local_fs +# Required-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: +# Description: +# <...> +# <...> +### END INIT INFO + +# Author: Shell.Xu + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC=shadowsocks # Introduce a short description here +NAME=shadowsocks # Introduce the short server's name here +DAEMON=/usr/sbin/shadowsocks # Introduce the server's location here +DAEMON_ARGS="" # Arguments to run the daemon with +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME + +# Exit if the package is not installed +[ -x $DAEMON ] || exit 0 + +# Read configuration variable file if it is present +[ -r /etc/default/$NAME ] && . /etc/default/$NAME + +# Load the VERBOSE setting and other rcS variables +. /lib/init/vars.sh + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.0-6) to ensure that this file is present. +. /lib/lsb/init-functions + +# +# Function that starts the daemon/service +# +do_start() +{ + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON --test > /dev/null \ + || return 1 + start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON -- \ + $DAEMON_ARGS \ + || return 2 + # Add code here, if necessary, that waits for the process to be ready + # to handle requests from services started subsequently which depend + # on this one. As a last resort, sleep for some time. +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + # Return + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME + RETVAL="$?" + [ "$RETVAL" = 2 ] && return 2 + # Wait for children to finish too if this is a daemon that forks + # and if the daemon is only ever run from this initscript. + # If the above conditions are not satisfied then add some other code + # that waits for the process to drop all resources that could be + # needed by services started subsequently. A last resort is to + # sleep for some time. + start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --exec $DAEMON + [ "$?" = 2 ] && return 2 + # Many daemons don't delete their pidfiles when they exit. + rm -f $PIDFILE + return "$RETVAL" +} + +# +# Function that sends a SIGHUP to the daemon/service +# +do_reload() { + # + # If the daemon can reload its configuration without + # restarting (for example, when it is sent a SIGHUP), + # then implement that here. + # + start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name $NAME + return 0 +} + +case "$1" in + start) + [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC " "$NAME" + do_start + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + stop) + [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + status) + status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? + ;; + #reload|force-reload) + # + # If do_reload() is not implemented then leave this commented out + # and leave 'force-reload' as an alias for 'restart'. + # + #log_daemon_msg "Reloading $DESC" "$NAME" + #do_reload + #log_end_msg $? + #;; + restart|force-reload) + # + # If the "reload" option is implemented then remove the + # 'force-reload' alias + # + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac + +: diff --git a/debian/manpage.1.ex b/debian/manpage.1.ex new file mode 100644 index 0000000..d590040 --- /dev/null +++ b/debian/manpage.1.ex @@ -0,0 +1,56 @@ +.\" Hey, EMACS: -*- nroff -*- +.\" (C) Copyright 2014 Shell.Xu , +.\" +.\" First parameter, NAME, should be all caps +.\" Second parameter, SECTION, should be 1-8, maybe w/ subsection +.\" other parameters are allowed: see man(7), man(1) +.TH SHADOWSOCKS SECTION "August 23, 2014" +.\" Please adjust this date whenever revising the manpage. +.\" +.\" Some roff macros, for reference: +.\" .nh disable hyphenation +.\" .hy enable hyphenation +.\" .ad l left justify +.\" .ad b justify to both left and right margins +.\" .nf disable filling +.\" .fi enable filling +.\" .br insert line break +.\" .sp insert n+1 empty lines +.\" for manpage-specific macros, see man(7) +.SH NAME +shadowsocks \- program to do something +.SH SYNOPSIS +.B shadowsocks +.RI [ options ] " files" ... +.br +.B bar +.RI [ options ] " files" ... +.SH DESCRIPTION +This manual page documents briefly the +.B shadowsocks +and +.B bar +commands. +.PP +.\" TeX users may be more comfortable with the \fB\fP and +.\" \fI\fP escape sequences to invode bold face and italics, +.\" respectively. +\fBshadowsocks\fP is a program that... +.SH OPTIONS +These programs follow the usual GNU command line syntax, with long +options starting with two dashes (`-'). +A summary of options is included below. +For a complete description, see the Info files. +.TP +.B \-h, \-\-help +Show summary of options. +.TP +.B \-v, \-\-version +Show version of program. +.SH SEE ALSO +.BR bar (1), +.BR baz (1). +.br +The programs are documented fully by +.IR "The Rise and Fall of a Fooish Bar" , +available via the Info system. diff --git a/debian/manpage.sgml.ex b/debian/manpage.sgml.ex new file mode 100644 index 0000000..7e4df62 --- /dev/null +++ b/debian/manpage.sgml.ex @@ -0,0 +1,154 @@ + manpage.1'. You may view + the manual page with: `docbook-to-man manpage.sgml | nroff -man | + less'. A typical entry in a Makefile or Makefile.am is: + +manpage.1: manpage.sgml + docbook-to-man $< > $@ + + + The docbook-to-man binary is found in the docbook-to-man package. + Please remember that if you create the nroff version in one of the + debian/rules file targets (such as build), you will need to include + docbook-to-man in your Build-Depends control field. + + --> + + + FIRSTNAME"> + SURNAME"> + + August 23, 2014"> + + SECTION"> + shell909090@gmail.com"> + + SHADOWSOCKS"> + + + Debian"> + GNU"> + GPL"> +]> + + + +
+ &dhemail; +
+ + &dhfirstname; + &dhsurname; + + + 2003 + &dhusername; + + &dhdate; + + + &dhucpackage; + + &dhsection; + + + &dhpackage; + + program to do something + + + + &dhpackage; + + + + + + + + DESCRIPTION + + This manual page documents briefly the + &dhpackage; and bar + commands. + + This manual page was written for the &debian; distribution + because the original program does not have a manual page. + Instead, it has documentation in the &gnu; + Info format; see below. + + &dhpackage; is a program that... + + + + OPTIONS + + These programs follow the usual &gnu; command line syntax, + with long options starting with two dashes (`-'). A summary of + options is included below. For a complete description, see the + Info files. + + + + + + + + Show summary of options. + + + + + + + + Show version of program. + + + + + + SEE ALSO + + bar (1), baz (1). + + The programs are documented fully by The Rise and + Fall of a Fooish Bar available via the + Info system. + + + AUTHOR + + This manual page was written by &dhusername; &dhemail; for + the &debian; system (and may be used by others). Permission is + granted to copy, distribute and/or modify this document under + the terms of the &gnu; General Public License, Version 2 any + later version published by the Free Software Foundation. + + + On Debian systems, the complete text of the GNU General Public + License can be found in /usr/share/common-licenses/GPL. + + + + + + diff --git a/debian/manpage.xml.ex b/debian/manpage.xml.ex new file mode 100644 index 0000000..b806072 --- /dev/null +++ b/debian/manpage.xml.ex @@ -0,0 +1,291 @@ + +.
will be generated. You may view the +manual page with: nroff -man .
| less'. A typical entry +in a Makefile or Makefile.am is: + +DB2MAN = /usr/share/sgml/docbook/stylesheet/xsl/docbook-xsl/manpages/docbook.xsl +XP = xsltproc -''-nonet -''-param man.charmap.use.subset "0" + +manpage.1: manpage.xml + $(XP) $(DB2MAN) $< + +The xsltproc binary is found in the xsltproc package. The XSL files are in +docbook-xsl. A description of the parameters you can use can be found in the +docbook-xsl-doc-* packages. Please remember that if you create the nroff +version in one of the debian/rules file targets (such as build), you will need +to include xsltproc and docbook-xsl in your Build-Depends control field. +Alternatively use the xmlto command/package. That will also automatically +pull in xsltproc and docbook-xsl. + +Notes for using docbook2x: docbook2x-man does not automatically create the +AUTHOR(S) and COPYRIGHT sections. In this case, please add them manually as + ... . + +To disable the automatic creation of the AUTHOR(S) and COPYRIGHT sections +read /usr/share/doc/docbook-xsl/doc/manpages/authors.html. This file can be +found in the docbook-xsl-doc-html package. + +Validation can be done using: `xmllint -''-noout -''-valid manpage.xml` + +General documentation about man-pages and man-page-formatting: +man(1), man(7), http://www.tldp.org/HOWTO/Man-Page/ + +--> + + + + + + + + + + + + + +]> + + + + &dhtitle; + &dhpackage; + + + &dhfirstname; + &dhsurname; + Wrote this manpage for the Debian system. +
+ &dhemail; +
+ + + + 2007 + &dhusername; + + + This manual page was written for the Debian system + (and may be used by others). + Permission is granted to copy, distribute and/or modify this + document under the terms of the GNU General Public License, + Version 2 or (at your option) any later version published by + the Free Software Foundation. + On Debian systems, the complete text of the GNU General Public + License can be found in + /usr/share/common-licenses/GPL. + + + + &dhucpackage; + &dhsection; + + + &dhpackage; + program to do something + + + + &dhpackage; + + + + + + + + + this + + + + + + + + this + that + + + + + &dhpackage; + + + + + + + + + + + + + + + + + + + DESCRIPTION + This manual page documents briefly the + &dhpackage; and bar + commands. + This manual page was written for the Debian distribution + because the original program does not have a manual page. + Instead, it has documentation in the GNU + info + 1 + format; see below. + &dhpackage; is a program that... + + + OPTIONS + The program follows the usual GNU command line syntax, + with long options starting with two dashes (`-'). A summary of + options is included below. For a complete description, see the + + info + 1 + files. + + + + + + + Does this and that. + + + + + + + Show summary of options. + + + + + + + Show version of program. + + + + + + FILES + + + /etc/foo.conf + + The system-wide configuration file to control the + behaviour of &dhpackage;. See + + foo.conf + 5 + for further details. + + + + ${HOME}/.foo.conf + + The per-user configuration file to control the + behaviour of &dhpackage;. See + + foo.conf + 5 + for further details. + + + + + + ENVIRONMENT + + + FOO_CONF + + If used, the defined file is used as configuration + file (see also ). + + + + + + DIAGNOSTICS + The following diagnostics may be issued + on stderr: + + + Bad configuration file. Exiting. + + The configuration file seems to contain a broken configuration + line. Use the option, to get more info. + + + + + &dhpackage; provides some return codes, that can + be used in scripts: + + Code + Diagnostic + + 0 + Program exited successfully. + + + 1 + The configuration file seems to be broken. + + + + + + BUGS + The program is currently limited to only work + with the foobar library. + The upstreams BTS can be found + at . + + + SEE ALSO + + + bar + 1 + , + baz + 1 + , + foo.conf + 5 + + The programs are documented fully by The Rise and + Fall of a Fooish Bar available via the + info + 1 + system. + + + diff --git a/debian/menu.ex b/debian/menu.ex new file mode 100644 index 0000000..52519c7 --- /dev/null +++ b/debian/menu.ex @@ -0,0 +1,2 @@ +?package(shadowsocks):needs="X11|text|vc|wm" section="Applications/see-menu-manual"\ + title="shadowsocks" command="/usr/bin/shadowsocks" diff --git a/debian/postinst.ex b/debian/postinst.ex new file mode 100644 index 0000000..c520dc5 --- /dev/null +++ b/debian/postinst.ex @@ -0,0 +1,39 @@ +#!/bin/sh +# postinst script for shadowsocks +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `configure' +# * `abort-upgrade' +# * `abort-remove' `in-favour' +# +# * `abort-remove' +# * `abort-deconfigure' `in-favour' +# `removing' +# +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + configure) + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/debian/postrm.ex b/debian/postrm.ex new file mode 100644 index 0000000..180abc0 --- /dev/null +++ b/debian/postrm.ex @@ -0,0 +1,37 @@ +#!/bin/sh +# postrm script for shadowsocks +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `remove' +# * `purge' +# * `upgrade' +# * `failed-upgrade' +# * `abort-install' +# * `abort-install' +# * `abort-upgrade' +# * `disappear' +# +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + purge|remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) + ;; + + *) + echo "postrm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/debian/preinst.ex b/debian/preinst.ex new file mode 100644 index 0000000..2306c07 --- /dev/null +++ b/debian/preinst.ex @@ -0,0 +1,35 @@ +#!/bin/sh +# preinst script for shadowsocks +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `install' +# * `install' +# * `upgrade' +# * `abort-upgrade' +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + install|upgrade) + ;; + + abort-upgrade) + ;; + + *) + echo "preinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/debian/prerm.ex b/debian/prerm.ex new file mode 100644 index 0000000..13a5919 --- /dev/null +++ b/debian/prerm.ex @@ -0,0 +1,38 @@ +#!/bin/sh +# prerm script for shadowsocks +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `remove' +# * `upgrade' +# * `failed-upgrade' +# * `remove' `in-favour' +# * `deconfigure' `in-favour' +# `removing' +# +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + remove|upgrade|deconfigure) + ;; + + failed-upgrade) + ;; + + *) + echo "prerm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..4556a26 --- /dev/null +++ b/debian/rules @@ -0,0 +1,13 @@ +#!/usr/bin/make -f +# -*- makefile -*- +# Sample debian/rules that uses debhelper. +# This file was originally written by Joey Hess and Craig Small. +# As a special exception, when this file is copied by dh-make into a +# dh-make output file, you may use that output file without restriction. +# This special exception was added by Craig Small in version 0.37 of dh-make. + +# Uncomment this to turn on verbose mode. +#export DH_VERBOSE=1 + +%: + dh --with python2 --buildsystem=python_distutils $@ diff --git a/debian/shadowsocks.default.ex b/debian/shadowsocks.default.ex new file mode 100644 index 0000000..24334b7 --- /dev/null +++ b/debian/shadowsocks.default.ex @@ -0,0 +1,10 @@ +# Defaults for shadowsocks initscript +# sourced by /etc/init.d/shadowsocks +# installed at /etc/default/shadowsocks by the maintainer scripts + +# +# This is a POSIX shell fragment +# + +# Additional options that are passed to the Daemon. +DAEMON_OPTS="" diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/debian/watch.ex b/debian/watch.ex new file mode 100644 index 0000000..df7c3be --- /dev/null +++ b/debian/watch.ex @@ -0,0 +1,23 @@ +# Example watch control file for uscan +# Rename this file to "watch" and then you can run the "uscan" command +# to check for upstream updates and more. +# See uscan(1) for format + +# Compulsory line, this is a version 3 file +version=3 + +# Uncomment to examine a Webpage +# +#http://www.example.com/downloads.php shadowsocks-(.*)\.tar\.gz + +# Uncomment to examine a Webserver directory +#http://www.example.com/pub/shadowsocks-(.*)\.tar\.gz + +# Uncommment to examine a FTP server +#ftp://ftp.example.com/pub/shadowsocks-(.*)\.tar\.gz debian uupdate + +# Uncomment to find new files on sourceforge, for devscripts >= 2.9 +# http://sf.net/shadowsocks/shadowsocks-(.*)\.tar\.gz + +# Uncomment to find new files on GooglePages +# http://example.googlepages.com/foo.html shadowsocks-(.*)\.tar\.gz From 48780fed2422ca79ca4a078f702460647f68c0f4 Mon Sep 17 00:00:00 2001 From: "Shell.Xu" Date: Sat, 23 Aug 2014 01:51:46 +0800 Subject: [PATCH 154/344] * adjust some config, look almostly done. --- debian/README.Debian | 6 --- debian/README.source | 9 ----- debian/changelog | 2 +- debian/config.json | 11 ++++++ debian/control | 9 +++-- debian/{init.d.ex => init.d} | 16 ++++---- debian/install | 1 + debian/menu.ex | 2 - debian/postinst.ex | 39 ------------------- debian/postrm.ex | 37 ------------------ debian/preinst.ex | 35 ----------------- debian/prerm.ex | 38 ------------------ debian/rules | 2 +- ...owsocks.default.ex => shadowsocks.default} | 2 +- 14 files changed, 29 insertions(+), 180 deletions(-) delete mode 100644 debian/README.Debian delete mode 100644 debian/README.source create mode 100644 debian/config.json rename debian/{init.d.ex => init.d} (88%) create mode 100644 debian/install delete mode 100644 debian/menu.ex delete mode 100644 debian/postinst.ex delete mode 100644 debian/postrm.ex delete mode 100644 debian/preinst.ex delete mode 100644 debian/prerm.ex rename debian/{shadowsocks.default.ex => shadowsocks.default} (83%) diff --git a/debian/README.Debian b/debian/README.Debian deleted file mode 100644 index c3a2cad..0000000 --- a/debian/README.Debian +++ /dev/null @@ -1,6 +0,0 @@ -shadowsocks for Debian ----------------------- - - - - -- Shell.Xu Sat, 23 Aug 2014 00:56:04 +0800 diff --git a/debian/README.source b/debian/README.source deleted file mode 100644 index 50cd37d..0000000 --- a/debian/README.source +++ /dev/null @@ -1,9 +0,0 @@ -shadowsocks for Debian ----------------------- - - - - - - diff --git a/debian/changelog b/debian/changelog index 445c1b6..323b8f1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -shadowsocks (2.1.0-1) unstable; urgency=low +python-shadowsocks (2.1.0-1) unstable; urgency=low * Initial release (Closes: 758900) diff --git a/debian/config.json b/debian/config.json new file mode 100644 index 0000000..35cb14a --- /dev/null +++ b/debian/config.json @@ -0,0 +1,11 @@ +{ + "server":"my_server_ip", + "server_port":8388, + "local_address": "127.0.0.1", + "local_port":1080, + "password":"mypassword", + "timeout":300, + "method":"aes-256-cfb", + "fast_open": false, + "workers": 1 +} \ No newline at end of file diff --git a/debian/control b/debian/control index 3e6cad6..85f144c 100644 --- a/debian/control +++ b/debian/control @@ -2,7 +2,7 @@ Source: python-shadowsocks Section: python Priority: extra Maintainer: Shell.Xu -Build-Depends: debhelper (>= 8), python-setuptools +Build-Depends: debhelper (>= 8), python, python-setuptools Standards-Version: 3.9.3 Homepage: https://github.com/clowwindy/shadowsocks #Vcs-Git: git://git.debian.org/collab-maint/shadowsocks.git @@ -10,6 +10,9 @@ Homepage: https://github.com/clowwindy/shadowsocks Package: python-shadowsocks Architecture: all -Depends: ${shlibs:Depends}, ${misc:Depends}, python-m2crypto -Description: A fast tunnel proxy that helps you bypass firewalls +Depends: ${shlibs:Depends}, ${misc:Depends}, python, python-m2crypto +Description: Fast tunnel proxy that helps you bypass firewalls A secure socks5 proxy, designed to protect your Internet traffic. + . + This package contain local and server part of shadowsocks, a fast, + powerful tunnel proxy to bypass firewalls. \ No newline at end of file diff --git a/debian/init.d.ex b/debian/init.d similarity index 88% rename from debian/init.d.ex rename to debian/init.d index 8a14b5b..1413cbd 100644 --- a/debian/init.d.ex +++ b/debian/init.d @@ -1,14 +1,14 @@ #!/bin/sh ### BEGIN INIT INFO # Provides: shadowsocks -# Required-Start: $network $local_fs -# Required-Stop: +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 -# Short-Description: -# Description: -# <...> -# <...> +# Short-Description: Fast tunnel proxy that helps you bypass firewalls +# Description: A secure socks5 proxy, designed to protect your Internet traffic. +# This package contain local and server part of shadowsocks, a fast, +# powerful tunnel proxy to bypass firewalls. ### END INIT INFO # Author: Shell.Xu @@ -17,8 +17,8 @@ PATH=/sbin:/usr/sbin:/bin:/usr/bin DESC=shadowsocks # Introduce a short description here NAME=shadowsocks # Introduce the short server's name here -DAEMON=/usr/sbin/shadowsocks # Introduce the server's location here -DAEMON_ARGS="" # Arguments to run the daemon with +DAEMON=/usr/bin/ssserver # Introduce the server's location here +DAEMON_ARGS="" # Arguments to run the daemon with PIDFILE=/var/run/$NAME.pid SCRIPTNAME=/etc/init.d/$NAME diff --git a/debian/install b/debian/install new file mode 100644 index 0000000..a614864 --- /dev/null +++ b/debian/install @@ -0,0 +1 @@ +debian/config.json etc/shadowsocks/ \ No newline at end of file diff --git a/debian/menu.ex b/debian/menu.ex deleted file mode 100644 index 52519c7..0000000 --- a/debian/menu.ex +++ /dev/null @@ -1,2 +0,0 @@ -?package(shadowsocks):needs="X11|text|vc|wm" section="Applications/see-menu-manual"\ - title="shadowsocks" command="/usr/bin/shadowsocks" diff --git a/debian/postinst.ex b/debian/postinst.ex deleted file mode 100644 index c520dc5..0000000 --- a/debian/postinst.ex +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh -# postinst script for shadowsocks -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `configure' -# * `abort-upgrade' -# * `abort-remove' `in-favour' -# -# * `abort-remove' -# * `abort-deconfigure' `in-favour' -# `removing' -# -# for details, see http://www.debian.org/doc/debian-policy/ or -# the debian-policy package - - -case "$1" in - configure) - ;; - - abort-upgrade|abort-remove|abort-deconfigure) - ;; - - *) - echo "postinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/postrm.ex b/debian/postrm.ex deleted file mode 100644 index 180abc0..0000000 --- a/debian/postrm.ex +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# postrm script for shadowsocks -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `remove' -# * `purge' -# * `upgrade' -# * `failed-upgrade' -# * `abort-install' -# * `abort-install' -# * `abort-upgrade' -# * `disappear' -# -# for details, see http://www.debian.org/doc/debian-policy/ or -# the debian-policy package - - -case "$1" in - purge|remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) - ;; - - *) - echo "postrm called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/preinst.ex b/debian/preinst.ex deleted file mode 100644 index 2306c07..0000000 --- a/debian/preinst.ex +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# preinst script for shadowsocks -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `install' -# * `install' -# * `upgrade' -# * `abort-upgrade' -# for details, see http://www.debian.org/doc/debian-policy/ or -# the debian-policy package - - -case "$1" in - install|upgrade) - ;; - - abort-upgrade) - ;; - - *) - echo "preinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/prerm.ex b/debian/prerm.ex deleted file mode 100644 index 13a5919..0000000 --- a/debian/prerm.ex +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/sh -# prerm script for shadowsocks -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `remove' -# * `upgrade' -# * `failed-upgrade' -# * `remove' `in-favour' -# * `deconfigure' `in-favour' -# `removing' -# -# for details, see http://www.debian.org/doc/debian-policy/ or -# the debian-policy package - - -case "$1" in - remove|upgrade|deconfigure) - ;; - - failed-upgrade) - ;; - - *) - echo "prerm called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/rules b/debian/rules index 4556a26..01e5f9b 100755 --- a/debian/rules +++ b/debian/rules @@ -10,4 +10,4 @@ #export DH_VERBOSE=1 %: - dh --with python2 --buildsystem=python_distutils $@ + dh $@ --with python2 --buildsystem=python_distutils diff --git a/debian/shadowsocks.default.ex b/debian/shadowsocks.default similarity index 83% rename from debian/shadowsocks.default.ex rename to debian/shadowsocks.default index 24334b7..62831c2 100644 --- a/debian/shadowsocks.default.ex +++ b/debian/shadowsocks.default @@ -7,4 +7,4 @@ # # Additional options that are passed to the Daemon. -DAEMON_OPTS="" +DAEMON_OPTS="-c /etc/shadowsocks/config.json" From 8d74b6276ece4f95e80d3dede38f95717c4e744b Mon Sep 17 00:00:00 2001 From: "Shell.Xu" Date: Sat, 23 Aug 2014 14:41:45 +0800 Subject: [PATCH 155/344] + add manpage --- debian/changelog | 2 +- debian/control | 4 +- debian/manpage.1 | 59 +++++++++ debian/manpage.1.ex | 56 -------- debian/manpage.sgml.ex | 154 ---------------------- debian/manpage.xml.ex | 291 ----------------------------------------- 6 files changed, 62 insertions(+), 504 deletions(-) create mode 100644 debian/manpage.1 delete mode 100644 debian/manpage.1.ex delete mode 100644 debian/manpage.sgml.ex delete mode 100644 debian/manpage.xml.ex diff --git a/debian/changelog b/debian/changelog index 323b8f1..445c1b6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -python-shadowsocks (2.1.0-1) unstable; urgency=low +shadowsocks (2.1.0-1) unstable; urgency=low * Initial release (Closes: 758900) diff --git a/debian/control b/debian/control index 85f144c..c8c1549 100644 --- a/debian/control +++ b/debian/control @@ -1,4 +1,4 @@ -Source: python-shadowsocks +Source: shadowsocks Section: python Priority: extra Maintainer: Shell.Xu @@ -8,7 +8,7 @@ Homepage: https://github.com/clowwindy/shadowsocks #Vcs-Git: git://git.debian.org/collab-maint/shadowsocks.git #Vcs-Browser: http://git.debian.org/?p=collab-maint/shadowsocks.git;a=summary -Package: python-shadowsocks +Package: shadowsocks Architecture: all Depends: ${shlibs:Depends}, ${misc:Depends}, python, python-m2crypto Description: Fast tunnel proxy that helps you bypass firewalls diff --git a/debian/manpage.1 b/debian/manpage.1 new file mode 100644 index 0000000..f0a9978 --- /dev/null +++ b/debian/manpage.1 @@ -0,0 +1,59 @@ +.\" Hey, EMACS: -*- nroff -*- +.\" (C) Copyright 2014 Shell.Xu , +.\" +.TH SHADOWSOCKS SECTION "August 23, 2014" +.SH NAME +shadowsocks \- Fast tunnel proxy that helps you bypass firewalls +.SH SYNOPSIS +.B ssserver +.RI [ options ] +.br +.B sslocal +.RI [ options ] +.SH DESCRIPTION +shadowsocks is a tunnel proxy helps you bypass firewall. +.B ssserver +is the server part, and +.B sslocal +is the local part. +.SH OPTIONS +.TP +.B \-h, \-\-help +Show this help message and exit. +.TP +.B \-s SERVER_ADDR +Server address, default: 0.0.0.0. +.TP +.B \-p SERVER_PORT +Server port, default: 8388. +.TP +.B \-k PASSWORD +Password. +.TP +.B \-m METHOD +Encryption method, default: aes-256-cfb. +.TP +.B \-t TIMEOUT +Timeout in seconds, default: 300. +.TP +.B \-c CONFIG +Path to config file. +.TP +.B \-\-fast-open +Use TCP_FASTOPEN, requires Linux 3.7+. +.TP +.B \-\-workers WORKERS +Number of workers, available on Unix/Linux. +.TP +.B \-v, \-vv +Verbose mode. +.TP +.B \-q, \-qq +Quiet mode, only show warnings/errors. +.SH SEE ALSO +.br +The programs are documented fully by +.IR "Shell Xu " +and +.IR "Clowwindy ", +available via the Info system. diff --git a/debian/manpage.1.ex b/debian/manpage.1.ex deleted file mode 100644 index d590040..0000000 --- a/debian/manpage.1.ex +++ /dev/null @@ -1,56 +0,0 @@ -.\" Hey, EMACS: -*- nroff -*- -.\" (C) Copyright 2014 Shell.Xu , -.\" -.\" First parameter, NAME, should be all caps -.\" Second parameter, SECTION, should be 1-8, maybe w/ subsection -.\" other parameters are allowed: see man(7), man(1) -.TH SHADOWSOCKS SECTION "August 23, 2014" -.\" Please adjust this date whenever revising the manpage. -.\" -.\" Some roff macros, for reference: -.\" .nh disable hyphenation -.\" .hy enable hyphenation -.\" .ad l left justify -.\" .ad b justify to both left and right margins -.\" .nf disable filling -.\" .fi enable filling -.\" .br insert line break -.\" .sp insert n+1 empty lines -.\" for manpage-specific macros, see man(7) -.SH NAME -shadowsocks \- program to do something -.SH SYNOPSIS -.B shadowsocks -.RI [ options ] " files" ... -.br -.B bar -.RI [ options ] " files" ... -.SH DESCRIPTION -This manual page documents briefly the -.B shadowsocks -and -.B bar -commands. -.PP -.\" TeX users may be more comfortable with the \fB\fP and -.\" \fI\fP escape sequences to invode bold face and italics, -.\" respectively. -\fBshadowsocks\fP is a program that... -.SH OPTIONS -These programs follow the usual GNU command line syntax, with long -options starting with two dashes (`-'). -A summary of options is included below. -For a complete description, see the Info files. -.TP -.B \-h, \-\-help -Show summary of options. -.TP -.B \-v, \-\-version -Show version of program. -.SH SEE ALSO -.BR bar (1), -.BR baz (1). -.br -The programs are documented fully by -.IR "The Rise and Fall of a Fooish Bar" , -available via the Info system. diff --git a/debian/manpage.sgml.ex b/debian/manpage.sgml.ex deleted file mode 100644 index 7e4df62..0000000 --- a/debian/manpage.sgml.ex +++ /dev/null @@ -1,154 +0,0 @@ - manpage.1'. You may view - the manual page with: `docbook-to-man manpage.sgml | nroff -man | - less'. A typical entry in a Makefile or Makefile.am is: - -manpage.1: manpage.sgml - docbook-to-man $< > $@ - - - The docbook-to-man binary is found in the docbook-to-man package. - Please remember that if you create the nroff version in one of the - debian/rules file targets (such as build), you will need to include - docbook-to-man in your Build-Depends control field. - - --> - - - FIRSTNAME"> - SURNAME"> - - August 23, 2014"> - - SECTION"> - shell909090@gmail.com"> - - SHADOWSOCKS"> - - - Debian"> - GNU"> - GPL"> -]> - - - -
- &dhemail; -
- - &dhfirstname; - &dhsurname; - - - 2003 - &dhusername; - - &dhdate; - - - &dhucpackage; - - &dhsection; - - - &dhpackage; - - program to do something - - - - &dhpackage; - - - - - - - - DESCRIPTION - - This manual page documents briefly the - &dhpackage; and bar - commands. - - This manual page was written for the &debian; distribution - because the original program does not have a manual page. - Instead, it has documentation in the &gnu; - Info format; see below. - - &dhpackage; is a program that... - - - - OPTIONS - - These programs follow the usual &gnu; command line syntax, - with long options starting with two dashes (`-'). A summary of - options is included below. For a complete description, see the - Info files. - - - - - - - - Show summary of options. - - - - - - - - Show version of program. - - - - - - SEE ALSO - - bar (1), baz (1). - - The programs are documented fully by The Rise and - Fall of a Fooish Bar available via the - Info system. - - - AUTHOR - - This manual page was written by &dhusername; &dhemail; for - the &debian; system (and may be used by others). Permission is - granted to copy, distribute and/or modify this document under - the terms of the &gnu; General Public License, Version 2 any - later version published by the Free Software Foundation. - - - On Debian systems, the complete text of the GNU General Public - License can be found in /usr/share/common-licenses/GPL. - - - - - - diff --git a/debian/manpage.xml.ex b/debian/manpage.xml.ex deleted file mode 100644 index b806072..0000000 --- a/debian/manpage.xml.ex +++ /dev/null @@ -1,291 +0,0 @@ - -.
will be generated. You may view the -manual page with: nroff -man .
| less'. A typical entry -in a Makefile or Makefile.am is: - -DB2MAN = /usr/share/sgml/docbook/stylesheet/xsl/docbook-xsl/manpages/docbook.xsl -XP = xsltproc -''-nonet -''-param man.charmap.use.subset "0" - -manpage.1: manpage.xml - $(XP) $(DB2MAN) $< - -The xsltproc binary is found in the xsltproc package. The XSL files are in -docbook-xsl. A description of the parameters you can use can be found in the -docbook-xsl-doc-* packages. Please remember that if you create the nroff -version in one of the debian/rules file targets (such as build), you will need -to include xsltproc and docbook-xsl in your Build-Depends control field. -Alternatively use the xmlto command/package. That will also automatically -pull in xsltproc and docbook-xsl. - -Notes for using docbook2x: docbook2x-man does not automatically create the -AUTHOR(S) and COPYRIGHT sections. In this case, please add them manually as - ... . - -To disable the automatic creation of the AUTHOR(S) and COPYRIGHT sections -read /usr/share/doc/docbook-xsl/doc/manpages/authors.html. This file can be -found in the docbook-xsl-doc-html package. - -Validation can be done using: `xmllint -''-noout -''-valid manpage.xml` - -General documentation about man-pages and man-page-formatting: -man(1), man(7), http://www.tldp.org/HOWTO/Man-Page/ - ---> - - - - - - - - - - - - - -]> - - - - &dhtitle; - &dhpackage; - - - &dhfirstname; - &dhsurname; - Wrote this manpage for the Debian system. -
- &dhemail; -
- - - - 2007 - &dhusername; - - - This manual page was written for the Debian system - (and may be used by others). - Permission is granted to copy, distribute and/or modify this - document under the terms of the GNU General Public License, - Version 2 or (at your option) any later version published by - the Free Software Foundation. - On Debian systems, the complete text of the GNU General Public - License can be found in - /usr/share/common-licenses/GPL. - - - - &dhucpackage; - &dhsection; - - - &dhpackage; - program to do something - - - - &dhpackage; - - - - - - - - - this - - - - - - - - this - that - - - - - &dhpackage; - - - - - - - - - - - - - - - - - - - DESCRIPTION - This manual page documents briefly the - &dhpackage; and bar - commands. - This manual page was written for the Debian distribution - because the original program does not have a manual page. - Instead, it has documentation in the GNU - info - 1 - format; see below. - &dhpackage; is a program that... - - - OPTIONS - The program follows the usual GNU command line syntax, - with long options starting with two dashes (`-'). A summary of - options is included below. For a complete description, see the - - info - 1 - files. - - - - - - - Does this and that. - - - - - - - Show summary of options. - - - - - - - Show version of program. - - - - - - FILES - - - /etc/foo.conf - - The system-wide configuration file to control the - behaviour of &dhpackage;. See - - foo.conf - 5 - for further details. - - - - ${HOME}/.foo.conf - - The per-user configuration file to control the - behaviour of &dhpackage;. See - - foo.conf - 5 - for further details. - - - - - - ENVIRONMENT - - - FOO_CONF - - If used, the defined file is used as configuration - file (see also ). - - - - - - DIAGNOSTICS - The following diagnostics may be issued - on stderr: - - - Bad configuration file. Exiting. - - The configuration file seems to contain a broken configuration - line. Use the option, to get more info. - - - - - &dhpackage; provides some return codes, that can - be used in scripts: - - Code - Diagnostic - - 0 - Program exited successfully. - - - 1 - The configuration file seems to be broken. - - - - - - BUGS - The program is currently limited to only work - with the foobar library. - The upstreams BTS can be found - at . - - - SEE ALSO - - - bar - 1 - , - baz - 1 - , - foo.conf - 5 - - The programs are documented fully by The Rise and - Fall of a Fooish Bar available via the - info - 1 - system. - - - From eafc3ab5472f6e909effa4ef8853ca3c5dc38b5e Mon Sep 17 00:00:00 2001 From: "Shell.Xu" Date: Sat, 23 Aug 2014 14:50:57 +0800 Subject: [PATCH 156/344] * clean all lintian errors and warnings --- debian/shadowsocks.manpages | 2 ++ debian/{manpage.1 => sslocal.1} | 2 +- debian/ssserver.1 | 59 +++++++++++++++++++++++++++++++++ debian/watch.ex | 23 ------------- 4 files changed, 62 insertions(+), 24 deletions(-) create mode 100644 debian/shadowsocks.manpages rename debian/{manpage.1 => sslocal.1} (96%) create mode 100644 debian/ssserver.1 delete mode 100644 debian/watch.ex diff --git a/debian/shadowsocks.manpages b/debian/shadowsocks.manpages new file mode 100644 index 0000000..3df8a33 --- /dev/null +++ b/debian/shadowsocks.manpages @@ -0,0 +1,2 @@ +debian/sslocal.1 +debian/ssserver.1 \ No newline at end of file diff --git a/debian/manpage.1 b/debian/sslocal.1 similarity index 96% rename from debian/manpage.1 rename to debian/sslocal.1 index f0a9978..0c2cf51 100644 --- a/debian/manpage.1 +++ b/debian/sslocal.1 @@ -1,7 +1,7 @@ .\" Hey, EMACS: -*- nroff -*- .\" (C) Copyright 2014 Shell.Xu , .\" -.TH SHADOWSOCKS SECTION "August 23, 2014" +.TH SHADOWSOCKS 1 "August 23, 2014" .SH NAME shadowsocks \- Fast tunnel proxy that helps you bypass firewalls .SH SYNOPSIS diff --git a/debian/ssserver.1 b/debian/ssserver.1 new file mode 100644 index 0000000..0c2cf51 --- /dev/null +++ b/debian/ssserver.1 @@ -0,0 +1,59 @@ +.\" Hey, EMACS: -*- nroff -*- +.\" (C) Copyright 2014 Shell.Xu , +.\" +.TH SHADOWSOCKS 1 "August 23, 2014" +.SH NAME +shadowsocks \- Fast tunnel proxy that helps you bypass firewalls +.SH SYNOPSIS +.B ssserver +.RI [ options ] +.br +.B sslocal +.RI [ options ] +.SH DESCRIPTION +shadowsocks is a tunnel proxy helps you bypass firewall. +.B ssserver +is the server part, and +.B sslocal +is the local part. +.SH OPTIONS +.TP +.B \-h, \-\-help +Show this help message and exit. +.TP +.B \-s SERVER_ADDR +Server address, default: 0.0.0.0. +.TP +.B \-p SERVER_PORT +Server port, default: 8388. +.TP +.B \-k PASSWORD +Password. +.TP +.B \-m METHOD +Encryption method, default: aes-256-cfb. +.TP +.B \-t TIMEOUT +Timeout in seconds, default: 300. +.TP +.B \-c CONFIG +Path to config file. +.TP +.B \-\-fast-open +Use TCP_FASTOPEN, requires Linux 3.7+. +.TP +.B \-\-workers WORKERS +Number of workers, available on Unix/Linux. +.TP +.B \-v, \-vv +Verbose mode. +.TP +.B \-q, \-qq +Quiet mode, only show warnings/errors. +.SH SEE ALSO +.br +The programs are documented fully by +.IR "Shell Xu " +and +.IR "Clowwindy ", +available via the Info system. diff --git a/debian/watch.ex b/debian/watch.ex deleted file mode 100644 index df7c3be..0000000 --- a/debian/watch.ex +++ /dev/null @@ -1,23 +0,0 @@ -# Example watch control file for uscan -# Rename this file to "watch" and then you can run the "uscan" command -# to check for upstream updates and more. -# See uscan(1) for format - -# Compulsory line, this is a version 3 file -version=3 - -# Uncomment to examine a Webpage -# -#http://www.example.com/downloads.php shadowsocks-(.*)\.tar\.gz - -# Uncomment to examine a Webserver directory -#http://www.example.com/pub/shadowsocks-(.*)\.tar\.gz - -# Uncommment to examine a FTP server -#ftp://ftp.example.com/pub/shadowsocks-(.*)\.tar\.gz debian uupdate - -# Uncomment to find new files on sourceforge, for devscripts >= 2.9 -# http://sf.net/shadowsocks/shadowsocks-(.*)\.tar\.gz - -# Uncomment to find new files on GooglePages -# http://example.googlepages.com/foo.html shadowsocks-(.*)\.tar\.gz From 858dd5a62e2651178113de58d3e7128b738b70b3 Mon Sep 17 00:00:00 2001 From: "Shell.Xu" Date: Sat, 23 Aug 2014 23:04:54 +0800 Subject: [PATCH 157/344] * fix init.d issue, pass test on server --- debian/control | 2 +- debian/init.d | 19 +++++++------------ debian/shadowsocks.default | 2 ++ 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/debian/control b/debian/control index c8c1549..ad22e42 100644 --- a/debian/control +++ b/debian/control @@ -10,7 +10,7 @@ Homepage: https://github.com/clowwindy/shadowsocks Package: shadowsocks Architecture: all -Depends: ${shlibs:Depends}, ${misc:Depends}, python, python-m2crypto +Depends: ${shlibs:Depends}, ${misc:Depends}, python, python-setuptools, python-m2crypto Description: Fast tunnel proxy that helps you bypass firewalls A secure socks5 proxy, designed to protect your Internet traffic. . diff --git a/debian/init.d b/debian/init.d index 1413cbd..2f4f352 100644 --- a/debian/init.d +++ b/debian/init.d @@ -21,6 +21,7 @@ DAEMON=/usr/bin/ssserver # Introduce the server's location here DAEMON_ARGS="" # Arguments to run the daemon with PIDFILE=/var/run/$NAME.pid SCRIPTNAME=/etc/init.d/$NAME +LOGFILE=/var/log/$NAME.log # Exit if the package is not installed [ -x $DAEMON ] || exit 0 @@ -44,10 +45,12 @@ do_start() # 0 if daemon has been started # 1 if daemon was already running # 2 if daemon could not be started - start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON --test > /dev/null \ + start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON \ + --background --make-pidfile --chdir / --chuid $USERID --no-close --test > /dev/null \ || return 1 - start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON -- \ - $DAEMON_ARGS \ + start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON \ + --background --make-pidfile --chdir / --chuid $USERID --no-close -- \ + $DAEMON_ARGS $DAEMON_OPTS >> $LOGFILE 2>&1 \ || return 2 # Add code here, if necessary, that waits for the process to be ready # to handle requests from services started subsequently which depend @@ -64,17 +67,9 @@ do_stop() # 1 if daemon was already stopped # 2 if daemon could not be stopped # other if a failure occurred - start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME + start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE RETVAL="$?" [ "$RETVAL" = 2 ] && return 2 - # Wait for children to finish too if this is a daemon that forks - # and if the daemon is only ever run from this initscript. - # If the above conditions are not satisfied then add some other code - # that waits for the process to drop all resources that could be - # needed by services started subsequently. A last resort is to - # sleep for some time. - start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --exec $DAEMON - [ "$?" = 2 ] && return 2 # Many daemons don't delete their pidfiles when they exit. rm -f $PIDFILE return "$RETVAL" diff --git a/debian/shadowsocks.default b/debian/shadowsocks.default index 62831c2..f82b17a 100644 --- a/debian/shadowsocks.default +++ b/debian/shadowsocks.default @@ -2,6 +2,8 @@ # sourced by /etc/init.d/shadowsocks # installed at /etc/default/shadowsocks by the maintainer scripts +USERID="nobody" + # # This is a POSIX shell fragment # From b0be74ad48df3aa748637fbf60fb1ab17ca8cfa6 Mon Sep 17 00:00:00 2001 From: "Shell.Xu" Date: Sun, 24 Aug 2014 02:55:58 +0800 Subject: [PATCH 158/344] * fix base version and quite mode --- debian/control | 2 +- debian/shadowsocks.default | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index ad22e42..bd7f3ac 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Section: python Priority: extra Maintainer: Shell.Xu Build-Depends: debhelper (>= 8), python, python-setuptools -Standards-Version: 3.9.3 +Standards-Version: 3.9.5 Homepage: https://github.com/clowwindy/shadowsocks #Vcs-Git: git://git.debian.org/collab-maint/shadowsocks.git #Vcs-Browser: http://git.debian.org/?p=collab-maint/shadowsocks.git;a=summary diff --git a/debian/shadowsocks.default b/debian/shadowsocks.default index f82b17a..a520602 100644 --- a/debian/shadowsocks.default +++ b/debian/shadowsocks.default @@ -9,4 +9,4 @@ USERID="nobody" # # Additional options that are passed to the Daemon. -DAEMON_OPTS="-c /etc/shadowsocks/config.json" +DAEMON_OPTS="-q -c /etc/shadowsocks/config.json" From a053b1e1108132786603429c0f8b03e10edf5e01 Mon Sep 17 00:00:00 2001 From: "Shell.Xu" Date: Sun, 24 Aug 2014 03:00:48 +0800 Subject: [PATCH 159/344] * add vcs info in control file --- debian/control | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index bd7f3ac..b929c4d 100644 --- a/debian/control +++ b/debian/control @@ -5,8 +5,8 @@ Maintainer: Shell.Xu Build-Depends: debhelper (>= 8), python, python-setuptools Standards-Version: 3.9.5 Homepage: https://github.com/clowwindy/shadowsocks -#Vcs-Git: git://git.debian.org/collab-maint/shadowsocks.git -#Vcs-Browser: http://git.debian.org/?p=collab-maint/shadowsocks.git;a=summary +Vcs-Git: git://github.com/shell909090/shadowsocks.git +Vcs-Browser: http://github.com/shell909090/shadowsocks Package: shadowsocks Architecture: all From 47d0183b80fb01fb56c99edda4cc90edb847c7b4 Mon Sep 17 00:00:00 2001 From: "Shell.Xu" Date: Sun, 24 Aug 2014 03:03:08 +0800 Subject: [PATCH 160/344] * 3.9.5 is not passed in stable system --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index b929c4d..5edf525 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Section: python Priority: extra Maintainer: Shell.Xu Build-Depends: debhelper (>= 8), python, python-setuptools -Standards-Version: 3.9.5 +Standards-Version: 3.9.4 Homepage: https://github.com/clowwindy/shadowsocks Vcs-Git: git://github.com/shell909090/shadowsocks.git Vcs-Browser: http://github.com/shell909090/shadowsocks From 78f7ee2ca135e3da38ca35e01e2fceddb772fb68 Mon Sep 17 00:00:00 2001 From: "Shell.Xu" Date: Sun, 24 Aug 2014 13:48:58 +0800 Subject: [PATCH 161/344] * fix build issue --- debian/changelog | 2 +- debian/control | 2 +- debian/copyright | 29 +++++++---------------------- debian/gbp.conf | 3 +++ debian/rules | 5 ----- 5 files changed, 12 insertions(+), 29 deletions(-) create mode 100644 debian/gbp.conf diff --git a/debian/changelog b/debian/changelog index 445c1b6..4e7ad16 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ shadowsocks (2.1.0-1) unstable; urgency=low - * Initial release (Closes: 758900) + * Initial release (Closes: #758900) -- Shell.Xu Sat, 23 Aug 2014 00:56:04 +0800 diff --git a/debian/control b/debian/control index 5edf525..349c19d 100644 --- a/debian/control +++ b/debian/control @@ -10,7 +10,7 @@ Vcs-Browser: http://github.com/shell909090/shadowsocks Package: shadowsocks Architecture: all -Depends: ${shlibs:Depends}, ${misc:Depends}, python, python-setuptools, python-m2crypto +Depends: ${shlibs:Depends}, ${misc:Depends}, ${python:Depends}, python-setuptools, python-m2crypto Description: Fast tunnel proxy that helps you bypass firewalls A secure socks5 proxy, designed to protect your Internet traffic. . diff --git a/debian/copyright b/debian/copyright index 7be1b17..7be8162 100644 --- a/debian/copyright +++ b/debian/copyright @@ -2,30 +2,15 @@ Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: shadowsocks Source: https://github.com/clowwindy/shadowsocks -Files: * -Copyright: 2014 clowwindy -License: MIT - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - . - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - . - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. - Files: debian/* Copyright: 2014 Shell.Xu -License: MIT +License: Expat + +Files: * +Copyright: 2014 clowwindy +License: Expat + +License: Expat Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights diff --git a/debian/gbp.conf b/debian/gbp.conf new file mode 100644 index 0000000..2929e3b --- /dev/null +++ b/debian/gbp.conf @@ -0,0 +1,3 @@ +[DEFAULT] +upstream-branch = master +debian-branch = master \ No newline at end of file diff --git a/debian/rules b/debian/rules index 01e5f9b..dafca02 100755 --- a/debian/rules +++ b/debian/rules @@ -1,10 +1,5 @@ #!/usr/bin/make -f # -*- makefile -*- -# Sample debian/rules that uses debhelper. -# This file was originally written by Joey Hess and Craig Small. -# As a special exception, when this file is copied by dh-make into a -# dh-make output file, you may use that output file without restriction. -# This special exception was added by Craig Small in version 0.37 of dh-make. # Uncomment this to turn on verbose mode. #export DH_VERBOSE=1 From fcdb1a34023b349bbd147d5c10c64c8048e44ed0 Mon Sep 17 00:00:00 2001 From: "Shell.Xu" Date: Sun, 24 Aug 2014 15:56:21 +0800 Subject: [PATCH 162/344] * not sure why, but gbp.conf made python source code disappear --- debian/control | 2 +- debian/gbp.conf | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 debian/gbp.conf diff --git a/debian/control b/debian/control index 349c19d..5edf525 100644 --- a/debian/control +++ b/debian/control @@ -10,7 +10,7 @@ Vcs-Browser: http://github.com/shell909090/shadowsocks Package: shadowsocks Architecture: all -Depends: ${shlibs:Depends}, ${misc:Depends}, ${python:Depends}, python-setuptools, python-m2crypto +Depends: ${shlibs:Depends}, ${misc:Depends}, python, python-setuptools, python-m2crypto Description: Fast tunnel proxy that helps you bypass firewalls A secure socks5 proxy, designed to protect your Internet traffic. . diff --git a/debian/gbp.conf b/debian/gbp.conf deleted file mode 100644 index 2929e3b..0000000 --- a/debian/gbp.conf +++ /dev/null @@ -1,3 +0,0 @@ -[DEFAULT] -upstream-branch = master -debian-branch = master \ No newline at end of file From 6d161756366b413e4ed39bd38723a2895a468cca Mon Sep 17 00:00:00 2001 From: "Shell.Xu" Date: Mon, 25 Aug 2014 09:13:51 +0800 Subject: [PATCH 163/344] * issue with package --- debian/control | 5 +++-- debian/rules | 3 --- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/debian/control b/debian/control index 5edf525..4468dc7 100644 --- a/debian/control +++ b/debian/control @@ -2,7 +2,7 @@ Source: shadowsocks Section: python Priority: extra Maintainer: Shell.Xu -Build-Depends: debhelper (>= 8), python, python-setuptools +Build-Depends: debhelper (>= 8), python-all (>= 2.6.6-3~), python-setuptools Standards-Version: 3.9.4 Homepage: https://github.com/clowwindy/shadowsocks Vcs-Git: git://github.com/shell909090/shadowsocks.git @@ -10,7 +10,8 @@ Vcs-Browser: http://github.com/shell909090/shadowsocks Package: shadowsocks Architecture: all -Depends: ${shlibs:Depends}, ${misc:Depends}, python, python-setuptools, python-m2crypto +Pre-Depends: dpkg (>= 1.15.6~) +Depends: ${misc:Depends}, ${python:Depends}, python-setuptools, python-m2crypto Description: Fast tunnel proxy that helps you bypass firewalls A secure socks5 proxy, designed to protect your Internet traffic. . diff --git a/debian/rules b/debian/rules index dafca02..62e2bb6 100755 --- a/debian/rules +++ b/debian/rules @@ -1,8 +1,5 @@ #!/usr/bin/make -f # -*- makefile -*- -# Uncomment this to turn on verbose mode. -#export DH_VERBOSE=1 - %: dh $@ --with python2 --buildsystem=python_distutils From 7ac2967d2917ad78aed0e621237222bd4973af3b Mon Sep 17 00:00:00 2001 From: "Shell.Xu" Date: Mon, 25 Aug 2014 12:30:23 +0800 Subject: [PATCH 164/344] * still package issue --- debian/control | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index 4468dc7..da00920 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Section: python Priority: extra Maintainer: Shell.Xu Build-Depends: debhelper (>= 8), python-all (>= 2.6.6-3~), python-setuptools -Standards-Version: 3.9.4 +Standards-Version: 3.9.5 Homepage: https://github.com/clowwindy/shadowsocks Vcs-Git: git://github.com/shell909090/shadowsocks.git Vcs-Browser: http://github.com/shell909090/shadowsocks @@ -11,7 +11,7 @@ Vcs-Browser: http://github.com/shell909090/shadowsocks Package: shadowsocks Architecture: all Pre-Depends: dpkg (>= 1.15.6~) -Depends: ${misc:Depends}, ${python:Depends}, python-setuptools, python-m2crypto +Depends: ${misc:Depends}, ${python:Depends}, python-pkg-resources, python-m2crypto Description: Fast tunnel proxy that helps you bypass firewalls A secure socks5 proxy, designed to protect your Internet traffic. . From eb7ef6e6e313d56f2662285cf70ce95936f3616e Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 26 Aug 2014 23:15:35 +0800 Subject: [PATCH 165/344] remove py2exe files --- packaging/py2exe/setup.py | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 packaging/py2exe/setup.py diff --git a/packaging/py2exe/setup.py b/packaging/py2exe/setup.py deleted file mode 100644 index 30340aa..0000000 --- a/packaging/py2exe/setup.py +++ /dev/null @@ -1,19 +0,0 @@ -from distutils.core import setup -# NOTICE!! -# This setup.py is written for py2exe -# Don't make a python package using this file! - -try: - import py2exe -except ImportError: - pass - -setup(name='shadowsocks', - version='1.2.3', - description='a lightweight tunnel proxy which can help you get through firewalls', - author='clowwindy', - author_email='clowwindy42@gmail.com', - url='https://github.com/clowwindy/shadowsocks', - options = {'py2exe': {'bundle_files': 1, 'compressed': True}}, - windows = [{"script":"local.py", "dest_base": "shadowsocks_local",}], - zipfile = None) From b468476c9c69bdc772762ce720f61f23ab38da2f Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 27 Aug 2014 00:02:24 +0800 Subject: [PATCH 166/344] add PEP8 check in travis CI --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 85d7939..193a0a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,10 @@ cache: before_install: - sudo apt-get update -qq - sudo apt-get install -qq build-essential libssl-dev swig python-m2crypto python-numpy dnsutils - - pip install m2crypto salsa20 + - pip install m2crypto salsa20 pep8 - sudo tests/socksify/install.sh script: + - pep8 . - python tests/test.py -c tests/table.json - python tests/test.py -c tests/aes.json - python tests/test.py -c tests/salsa20.json From 972bf22e6d214cbc107e702ba1c156b54ea5a273 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 27 Aug 2014 00:17:26 +0800 Subject: [PATCH 167/344] fix PEP8 --- shadowsocks/asyncdns.py | 4 ++-- shadowsocks/common.py | 3 ++- shadowsocks/encrypt.py | 3 ++- shadowsocks/encrypt_salsa20.py | 5 +++-- shadowsocks/tcprelay.py | 15 ++++++++------- shadowsocks/utils.py | 2 +- tests/test.py | 2 +- tests/test_latency.py | 1 - 8 files changed, 19 insertions(+), 16 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index f745be7..3cf1625 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -403,8 +403,8 @@ class DNSResolver(object): self._request_id = 1 req = build_request(hostname, qtype, self._request_id) for server in self._servers: - logging.debug('resolving %s with type %d using server %s', hostname, - qtype, server) + logging.debug('resolving %s with type %d using server %s', + hostname, qtype, server) self._sock.sendto(req, (server, 53)) def resolve(self, hostname, callback): diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 7548ff3..6307ea7 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -126,7 +126,8 @@ def parse_header(data): else: logging.warn('header is too short') else: - logging.warn('unsupported addrtype %d, maybe wrong password' % addrtype) + logging.warn('unsupported addrtype %d, maybe wrong password' % + addrtype) if dest_addr is None: return None return addrtype, dest_addr, dest_port, header_length diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 86c5d6d..952fd72 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -154,7 +154,8 @@ class Encryptor(object): iv = iv_ iv = iv[:m[1]] if op == 1: - self.cipher_iv = iv[:m[1]] # this iv is for cipher not decipher + # this iv is for cipher not decipher + self.cipher_iv = iv[:m[1]] if method != 'salsa20-ctr': import M2Crypto.EVP return M2Crypto.EVP.Cipher(method.replace('-', '_'), key, iv, diff --git a/shadowsocks/encrypt_salsa20.py b/shadowsocks/encrypt_salsa20.py index 62bca65..823ae65 100644 --- a/shadowsocks/encrypt_salsa20.py +++ b/shadowsocks/encrypt_salsa20.py @@ -67,7 +67,8 @@ class Salsa20Cipher(object): def _next_stream(self): self._nonce &= 0xFFFFFFFFFFFFFFFF self._stream = salsa20.Salsa20_keystream(BLOCK_SIZE, - struct.pack(' Date: Wed, 27 Aug 2014 23:24:34 +0800 Subject: [PATCH 168/344] fix CHANGES --- CHANGES | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGES b/CHANGES index 55f4f4a..cacf0d5 100644 --- a/CHANGES +++ b/CHANGES @@ -1,7 +1,5 @@ 2.1.0 2014-08-10 - Use only IPv4 DNS server - -2.1.0 2014-08-01 - Does not ship config.json - Better error message From 63fae5119dff3e45e664ac2eb3bbadbc83689123 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 6 Sep 2014 20:58:56 +0800 Subject: [PATCH 169/344] add debian sid instructions --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 4784b9b..e000ef4 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,9 @@ remote server. apt-get install python-pip python-m2crypto pip install shadowsocks +Or simply `apt-get install shadowsocks` if you have [Debian sid] in your +source list. + #### CentOS: yum install m2crypto python-setuptools @@ -117,6 +120,7 @@ Bugs and Issues [Build Status]: https://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat [Chinese Readme]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E [Chrome Standalone]: https://support.google.com/installer/answer/126299 +[Debian sid]: https://packages.debian.org/unstable/python/shadowsocks [iOS]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Help [Issue Tracker]: https://github.com/clowwindy/shadowsocks/issues?state=open [Mailing list]: http://groups.google.com/group/shadowsocks From 1b4e885818a44e41961156554f9b36ec1ab1012d Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 9 Sep 2014 14:26:34 +0800 Subject: [PATCH 170/344] add rc4-sha256; close #178 --- setup.py | 2 +- shadowsocks/encrypt.py | 18 ++++++++++------ shadowsocks/encrypt_rc4_sha256.py | 35 +++++++++++++++++++++++++++++++ shadowsocks/encrypt_salsa20.py | 22 ++++++++++++++++++- tests/rc4-sha256.json | 10 +++++++++ 5 files changed, 79 insertions(+), 8 deletions(-) create mode 100644 shadowsocks/encrypt_rc4_sha256.py create mode 100644 tests/rc4-sha256.json diff --git a/setup.py b/setup.py index be07821..9cb19d6 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.1.0", + version="2.2.0", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 952fd72..1156d89 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -27,6 +27,7 @@ import string import struct import logging import encrypt_salsa20 +import encrypt_rc4_sha256 def random_string(length): @@ -115,6 +116,7 @@ method_supported = { 'idea-cfb': (16, 8), 'rc2-cfb': (16, 8), 'rc4': (16, 0), + 'rc4-sha256': (32, 16), 'seed-cfb': (16, 16), 'salsa20-ctr': (32, 8), } @@ -156,13 +158,15 @@ class Encryptor(object): if op == 1: # this iv is for cipher not decipher self.cipher_iv = iv[:m[1]] - if method != 'salsa20-ctr': + if method == 'salsa20-ctr': + return encrypt_salsa20.Salsa20Cipher(method, key, iv, op) + elif method == 'rc4-sha256': + return encrypt_rc4_sha256.create_cipher(method, key, iv, op) + else: import M2Crypto.EVP return M2Crypto.EVP.Cipher(method.replace('-', '_'), key, iv, op, key_as_bytes=0, d='md5', salt=None, i=1, padding=1) - else: - return encrypt_salsa20.Salsa20Cipher(method, key, iv, op) logging.error('method %s not supported' % method) sys.exit(1) @@ -217,11 +221,13 @@ def encrypt_all(password, method, op, data): else: iv = data[:iv_len] data = data[iv_len:] - if method != 'salsa20-ctr': + if method == 'salsa20-ctr': + cipher = encrypt_salsa20.Salsa20Cipher(method, key, iv, op) + elif method == 'rc4-sha256': + return encrypt_rc4_sha256.create_cipher(method, key, iv, op) + else: cipher = M2Crypto.EVP.Cipher(method.replace('-', '_'), key, iv, op, key_as_bytes=0, d='md5', salt=None, i=1, padding=1) - else: - cipher = encrypt_salsa20.Salsa20Cipher(method, key, iv, op) result.append(cipher.update(data)) return ''.join(result) diff --git a/shadowsocks/encrypt_rc4_sha256.py b/shadowsocks/encrypt_rc4_sha256.py new file mode 100644 index 0000000..968f3e2 --- /dev/null +++ b/shadowsocks/encrypt_rc4_sha256.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import hashlib + + +def create_cipher(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, + i=1, padding=1): + sha256 = hashlib.sha256() + sha256.update(key) + sha256.update(iv) + rc4_key = sha256.digest() + + import M2Crypto.EVP + return M2Crypto.EVP.Cipher('rc4', rc4_key, '', op, key_as_bytes=0, + d='md5', salt=None, i=1, padding=1) diff --git a/shadowsocks/encrypt_salsa20.py b/shadowsocks/encrypt_salsa20.py index 823ae65..be8cb8b 100644 --- a/shadowsocks/encrypt_salsa20.py +++ b/shadowsocks/encrypt_salsa20.py @@ -1,4 +1,24 @@ -#!/usr/bin/python +#!/usr/bin/env python + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. import time import struct diff --git a/tests/rc4-sha256.json b/tests/rc4-sha256.json new file mode 100644 index 0000000..7119c09 --- /dev/null +++ b/tests/rc4-sha256.json @@ -0,0 +1,10 @@ +{ + "server":"127.0.0.1", + "server_port":8388, + "local_port":1081, + "password":"aes_password", + "timeout":60, + "method":"rc4-sha256", + "local_address":"127.0.0.1", + "fast_open":false +} From 1e1b1adba98c66da48acaa663fe59a03a44db686 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 9 Sep 2014 14:34:35 +0800 Subject: [PATCH 171/344] add RC4-SHA256 unit test --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 193a0a4..24b234b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ script: - pep8 . - python tests/test.py -c tests/table.json - python tests/test.py -c tests/aes.json + - python tests/test.py -c tests/rc4-sha256.json - python tests/test.py -c tests/salsa20.json - python tests/test.py -c tests/server-multi-ports.json - python tests/test.py -c tests/server-multi-passwd.json From c9b489b5e316bfb226582c990b262469c5645fb8 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 9 Sep 2014 14:43:32 +0800 Subject: [PATCH 172/344] fix UDP --- shadowsocks/encrypt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 1156d89..3537d00 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -224,7 +224,7 @@ def encrypt_all(password, method, op, data): if method == 'salsa20-ctr': cipher = encrypt_salsa20.Salsa20Cipher(method, key, iv, op) elif method == 'rc4-sha256': - return encrypt_rc4_sha256.create_cipher(method, key, iv, op) + cipher = encrypt_rc4_sha256.create_cipher(method, key, iv, op) else: cipher = M2Crypto.EVP.Cipher(method.replace('-', '_'), key, iv, op, key_as_bytes=0, d='md5', From 1fea9dca8d6cf03d25672d573203b0aa5d794a6c Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 9 Sep 2014 14:49:38 +0800 Subject: [PATCH 173/344] update README: add link to encryption methods --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e000ef4..9f78680 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Explanation of the fields: | local_port | local port | | password | password used for encryption | | timeout | in seconds | -| method | encryption method, "aes-256-cfb" is recommended | +| method | default: "aes-256-cfb", see [Encryption] | | fast_open | use [TCP_FASTOPEN], true / false | | workers | number of workers, available on Unix/Linux | @@ -121,6 +121,7 @@ Bugs and Issues [Chinese Readme]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E [Chrome Standalone]: https://support.google.com/installer/answer/126299 [Debian sid]: https://packages.debian.org/unstable/python/shadowsocks +[Encryption]: https://github.com/clowwindy/shadowsocks/wiki/Encryption [iOS]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Help [Issue Tracker]: https://github.com/clowwindy/shadowsocks/issues?state=open [Mailing list]: http://groups.google.com/group/shadowsocks From 1044358a6bf6b3f275d8ebce08bfaf325042b5b4 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 9 Sep 2014 16:17:33 +0800 Subject: [PATCH 174/344] use md5 instead of sha256; #178 --- .travis.yml | 2 +- shadowsocks/encrypt.py | 12 ++++++------ .../{encrypt_rc4_sha256.py => encrypt_rc4_md5.py} | 9 +++++---- tests/{rc4-sha256.json => rc4-md5.json} | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) rename shadowsocks/{encrypt_rc4_sha256.py => encrypt_rc4_md5.py} (92%) rename tests/{rc4-sha256.json => rc4-md5.json} (83%) diff --git a/.travis.yml b/.travis.yml index 24b234b..1d716f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ script: - pep8 . - python tests/test.py -c tests/table.json - python tests/test.py -c tests/aes.json - - python tests/test.py -c tests/rc4-sha256.json + - python tests/test.py -c tests/rc4-md5.json - python tests/test.py -c tests/salsa20.json - python tests/test.py -c tests/server-multi-ports.json - python tests/test.py -c tests/server-multi-passwd.json diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 3537d00..d144e64 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -27,7 +27,7 @@ import string import struct import logging import encrypt_salsa20 -import encrypt_rc4_sha256 +import encrypt_rc4_md5 def random_string(length): @@ -116,7 +116,7 @@ method_supported = { 'idea-cfb': (16, 8), 'rc2-cfb': (16, 8), 'rc4': (16, 0), - 'rc4-sha256': (32, 16), + 'rc4-md5': (16, 16), 'seed-cfb': (16, 16), 'salsa20-ctr': (32, 8), } @@ -160,8 +160,8 @@ class Encryptor(object): self.cipher_iv = iv[:m[1]] if method == 'salsa20-ctr': return encrypt_salsa20.Salsa20Cipher(method, key, iv, op) - elif method == 'rc4-sha256': - return encrypt_rc4_sha256.create_cipher(method, key, iv, op) + elif method == 'rc4-md5': + return encrypt_rc4_md5.create_cipher(method, key, iv, op) else: import M2Crypto.EVP return M2Crypto.EVP.Cipher(method.replace('-', '_'), key, iv, @@ -223,8 +223,8 @@ def encrypt_all(password, method, op, data): data = data[iv_len:] if method == 'salsa20-ctr': cipher = encrypt_salsa20.Salsa20Cipher(method, key, iv, op) - elif method == 'rc4-sha256': - cipher = encrypt_rc4_sha256.create_cipher(method, key, iv, op) + elif method == 'rc4-md5': + cipher = encrypt_rc4_md5.create_cipher(method, key, iv, op) else: cipher = M2Crypto.EVP.Cipher(method.replace('-', '_'), key, iv, op, key_as_bytes=0, d='md5', diff --git a/shadowsocks/encrypt_rc4_sha256.py b/shadowsocks/encrypt_rc4_md5.py similarity index 92% rename from shadowsocks/encrypt_rc4_sha256.py rename to shadowsocks/encrypt_rc4_md5.py index 968f3e2..f71bad3 100644 --- a/shadowsocks/encrypt_rc4_sha256.py +++ b/shadowsocks/encrypt_rc4_md5.py @@ -25,10 +25,11 @@ import hashlib def create_cipher(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, i=1, padding=1): - sha256 = hashlib.sha256() - sha256.update(key) - sha256.update(iv) - rc4_key = sha256.digest() + md5 = hashlib.md5() + md5.update(key) + md5.update(iv) + rc4_key = md5.digest() + print len(rc4_key) import M2Crypto.EVP return M2Crypto.EVP.Cipher('rc4', rc4_key, '', op, key_as_bytes=0, diff --git a/tests/rc4-sha256.json b/tests/rc4-md5.json similarity index 83% rename from tests/rc4-sha256.json rename to tests/rc4-md5.json index 7119c09..26ba0df 100644 --- a/tests/rc4-sha256.json +++ b/tests/rc4-md5.json @@ -4,7 +4,7 @@ "local_port":1081, "password":"aes_password", "timeout":60, - "method":"rc4-sha256", + "method":"rc4-md5", "local_address":"127.0.0.1", "fast_open":false } From 3e3c885a42ac1422e6e1be763c4fb8cb58d51595 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 9 Sep 2014 16:18:39 +0800 Subject: [PATCH 175/344] remove print --- shadowsocks/encrypt_rc4_md5.py | 1 - 1 file changed, 1 deletion(-) diff --git a/shadowsocks/encrypt_rc4_md5.py b/shadowsocks/encrypt_rc4_md5.py index f71bad3..8295153 100644 --- a/shadowsocks/encrypt_rc4_md5.py +++ b/shadowsocks/encrypt_rc4_md5.py @@ -29,7 +29,6 @@ def create_cipher(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, md5.update(key) md5.update(iv) rc4_key = md5.digest() - print len(rc4_key) import M2Crypto.EVP return M2Crypto.EVP.Cipher('rc4', rc4_key, '', op, key_as_bytes=0, From 964d7613cbab9043775c1712601a04be2ea7ce0b Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 9 Sep 2014 16:32:18 +0800 Subject: [PATCH 176/344] update changes --- CHANGES | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES b/CHANGES index cacf0d5..4558c9a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +2.2.0 2014-09-09 +- Add RC4-MD5 encryption + 2.1.0 2014-08-10 - Use only IPv4 DNS server - Does not ship config.json From 327c70e353e2761934e14b422dfa198d5d2a0cb9 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 12 Sep 2014 00:51:25 +0800 Subject: [PATCH 177/344] Graceful shutdown; close #179 --- setup.py | 2 +- shadowsocks/asyncdns.py | 2 +- shadowsocks/eventloop.py | 17 ++++++++++++----- shadowsocks/server.py | 18 +++++++++++------- shadowsocks/tcprelay.py | 12 ++++++++++-- shadowsocks/udprelay.py | 13 +++++++++---- shadowsocks/utils.py | 4 ++-- 7 files changed, 46 insertions(+), 22 deletions(-) diff --git a/setup.py b/setup.py index 9cb19d6..dd1ca21 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.2.0", + version="2.2.1", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 3cf1625..d74d5e8 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -324,7 +324,7 @@ class DNSResolver(object): socket.SOL_UDP) self._sock.setblocking(False) loop.add(self._sock, eventloop.POLL_IN) - loop.add_handler(self.handle_events) + loop.add_handler(self.handle_events, ref=False) def _call_callback(self, hostname, ip, error=None): callbacks = self._hostname_to_cb.get(hostname, []) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index 0b628e0..4b2469a 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -168,7 +168,7 @@ class EventLoop(object): 'package') self._fd_to_f = {} self._handlers = [] - self.stopping = False + self._ref_handlers = [] logging.debug('using event model: %s', model) def poll(self, timeout=None): @@ -189,17 +189,24 @@ class EventLoop(object): fd = f.fileno() self._impl.modify_fd(fd, mode) - def add_handler(self, handler): + def add_handler(self, handler, ref=True): self._handlers.append(handler) + if ref: + # when all ref handlers are removed, loop stops + self._ref_handlers.append(handler) + + def remove_handler(self, handler): + self._handlers.remove(handler) + if handler in self._ref_handlers: + self._ref_handlers.remove(handler) def run(self): - while not self.stopping: + while self._ref_handlers: try: events = self.poll(1) except (OSError, IOError) as e: - if errno_from_exception(e) == errno.EPIPE: + if errno_from_exception(e) in (errno.EPIPE, errno.EINTR): # Happens when the client closes the connection - logging.error('poll:%s', e) continue else: logging.error('poll:%s', e) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index d5648bc..5f92b58 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -30,6 +30,7 @@ import eventloop import tcprelay import udprelay import asyncdns +import signal def main(): @@ -67,13 +68,14 @@ def main(): udp_servers.append(udprelay.UDPRelay(a_config, dns_resolver, False)) def run_server(): + def child_handler(signum, _): + logging.warn('received SIGQUIT, doing graceful shutting down..') + map(lambda s: s.close(next_tick=True), tcp_servers + udp_servers) + signal.signal(signal.SIGQUIT, child_handler) try: loop = eventloop.EventLoop() dns_resolver.add_to_loop(loop) - for tcp_server in tcp_servers: - tcp_server.add_to_loop(loop) - for udp_server in udp_servers: - udp_server.add_to_loop(loop) + map(lambda s: s.add_to_loop(loop), tcp_servers + udp_servers) loop.run() except (KeyboardInterrupt, IOError, OSError) as e: logging.error(e) @@ -97,11 +99,13 @@ def main(): if not is_child: def handler(signum, _): for pid in children: - os.kill(pid, signum) - os.waitpid(pid, 0) + try: + os.kill(pid, signum) + except OSError: # child may already exited + pass sys.exit() - import signal signal.signal(signal.SIGTERM, handler) + signal.signal(signal.SIGQUIT, handler) # master for a_tcp_server in tcp_servers: diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 36b1c56..3e248a1 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -631,7 +631,15 @@ class TCPRelay(object): if now - self._last_time > TIMEOUT_PRECISION: self._sweep_timeout() self._last_time = now + if self._closed: + if self._server_socket: + self._eventloop.remove(self._server_socket) + self._server_socket.close() + self._server_socket = None + if not self._fd_to_handlers: + self._eventloop.remove_handler(self._handle_events) - def close(self): + def close(self, next_tick=False): self._closed = True - self._server_socket.close() + if not next_tick: + self._server_socket.close() diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index d7e3d4c..89f3593 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -260,12 +260,17 @@ class UDPRelay(object): logging.error('UDP client_socket err') self._handle_client(sock) now = time.time() - if now - self._last_time > 3.5: + if now - self._last_time > 3: self._cache.sweep() - if now - self._last_time > 7: self._client_fd_to_server_addr.sweep() self._last_time = now + if self._closed: + self._server_socket.close() + for sock in self._sockets: + sock.close() + self._eventloop.remove_handler(self._handle_events) - def close(self): + def close(self, next_tick=False): self._closed = True - self._server_socket.close() + if not next_tick: + self._server_socket.close() diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 785f18c..2f2c0b7 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -90,7 +90,7 @@ def get_config(is_local): longopts = ['fast-open'] else: shortopts = 'hs:p:k:m:c:t:vq' - longopts = ['fast-open', 'workers:'] + longopts = ['fast-open', 'workers='] try: config_path = find_config() optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) @@ -134,7 +134,7 @@ def get_config(is_local): elif key == '--fast-open': config['fast_open'] = True elif key == '--workers': - config['workers'] = value + config['workers'] = int(value) elif key == '-h': if is_local: print_local_help() From a6b8a5eef910b75829487a506202f1df68d358af Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 12 Sep 2014 00:53:12 +0800 Subject: [PATCH 178/344] add some debug logs --- shadowsocks/eventloop.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index 4b2469a..65f6473 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -207,6 +207,7 @@ class EventLoop(object): except (OSError, IOError) as e: if errno_from_exception(e) in (errno.EPIPE, errno.EINTR): # Happens when the client closes the connection + logging.debug('poll:%s', e) continue else: logging.error('poll:%s', e) From fdb3a36170031117e17518e5ee2515221472e5e0 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 12 Sep 2014 20:47:04 +0800 Subject: [PATCH 179/344] implement graceful shutdown in sslocal; close #179 --- shadowsocks/local.py | 7 +++++++ shadowsocks/server.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 0a00136..e480b58 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -24,6 +24,7 @@ import sys import os import logging +import signal import utils import encrypt import eventloop @@ -58,6 +59,12 @@ def main(): dns_resolver.add_to_loop(loop) tcp_server.add_to_loop(loop) udp_server.add_to_loop(loop) + + def handler(signum, _): + logging.warn('received SIGQUIT, doing graceful shutting down..') + tcp_server.close(next_tick=True) + udp_server.close(next_tick=True) + signal.signal(signal.SIGQUIT, handler) loop.run() except (KeyboardInterrupt, IOError, OSError) as e: logging.error(e) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 5f92b58..3adc4fd 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -24,13 +24,13 @@ import sys import os import logging +import signal import utils import encrypt import eventloop import tcprelay import udprelay import asyncdns -import signal def main(): From e43520da58d1630bf12d7997847639f11d0e8f80 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 12 Sep 2014 20:59:47 +0800 Subject: [PATCH 180/344] log when closing port --- shadowsocks/tcprelay.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 3e248a1..ccb74ee 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -508,6 +508,7 @@ class TCPRelay(object): else: listen_addr = config['server'] listen_port = config['server_port'] + self._listen_port = listen_port addrs = socket.getaddrinfo(listen_addr, listen_port, 0, socket.SOCK_STREAM, socket.SOL_TCP) @@ -636,6 +637,7 @@ class TCPRelay(object): self._eventloop.remove(self._server_socket) self._server_socket.close() self._server_socket = None + logging.info('closed listen port %d', self._listen_port) if not self._fd_to_handlers: self._eventloop.remove_handler(self._handle_events) From 380a646f095e26bd1b27f557d6c7661813df2a6d Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 12 Sep 2014 22:18:08 +0800 Subject: [PATCH 181/344] handle signal soon; #179 --- shadowsocks/eventloop.py | 5 +++-- shadowsocks/tcprelay.py | 16 ++++++++-------- shadowsocks/udprelay.py | 10 +++++----- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index 65f6473..a80a525 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -206,9 +206,10 @@ class EventLoop(object): events = self.poll(1) except (OSError, IOError) as e: if errno_from_exception(e) in (errno.EPIPE, errno.EINTR): - # Happens when the client closes the connection + # EPIPE: Happens when the client closes the connection + # EINTR: Happens when received a signal + # handles them as soon as possible logging.debug('poll:%s', e) - continue else: logging.error('poll:%s', e) import traceback diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index ccb74ee..372f1e2 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -632,14 +632,14 @@ class TCPRelay(object): if now - self._last_time > TIMEOUT_PRECISION: self._sweep_timeout() self._last_time = now - if self._closed: - if self._server_socket: - self._eventloop.remove(self._server_socket) - self._server_socket.close() - self._server_socket = None - logging.info('closed listen port %d', self._listen_port) - if not self._fd_to_handlers: - self._eventloop.remove_handler(self._handle_events) + if self._closed: + if self._server_socket: + self._eventloop.remove(self._server_socket) + self._server_socket.close() + self._server_socket = None + logging.info('closed listen port %d', self._listen_port) + if not self._fd_to_handlers: + self._eventloop.remove_handler(self._handle_events) def close(self, next_tick=False): self._closed = True diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index 89f3593..64d6a43 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -264,11 +264,11 @@ class UDPRelay(object): self._cache.sweep() self._client_fd_to_server_addr.sweep() self._last_time = now - if self._closed: - self._server_socket.close() - for sock in self._sockets: - sock.close() - self._eventloop.remove_handler(self._handle_events) + if self._closed: + self._server_socket.close() + for sock in self._sockets: + sock.close() + self._eventloop.remove_handler(self._handle_events) def close(self, next_tick=False): self._closed = True From ea70a5bd790eaa8e7bb5706f50f32f3aca1be06e Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 12 Sep 2014 22:20:33 +0800 Subject: [PATCH 182/344] exit 1 on error; #179 --- shadowsocks/local.py | 2 +- shadowsocks/server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/local.py b/shadowsocks/local.py index e480b58..e7bf48d 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -70,7 +70,7 @@ def main(): logging.error(e) import traceback traceback.print_exc() - os._exit(0) + os._exit(1) if __name__ == '__main__': main() diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 3adc4fd..c896a25 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -81,7 +81,7 @@ def main(): logging.error(e) import traceback traceback.print_exc() - os._exit(0) + os._exit(1) if int(config['workers']) > 1: if os.name == 'posix': From 2a61e80071b28925d41c335eab569aad8d455ab0 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 12 Sep 2014 23:06:18 +0800 Subject: [PATCH 183/344] only output exception in verbose mode --- shadowsocks/local.py | 5 +++-- shadowsocks/server.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/shadowsocks/local.py b/shadowsocks/local.py index e7bf48d..5aeee3b 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -68,8 +68,9 @@ def main(): loop.run() except (KeyboardInterrupt, IOError, OSError) as e: logging.error(e) - import traceback - traceback.print_exc() + if config['verbose']: + import traceback + traceback.print_exc() os._exit(1) if __name__ == '__main__': diff --git a/shadowsocks/server.py b/shadowsocks/server.py index c896a25..0ad46fb 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -79,8 +79,9 @@ def main(): loop.run() except (KeyboardInterrupt, IOError, OSError) as e: logging.error(e) - import traceback - traceback.print_exc() + if config['verbose']: + import traceback + traceback.print_exc() os._exit(1) if int(config['workers']) > 1: From 21f61c7d91985b71bf37eb64ab62d044265863df Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 12 Sep 2014 23:56:37 +0800 Subject: [PATCH 184/344] avoid removing items of the handler list while iterating over it; #179 --- shadowsocks/eventloop.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index a80a525..de0039e 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -154,6 +154,7 @@ class SelectLoop(object): class EventLoop(object): def __init__(self): + self._iterating = False if hasattr(select, 'epoll'): self._impl = EpollLoop() model = 'epoll' @@ -169,6 +170,7 @@ class EventLoop(object): self._fd_to_f = {} self._handlers = [] self._ref_handlers = [] + self._handlers_to_remove = [] logging.debug('using event model: %s', model) def poll(self, timeout=None): @@ -196,11 +198,15 @@ class EventLoop(object): self._ref_handlers.append(handler) def remove_handler(self, handler): - self._handlers.remove(handler) if handler in self._ref_handlers: self._ref_handlers.remove(handler) + if self._iterating: + self._handlers_to_remove.append(handler) + else: + self._handlers.remove(handler) def run(self): + events = [] while self._ref_handlers: try: events = self.poll(1) @@ -215,6 +221,7 @@ class EventLoop(object): import traceback traceback.print_exc() continue + self._iterating = True for handler in self._handlers: # TODO when there are a lot of handlers try: @@ -223,6 +230,10 @@ class EventLoop(object): logging.error(e) import traceback traceback.print_exc() + for handler in self._handlers_to_remove: + self._handlers.remove(handler) + self._handlers_to_remove = [] + self._iterating = False # from tornado From ebf5baa3112135e30173cce5cf47c7a95eadb485 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 13 Sep 2014 15:54:37 +0800 Subject: [PATCH 185/344] udpate changes --- CHANGES | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES b/CHANGES index 4558c9a..7386aac 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +2.2.1 2014-09-09 +- Support graceful shutdown +- Fix some bugs + 2.2.0 2014-09-09 - Add RC4-MD5 encryption From 3ce7c18dfbe12c4c348cfc6f9bfe144f6238d7af Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 14 Sep 2014 10:06:16 +0800 Subject: [PATCH 186/344] close #180 --- setup.py | 2 +- shadowsocks/asyncdns.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index dd1ca21..8d12440 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.2.1", + version="2.2.2", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index d74d5e8..a103f8c 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -218,6 +218,8 @@ def parse_response(data): response = DNSResponse() if qds: response.hostname = qds[0][0] + for an in qds: + response.questions.append((an[1], an[2], an[3])) for an in ans: response.answers.append((an[1], an[2], an[3])) return response @@ -249,6 +251,7 @@ def is_valid_hostname(hostname): class DNSResponse(object): def __init__(self): self.hostname = None + self.questions = [] # each: (addr, type, class) self.answers = [] # each: (addr, type, class) def __str__(self): @@ -358,7 +361,9 @@ class DNSResolver(object): else: if ip: self._cache[hostname] = ip - self._call_callback(hostname, ip) + self._call_callback(hostname, ip) + elif self._hostname_status.get(hostname, None) == STATUS_IPV6: + self._call_callback(hostname, None) def handle_events(self, events): for sock, fd, event in events: From e72594ffbe92b80ddb1770268ca3effb3b64a0ee Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 14 Sep 2014 10:16:40 +0800 Subject: [PATCH 187/344] only end querying when response is of query type AAAA --- shadowsocks/asyncdns.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index a103f8c..55fbb11 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -363,7 +363,10 @@ class DNSResolver(object): self._cache[hostname] = ip self._call_callback(hostname, ip) elif self._hostname_status.get(hostname, None) == STATUS_IPV6: - self._call_callback(hostname, None) + for question in response.questions: + if question[1] == QTYPE_AAAA: + self._call_callback(hostname, None) + break def handle_events(self, events): for sock, fd, event in events: From 5aece6f46423c0c2508d7ead831e2a4eb55dd1d3 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 14 Sep 2014 10:17:45 +0800 Subject: [PATCH 188/344] remove test in asyncdns --- shadowsocks/asyncdns.py | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 55fbb11..009c2d3 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -447,40 +447,3 @@ class DNSResolver(object): if self._sock: self._sock.close() self._sock = None - - -def test(): - logging.getLogger('').handlers = [] - logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s %(levelname)-8s %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', filemode='a+') - - def _callback(address, error): - print error, address - - loop = eventloop.EventLoop() - resolver = DNSResolver() - resolver.add_to_loop(loop) - - for hostname in ['www.google.com', - '8.8.8.8', - 'localhost', - 'activate.adobe.com', - 'www.twitter.com', - 'ipv6.google.com', - 'ipv6.l.google.com', - 'www.gmail.com', - 'r4---sn-3qqp-ioql.googlevideo.com', - 'www.baidu.com', - 'www.a.shifen.com', - 'm.baidu.jp', - 'www.youku.com', - 'www.twitter.com', - 'ipv6.google.com']: - resolver.resolve(hostname, _callback) - - loop.run() - - -if __name__ == '__main__': - test() From 64503498d0843dfd44e547cfef49aac639616c20 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 14 Sep 2014 10:24:09 +0800 Subject: [PATCH 189/344] update changes --- CHANGES | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 7386aac..f6e8172 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,7 @@ -2.2.1 2014-09-09 +2.2.2 2014-09-14 +- Fix when multiple DNS set, IPv6 only sites are broken + +2.2.1 2014-09-10 - Support graceful shutdown - Fix some bugs From a2ed12611c3792f6056475569f818426c5842248 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 16 Sep 2014 09:03:31 +0800 Subject: [PATCH 190/344] bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8d12440..460dc69 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.2.2", + version="2.3", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From 76fed23b348b0263e2f6e46e0558deb5ab890a8e Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 16 Sep 2014 10:58:45 +0800 Subject: [PATCH 191/344] refactor encrypt; add cfb1 and cfb8 modes --- .travis.yml | 2 + setup.py | 2 +- shadowsocks/crypto/__init__.py | 21 ++++ shadowsocks/crypto/ctypes_openssl.py | 100 ++++++++++++++++++ shadowsocks/crypto/m2.py | 66 ++++++++++++ .../{encrypt_rc4_md5.py => crypto/rc4_md5.py} | 9 ++ .../salsa20_ctr.py} | 5 + shadowsocks/encrypt.py | 69 ++++-------- tests/aes-cfb1.json | 10 ++ tests/aes-cfb8.json | 10 ++ tests/test.py | 4 +- 11 files changed, 245 insertions(+), 53 deletions(-) create mode 100644 shadowsocks/crypto/__init__.py create mode 100644 shadowsocks/crypto/ctypes_openssl.py create mode 100644 shadowsocks/crypto/m2.py rename shadowsocks/{encrypt_rc4_md5.py => crypto/rc4_md5.py} (94%) rename shadowsocks/{encrypt_salsa20.py => crypto/salsa20_ctr.py} (98%) create mode 100644 tests/aes-cfb1.json create mode 100644 tests/aes-cfb8.json diff --git a/.travis.yml b/.travis.yml index 1d716f2..b10edd9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,8 @@ script: - pep8 . - python tests/test.py -c tests/table.json - python tests/test.py -c tests/aes.json + - python tests/test.py -c tests/aes-cfb1.json + - python tests/test.py -c tests/aes-cfb8.json - python tests/test.py -c tests/rc4-md5.json - python tests/test.py -c tests/salsa20.json - python tests/test.py -c tests/server-multi-ports.json diff --git a/setup.py b/setup.py index 460dc69..8ddf3c5 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( author='clowwindy', author_email='clowwindy42@gmail.com', url='https://github.com/clowwindy/shadowsocks', - packages=['shadowsocks'], + packages=['shadowsocks', 'shadowsocks.crypto'], package_data={ 'shadowsocks': ['README.rst', 'LICENSE'] }, diff --git a/shadowsocks/crypto/__init__.py b/shadowsocks/crypto/__init__.py new file mode 100644 index 0000000..bd3a926 --- /dev/null +++ b/shadowsocks/crypto/__init__.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. diff --git a/shadowsocks/crypto/ctypes_openssl.py b/shadowsocks/crypto/ctypes_openssl.py new file mode 100644 index 0000000..8cbd9e5 --- /dev/null +++ b/shadowsocks/crypto/ctypes_openssl.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import logging + +__all__ = ['ciphers'] + +loaded = False + + +def load_openssl(): + global loaded, libcrypto, CDLL, c_char_p, c_int, c_long, byref,\ + create_string_buffer, c_void_p, buf + from ctypes import CDLL, c_char_p, c_int, c_long, byref,\ + create_string_buffer, c_void_p + from ctypes.util import find_library + libcrypto_path = find_library('crypto') + logging.info('loading libcrypto from %s', libcrypto_path) + libcrypto = CDLL(libcrypto_path) + libcrypto.EVP_get_cipherbyname.restype = c_void_p + libcrypto.EVP_CIPHER_CTX_new.restype = c_void_p + libcrypto.EVP_CIPHER_CTX_new.argtypes = (c_void_p, c_void_p, c_char_p, + c_char_p) + + libcrypto.EVP_CipherInit_ex.argtypes = (c_void_p, c_void_p, c_char_p, + c_char_p, c_char_p, c_int) + + libcrypto.EVP_CipherUpdate.argtypes = (c_void_p, c_void_p, c_void_p, + c_char_p, c_int) + + libcrypto.EVP_CIPHER_CTX_cleanup.argtypes = (c_void_p,) + libcrypto.EVP_CIPHER_CTX_free.argtypes = (c_void_p,) + + buf = create_string_buffer(65536) + loaded = True + + +class CtypesCrypto(object): + def __init__(self, cipher_name, key, iv, op): + if not loaded: + load_openssl() + self._ctx = None + cipher = libcrypto.EVP_get_cipherbyname(cipher_name) + if not cipher: + raise Exception('cipher %s not found in libcrypto' % cipher_name) + key_ptr = c_char_p(key) + iv_ptr = c_char_p(iv) + self._ctx = libcrypto.EVP_CIPHER_CTX_new(cipher, None, + key_ptr, iv_ptr) + if not self._ctx: + raise Exception('can not create cipher context') + r = libcrypto.EVP_CipherInit_ex(self._ctx, cipher, None, + key_ptr, iv_ptr, c_int(op)) + if not r: + self.clean() + raise Exception('can not initialize cipher context') + + def update(self, data): + cipher_out_len = c_long(0) + libcrypto.EVP_CipherUpdate(self._ctx, byref(buf), + byref(cipher_out_len), c_char_p(data), + len(data)) + return buf.raw[:cipher_out_len.value] + + def __del__(self): + self.clean() + + def clean(self): + if self._ctx: + libcrypto.EVP_CIPHER_CTX_cleanup(self._ctx) + libcrypto.EVP_CIPHER_CTX_free(self._ctx) + + +ciphers = { + 'aes-128-cfb8': (16, 16, CtypesCrypto), + 'aes-192-cfb8': (24, 16, CtypesCrypto), + 'aes-256-cfb8': (32, 16, CtypesCrypto), + 'aes-128-cfb1': (16, 16, CtypesCrypto), + 'aes-192-cfb1': (24, 16, CtypesCrypto), + 'aes-256-cfb1': (32, 16, CtypesCrypto), +} diff --git a/shadowsocks/crypto/m2.py b/shadowsocks/crypto/m2.py new file mode 100644 index 0000000..cff3682 --- /dev/null +++ b/shadowsocks/crypto/m2.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import sys +import logging + +__all__ = ['ciphers'] + +has_m2 = True +try: + __import__('M2Crypto') +except ImportError: + has_m2 = False + + +def create_cipher(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, i=1, + padding=1): + + import M2Crypto.EVP + return M2Crypto.EVP.Cipher('rc4', key, iv, op, key_as_bytes=0, + d='md5', salt=None, i=1, padding=1) + + +def err(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, i=1, padding=1): + logging.error(('M2Crypto is required to use %s, please run' + ' `apt-get install python-m2crypto`') % alg) + sys.exit(1) + + +if not has_m2: + create_cipher = err + +ciphers = { + 'aes-128-cfb': (16, 16, create_cipher), + 'aes-192-cfb': (24, 16, create_cipher), + 'aes-256-cfb': (32, 16, create_cipher), + 'bf-cfb': (16, 8, create_cipher), + 'camellia-128-cfb': (16, 16, create_cipher), + 'camellia-192-cfb': (24, 16, create_cipher), + 'camellia-256-cfb': (32, 16, create_cipher), + 'cast5-cfb': (16, 8, create_cipher), + 'des-cfb': (8, 8, create_cipher), + 'idea-cfb': (16, 8, create_cipher), + 'rc2-cfb': (16, 8, create_cipher), + 'rc4': (16, 0, create_cipher), + 'seed-cfb': (16, 16, create_cipher), +} diff --git a/shadowsocks/encrypt_rc4_md5.py b/shadowsocks/crypto/rc4_md5.py similarity index 94% rename from shadowsocks/encrypt_rc4_md5.py rename to shadowsocks/crypto/rc4_md5.py index 8295153..571f666 100644 --- a/shadowsocks/encrypt_rc4_md5.py +++ b/shadowsocks/crypto/rc4_md5.py @@ -20,9 +20,13 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. + import hashlib +__all__ = ['ciphers'] + + def create_cipher(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, i=1, padding=1): md5 = hashlib.md5() @@ -33,3 +37,8 @@ def create_cipher(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, import M2Crypto.EVP return M2Crypto.EVP.Cipher('rc4', rc4_key, '', op, key_as_bytes=0, d='md5', salt=None, i=1, padding=1) + + +ciphers = { + 'rc4-md5': (16, 16, create_cipher), +} diff --git a/shadowsocks/encrypt_salsa20.py b/shadowsocks/crypto/salsa20_ctr.py similarity index 98% rename from shadowsocks/encrypt_salsa20.py rename to shadowsocks/crypto/salsa20_ctr.py index be8cb8b..22a8f35 100644 --- a/shadowsocks/encrypt_salsa20.py +++ b/shadowsocks/crypto/salsa20_ctr.py @@ -149,5 +149,10 @@ def test(): assert ''.join(results) == plain +ciphers = { + 'salsa20-ctr': (32, 8, Salsa20Cipher), +} + + if __name__ == '__main__': test() diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index d144e64..9675e85 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -26,8 +26,19 @@ import hashlib import string import struct import logging -import encrypt_salsa20 -import encrypt_rc4_md5 +import crypto.m2 +import crypto.rc4_md5 +import crypto.salsa20_ctr +import crypto.ctypes_openssl + + +method_supported = { +} + +method_supported.update(crypto.m2.ciphers) +method_supported.update(crypto.rc4_md5.ciphers) +method_supported.update(crypto.salsa20_ctr.ciphers) +method_supported.update(crypto.ctypes_openssl.ciphers) def random_string(length): @@ -57,13 +68,6 @@ def get_table(key): def init_table(key, method=None): if method is not None and method == 'table': method = None - if method: - try: - __import__('M2Crypto') - except ImportError: - logging.error(('M2Crypto is required to use %s, please run' - ' `apt-get install python-m2crypto`') % method) - sys.exit(1) if not method: if key in cached_tables: return cached_tables[key] @@ -103,25 +107,6 @@ def EVP_BytesToKey(password, key_len, iv_len): return (key, iv) -method_supported = { - 'aes-128-cfb': (16, 16), - 'aes-192-cfb': (24, 16), - 'aes-256-cfb': (32, 16), - 'bf-cfb': (16, 8), - 'camellia-128-cfb': (16, 16), - 'camellia-192-cfb': (24, 16), - 'camellia-256-cfb': (32, 16), - 'cast5-cfb': (16, 8), - 'des-cfb': (8, 8), - 'idea-cfb': (16, 8), - 'rc2-cfb': (16, 8), - 'rc4': (16, 0), - 'rc4-md5': (16, 16), - 'seed-cfb': (16, 16), - 'salsa20-ctr': (32, 8), -} - - class Encryptor(object): def __init__(self, key, method=None): if method == 'table': @@ -138,7 +123,7 @@ class Encryptor(object): self.encrypt_table, self.decrypt_table = init_table(key) self.cipher = None - def get_cipher_len(self, method): + def get_cipher_param(self, method): method = method.lower() m = method_supported.get(method, None) return m @@ -149,7 +134,7 @@ class Encryptor(object): def get_cipher(self, password, method, op, iv=None): password = password.encode('utf-8') method = method.lower() - m = self.get_cipher_len(method) + m = self.get_cipher_param(method) if m: key, iv_ = EVP_BytesToKey(password, m[0], m[1]) if iv is None: @@ -158,15 +143,7 @@ class Encryptor(object): if op == 1: # this iv is for cipher not decipher self.cipher_iv = iv[:m[1]] - if method == 'salsa20-ctr': - return encrypt_salsa20.Salsa20Cipher(method, key, iv, op) - elif method == 'rc4-md5': - return encrypt_rc4_md5.create_cipher(method, key, iv, op) - else: - import M2Crypto.EVP - return M2Crypto.EVP.Cipher(method.replace('-', '_'), key, iv, - op, key_as_bytes=0, d='md5', - salt=None, i=1, padding=1) + return m[2](method, key, iv, op) logging.error('method %s not supported' % method) sys.exit(1) @@ -190,7 +167,7 @@ class Encryptor(object): return string.translate(buf, self.decrypt_table) else: if self.decipher is None: - decipher_iv_len = self.get_cipher_len(self.method)[1] + decipher_iv_len = self.get_cipher_param(self.method)[1] decipher_iv = buf[:decipher_iv_len] self.decipher = self.get_cipher(self.key, self.method, 0, iv=decipher_iv) @@ -210,10 +187,9 @@ def encrypt_all(password, method, op, data): else: return string.translate(data, decrypt_table) else: - import M2Crypto.EVP result = [] method = method.lower() - (key_len, iv_len) = method_supported[method] + (key_len, iv_len, m) = method_supported[method] (key, _) = EVP_BytesToKey(password, key_len, iv_len) if op: iv = random_string(iv_len) @@ -221,13 +197,6 @@ def encrypt_all(password, method, op, data): else: iv = data[:iv_len] data = data[iv_len:] - if method == 'salsa20-ctr': - cipher = encrypt_salsa20.Salsa20Cipher(method, key, iv, op) - elif method == 'rc4-md5': - cipher = encrypt_rc4_md5.create_cipher(method, key, iv, op) - else: - cipher = M2Crypto.EVP.Cipher(method.replace('-', '_'), key, iv, - op, key_as_bytes=0, d='md5', - salt=None, i=1, padding=1) + cipher = m(method, key, iv, op) result.append(cipher.update(data)) return ''.join(result) diff --git a/tests/aes-cfb1.json b/tests/aes-cfb1.json new file mode 100644 index 0000000..40d0b21 --- /dev/null +++ b/tests/aes-cfb1.json @@ -0,0 +1,10 @@ +{ + "server":"127.0.0.1", + "server_port":8388, + "local_port":1081, + "password":"aes_password", + "timeout":60, + "method":"aes-256-cfb1", + "local_address":"127.0.0.1", + "fast_open":false +} diff --git a/tests/aes-cfb8.json b/tests/aes-cfb8.json new file mode 100644 index 0000000..fb7014b --- /dev/null +++ b/tests/aes-cfb8.json @@ -0,0 +1,10 @@ +{ + "server":"127.0.0.1", + "server_port":8388, + "local_port":1081, + "password":"aes_password", + "timeout":60, + "method":"aes-256-cfb8", + "local_address":"127.0.0.1", + "fast_open":false +} diff --git a/tests/test.py b/tests/test.py index 18aff4b..632312f 100755 --- a/tests/test.py +++ b/tests/test.py @@ -11,8 +11,8 @@ from subprocess import Popen, PIPE sys.path.insert(0, './') if 'salsa20' in sys.argv[-1]: - from shadowsocks import encrypt_salsa20 - encrypt_salsa20.test() + from shadowsocks.crypto import salsa20_ctr + salsa20_ctr.test() print 'encryption test passed' p1 = Popen(['python', 'shadowsocks/server.py', '-c', sys.argv[-1]], stdin=PIPE, From 2122f96aeb38181cdb35f3dcbc77fcf969a63ab8 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 16 Sep 2014 11:03:36 +0800 Subject: [PATCH 192/344] lint code --- shadowsocks/encrypt.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 9675e85..2f9872a 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -32,8 +32,7 @@ import crypto.salsa20_ctr import crypto.ctypes_openssl -method_supported = { -} +method_supported = {} method_supported.update(crypto.m2.ciphers) method_supported.update(crypto.rc4_md5.ciphers) From 0e649453edb26905f0ad5006593a5de9a272ff0a Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 16 Sep 2014 23:55:50 +0800 Subject: [PATCH 193/344] support CTR mode --- shadowsocks/crypto/ctypes_openssl.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/shadowsocks/crypto/ctypes_openssl.py b/shadowsocks/crypto/ctypes_openssl.py index 8cbd9e5..b688765 100644 --- a/shadowsocks/crypto/ctypes_openssl.py +++ b/shadowsocks/crypto/ctypes_openssl.py @@ -54,12 +54,24 @@ def load_openssl(): loaded = True +def load_ctr_cipher(cipher_name): + func_name = 'EVP_' + cipher_name.replace('-', '_') + cipher = getattr(libcrypto, func_name, None) + if cipher: + cipher.restype = c_void_p + return cipher() + return None + + class CtypesCrypto(object): def __init__(self, cipher_name, key, iv, op): if not loaded: load_openssl() self._ctx = None - cipher = libcrypto.EVP_get_cipherbyname(cipher_name) + if 'ctr' in cipher_name: + cipher = load_ctr_cipher(cipher_name) + else: + cipher = libcrypto.EVP_get_cipherbyname(cipher_name) if not cipher: raise Exception('cipher %s not found in libcrypto' % cipher_name) key_ptr = c_char_p(key) @@ -91,6 +103,9 @@ class CtypesCrypto(object): ciphers = { + 'aes-128-ctr': (16, 16, CtypesCrypto), + 'aes-192-ctr': (24, 16, CtypesCrypto), + 'aes-256-ctr': (32, 16, CtypesCrypto), 'aes-128-cfb8': (16, 16, CtypesCrypto), 'aes-192-cfb8': (24, 16, CtypesCrypto), 'aes-256-cfb8': (32, 16, CtypesCrypto), From 8114f7043a0b12c10f3ef0289f8d8de1c30def7c Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 16 Sep 2014 23:58:21 +0800 Subject: [PATCH 194/344] add CTR test --- tests/aes-ctr.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 tests/aes-ctr.json diff --git a/tests/aes-ctr.json b/tests/aes-ctr.json new file mode 100644 index 0000000..1fed8a8 --- /dev/null +++ b/tests/aes-ctr.json @@ -0,0 +1,10 @@ +{ + "server":"127.0.0.1", + "server_port":8388, + "local_port":1081, + "password":"aes_password", + "timeout":60, + "method":"aes-256-ctr", + "local_address":"127.0.0.1", + "fast_open":false +} From 4ddc20945ef788614b275908ebb29f0acf4b41d0 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 17 Sep 2014 00:05:12 +0800 Subject: [PATCH 195/344] add travis test --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b10edd9..70a3cf9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ script: - pep8 . - python tests/test.py -c tests/table.json - python tests/test.py -c tests/aes.json + - python tests/test.py -c tests/aes-ctr.json - python tests/test.py -c tests/aes-cfb1.json - python tests/test.py -c tests/aes-cfb8.json - python tests/test.py -c tests/rc4-md5.json From 402bc5743be6ab1420cabe418d26ec46ebbf563d Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 17 Sep 2014 00:11:50 +0800 Subject: [PATCH 196/344] add comment about buf --- shadowsocks/crypto/ctypes_openssl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shadowsocks/crypto/ctypes_openssl.py b/shadowsocks/crypto/ctypes_openssl.py index b688765..88d9392 100644 --- a/shadowsocks/crypto/ctypes_openssl.py +++ b/shadowsocks/crypto/ctypes_openssl.py @@ -91,6 +91,7 @@ class CtypesCrypto(object): libcrypto.EVP_CipherUpdate(self._ctx, byref(buf), byref(cipher_out_len), c_char_p(data), len(data)) + # buf is copied to a str object when we access buf.raw return buf.raw[:cipher_out_len.value] def __del__(self): From d6a5c567aebff58600d452469c9dcc8a681ab6fc Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 17 Sep 2014 13:52:28 +0800 Subject: [PATCH 197/344] lint --- shadowsocks/crypto/ctypes_openssl.py | 45 ++++++++++++++++++++++++++++ shadowsocks/crypto/salsa20_ctr.py | 10 +++---- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/shadowsocks/crypto/ctypes_openssl.py b/shadowsocks/crypto/ctypes_openssl.py index 88d9392..823dd90 100644 --- a/shadowsocks/crypto/ctypes_openssl.py +++ b/shadowsocks/crypto/ctypes_openssl.py @@ -114,3 +114,48 @@ ciphers = { 'aes-192-cfb1': (24, 16, CtypesCrypto), 'aes-256-cfb1': (32, 16, CtypesCrypto), } + + +def test(): + from os import urandom + import random + import time + + BLOCK_SIZE = 16384 + rounds = 1 * 1024 + plain = urandom(BLOCK_SIZE * rounds) + import M2Crypto.EVP + # cipher = M2Crypto.EVP.Cipher('aes_128_cfb', 'k' * 32, 'i' * 16, 1, + # key_as_bytes=0, d='md5', salt=None, i=1, + # padding=1) + # decipher = M2Crypto.EVP.Cipher('aes_128_cfb', 'k' * 32, 'i' * 16, 0, + # key_as_bytes=0, d='md5', salt=None, i=1, + # padding=1) + cipher = CtypesCrypto('aes-128-cfb', 'k' * 32, 'i' * 16, 1) + decipher = CtypesCrypto('aes-128-cfb', 'k' * 32, 'i' * 16, 0) + + # cipher = Salsa20Cipher('salsa20-ctr', 'k' * 32, 'i' * 8, 1) + # decipher = Salsa20Cipher('salsa20-ctr', 'k' * 32, 'i' * 8, 1) + results = [] + pos = 0 + print 'salsa20 test start' + start = time.time() + while pos < len(plain): + l = random.randint(100, 32768) + c = cipher.update(plain[pos:pos + l]) + results.append(c) + pos += l + pos = 0 + c = ''.join(results) + results = [] + while pos < len(plain): + l = random.randint(100, 32768) + results.append(decipher.update(c[pos:pos + l])) + pos += l + end = time.time() + print 'speed: %d bytes/s' % (BLOCK_SIZE * rounds / (end - start)) + assert ''.join(results) == plain + + +if __name__ == '__main__': + test() \ No newline at end of file diff --git a/shadowsocks/crypto/salsa20_ctr.py b/shadowsocks/crypto/salsa20_ctr.py index 22a8f35..3f74d59 100644 --- a/shadowsocks/crypto/salsa20_ctr.py +++ b/shadowsocks/crypto/salsa20_ctr.py @@ -112,6 +112,11 @@ class Salsa20Cipher(object): return ''.join(results) +ciphers = { + 'salsa20-ctr': (32, 8, Salsa20Cipher), +} + + def test(): from os import urandom import random @@ -149,10 +154,5 @@ def test(): assert ''.join(results) == plain -ciphers = { - 'salsa20-ctr': (32, 8, Salsa20Cipher), -} - - if __name__ == '__main__': test() From f6951c963303e3ecfbb4194dafe94ad9400e2284 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 17 Sep 2014 13:54:33 +0800 Subject: [PATCH 198/344] fix m2 --- shadowsocks/crypto/m2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/crypto/m2.py b/shadowsocks/crypto/m2.py index cff3682..d69a468 100644 --- a/shadowsocks/crypto/m2.py +++ b/shadowsocks/crypto/m2.py @@ -36,7 +36,7 @@ def create_cipher(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, i=1, padding=1): import M2Crypto.EVP - return M2Crypto.EVP.Cipher('rc4', key, iv, op, key_as_bytes=0, + return M2Crypto.EVP.Cipher(alg.replace('-', '_'), key, iv, op, key_as_bytes=0, d='md5', salt=None, i=1, padding=1) From 4be27312932475d05517d2ec5f63b6cf3d883e82 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 17 Sep 2014 14:15:24 +0800 Subject: [PATCH 199/344] fix PEP8 --- shadowsocks/crypto/ctypes_openssl.py | 2 +- shadowsocks/crypto/m2.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/shadowsocks/crypto/ctypes_openssl.py b/shadowsocks/crypto/ctypes_openssl.py index 823dd90..696952a 100644 --- a/shadowsocks/crypto/ctypes_openssl.py +++ b/shadowsocks/crypto/ctypes_openssl.py @@ -158,4 +158,4 @@ def test(): if __name__ == '__main__': - test() \ No newline at end of file + test() diff --git a/shadowsocks/crypto/m2.py b/shadowsocks/crypto/m2.py index d69a468..d2a4927 100644 --- a/shadowsocks/crypto/m2.py +++ b/shadowsocks/crypto/m2.py @@ -36,8 +36,9 @@ def create_cipher(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, i=1, padding=1): import M2Crypto.EVP - return M2Crypto.EVP.Cipher(alg.replace('-', '_'), key, iv, op, key_as_bytes=0, - d='md5', salt=None, i=1, padding=1) + return M2Crypto.EVP.Cipher(alg.replace('-', '_'), key, iv, op, + key_as_bytes=0, d='md5', salt=None, i=1, + padding=1) def err(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, i=1, padding=1): From b3420f24c7275cf636afa024910a5157b7870789 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 18 Sep 2014 09:20:57 +0800 Subject: [PATCH 200/344] update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9f78680..f114bb2 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ List all available args with `-h`. Wiki ---- +You can find all the documentation in the wiki: https://github.com/clowwindy/shadowsocks/wiki License From 81223902d09da1dc0bad8e82768832df00b12b6c Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 19 Sep 2014 01:07:10 +0800 Subject: [PATCH 201/344] fix a potential BOF --- shadowsocks/crypto/ctypes_openssl.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/shadowsocks/crypto/ctypes_openssl.py b/shadowsocks/crypto/ctypes_openssl.py index 696952a..de3514c 100644 --- a/shadowsocks/crypto/ctypes_openssl.py +++ b/shadowsocks/crypto/ctypes_openssl.py @@ -26,6 +26,8 @@ __all__ = ['ciphers'] loaded = False +buf_size = 2048 + def load_openssl(): global loaded, libcrypto, CDLL, c_char_p, c_int, c_long, byref,\ @@ -50,7 +52,7 @@ def load_openssl(): libcrypto.EVP_CIPHER_CTX_cleanup.argtypes = (c_void_p,) libcrypto.EVP_CIPHER_CTX_free.argtypes = (c_void_p,) - buf = create_string_buffer(65536) + buf = create_string_buffer(buf_size) loaded = True @@ -87,10 +89,14 @@ class CtypesCrypto(object): raise Exception('can not initialize cipher context') def update(self, data): + global buf_size, buf cipher_out_len = c_long(0) + l = len(data) + if buf_size < l: + buf_size = l * 2 + buf = create_string_buffer(buf_size) libcrypto.EVP_CipherUpdate(self._ctx, byref(buf), - byref(cipher_out_len), c_char_p(data), - len(data)) + byref(cipher_out_len), c_char_p(data), l) # buf is copied to a str object when we access buf.raw return buf.raw[:cipher_out_len.value] From f0b0f6edffcfff1e82cbde49063d1a3c5e480da7 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 20 Sep 2014 18:40:35 +0800 Subject: [PATCH 202/344] do not mix config of client and server for server-multi-passwd --- .travis.yml | 2 +- tests/server-multi-passwd-client-side.json | 8 ++++++++ tests/server-multi-passwd.json | 2 -- tests/test.py | 17 +++++++++++++---- 4 files changed, 22 insertions(+), 7 deletions(-) create mode 100644 tests/server-multi-passwd-client-side.json diff --git a/.travis.yml b/.travis.yml index 70a3cf9..e36c641 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,6 @@ script: - python tests/test.py -c tests/rc4-md5.json - python tests/test.py -c tests/salsa20.json - python tests/test.py -c tests/server-multi-ports.json - - python tests/test.py -c tests/server-multi-passwd.json + - python tests/test.py -c tests/server-multi-passwd.json tests/server-multi-passwd-client-side.json - python tests/test.py -c tests/server-multi-passwd-table.json - python tests/test.py -c tests/workers.json diff --git a/tests/server-multi-passwd-client-side.json b/tests/server-multi-passwd-client-side.json new file mode 100644 index 0000000..c822c98 --- /dev/null +++ b/tests/server-multi-passwd-client-side.json @@ -0,0 +1,8 @@ +{ + "server": "127.0.0.1", + "server_port": "8385", + "local_port": 1081, + "password": "foobar5", + "timeout": 60, + "method": "aes-256-cfb" +} diff --git a/tests/server-multi-passwd.json b/tests/server-multi-passwd.json index eff5ed8..b1407f0 100644 --- a/tests/server-multi-passwd.json +++ b/tests/server-multi-passwd.json @@ -1,8 +1,6 @@ { "server": "127.0.0.1", - "server_port": 8384, "local_port": 1081, - "password": "foobar4", "port_password": { "8381": "foobar1", "8382": "foobar2", diff --git a/tests/test.py b/tests/test.py index 632312f..83c2332 100755 --- a/tests/test.py +++ b/tests/test.py @@ -10,15 +10,24 @@ from subprocess import Popen, PIPE sys.path.insert(0, './') +if sys.argv[-3] == '-c': + client_config = sys.argv[-1] + server_config = sys.argv[-2] +elif sys.argv[-2] == '-c': + client_config = sys.argv[-1] + server_config = sys.argv[-1] +else: + raise Exception('usage: test.py -c server_conf [client_conf]') + if 'salsa20' in sys.argv[-1]: from shadowsocks.crypto import salsa20_ctr salsa20_ctr.test() print 'encryption test passed' -p1 = Popen(['python', 'shadowsocks/server.py', '-c', sys.argv[-1]], stdin=PIPE, - stdout=PIPE, stderr=PIPE, close_fds=True) -p2 = Popen(['python', 'shadowsocks/local.py', '-c', sys.argv[-1]], stdin=PIPE, - stdout=PIPE, stderr=PIPE, close_fds=True) +p1 = Popen(['python', 'shadowsocks/server.py', '-c', server_config], + stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) +p2 = Popen(['python', 'shadowsocks/local.py', '-c', client_config], + stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) p3 = None try: From ab74c010fd87e1b38f0794dbd90654f9f935aacb Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 20 Sep 2014 19:01:52 +0800 Subject: [PATCH 203/344] close #183 --- shadowsocks/server.py | 2 +- shadowsocks/utils.py | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 0ad46fb..0b1bfd8 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -41,7 +41,7 @@ def main(): utils.print_shadowsocks() if config['port_password']: - if config['server_port'] or config['password']: + if config['password']: logging.warn('warning: port_password should not be used with ' 'server_port and password. server_port and password ' 'will be ignored') diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 2f2c0b7..252c564 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -154,7 +154,7 @@ def get_config(is_local): print_help(is_local) sys.exit(2) - config['password'] = config.get('password', None) + config['password'] = config.get('password', '') config['method'] = config.get('method', 'aes-256-cfb') config['port_password'] = config.get('port_password', None) config['timeout'] = int(config.get('timeout', 300)) @@ -172,11 +172,23 @@ def get_config(is_local): config['server'] = config.get('server', '0.0.0.0') config['server_port'] = config.get('server_port', 8388) - if not ('password' in config and config['password']): + if is_local and not config.get('password', None): logging.error('password not specified') print_help(is_local) sys.exit(2) + if not is_local and not config.get('password', None) \ + and not config.get('port_password', None): + logging.error('password or port_password not specified') + print_help(is_local) + sys.exit(2) + + if 'local_port' in config: + config['local_port'] = int(config['local_port']) + + if 'server_port' in config: + config['server_port'] = int(config['server_port']) + logging.getLogger('').handlers = [] logging.addLevelName(VERBOSE_LEVEL, 'VERBOSE') if config['verbose'] >= 2: From f677c8152d6d399651f9f499ac7a8d487073309c Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 20 Sep 2014 19:11:33 +0800 Subject: [PATCH 204/344] close #185 --- shadowsocks/local.py | 2 +- shadowsocks/server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 5aeee3b..be4d66f 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -64,7 +64,7 @@ def main(): logging.warn('received SIGQUIT, doing graceful shutting down..') tcp_server.close(next_tick=True) udp_server.close(next_tick=True) - signal.signal(signal.SIGQUIT, handler) + signal.signal(getattr(signal, 'SIGQUIT', signal.SIGTERM), handler) loop.run() except (KeyboardInterrupt, IOError, OSError) as e: logging.error(e) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 0b1bfd8..16d39f0 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -71,7 +71,7 @@ def main(): def child_handler(signum, _): logging.warn('received SIGQUIT, doing graceful shutting down..') map(lambda s: s.close(next_tick=True), tcp_servers + udp_servers) - signal.signal(signal.SIGQUIT, child_handler) + signal.signal(getattr(signal, 'SIGQUIT', signal.SIGTERM), child_handler) try: loop = eventloop.EventLoop() dns_resolver.add_to_loop(loop) From c668ddbc1b37338c73ece403dcf56fb02ed2aa88 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 20 Sep 2014 19:57:55 +0800 Subject: [PATCH 205/344] fix multiple ports --- shadowsocks/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 252c564..fee4afc 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -186,7 +186,7 @@ def get_config(is_local): if 'local_port' in config: config['local_port'] = int(config['local_port']) - if 'server_port' in config: + if 'server_port' in config and type(config['server_port']) != list: config['server_port'] = int(config['server_port']) logging.getLogger('').handlers = [] From dda8f31bd3c9c41daf68cc46b17e894f17b5e058 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 20 Sep 2014 20:21:09 +0800 Subject: [PATCH 206/344] fix PEP8 --- shadowsocks/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 16d39f0..fd38dd9 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -71,7 +71,8 @@ def main(): def child_handler(signum, _): logging.warn('received SIGQUIT, doing graceful shutting down..') map(lambda s: s.close(next_tick=True), tcp_servers + udp_servers) - signal.signal(getattr(signal, 'SIGQUIT', signal.SIGTERM), child_handler) + signal.signal(getattr(signal, 'SIGQUIT', signal.SIGTERM), + child_handler) try: loop = eventloop.EventLoop() dns_resolver.add_to_loop(loop) From 1af7f84f415ea68cc583e30d99f1ff4333e684dc Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 23 Sep 2014 13:52:52 +0800 Subject: [PATCH 207/344] update changes --- CHANGES | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES b/CHANGES index f6e8172..fb483ea 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,8 @@ +2.3 2014-09-23 +- Support CFB1, CFB8 and CTR mode of AES +- Do not require password config when using port_password +- Use SIGTERM instead of SIGQUIT on Windows + 2.2.2 2014-09-14 - Fix when multiple DNS set, IPv6 only sites are broken From dfd4cdd75835b25cc702e5728dcc613803d4680e Mon Sep 17 00:00:00 2001 From: sybblow Date: Thu, 25 Sep 2014 20:18:02 +0800 Subject: [PATCH 208/344] avoid memory leak Although fd is always the smallest number which haven't been allocated, which means memory leak is quite small. but the worst situation may consume lots of memory. --- shadowsocks/eventloop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index de0039e..4a35950 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -184,7 +184,7 @@ class EventLoop(object): def remove(self, f): fd = f.fileno() - self._fd_to_f[fd] = None + del self._fd_to_f[fd] self._impl.remove_fd(fd) def modify(self, f, mode): From eb9b6f0a9b716423202fc390672bfb78f9f0eb8d Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 8 Oct 2014 21:03:01 +0800 Subject: [PATCH 209/344] does not require M2Crypto anymore --- setup.py | 2 +- shadowsocks/crypto/ctypes_openssl.py | 32 ++++++++++++++++++++----- shadowsocks/crypto/m2.py | 36 ++++++++++++++-------------- shadowsocks/crypto/rc4_md5.py | 16 ++++++++++--- shadowsocks/encrypt.py | 3 ++- 5 files changed, 60 insertions(+), 29 deletions(-) diff --git a/setup.py b/setup.py index 8ddf3c5..8388e3f 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.3", + version="2.3.1", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', diff --git a/shadowsocks/crypto/ctypes_openssl.py b/shadowsocks/crypto/ctypes_openssl.py index de3514c..7821d5b 100644 --- a/shadowsocks/crypto/ctypes_openssl.py +++ b/shadowsocks/crypto/ctypes_openssl.py @@ -35,7 +35,12 @@ def load_openssl(): from ctypes import CDLL, c_char_p, c_int, c_long, byref,\ create_string_buffer, c_void_p from ctypes.util import find_library - libcrypto_path = find_library('crypto') + for p in ('crypto', 'eay32', 'libeay32'): + libcrypto_path = find_library(p) + if libcrypto_path: + break + else: + raise Exception('libcrypto(OpenSSL) not found') logging.info('loading libcrypto from %s', libcrypto_path) libcrypto = CDLL(libcrypto_path) libcrypto.EVP_get_cipherbyname.restype = c_void_p @@ -56,7 +61,7 @@ def load_openssl(): loaded = True -def load_ctr_cipher(cipher_name): +def load_cipher(cipher_name): func_name = 'EVP_' + cipher_name.replace('-', '_') cipher = getattr(libcrypto, func_name, None) if cipher: @@ -70,10 +75,9 @@ class CtypesCrypto(object): if not loaded: load_openssl() self._ctx = None - if 'ctr' in cipher_name: - cipher = load_ctr_cipher(cipher_name) - else: - cipher = libcrypto.EVP_get_cipherbyname(cipher_name) + cipher = libcrypto.EVP_get_cipherbyname(cipher_name) + if not cipher: + cipher = load_cipher(cipher_name) if not cipher: raise Exception('cipher %s not found in libcrypto' % cipher_name) key_ptr = c_char_p(key) @@ -110,6 +114,12 @@ class CtypesCrypto(object): ciphers = { + 'aes-128-cfb': (16, 16, CtypesCrypto), + 'aes-192-cfb': (24, 16, CtypesCrypto), + 'aes-256-cfb': (32, 16, CtypesCrypto), + 'aes-128-ofb': (16, 16, CtypesCrypto), + 'aes-192-ofb': (24, 16, CtypesCrypto), + 'aes-256-ofb': (32, 16, CtypesCrypto), 'aes-128-ctr': (16, 16, CtypesCrypto), 'aes-192-ctr': (24, 16, CtypesCrypto), 'aes-256-ctr': (32, 16, CtypesCrypto), @@ -119,6 +129,16 @@ ciphers = { 'aes-128-cfb1': (16, 16, CtypesCrypto), 'aes-192-cfb1': (24, 16, CtypesCrypto), 'aes-256-cfb1': (32, 16, CtypesCrypto), + 'bf-cfb': (16, 8, CtypesCrypto), + 'camellia-128-cfb': (16, 16, CtypesCrypto), + 'camellia-192-cfb': (24, 16, CtypesCrypto), + 'camellia-256-cfb': (32, 16, CtypesCrypto), + 'cast5-cfb': (16, 8, CtypesCrypto), + 'des-cfb': (8, 8, CtypesCrypto), + 'idea-cfb': (16, 8, CtypesCrypto), + 'rc2-cfb': (16, 8, CtypesCrypto), + 'rc4': (16, 0, CtypesCrypto), + 'seed-cfb': (16, 16, CtypesCrypto), } diff --git a/shadowsocks/crypto/m2.py b/shadowsocks/crypto/m2.py index d2a4927..c140061 100644 --- a/shadowsocks/crypto/m2.py +++ b/shadowsocks/crypto/m2.py @@ -47,21 +47,21 @@ def err(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, i=1, padding=1): sys.exit(1) -if not has_m2: - create_cipher = err - -ciphers = { - 'aes-128-cfb': (16, 16, create_cipher), - 'aes-192-cfb': (24, 16, create_cipher), - 'aes-256-cfb': (32, 16, create_cipher), - 'bf-cfb': (16, 8, create_cipher), - 'camellia-128-cfb': (16, 16, create_cipher), - 'camellia-192-cfb': (24, 16, create_cipher), - 'camellia-256-cfb': (32, 16, create_cipher), - 'cast5-cfb': (16, 8, create_cipher), - 'des-cfb': (8, 8, create_cipher), - 'idea-cfb': (16, 8, create_cipher), - 'rc2-cfb': (16, 8, create_cipher), - 'rc4': (16, 0, create_cipher), - 'seed-cfb': (16, 16, create_cipher), -} +if has_m2: + ciphers = { + 'aes-128-cfb': (16, 16, create_cipher), + 'aes-192-cfb': (24, 16, create_cipher), + 'aes-256-cfb': (32, 16, create_cipher), + 'bf-cfb': (16, 8, create_cipher), + 'camellia-128-cfb': (16, 16, create_cipher), + 'camellia-192-cfb': (24, 16, create_cipher), + 'camellia-256-cfb': (32, 16, create_cipher), + 'cast5-cfb': (16, 8, create_cipher), + 'des-cfb': (8, 8, create_cipher), + 'idea-cfb': (16, 8, create_cipher), + 'rc2-cfb': (16, 8, create_cipher), + 'rc4': (16, 0, create_cipher), + 'seed-cfb': (16, 16, create_cipher), + } +else: + ciphers = {} diff --git a/shadowsocks/crypto/rc4_md5.py b/shadowsocks/crypto/rc4_md5.py index 571f666..3dac6d5 100644 --- a/shadowsocks/crypto/rc4_md5.py +++ b/shadowsocks/crypto/rc4_md5.py @@ -26,17 +26,27 @@ import hashlib __all__ = ['ciphers'] +m2_not_found = False + def create_cipher(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, i=1, padding=1): + global m2_not_found + md5 = hashlib.md5() md5.update(key) md5.update(iv) rc4_key = md5.digest() - import M2Crypto.EVP - return M2Crypto.EVP.Cipher('rc4', rc4_key, '', op, key_as_bytes=0, - d='md5', salt=None, i=1, padding=1) + if not m2_not_found: + try: + import M2Crypto.EVP + return M2Crypto.EVP.Cipher('rc4', rc4_key, '', op, key_as_bytes=0, + d='md5', salt=None, i=1, padding=1) + except: + m2_not_found = True + import ctypes_openssl + return ctypes_openssl.CtypesCrypto('rc4', rc4_key, '', op) ciphers = { diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 2f9872a..2e66175 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -34,10 +34,11 @@ import crypto.ctypes_openssl method_supported = {} -method_supported.update(crypto.m2.ciphers) method_supported.update(crypto.rc4_md5.ciphers) method_supported.update(crypto.salsa20_ctr.ciphers) method_supported.update(crypto.ctypes_openssl.ciphers) +# let M2Crypto override ctypes_openssl +method_supported.update(crypto.m2.ciphers) def random_string(length): From 0b64f7670f108baf7d42e9a86bf9d1b5cf881e61 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 8 Oct 2014 21:07:05 +0800 Subject: [PATCH 210/344] add help for Windows --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index f114bb2..e7697a7 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,12 @@ source list. easy_install pip pip install shadowsocks +#### Windows: + +Download OpenSSL for Windows and install. Then + + pip install shadowsocks + Configuration ------------- From b47de1dcf69fd06784d205972126586c165b8dbd Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 9 Oct 2014 21:25:24 +0800 Subject: [PATCH 211/344] update CHANGES --- CHANGES | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES b/CHANGES index fb483ea..642e0d5 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +2.3.1 2014-10-09 +- Does not require M2Crypto any more + 2.3 2014-09-23 - Support CFB1, CFB8 and CTR mode of AES - Do not require password config when using port_password From 9e8aa1c56ae4630fb3fce89cafe6cae78cdd8d82 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 11 Oct 2014 00:37:29 +0800 Subject: [PATCH 212/344] fix libcrypto on Windows --- shadowsocks/crypto/ctypes_openssl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shadowsocks/crypto/ctypes_openssl.py b/shadowsocks/crypto/ctypes_openssl.py index 7821d5b..9c8386b 100644 --- a/shadowsocks/crypto/ctypes_openssl.py +++ b/shadowsocks/crypto/ctypes_openssl.py @@ -56,6 +56,8 @@ def load_openssl(): libcrypto.EVP_CIPHER_CTX_cleanup.argtypes = (c_void_p,) libcrypto.EVP_CIPHER_CTX_free.argtypes = (c_void_p,) + if hasattr(libcrypto, 'OpenSSL_add_all_ciphers'): + libcrypto.OpenSSL_add_all_ciphers() buf = create_string_buffer(buf_size) loaded = True From cd0963802365468fa1134be940551405b80ef449 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 11 Oct 2014 12:36:23 +0800 Subject: [PATCH 213/344] bump --- CHANGES | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 642e0d5..045bae0 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +2.3.2 2014-10-11 +- Fix OpenSSL on Windows + 2.3.1 2014-10-09 - Does not require M2Crypto any more diff --git a/setup.py b/setup.py index 8388e3f..cdc17aa 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.3.1", + version="2.3.2", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From 89a9872308150bca4c96fd575ad38d68c541afb2 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 11 Oct 2014 12:42:59 +0800 Subject: [PATCH 214/344] remove m2crypto from readme --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e7697a7..3d0ea91 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ remote server. #### Debian / Ubuntu: - apt-get install python-pip python-m2crypto + apt-get install python-pip pip install shadowsocks Or simply `apt-get install shadowsocks` if you have [Debian sid] in your @@ -31,15 +31,16 @@ source list. #### CentOS: - yum install m2crypto python-setuptools + yum install python-setuptools easy_install pip pip install shadowsocks #### Windows: -Download OpenSSL for Windows and install. Then - - pip install shadowsocks +Download OpenSSL for Windows and install. Then install shadowsocks via +easy_install and pip as Linux. If you don't know how to use them, you can +directly download [the package], and use `python shadowsocks/server.py` +instead of `ssserver` command below. Configuration ------------- @@ -128,6 +129,7 @@ Bugs and Issues [Chinese Readme]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E [Chrome Standalone]: https://support.google.com/installer/answer/126299 [Debian sid]: https://packages.debian.org/unstable/python/shadowsocks +[the package]: https://pypi.python.org/pypi/shadowsocks [Encryption]: https://github.com/clowwindy/shadowsocks/wiki/Encryption [iOS]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Help [Issue Tracker]: https://github.com/clowwindy/shadowsocks/issues?state=open From 2461f623ddc08fb5e442bf16e83ede7b01ef682b Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 14:33:43 +0800 Subject: [PATCH 215/344] change lru behaviour; #202 --- shadowsocks/lru_cache.py | 48 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/shadowsocks/lru_cache.py b/shadowsocks/lru_cache.py index 1313af5..473a5fd 100644 --- a/shadowsocks/lru_cache.py +++ b/shadowsocks/lru_cache.py @@ -15,12 +15,14 @@ class LRUCache(collections.MutableMapping): self.close_callback = close_callback self._store = {} self._time_to_keys = collections.defaultdict(list) + self._keys_to_last_time = {} self._last_visits = [] self.update(dict(*args, **kwargs)) # use the free update to set keys def __getitem__(self, key): # O(logm) t = time.time() + self._keys_to_last_time[key] = t self._time_to_keys[t].append(key) heapq.heappush(self._last_visits, t) return self._store[key] @@ -28,6 +30,7 @@ class LRUCache(collections.MutableMapping): def __setitem__(self, key, value): # O(logm) t = time.time() + self._keys_to_last_time[key] = t self._store[key] = value self._time_to_keys[t].append(key) heapq.heappush(self._last_visits, t) @@ -35,6 +38,7 @@ class LRUCache(collections.MutableMapping): def __delitem__(self, key): # O(1) del self._store[key] + del self._keys_to_last_time[key] def __iter__(self): return iter(self._store) @@ -53,13 +57,49 @@ class LRUCache(collections.MutableMapping): if self.close_callback is not None: for key in self._time_to_keys[least]: if self._store.__contains__(key): - value = self._store[key] - self.close_callback(value) + if now - self._keys_to_last_time[key] > self.timeout: + value = self._store[key] + self.close_callback(value) for key in self._time_to_keys[least]: heapq.heappop(self._last_visits) if self._store.__contains__(key): - del self._store[key] - c += 1 + if now - self._keys_to_last_time[key] > self.timeout: + del self._store[key] + c += 1 del self._time_to_keys[least] if c: logging.debug('%d keys swept' % c) + + +def test(): + c = LRUCache(timeout=0.3) + + c['a'] = 1 + assert c['a'] == 1 + + time.sleep(0.5) + c.sweep() + assert 'a' not in c + + c['a'] = 2 + c['b'] = 3 + time.sleep(0.2) + c.sweep() + assert c['a'] == 2 + assert c['b'] == 3 + + time.sleep(0.2) + c.sweep() + c['b'] + time.sleep(0.2) + c.sweep() + assert 'a' not in c + assert c['b'] == 3 + + time.sleep(0.5) + c.sweep() + assert 'a' not in c + assert 'b' not in c + +if __name__ == '__main__': + test() From 3fc543cb7f2963a2399e9ab6a093613b89b42562 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 14:49:35 +0800 Subject: [PATCH 216/344] add unit test for lru cache --- .travis.yml | 3 ++- tests/test_latency.py | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index e36c641..46419ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,10 +9,11 @@ cache: before_install: - sudo apt-get update -qq - sudo apt-get install -qq build-essential libssl-dev swig python-m2crypto python-numpy dnsutils - - pip install m2crypto salsa20 pep8 + - pip install m2crypto salsa20 pep8 nose - sudo tests/socksify/install.sh script: - pep8 . + - nosetests shadowsocks/* - python tests/test.py -c tests/table.json - python tests/test.py -c tests/aes.json - python tests/test.py -c tests/aes-ctr.json diff --git a/tests/test_latency.py b/tests/test_latency.py index 26df542..5d44644 100644 --- a/tests/test_latency.py +++ b/tests/test_latency.py @@ -4,9 +4,10 @@ import sys import time -before = time.time() +if __name__ == '__main__': + before = time.time() -for line in sys.stdin: - if 'HTTP/1.1 ' in line: - diff = time.time() - before - print 'headline %dms' % (diff * 1000) + for line in sys.stdin: + if 'HTTP/1.1 ' in line: + diff = time.time() - before + print 'headline %dms' % (diff * 1000) From 8454220adcb1ea47cb8a25e359e310513ee8913a Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 15:02:12 +0800 Subject: [PATCH 217/344] update unit test --- .travis.yml | 2 +- tests/nose_plugin.py | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 tests/nose_plugin.py diff --git a/.travis.yml b/.travis.yml index 46419ea..a430a71 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ before_install: - sudo tests/socksify/install.sh script: - pep8 . - - nosetests shadowsocks/* + - python tests/nose_plugin.py - python tests/test.py -c tests/table.json - python tests/test.py -c tests/aes.json - python tests/test.py -c tests/aes-ctr.json diff --git a/tests/nose_plugin.py b/tests/nose_plugin.py new file mode 100644 index 0000000..1db6351 --- /dev/null +++ b/tests/nose_plugin.py @@ -0,0 +1,27 @@ +import sys +import nose +from nose.plugins.base import Plugin + +class ExtensionPlugin(Plugin): + + name = "ExtensionPlugin" + + def options(self, parser, env): + Plugin.options(self,parser,env) + + def configure(self, options, config): + Plugin.configure(self, options, config) + self.enabled = True + + def wantFile(self, file): + return file.endswith('.py') + + def wantDirectory(self,directory): + return True + + def wantModule(self,file): + return True + + +if __name__ == '__main__': + nose.main(addplugins=[ExtensionPlugin()]) \ No newline at end of file From 03eecf1206f1cb0da26d6f96443fa2dbcd0df8cb Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 15:03:40 +0800 Subject: [PATCH 218/344] fix pep8 --- tests/nose_plugin.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/nose_plugin.py b/tests/nose_plugin.py index 1db6351..ad32cf0 100644 --- a/tests/nose_plugin.py +++ b/tests/nose_plugin.py @@ -1,13 +1,13 @@ -import sys import nose from nose.plugins.base import Plugin + class ExtensionPlugin(Plugin): name = "ExtensionPlugin" def options(self, parser, env): - Plugin.options(self,parser,env) + Plugin.options(self, parser, env) def configure(self, options, config): Plugin.configure(self, options, config) @@ -16,12 +16,12 @@ class ExtensionPlugin(Plugin): def wantFile(self, file): return file.endswith('.py') - def wantDirectory(self,directory): + def wantDirectory(self, directory): return True - def wantModule(self,file): + def wantModule(self, file): return True if __name__ == '__main__': - nose.main(addplugins=[ExtensionPlugin()]) \ No newline at end of file + nose.main(addplugins=[ExtensionPlugin()]) From da11d027061cd9c239dd3ca9978c88eb93e3d86d Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 15:14:28 +0800 Subject: [PATCH 219/344] fix memory leak --- shadowsocks/lru_cache.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shadowsocks/lru_cache.py b/shadowsocks/lru_cache.py index 473a5fd..6f7a6f3 100644 --- a/shadowsocks/lru_cache.py +++ b/shadowsocks/lru_cache.py @@ -65,6 +65,7 @@ class LRUCache(collections.MutableMapping): if self._store.__contains__(key): if now - self._keys_to_last_time[key] > self.timeout: del self._store[key] + del self._keys_to_last_time[key] c += 1 del self._time_to_keys[least] if c: From ac94104ac03d376fe920eac46956f5dcc0837210 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 15:49:24 +0800 Subject: [PATCH 220/344] use deque instead of heapq --- shadowsocks/lru_cache.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/shadowsocks/lru_cache.py b/shadowsocks/lru_cache.py index 6f7a6f3..1815320 100644 --- a/shadowsocks/lru_cache.py +++ b/shadowsocks/lru_cache.py @@ -3,10 +3,17 @@ import collections import logging -import heapq import time +# this LRUCache is optimized for concurrency, not QPS +# n: concurrency, keys stored in the cache +# m: visits not timed out, proportional to QPS * timeout +# get & set is O(1), not O(n). thus we can support very large n +# TODO: if timeout or QPS is too large, then this cache is not very efficient, +# as sweep() causes long pause + + class LRUCache(collections.MutableMapping): """This class is not thread safe""" @@ -16,24 +23,24 @@ class LRUCache(collections.MutableMapping): self._store = {} self._time_to_keys = collections.defaultdict(list) self._keys_to_last_time = {} - self._last_visits = [] + self._last_visits = collections.deque() self.update(dict(*args, **kwargs)) # use the free update to set keys def __getitem__(self, key): - # O(logm) + # O(1) t = time.time() self._keys_to_last_time[key] = t self._time_to_keys[t].append(key) - heapq.heappush(self._last_visits, t) + self._last_visits.append(t) return self._store[key] def __setitem__(self, key, value): - # O(logm) + # O(1) t = time.time() self._keys_to_last_time[key] = t self._store[key] = value self._time_to_keys[t].append(key) - heapq.heappush(self._last_visits, t) + self._last_visits.append(t) def __delitem__(self, key): # O(1) @@ -61,7 +68,7 @@ class LRUCache(collections.MutableMapping): value = self._store[key] self.close_callback(value) for key in self._time_to_keys[least]: - heapq.heappop(self._last_visits) + self._last_visits.popleft() if self._store.__contains__(key): if now - self._keys_to_last_time[key] > self.timeout: del self._store[key] From c553e7a725101ae084e07857c5a6e4f244effbcc Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 16:10:41 +0800 Subject: [PATCH 221/344] lint code --- shadowsocks/crypto/salsa20_ctr.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/shadowsocks/crypto/salsa20_ctr.py b/shadowsocks/crypto/salsa20_ctr.py index 3f74d59..0a89d6b 100644 --- a/shadowsocks/crypto/salsa20_ctr.py +++ b/shadowsocks/crypto/salsa20_ctr.py @@ -28,6 +28,9 @@ import sys slow_xor = False imported = False +salsa20 = None +numpy = None + BLOCK_SIZE = 16384 @@ -36,13 +39,13 @@ def run_imports(): if not imported: imported = True try: - import numpy + __import__('numpy') except ImportError: logging.error('can not import numpy, using SLOW XOR') logging.error('please install numpy if you use salsa20') slow_xor = True try: - import salsa20 + __import__('salsa20') except ImportError: logging.error('you have to install salsa20 before you use salsa20') sys.exit(1) @@ -123,7 +126,7 @@ def test(): rounds = 1 * 1024 plain = urandom(BLOCK_SIZE * rounds) - import M2Crypto.EVP + # import M2Crypto.EVP # cipher = M2Crypto.EVP.Cipher('aes_128_cfb', 'k' * 32, 'i' * 16, 1, # key_as_bytes=0, d='md5', salt=None, i=1, # padding=1) From 89dc6031e8d52301af128829f38038c61ada3da5 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 16:14:28 +0800 Subject: [PATCH 222/344] lint code --- .travis.yml | 3 ++- shadowsocks/crypto/ctypes_openssl.py | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index a430a71..9dcb1bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,10 +9,11 @@ cache: before_install: - sudo apt-get update -qq - sudo apt-get install -qq build-essential libssl-dev swig python-m2crypto python-numpy dnsutils - - pip install m2crypto salsa20 pep8 nose + - pip install m2crypto salsa20 pep8 pyflakes nose - sudo tests/socksify/install.sh script: - pep8 . + - pyflakes . - python tests/nose_plugin.py - python tests/test.py -c tests/table.json - python tests/test.py -c tests/aes.json diff --git a/shadowsocks/crypto/ctypes_openssl.py b/shadowsocks/crypto/ctypes_openssl.py index 9c8386b..2f4947a 100644 --- a/shadowsocks/crypto/ctypes_openssl.py +++ b/shadowsocks/crypto/ctypes_openssl.py @@ -21,19 +21,20 @@ # SOFTWARE. import logging +from ctypes import CDLL, c_char_p, c_int, c_long, byref,\ + create_string_buffer, c_void_p __all__ = ['ciphers'] +libcrypto = None loaded = False buf_size = 2048 def load_openssl(): - global loaded, libcrypto, CDLL, c_char_p, c_int, c_long, byref,\ - create_string_buffer, c_void_p, buf - from ctypes import CDLL, c_char_p, c_int, c_long, byref,\ - create_string_buffer, c_void_p + global loaded, libcrypto, buf + from ctypes.util import find_library for p in ('crypto', 'eay32', 'libeay32'): libcrypto_path = find_library(p) @@ -152,7 +153,7 @@ def test(): BLOCK_SIZE = 16384 rounds = 1 * 1024 plain = urandom(BLOCK_SIZE * rounds) - import M2Crypto.EVP + # import M2Crypto.EVP # cipher = M2Crypto.EVP.Cipher('aes_128_cfb', 'k' * 32, 'i' * 16, 1, # key_as_bytes=0, d='md5', salt=None, i=1, # padding=1) From 0814888ba5662de479996dd80a3b52e4c293449b Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 16:29:19 +0800 Subject: [PATCH 223/344] use absolute import --- shadowsocks/asyncdns.py | 7 +++---- shadowsocks/encrypt.py | 14 ++++++-------- shadowsocks/local.py | 8 ++------ shadowsocks/server.py | 8 ++------ shadowsocks/tcprelay.py | 7 +++---- shadowsocks/udprelay.py | 7 +++---- 6 files changed, 19 insertions(+), 32 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 009c2d3..b639dec 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -27,14 +27,13 @@ import socket import struct import re import logging -import common -import lru_cache -import eventloop + +from shadowsocks import common, lru_cache, eventloop CACHE_SWEEP_INTERVAL = 30 -VALID_HOSTNAME = re.compile("(?!-)[A-Z\d-]{1,63}(? Date: Fri, 31 Oct 2014 18:28:22 +0800 Subject: [PATCH 224/344] python 3 support; not stable yet --- shadowsocks/__init__.py | 23 +++++++++++ shadowsocks/asyncdns.py | 43 ++++++++++++--------- shadowsocks/common.py | 37 ++++++++++++++---- shadowsocks/crypto/__init__.py | 3 ++ shadowsocks/crypto/ctypes_openssl.py | 57 +++++++++++++++------------- shadowsocks/crypto/m2.py | 29 +++++++------- shadowsocks/crypto/rc4_md5.py | 4 +- shadowsocks/crypto/salsa20_ctr.py | 13 ++++--- shadowsocks/encrypt.py | 25 ++++++------ shadowsocks/eventloop.py | 4 +- shadowsocks/local.py | 4 ++ shadowsocks/lru_cache.py | 3 ++ shadowsocks/server.py | 10 +++-- shadowsocks/tcprelay.py | 17 +++++---- shadowsocks/udprelay.py | 6 ++- shadowsocks/utils.py | 34 +++++++++-------- tests/test.py | 7 +++- 17 files changed, 204 insertions(+), 115 deletions(-) diff --git a/shadowsocks/__init__.py b/shadowsocks/__init__.py index 013e4b7..5ba5908 100644 --- a/shadowsocks/__init__.py +++ b/shadowsocks/__init__.py @@ -1 +1,24 @@ #!/usr/bin/python + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import absolute_import, division, print_function, \ + with_statement diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index b639dec..4354b1d 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -21,6 +21,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import absolute_import, division, print_function, \ + with_statement + import time import os import socket @@ -33,7 +36,7 @@ from shadowsocks import common, lru_cache, eventloop CACHE_SWEEP_INTERVAL = 30 -VALID_HOSTNAME = re.compile(r"(?!-)[A-Z\d-]{1,63}(? 63: return None - results.append(chr(l)) + results.append(common.chr(l)) results.append(label) - results.append('\0') - return ''.join(results) + results.append(b'\0') + return b''.join(results) def build_request(address, qtype, request_id): @@ -111,7 +114,7 @@ def parse_ip(addrtype, data, length, offset): def parse_name(data, offset): p = offset labels = [] - l = ord(data[p]) + l = common.ord(data[p]) while l > 0: if (l & (128 + 64)) == (128 + 64): # pointer @@ -121,12 +124,12 @@ def parse_name(data, offset): labels.append(r[1]) p += 2 # pointer is the end - return p - offset, '.'.join(labels) + return p - offset, b'.'.join(labels) else: labels.append(data[p + 1:p + 1 + l]) p += 1 + l - l = ord(data[p]) - return p - offset + 1, '.'.join(labels) + l = common.ord(data[p]) + return p - offset + 1, b'.'.join(labels) # rfc1035 @@ -198,20 +201,20 @@ def parse_response(data): qds = [] ans = [] offset = 12 - for i in xrange(0, res_qdcount): + for i in range(0, res_qdcount): l, r = parse_record(data, offset, True) offset += l if r: qds.append(r) - for i in xrange(0, res_ancount): + for i in range(0, res_ancount): l, r = parse_record(data, offset) offset += l if r: ans.append(r) - for i in xrange(0, res_nscount): + for i in range(0, res_nscount): l, r = parse_record(data, offset) offset += l - for i in xrange(0, res_arcount): + for i in range(0, res_arcount): l, r = parse_record(data, offset) offset += l response = DNSResponse() @@ -232,6 +235,8 @@ def parse_response(data): def is_ip(address): for family in (socket.AF_INET, socket.AF_INET6): try: + if type(address) != str: + address = address.decode('utf8') socket.inet_pton(family, address) return family except (TypeError, ValueError, OSError, IOError): @@ -242,9 +247,9 @@ def is_ip(address): def is_valid_hostname(hostname): if len(hostname) > 255: return False - if hostname[-1] == ".": + if hostname[-1] == b'.': hostname = hostname[:-1] - return all(VALID_HOSTNAME.match(x) for x in hostname.split(".")) + return all(VALID_HOSTNAME.match(x) for x in hostname.split(b'.')) class DNSResponse(object): @@ -287,11 +292,13 @@ class DNSResolver(object): for line in content: line = line.strip() if line: - if line.startswith('nameserver'): + if line.startswith(b'nameserver'): parts = line.split() if len(parts) >= 2: server = parts[1] if is_ip(server) == socket.AF_INET: + if type(server) != str: + server = server.decode('utf8') self._servers.append(server) except IOError: pass @@ -310,7 +317,7 @@ class DNSResolver(object): if len(parts) >= 2: ip = parts[0] if is_ip(ip): - for i in xrange(1, len(parts)): + for i in range(1, len(parts)): hostname = parts[i] if hostname: self._hosts[hostname] = ip diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 6307ea7..cba287e 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -21,16 +21,37 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import absolute_import, division, print_function, \ + with_statement + import socket import struct import logging +def compat_ord(s): + if type(s) == int: + return s + return _ord(s) + + +def compat_chr(d): + if bytes == str: + return _chr(d) + return bytes([d]) + + +_ord = ord +_chr = chr +ord = compat_ord +chr = compat_chr + + def inet_ntop(family, ipstr): if family == socket.AF_INET: return socket.inet_ntoa(ipstr) elif family == socket.AF_INET6: - v6addr = ':'.join(('%02X%02X' % (ord(i), ord(j))) + v6addr = b':'.join((b'%02X%02X' % (ord(i), ord(j))) for i, j in zip(ipstr[::2], ipstr[1::2])) return v6addr @@ -39,15 +60,15 @@ def inet_pton(family, addr): if family == socket.AF_INET: return socket.inet_aton(addr) elif family == socket.AF_INET6: - if '.' in addr: # a v4 addr - v4addr = addr[addr.rindex(':') + 1:] + if b'.' in addr: # a v4 addr + v4addr = addr[addr.rindex(b':') + 1:] v4addr = socket.inet_aton(v4addr) - v4addr = map(lambda x: ('%02X' % ord(x)), v4addr) - v4addr.insert(2, ':') - newaddr = addr[:addr.rindex(':') + 1] + ''.join(v4addr) + v4addr = map(lambda x: (b'%02X' % ord(x)), v4addr) + v4addr.insert(2, b':') + newaddr = addr[:addr.rindex(b':') + 1] + b''.join(v4addr) return inet_pton(family, newaddr) dbyts = [0] * 8 # 8 groups - grps = addr.split(':') + grps = addr.split(b':') for i, v in enumerate(grps): if v: dbyts[i] = int(v, 16) @@ -58,7 +79,7 @@ def inet_pton(family, addr): else: break break - return ''.join((chr(i // 256) + chr(i % 256)) for i in dbyts) + return b''.join((chr(i // 256) + chr(i % 256)) for i in dbyts) else: raise RuntimeError("What family?") diff --git a/shadowsocks/crypto/__init__.py b/shadowsocks/crypto/__init__.py index bd3a926..6251321 100644 --- a/shadowsocks/crypto/__init__.py +++ b/shadowsocks/crypto/__init__.py @@ -19,3 +19,6 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. + +from __future__ import absolute_import, division, print_function, \ + with_statement diff --git a/shadowsocks/crypto/ctypes_openssl.py b/shadowsocks/crypto/ctypes_openssl.py index 2f4947a..807c54f 100644 --- a/shadowsocks/crypto/ctypes_openssl.py +++ b/shadowsocks/crypto/ctypes_openssl.py @@ -20,6 +20,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import absolute_import, division, print_function, \ + with_statement + import logging from ctypes import CDLL, c_char_p, c_int, c_long, byref,\ create_string_buffer, c_void_p @@ -117,31 +120,31 @@ class CtypesCrypto(object): ciphers = { - 'aes-128-cfb': (16, 16, CtypesCrypto), - 'aes-192-cfb': (24, 16, CtypesCrypto), - 'aes-256-cfb': (32, 16, CtypesCrypto), - 'aes-128-ofb': (16, 16, CtypesCrypto), - 'aes-192-ofb': (24, 16, CtypesCrypto), - 'aes-256-ofb': (32, 16, CtypesCrypto), - 'aes-128-ctr': (16, 16, CtypesCrypto), - 'aes-192-ctr': (24, 16, CtypesCrypto), - 'aes-256-ctr': (32, 16, CtypesCrypto), - 'aes-128-cfb8': (16, 16, CtypesCrypto), - 'aes-192-cfb8': (24, 16, CtypesCrypto), - 'aes-256-cfb8': (32, 16, CtypesCrypto), - 'aes-128-cfb1': (16, 16, CtypesCrypto), - 'aes-192-cfb1': (24, 16, CtypesCrypto), - 'aes-256-cfb1': (32, 16, CtypesCrypto), - 'bf-cfb': (16, 8, CtypesCrypto), - 'camellia-128-cfb': (16, 16, CtypesCrypto), - 'camellia-192-cfb': (24, 16, CtypesCrypto), - 'camellia-256-cfb': (32, 16, CtypesCrypto), - 'cast5-cfb': (16, 8, CtypesCrypto), - 'des-cfb': (8, 8, CtypesCrypto), - 'idea-cfb': (16, 8, CtypesCrypto), - 'rc2-cfb': (16, 8, CtypesCrypto), - 'rc4': (16, 0, CtypesCrypto), - 'seed-cfb': (16, 16, CtypesCrypto), + b'aes-128-cfb': (16, 16, CtypesCrypto), + b'aes-192-cfb': (24, 16, CtypesCrypto), + b'aes-256-cfb': (32, 16, CtypesCrypto), + b'aes-128-ofb': (16, 16, CtypesCrypto), + b'aes-192-ofb': (24, 16, CtypesCrypto), + b'aes-256-ofb': (32, 16, CtypesCrypto), + b'aes-128-ctr': (16, 16, CtypesCrypto), + b'aes-192-ctr': (24, 16, CtypesCrypto), + b'aes-256-ctr': (32, 16, CtypesCrypto), + b'aes-128-cfb8': (16, 16, CtypesCrypto), + b'aes-192-cfb8': (24, 16, CtypesCrypto), + b'aes-256-cfb8': (32, 16, CtypesCrypto), + b'aes-128-cfb1': (16, 16, CtypesCrypto), + b'aes-192-cfb1': (24, 16, CtypesCrypto), + b'aes-256-cfb1': (32, 16, CtypesCrypto), + b'bf-cfb': (16, 8, CtypesCrypto), + b'camellia-128-cfb': (16, 16, CtypesCrypto), + b'camellia-192-cfb': (24, 16, CtypesCrypto), + b'camellia-256-cfb': (32, 16, CtypesCrypto), + b'cast5-cfb': (16, 8, CtypesCrypto), + b'des-cfb': (8, 8, CtypesCrypto), + b'idea-cfb': (16, 8, CtypesCrypto), + b'rc2-cfb': (16, 8, CtypesCrypto), + b'rc4': (16, 0, CtypesCrypto), + b'seed-cfb': (16, 16, CtypesCrypto), } @@ -167,7 +170,7 @@ def test(): # decipher = Salsa20Cipher('salsa20-ctr', 'k' * 32, 'i' * 8, 1) results = [] pos = 0 - print 'salsa20 test start' + print('salsa20 test start') start = time.time() while pos < len(plain): l = random.randint(100, 32768) @@ -182,7 +185,7 @@ def test(): results.append(decipher.update(c[pos:pos + l])) pos += l end = time.time() - print 'speed: %d bytes/s' % (BLOCK_SIZE * rounds / (end - start)) + print('speed: %d bytes/s' % (BLOCK_SIZE * rounds / (end - start))) assert ''.join(results) == plain diff --git a/shadowsocks/crypto/m2.py b/shadowsocks/crypto/m2.py index c140061..2550a5c 100644 --- a/shadowsocks/crypto/m2.py +++ b/shadowsocks/crypto/m2.py @@ -20,6 +20,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import absolute_import, division, print_function, \ + with_statement + import sys import logging @@ -49,19 +52,19 @@ def err(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, i=1, padding=1): if has_m2: ciphers = { - 'aes-128-cfb': (16, 16, create_cipher), - 'aes-192-cfb': (24, 16, create_cipher), - 'aes-256-cfb': (32, 16, create_cipher), - 'bf-cfb': (16, 8, create_cipher), - 'camellia-128-cfb': (16, 16, create_cipher), - 'camellia-192-cfb': (24, 16, create_cipher), - 'camellia-256-cfb': (32, 16, create_cipher), - 'cast5-cfb': (16, 8, create_cipher), - 'des-cfb': (8, 8, create_cipher), - 'idea-cfb': (16, 8, create_cipher), - 'rc2-cfb': (16, 8, create_cipher), - 'rc4': (16, 0, create_cipher), - 'seed-cfb': (16, 16, create_cipher), + b'aes-128-cfb': (16, 16, create_cipher), + b'aes-192-cfb': (24, 16, create_cipher), + b'aes-256-cfb': (32, 16, create_cipher), + b'bf-cfb': (16, 8, create_cipher), + b'camellia-128-cfb': (16, 16, create_cipher), + b'camellia-192-cfb': (24, 16, create_cipher), + b'camellia-256-cfb': (32, 16, create_cipher), + b'cast5-cfb': (16, 8, create_cipher), + b'des-cfb': (8, 8, create_cipher), + b'idea-cfb': (16, 8, create_cipher), + b'rc2-cfb': (16, 8, create_cipher), + b'rc4': (16, 0, create_cipher), + b'seed-cfb': (16, 16, create_cipher), } else: ciphers = {} diff --git a/shadowsocks/crypto/rc4_md5.py b/shadowsocks/crypto/rc4_md5.py index 3dac6d5..e8888b1 100644 --- a/shadowsocks/crypto/rc4_md5.py +++ b/shadowsocks/crypto/rc4_md5.py @@ -20,6 +20,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import absolute_import, division, print_function, \ + with_statement import hashlib @@ -50,5 +52,5 @@ def create_cipher(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, ciphers = { - 'rc4-md5': (16, 16, create_cipher), + b'rc4-md5': (16, 16, create_cipher), } diff --git a/shadowsocks/crypto/salsa20_ctr.py b/shadowsocks/crypto/salsa20_ctr.py index 0a89d6b..d1910ea 100644 --- a/shadowsocks/crypto/salsa20_ctr.py +++ b/shadowsocks/crypto/salsa20_ctr.py @@ -20,6 +20,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import absolute_import, division, print_function, \ + with_statement + import time import struct import logging @@ -39,13 +42,13 @@ def run_imports(): if not imported: imported = True try: - __import__('numpy') + numpy = __import__('numpy') except ImportError: logging.error('can not import numpy, using SLOW XOR') logging.error('please install numpy if you use salsa20') slow_xor = True try: - __import__('salsa20') + salsa20 = __import__('salsa20') except ImportError: logging.error('you have to install salsa20 before you use salsa20') sys.exit(1) @@ -116,7 +119,7 @@ class Salsa20Cipher(object): ciphers = { - 'salsa20-ctr': (32, 8, Salsa20Cipher), + b'salsa20-ctr': (32, 8, Salsa20Cipher), } @@ -138,7 +141,7 @@ def test(): decipher = Salsa20Cipher('salsa20-ctr', 'k' * 32, 'i' * 8, 1) results = [] pos = 0 - print 'salsa20 test start' + print('salsa20 test start') start = time.time() while pos < len(plain): l = random.randint(100, 32768) @@ -153,7 +156,7 @@ def test(): results.append(decipher.update(c[pos:pos + l])) pos += l end = time.time() - print 'speed: %d bytes/s' % (BLOCK_SIZE * rounds / (end - start)) + print('speed: %d bytes/s' % (BLOCK_SIZE * rounds / (end - start))) assert ''.join(results) == plain diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 45059de..974785c 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -20,6 +20,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import absolute_import, division, print_function, \ + with_statement + import os import sys import hashlib @@ -74,23 +77,20 @@ def init_table(key, method=None): string.maketrans('', '')) cached_tables[key] = [encrypt_table, decrypt_table] else: - try: - Encryptor(key, method) # test if the settings if OK - except Exception as e: - logging.error(e) - sys.exit(1) + Encryptor(key, method) # test if the settings if OK def EVP_BytesToKey(password, key_len, iv_len): # equivalent to OpenSSL's EVP_BytesToKey() with count 1 # so that we make the same key and iv as nodejs version - password = str(password) + if hasattr(password, 'encode'): + password = password.encode('utf-8') r = cached_keys.get(password, None) if r: return r m = [] i = 0 - while len(''.join(m)) < (key_len + iv_len): + while len(b''.join(m)) < (key_len + iv_len): md5 = hashlib.md5() data = password if i > 0: @@ -98,7 +98,7 @@ def EVP_BytesToKey(password, key_len, iv_len): md5.update(data) m.append(md5.digest()) i += 1 - ms = ''.join(m) + ms = b''.join(m) key = ms[:key_len] iv = ms[key_len:key_len + iv_len] cached_keys[password] = (key, iv) @@ -107,13 +107,13 @@ def EVP_BytesToKey(password, key_len, iv_len): class Encryptor(object): def __init__(self, key, method=None): - if method == 'table': + if method == b'table': method = None self.key = key self.method = method self.iv = None self.iv_sent = False - self.cipher_iv = '' + self.cipher_iv = b'' self.decipher = None if method: self.cipher = self.get_cipher(key, method, 1, iv=random_string(32)) @@ -130,7 +130,8 @@ class Encryptor(object): return len(self.cipher_iv) def get_cipher(self, password, method, op, iv=None): - password = password.encode('utf-8') + if hasattr(password, 'encode'): + password = password.encode('utf-8') method = method.lower() m = self.get_cipher_param(method) if m: @@ -176,7 +177,7 @@ class Encryptor(object): def encrypt_all(password, method, op, data): - if method is not None and method.lower() == 'table': + if method is not None and method.lower() == b'table': method = None if not method: [encrypt_table, decrypt_table] = init_table(password) diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py index 4a35950..55c30bb 100644 --- a/shadowsocks/eventloop.py +++ b/shadowsocks/eventloop.py @@ -24,6 +24,8 @@ # from ssloop # https://github.com/clowwindy/ssloop +from __future__ import absolute_import, division, print_function, \ + with_statement import os import socket @@ -100,7 +102,7 @@ class KqueueLoop(object): results[fd] |= POLL_IN elif e.filter == select.KQ_FILTER_WRITE: results[fd] |= POLL_OUT - return results.iteritems() + return results.items() def add_fd(self, fd, mode): self._fds[fd] = mode diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 24e7ad7..0c627a5 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -21,11 +21,15 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import absolute_import, division, print_function, \ + with_statement + import sys import os import logging import signal +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../')) from shadowsocks import utils, encrypt, eventloop, tcprelay, udprelay, asyncdns diff --git a/shadowsocks/lru_cache.py b/shadowsocks/lru_cache.py index 1815320..20110ce 100644 --- a/shadowsocks/lru_cache.py +++ b/shadowsocks/lru_cache.py @@ -1,6 +1,9 @@ #!/usr/bin/python # -*- coding: utf-8 -*- +from __future__ import absolute_import, division, print_function, \ + with_statement + import collections import logging import time diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 2741ac1..0c4c489 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -21,11 +21,15 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import absolute_import, division, print_function, \ + with_statement + import sys import os import logging import signal +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../')) from shadowsocks import utils, encrypt, eventloop, tcprelay, udprelay, asyncdns @@ -66,13 +70,13 @@ def main(): def run_server(): def child_handler(signum, _): logging.warn('received SIGQUIT, doing graceful shutting down..') - map(lambda s: s.close(next_tick=True), tcp_servers + udp_servers) + list(map(lambda s: s.close(next_tick=True), tcp_servers + udp_servers)) signal.signal(getattr(signal, 'SIGQUIT', signal.SIGTERM), child_handler) try: loop = eventloop.EventLoop() dns_resolver.add_to_loop(loop) - map(lambda s: s.add_to_loop(loop), tcp_servers + udp_servers) + list(map(lambda s: s.add_to_loop(loop), tcp_servers + udp_servers)) loop.run() except (KeyboardInterrupt, IOError, OSError) as e: logging.error(e) @@ -85,7 +89,7 @@ def main(): if os.name == 'posix': children = [] is_child = False - for i in xrange(0, int(config['workers'])): + for i in range(0, int(config['workers'])): r = os.fork() if r == 0: logging.info('worker started') diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index ec6adbe..4145c84 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -21,6 +21,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import absolute_import, division, print_function, \ + with_statement + import time import socket import errno @@ -29,7 +32,7 @@ import logging import traceback import random -from shadowsocks import encrypt, eventloop, utils +from shadowsocks import encrypt, eventloop, utils, common from shadowsocks.common import parse_header @@ -231,13 +234,13 @@ class TCPRelayHandler(object): def _handle_stage_hello(self, data): try: if self._is_local: - cmd = ord(data[1]) + cmd = common.ord(data[1]) if cmd == CMD_UDP_ASSOCIATE: logging.debug('UDP associate') if self._local_sock.family == socket.AF_INET6: - header = '\x05\x00\x00\x04' + header = b'\x05\x00\x00\x04' else: - header = '\x05\x00\x00\x01' + header = b'\x05\x00\x00\x01' addr, port = self._local_sock.getsockname() addr_to_send = socket.inet_pton(self._local_sock.family, addr) @@ -265,7 +268,7 @@ class TCPRelayHandler(object): self._stage = STAGE_DNS if self._is_local: # forward address to remote - self._write_to_sock('\x05\x00\x00\x01\x00\x00\x00\x00\x10\x10', + self._write_to_sock(b'\x05\x00\x00\x01\x00\x00\x00\x00\x10\x10', self._local_sock) data_to_send = self._encryptor.encrypt(data) self._data_to_write_to_remote.append(data_to_send) @@ -366,7 +369,7 @@ class TCPRelayHandler(object): return elif is_local and self._stage == STAGE_INIT: # TODO check auth method - self._write_to_sock('\x05\00', self._local_sock) + self._write_to_sock(b'\x05\00', self._local_sock) self._stage = STAGE_HELLO return elif self._stage == STAGE_REPLY: @@ -411,7 +414,7 @@ class TCPRelayHandler(object): def _on_remote_write(self): self._stage = STAGE_STREAM if self._data_to_write_to_remote: - data = ''.join(self._data_to_write_to_remote) + data = b''.join(self._data_to_write_to_remote) self._data_to_write_to_remote = [] self._write_to_sock(data, self._remote_sock) else: diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index bfd688a..b4951be 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -65,6 +65,8 @@ # `client` means UDP clients that connects to other servers # `server` means the UDP server that handles user requests +from __future__ import absolute_import, division, print_function, \ + with_statement import time import socket @@ -73,7 +75,7 @@ import struct import errno import random -from shadowsocks import encrypt, eventloop, lru_cache +from shadowsocks import encrypt, eventloop, lru_cache, common from shadowsocks.common import parse_header, pack_addr @@ -146,7 +148,7 @@ class UDPRelay(object): if not data: logging.debug('UDP handle_server: data is empty') if self._is_local: - frag = ord(data[2]) + frag = common.ord(data[2]) if frag != 0: logging.warn('drop a message since frag is not 0') return diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index fee4afc..3edcae9 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -21,6 +21,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import absolute_import, division, print_function, \ + with_statement + import os import json import sys @@ -33,9 +36,9 @@ VERBOSE_LEVEL = 5 def check_python(): info = sys.version_info - if not (info[0] == 2 and info[1] >= 6): - print 'Python 2.6 or 2.7 required' - sys.exit(1) + # if not (info[0] == 2 and info[1] >= 6): + # print('Python 2.6 or 2.7 required') + # sys.exit(1) def print_shadowsocks(): @@ -45,7 +48,7 @@ def print_shadowsocks(): version = pkg_resources.get_distribution('shadowsocks').version except Exception: pass - print 'shadowsocks %s' % version + print('shadowsocks %s' % version) def find_config(): @@ -76,7 +79,7 @@ def check_config(config): if config.get('timeout', 300) > 600: logging.warn('warning: your timeout %d seems too long' % int(config.get('timeout'))) - if config.get('password') in ['mypassword', 'barfoo!']: + if config.get('password') in ['mypassword']: logging.error('DON\'T USE DEFAULT PASSWORD! Please change it in your ' 'config.json!') exit(1) @@ -102,7 +105,8 @@ def get_config(is_local): logging.info('loading config from %s' % config_path) with open(config_path, 'rb') as f: try: - config = json.load(f, object_hook=_decode_dict) + config = json.loads(f.read().decode('utf8'), + object_hook=_decode_dict) except ValueError as e: logging.error('found an error in config.json: %s', e.message) @@ -145,7 +149,7 @@ def get_config(is_local): v_count -= 1 config['verbose'] = v_count except getopt.GetoptError as e: - print >>sys.stderr, e + print(e, file=sys.stderr) print_help(is_local) sys.exit(2) @@ -218,7 +222,7 @@ def print_help(is_local): def print_local_help(): - print '''usage: sslocal [-h] -s SERVER_ADDR [-p SERVER_PORT] + print('''usage: sslocal [-h] -s SERVER_ADDR [-p SERVER_PORT] [-b LOCAL_ADDR] [-l LOCAL_PORT] -k PASSWORD [-m METHOD] [-t TIMEOUT] [-c CONFIG] [--fast-open] [-v] [-q] @@ -237,11 +241,11 @@ optional arguments: -q, -qq quiet mode, only show warnings/errors Online help: -''' +''') def print_server_help(): - print '''usage: ssserver [-h] [-s SERVER_ADDR] [-p SERVER_PORT] -k PASSWORD + print('''usage: ssserver [-h] [-s SERVER_ADDR] [-p SERVER_PORT] -k PASSWORD -m METHOD [-t TIMEOUT] [-c CONFIG] [--fast-open] [--workers WORKERS] [-v] [-q] @@ -259,13 +263,13 @@ optional arguments: -q, -qq quiet mode, only show warnings/errors Online help: -''' +''') def _decode_list(data): rv = [] for item in data: - if isinstance(item, unicode): + if hasattr(item, 'encode'): item = item.encode('utf-8') elif isinstance(item, list): item = _decode_list(item) @@ -277,10 +281,8 @@ def _decode_list(data): def _decode_dict(data): rv = {} - for key, value in data.iteritems(): - if isinstance(key, unicode): - key = key.encode('utf-8') - if isinstance(value, unicode): + for key, value in data.items(): + if hasattr(value, 'encode'): value = value.encode('utf-8') elif isinstance(value, list): value = _decode_list(value) diff --git a/tests/test.py b/tests/test.py index 83c2332..5475372 100755 --- a/tests/test.py +++ b/tests/test.py @@ -1,6 +1,9 @@ #!/usr/bin/python # -*- coding: utf-8 -*- +from __future__ import absolute_import, division, print_function, \ + with_statement + import sys import os import signal @@ -22,7 +25,7 @@ else: if 'salsa20' in sys.argv[-1]: from shadowsocks.crypto import salsa20_ctr salsa20_ctr.test() - print 'encryption test passed' + print('encryption test passed') p1 = Popen(['python', 'shadowsocks/server.py', '-c', server_config], stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) @@ -69,7 +72,7 @@ try: sys.exit(r) else: sys.exit(1) - print 'test passed' + print('test passed') finally: for p in [p1, p2]: From f98728a761fea7bd8390a3fd590f898b1f18dbd8 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 18:29:27 +0800 Subject: [PATCH 225/344] drop table support in python 3 --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9dcb1bc..517d9ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,9 @@ language: python python: - 2.6 - 2.7 + - 3.2 + - 3.3 + - 3.4 - pypy cache: directories: @@ -15,7 +18,6 @@ script: - pep8 . - pyflakes . - python tests/nose_plugin.py - - python tests/test.py -c tests/table.json - python tests/test.py -c tests/aes.json - python tests/test.py -c tests/aes-ctr.json - python tests/test.py -c tests/aes-cfb1.json From 53d1bfaecfd310b39a7f8bc8cf9eb5b2b98a2f02 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 18:32:04 +0800 Subject: [PATCH 226/344] update setup.py --- setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.py b/setup.py index cdc17aa..8161cc8 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,12 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: Proxy Servers', ], long_description=long_description, From b409c1887782e1d89abb8a303fec6461c868b873 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 18:39:39 +0800 Subject: [PATCH 227/344] fix python 3 tests --- tests/test.py | 7 ++----- tests/test_latency.py | 13 ------------- 2 files changed, 2 insertions(+), 18 deletions(-) delete mode 100644 tests/test_latency.py diff --git a/tests/test.py b/tests/test.py index 5475372..5d970c8 100755 --- a/tests/test.py +++ b/tests/test.py @@ -22,11 +22,6 @@ elif sys.argv[-2] == '-c': else: raise Exception('usage: test.py -c server_conf [client_conf]') -if 'salsa20' in sys.argv[-1]: - from shadowsocks.crypto import salsa20_ctr - salsa20_ctr.test() - print('encryption test passed') - p1 = Popen(['python', 'shadowsocks/server.py', '-c', server_config], stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) p2 = Popen(['python', 'shadowsocks/local.py', '-c', client_config], @@ -44,6 +39,8 @@ try: for fd in r: line = fd.readline() + if bytes != str: + line = str(line, 'utf8') sys.stdout.write(line) if line.find('starting local') >= 0: local_ready = True diff --git a/tests/test_latency.py b/tests/test_latency.py deleted file mode 100644 index 5d44644..0000000 --- a/tests/test_latency.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/python - -import sys -import time - - -if __name__ == '__main__': - before = time.time() - - for line in sys.stdin: - if 'HTTP/1.1 ' in line: - diff = time.time() - before - print 'headline %dms' % (diff * 1000) From 0fd2f30911f9419aa700b1822cc7754e71ee8f43 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 18:43:15 +0800 Subject: [PATCH 228/344] lint code --- shadowsocks/common.py | 2 +- shadowsocks/crypto/salsa20_ctr.py | 2 +- shadowsocks/encrypt.py | 2 +- shadowsocks/server.py | 3 ++- shadowsocks/tcprelay.py | 3 ++- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index cba287e..d585065 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -52,7 +52,7 @@ def inet_ntop(family, ipstr): return socket.inet_ntoa(ipstr) elif family == socket.AF_INET6: v6addr = b':'.join((b'%02X%02X' % (ord(i), ord(j))) - for i, j in zip(ipstr[::2], ipstr[1::2])) + for i, j in zip(ipstr[::2], ipstr[1::2])) return v6addr diff --git a/shadowsocks/crypto/salsa20_ctr.py b/shadowsocks/crypto/salsa20_ctr.py index d1910ea..0f5bfb8 100644 --- a/shadowsocks/crypto/salsa20_ctr.py +++ b/shadowsocks/crypto/salsa20_ctr.py @@ -72,7 +72,7 @@ def numpy_xor(a, b): def py_xor_str(a, b): c = [] - for i in xrange(0, len(a)): + for i in range(0, len(a)): c.append(chr(ord(a[i]) ^ ord(b[i]))) return ''.join(c) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 974785c..4997a6e 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -61,7 +61,7 @@ def get_table(key): s = m.digest() (a, b) = struct.unpack(' Date: Fri, 31 Oct 2014 18:45:21 +0800 Subject: [PATCH 229/344] update python requirement --- shadowsocks/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 3edcae9..b54a21b 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -36,9 +36,9 @@ VERBOSE_LEVEL = 5 def check_python(): info = sys.version_info - # if not (info[0] == 2 and info[1] >= 6): - # print('Python 2.6 or 2.7 required') - # sys.exit(1) + if info[0] == 2 and not info[1] >= 6: + print('Python 2.6+ required') + sys.exit(1) def print_shadowsocks(): From 753e46654dc3bc343670f91b82635d221e2a6d12 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 18:46:45 +0800 Subject: [PATCH 230/344] bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8161cc8..1520ad7 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ with open('README.rst') as f: setup( name="shadowsocks", - version="2.3.2", + version="2.4", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From ede2b1b1209a7ea74f9df917a5cf7ab0504837e1 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 18:56:24 +0800 Subject: [PATCH 231/344] fix salsa20 on python 3 --- shadowsocks/crypto/ctypes_openssl.py | 4 ++-- shadowsocks/crypto/salsa20_ctr.py | 25 ++++++++++++++++--------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/shadowsocks/crypto/ctypes_openssl.py b/shadowsocks/crypto/ctypes_openssl.py index 807c54f..7707639 100644 --- a/shadowsocks/crypto/ctypes_openssl.py +++ b/shadowsocks/crypto/ctypes_openssl.py @@ -163,8 +163,8 @@ def test(): # decipher = M2Crypto.EVP.Cipher('aes_128_cfb', 'k' * 32, 'i' * 16, 0, # key_as_bytes=0, d='md5', salt=None, i=1, # padding=1) - cipher = CtypesCrypto('aes-128-cfb', 'k' * 32, 'i' * 16, 1) - decipher = CtypesCrypto('aes-128-cfb', 'k' * 32, 'i' * 16, 0) + cipher = CtypesCrypto('aes-128-cfb', b'k' * 32, b'i' * 16, 1) + decipher = CtypesCrypto('aes-128-cfb', b'k' * 32, b'i' * 16, 0) # cipher = Salsa20Cipher('salsa20-ctr', 'k' * 32, 'i' * 8, 1) # decipher = Salsa20Cipher('salsa20-ctr', 'k' * 32, 'i' * 8, 1) diff --git a/shadowsocks/crypto/salsa20_ctr.py b/shadowsocks/crypto/salsa20_ctr.py index 0f5bfb8..e6c561c 100644 --- a/shadowsocks/crypto/salsa20_ctr.py +++ b/shadowsocks/crypto/salsa20_ctr.py @@ -72,9 +72,16 @@ def numpy_xor(a, b): def py_xor_str(a, b): c = [] - for i in range(0, len(a)): - c.append(chr(ord(a[i]) ^ ord(b[i]))) - return ''.join(c) + if bytes == str: + for i in range(0, len(a)): + c.append(chr(ord(a[i]) ^ ord(b[i]))) + else: + for i in range(0, len(a)): + c.append(a[i] ^ b[i]) + if bytes == str: + return ''.join(c) + else: + return bytes(c) class Salsa20Cipher(object): @@ -83,7 +90,7 @@ class Salsa20Cipher(object): def __init__(self, alg, key, iv, op, key_as_bytes=0, d=None, salt=None, i=1, padding=1): run_imports() - if alg != 'salsa20-ctr': + if alg != b'salsa20-ctr': raise Exception('unknown algorithm') self._key = key self._nonce = struct.unpack(' Date: Fri, 31 Oct 2014 18:57:06 +0800 Subject: [PATCH 232/344] lint code --- shadowsocks/crypto/salsa20_ctr.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/shadowsocks/crypto/salsa20_ctr.py b/shadowsocks/crypto/salsa20_ctr.py index e6c561c..7496975 100644 --- a/shadowsocks/crypto/salsa20_ctr.py +++ b/shadowsocks/crypto/salsa20_ctr.py @@ -75,12 +75,10 @@ def py_xor_str(a, b): if bytes == str: for i in range(0, len(a)): c.append(chr(ord(a[i]) ^ ord(b[i]))) + return ''.join(c) else: for i in range(0, len(a)): c.append(a[i] ^ b[i]) - if bytes == str: - return ''.join(c) - else: return bytes(c) From 497524b3136498b41800024d0e1c65afe0bda0bd Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 19:08:00 +0800 Subject: [PATCH 233/344] drop table on Python 3 --- shadowsocks/encrypt.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 4997a6e..91f7439 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -34,20 +34,25 @@ from shadowsocks.crypto import m2, rc4_md5, salsa20_ctr, ctypes_openssl method_supported = {} - method_supported.update(rc4_md5.ciphers) method_supported.update(salsa20_ctr.ciphers) method_supported.update(ctypes_openssl.ciphers) # let M2Crypto override ctypes_openssl method_supported.update(m2.ciphers) +if hasattr(string, 'maketrans'): + maketrans = string.maketrans + translate = string.translate +else: + maketrans = bytes.maketrans + translate = bytes.translate + def random_string(length): try: import M2Crypto.Rand return M2Crypto.Rand.rand_bytes(length) except ImportError: - # TODO really strong enough on Linux? return os.urandom(length) @@ -60,7 +65,7 @@ def get_table(key): m.update(key) s = m.digest() (a, b) = struct.unpack(' Date: Fri, 31 Oct 2014 19:10:29 +0800 Subject: [PATCH 234/344] fix import --- shadowsocks/crypto/rc4_md5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/crypto/rc4_md5.py b/shadowsocks/crypto/rc4_md5.py index e8888b1..daa6ba5 100644 --- a/shadowsocks/crypto/rc4_md5.py +++ b/shadowsocks/crypto/rc4_md5.py @@ -47,7 +47,7 @@ def create_cipher(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, d='md5', salt=None, i=1, padding=1) except: m2_not_found = True - import ctypes_openssl + from shadowsocks.crypto import ctypes_openssl return ctypes_openssl.CtypesCrypto('rc4', rc4_key, '', op) From 5024b300f16b012785e008336e22fa49c980ad86 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 19:21:13 +0800 Subject: [PATCH 235/344] fix rc4-md5 --- shadowsocks/crypto/rc4_md5.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/shadowsocks/crypto/rc4_md5.py b/shadowsocks/crypto/rc4_md5.py index daa6ba5..e2d8aa4 100644 --- a/shadowsocks/crypto/rc4_md5.py +++ b/shadowsocks/crypto/rc4_md5.py @@ -43,12 +43,13 @@ def create_cipher(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, if not m2_not_found: try: import M2Crypto.EVP - return M2Crypto.EVP.Cipher('rc4', rc4_key, '', op, key_as_bytes=0, - d='md5', salt=None, i=1, padding=1) + return M2Crypto.EVP.Cipher(b'rc4', rc4_key, b'', op, + key_as_bytes=0, d='md5', salt=None, i=1, + padding=1) except: m2_not_found = True from shadowsocks.crypto import ctypes_openssl - return ctypes_openssl.CtypesCrypto('rc4', rc4_key, '', op) + return ctypes_openssl.CtypesCrypto(b'rc4', rc4_key, b'', op) ciphers = { From a502dd32fc097cfc3ee1613dcb81e9188793db0d Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 21:49:22 +0800 Subject: [PATCH 236/344] fix UDP --- shadowsocks/encrypt.py | 2 +- shadowsocks/udprelay.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 91f7439..336f455 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -204,4 +204,4 @@ def encrypt_all(password, method, op, data): data = data[iv_len:] cipher = m(method, key, iv, op) result.append(cipher.update(data)) - return ''.join(result) + return b''.join(result) diff --git a/shadowsocks/udprelay.py b/shadowsocks/udprelay.py index b4951be..2b8b12f 100644 --- a/shadowsocks/udprelay.py +++ b/shadowsocks/udprelay.py @@ -229,7 +229,7 @@ class UDPRelay(object): if header_result is None: return # addrtype, dest_addr, dest_port, header_length = header_result - response = '\x00\x00\x00' + data + response = b'\x00\x00\x00' + data client_addr = self._client_fd_to_server_addr.get(sock.fileno()) if client_addr: self._server_socket.sendto(response, client_addr) From 4699de1dd9ebc06adb16fb077e143d0571ad7ad6 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 21:53:53 +0800 Subject: [PATCH 237/344] fix python 3.2 --- .travis.yml | 1 - shadowsocks/utils.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 517d9ad..37079f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,5 +26,4 @@ script: - python tests/test.py -c tests/salsa20.json - python tests/test.py -c tests/server-multi-ports.json - python tests/test.py -c tests/server-multi-passwd.json tests/server-multi-passwd-client-side.json - - python tests/test.py -c tests/server-multi-passwd-table.json - python tests/test.py -c tests/workers.json diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index b54a21b..87b2c92 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -207,7 +207,7 @@ def get_config(is_local): level = logging.INFO logging.basicConfig(level=level, format='%(asctime)s %(levelname)-8s %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', filemode='a+') + datefmt='%Y-%m-%d %H:%M:%S') check_config(config) From efd45ddfc68c7ad19b35ef4aaf89aa9e7a0515c8 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 21:59:22 +0800 Subject: [PATCH 238/344] fix UDP server --- shadowsocks/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index d585065..fbc977b 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -105,14 +105,14 @@ def pack_addr(address): try: r = socket.inet_pton(family, address) if family == socket.AF_INET6: - return '\x04' + r + return b'\x04' + r else: - return '\x01' + r + return b'\x01' + r except (TypeError, ValueError, OSError, IOError): pass if len(address) > 255: address = address[:255] # TODO - return '\x03' + chr(len(address)) + address + return b'\x03' + chr(len(address)) + address def parse_header(data): From 0719e80fbd3a37abb8c85da551772589e3d0b742 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 22:08:51 +0800 Subject: [PATCH 239/344] fix python 3.* --- shadowsocks/crypto/ctypes_openssl.py | 14 ++++++++------ shadowsocks/utils.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/shadowsocks/crypto/ctypes_openssl.py b/shadowsocks/crypto/ctypes_openssl.py index 7707639..7c9254a 100644 --- a/shadowsocks/crypto/ctypes_openssl.py +++ b/shadowsocks/crypto/ctypes_openssl.py @@ -68,7 +68,9 @@ def load_openssl(): def load_cipher(cipher_name): - func_name = 'EVP_' + cipher_name.replace('-', '_') + func_name = b'EVP_' + cipher_name.replace(b'-', b'_') + if bytes != str: + func_name = str(func_name, 'utf-8') cipher = getattr(libcrypto, func_name, None) if cipher: cipher.restype = c_void_p @@ -163,14 +165,14 @@ def test(): # decipher = M2Crypto.EVP.Cipher('aes_128_cfb', 'k' * 32, 'i' * 16, 0, # key_as_bytes=0, d='md5', salt=None, i=1, # padding=1) - cipher = CtypesCrypto('aes-128-cfb', b'k' * 32, b'i' * 16, 1) - decipher = CtypesCrypto('aes-128-cfb', b'k' * 32, b'i' * 16, 0) + cipher = CtypesCrypto(b'aes-128-cfb', b'k' * 32, b'i' * 16, 1) + decipher = CtypesCrypto(b'aes-128-cfb', b'k' * 32, b'i' * 16, 0) # cipher = Salsa20Cipher('salsa20-ctr', 'k' * 32, 'i' * 8, 1) # decipher = Salsa20Cipher('salsa20-ctr', 'k' * 32, 'i' * 8, 1) results = [] pos = 0 - print('salsa20 test start') + print('openssl test start') start = time.time() while pos < len(plain): l = random.randint(100, 32768) @@ -178,7 +180,7 @@ def test(): results.append(c) pos += l pos = 0 - c = ''.join(results) + c = b''.join(results) results = [] while pos < len(plain): l = random.randint(100, 32768) @@ -186,7 +188,7 @@ def test(): pos += l end = time.time() print('speed: %d bytes/s' % (BLOCK_SIZE * rounds / (end - start))) - assert ''.join(results) == plain + assert b''.join(results) == plain if __name__ == '__main__': diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 87b2c92..88b7f5b 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -87,7 +87,7 @@ def check_config(config): def get_config(is_local): logging.basicConfig(level=logging.INFO, - format='%(levelname)-s: %(message)s', filemode='a+') + format='%(levelname)-s: %(message)s') if is_local: shortopts = 'hs:b:p:k:l:m:c:t:vq' longopts = ['fast-open'] From aff411a4560a9771a5aae0bafcdc258d282b2538 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 22:16:44 +0800 Subject: [PATCH 240/344] seems that dependency requires 3.3+ --- .travis.yml | 1 - setup.py | 1 - shadowsocks/utils.py | 6 ++++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 37079f8..79a8922 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: python python: - 2.6 - 2.7 - - 3.2 - 3.3 - 3.4 - pypy diff --git a/setup.py b/setup.py index 1520ad7..63ef6de 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,6 @@ setup( 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: Implementation :: CPython', diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 88b7f5b..bb14391 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -39,6 +39,12 @@ def check_python(): if info[0] == 2 and not info[1] >= 6: print('Python 2.6+ required') sys.exit(1) + if info[0] == 3 and not info[1] >= 3: + print('Python 3.3+ required') + sys.exit(1) + else: + print('Python version not supported') + sys.exit(1) def print_shadowsocks(): From 50e5e620a3a0ec1b23adb83ade1224c0f4fa4591 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 31 Oct 2014 22:27:12 +0800 Subject: [PATCH 241/344] fix version detection --- shadowsocks/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index bb14391..9b6cd3a 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -39,10 +39,10 @@ def check_python(): if info[0] == 2 and not info[1] >= 6: print('Python 2.6+ required') sys.exit(1) - if info[0] == 3 and not info[1] >= 3: + elif info[0] == 3 and not info[1] >= 3: print('Python 3.3+ required') sys.exit(1) - else: + elif info[0] not in [2, 3]: print('Python version not supported') sys.exit(1) From 099323e28ea2a68d2412f8f63c30b7af94086cde Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 1 Nov 2014 12:03:38 +0800 Subject: [PATCH 242/344] update CHANGES --- CHANGES | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES b/CHANGES index 045bae0..353e18d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,8 @@ +2.4 2014-11-01 +- Python 3 support +- Performance improvement +- Fix LRU cache behavior + 2.3.2 2014-10-11 - Fix OpenSSL on Windows From c69a12777244001fee76d433435e73a75c2907dc Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 1 Nov 2014 12:25:44 +0800 Subject: [PATCH 243/344] add some tests --- shadowsocks/crypto/ctypes_openssl.py | 36 ++++++++++++++++++++++++---- shadowsocks/lru_cache.py | 20 ++++++++++++++++ tests/test.py | 20 ++++++++++++++++ 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/shadowsocks/crypto/ctypes_openssl.py b/shadowsocks/crypto/ctypes_openssl.py index 7c9254a..fd20832 100644 --- a/shadowsocks/crypto/ctypes_openssl.py +++ b/shadowsocks/crypto/ctypes_openssl.py @@ -150,7 +150,7 @@ ciphers = { } -def test(): +def run_method(method): from os import urandom import random import time @@ -165,8 +165,8 @@ def test(): # decipher = M2Crypto.EVP.Cipher('aes_128_cfb', 'k' * 32, 'i' * 16, 0, # key_as_bytes=0, d='md5', salt=None, i=1, # padding=1) - cipher = CtypesCrypto(b'aes-128-cfb', b'k' * 32, b'i' * 16, 1) - decipher = CtypesCrypto(b'aes-128-cfb', b'k' * 32, b'i' * 16, 0) + cipher = CtypesCrypto(method, b'k' * 32, b'i' * 16, 1) + decipher = CtypesCrypto(method, b'k' * 32, b'i' * 16, 0) # cipher = Salsa20Cipher('salsa20-ctr', 'k' * 32, 'i' * 8, 1) # decipher = Salsa20Cipher('salsa20-ctr', 'k' * 32, 'i' * 8, 1) @@ -191,5 +191,33 @@ def test(): assert b''.join(results) == plain +def test_aes_128_cfb(): + run_method(b'aes-128-cfb') + + +def test_aes_256_cfb(): + run_method(b'aes-256-cfb') + + +def test_aes_128_cfb8(): + run_method(b'aes-128-cfb8') + + +def test_aes_256_ofb(): + run_method(b'aes-256-ofb') + + +def test_aes_256_ctr(): + run_method(b'aes-256-ctr') + + +def test_bf_cfb(): + run_method(b'bf-cfb') + + +def test_rc4(): + run_method(b'rc4') + + if __name__ == '__main__': - test() + test_aes_128_cfb() diff --git a/shadowsocks/lru_cache.py b/shadowsocks/lru_cache.py index 20110ce..6400a2c 100644 --- a/shadowsocks/lru_cache.py +++ b/shadowsocks/lru_cache.py @@ -1,6 +1,26 @@ #!/usr/bin/python # -*- coding: utf-8 -*- +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + from __future__ import absolute_import, division, print_function, \ with_statement diff --git a/tests/test.py b/tests/test.py index 5d970c8..b04862f 100755 --- a/tests/test.py +++ b/tests/test.py @@ -1,6 +1,26 @@ #!/usr/bin/python # -*- coding: utf-8 -*- +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + from __future__ import absolute_import, division, print_function, \ with_statement From 7bff00220206089cbfbefc0d22be1d02e54a18d1 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 1 Nov 2014 12:40:06 +0800 Subject: [PATCH 244/344] more cipher tests --- shadowsocks/crypto/ctypes_openssl.py | 36 ++------------------ shadowsocks/crypto/m2.py | 29 ++++++++++++++++ shadowsocks/crypto/rc4_md5.py | 13 +++++++ shadowsocks/crypto/salsa20_ctr.py | 34 ++----------------- shadowsocks/crypto/util.py | 51 ++++++++++++++++++++++++++++ 5 files changed, 98 insertions(+), 65 deletions(-) create mode 100644 shadowsocks/crypto/util.py diff --git a/shadowsocks/crypto/ctypes_openssl.py b/shadowsocks/crypto/ctypes_openssl.py index fd20832..39a8edd 100644 --- a/shadowsocks/crypto/ctypes_openssl.py +++ b/shadowsocks/crypto/ctypes_openssl.py @@ -151,44 +151,12 @@ ciphers = { def run_method(method): - from os import urandom - import random - import time + from shadowsocks.crypto import util - BLOCK_SIZE = 16384 - rounds = 1 * 1024 - plain = urandom(BLOCK_SIZE * rounds) - # import M2Crypto.EVP - # cipher = M2Crypto.EVP.Cipher('aes_128_cfb', 'k' * 32, 'i' * 16, 1, - # key_as_bytes=0, d='md5', salt=None, i=1, - # padding=1) - # decipher = M2Crypto.EVP.Cipher('aes_128_cfb', 'k' * 32, 'i' * 16, 0, - # key_as_bytes=0, d='md5', salt=None, i=1, - # padding=1) cipher = CtypesCrypto(method, b'k' * 32, b'i' * 16, 1) decipher = CtypesCrypto(method, b'k' * 32, b'i' * 16, 0) - # cipher = Salsa20Cipher('salsa20-ctr', 'k' * 32, 'i' * 8, 1) - # decipher = Salsa20Cipher('salsa20-ctr', 'k' * 32, 'i' * 8, 1) - results = [] - pos = 0 - print('openssl test start') - start = time.time() - while pos < len(plain): - l = random.randint(100, 32768) - c = cipher.update(plain[pos:pos + l]) - results.append(c) - pos += l - pos = 0 - c = b''.join(results) - results = [] - while pos < len(plain): - l = random.randint(100, 32768) - results.append(decipher.update(c[pos:pos + l])) - pos += l - end = time.time() - print('speed: %d bytes/s' % (BLOCK_SIZE * rounds / (end - start))) - assert b''.join(results) == plain + util.run_cipher(cipher, decipher) def test_aes_128_cfb(): diff --git a/shadowsocks/crypto/m2.py b/shadowsocks/crypto/m2.py index 2550a5c..4082db0 100644 --- a/shadowsocks/crypto/m2.py +++ b/shadowsocks/crypto/m2.py @@ -68,3 +68,32 @@ if has_m2: } else: ciphers = {} + + +def run_method(method): + from shadowsocks.crypto import util + + cipher = create_cipher(method, b'k' * 32, b'i' * 16, 1) + decipher = create_cipher(method, b'k' * 32, b'i' * 16, 0) + + util.run_cipher(cipher, decipher) + + +def test_aes_128_cfb(): + run_method(b'aes-128-cfb') + + +def test_aes_256_cfb(): + run_method(b'aes-256-cfb') + + +def test_bf_cfb(): + run_method(b'bf-cfb') + + +def test_rc4(): + run_method(b'rc4') + + +if __name__ == '__main__': + test_aes_128_cfb() \ No newline at end of file diff --git a/shadowsocks/crypto/rc4_md5.py b/shadowsocks/crypto/rc4_md5.py index e2d8aa4..952962f 100644 --- a/shadowsocks/crypto/rc4_md5.py +++ b/shadowsocks/crypto/rc4_md5.py @@ -55,3 +55,16 @@ def create_cipher(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, ciphers = { b'rc4-md5': (16, 16, create_cipher), } + + +def test(): + from shadowsocks.crypto import util + + cipher = create_cipher(b'salsa20-ctr', b'k' * 32, b'i' * 8, 1) + decipher = create_cipher(b'salsa20-ctr', b'k' * 32, b'i' * 8, 1) + + util.run_cipher(cipher, decipher) + + +if __name__ == '__main__': + test() \ No newline at end of file diff --git a/shadowsocks/crypto/salsa20_ctr.py b/shadowsocks/crypto/salsa20_ctr.py index 7496975..759aa03 100644 --- a/shadowsocks/crypto/salsa20_ctr.py +++ b/shadowsocks/crypto/salsa20_ctr.py @@ -129,40 +129,12 @@ ciphers = { def test(): - from os import urandom - import random - - rounds = 1 * 1024 - plain = urandom(BLOCK_SIZE * rounds) - # import M2Crypto.EVP - # cipher = M2Crypto.EVP.Cipher('aes_128_cfb', 'k' * 32, 'i' * 16, 1, - # key_as_bytes=0, d='md5', salt=None, i=1, - # padding=1) - # decipher = M2Crypto.EVP.Cipher('aes_128_cfb', 'k' * 32, 'i' * 16, 0, - # key_as_bytes=0, d='md5', salt=None, i=1, - # padding=1) + from shadowsocks.crypto import util cipher = Salsa20Cipher(b'salsa20-ctr', b'k' * 32, b'i' * 8, 1) decipher = Salsa20Cipher(b'salsa20-ctr', b'k' * 32, b'i' * 8, 1) - results = [] - pos = 0 - print('salsa20 test start') - start = time.time() - while pos < len(plain): - l = random.randint(100, 32768) - c = cipher.update(plain[pos:pos + l]) - results.append(c) - pos += l - pos = 0 - c = b''.join(results) - results = [] - while pos < len(plain): - l = random.randint(100, 32768) - results.append(decipher.update(c[pos:pos + l])) - pos += l - end = time.time() - print('speed: %d bytes/s' % (BLOCK_SIZE * rounds / (end - start))) - assert b''.join(results) == plain + + util.run_cipher(cipher, decipher) if __name__ == '__main__': diff --git a/shadowsocks/crypto/util.py b/shadowsocks/crypto/util.py new file mode 100644 index 0000000..3bac1db --- /dev/null +++ b/shadowsocks/crypto/util.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +def run_cipher(cipher, decipher): + from os import urandom + import random + import time + + BLOCK_SIZE = 16384 + rounds = 1 * 1024 + plain = urandom(BLOCK_SIZE * rounds) + + results = [] + pos = 0 + print('test start') + start = time.time() + while pos < len(plain): + l = random.randint(100, 32768) + c = cipher.update(plain[pos:pos + l]) + results.append(c) + pos += l + pos = 0 + c = b''.join(results) + results = [] + while pos < len(plain): + l = random.randint(100, 32768) + results.append(decipher.update(c[pos:pos + l])) + pos += l + end = time.time() + print('speed: %d bytes/s' % (BLOCK_SIZE * rounds / (end - start))) + assert b''.join(results) == plain From 812a286f1294c97ee4a58399b52269e00cf88ea4 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 1 Nov 2014 12:44:22 +0800 Subject: [PATCH 245/344] lint code --- shadowsocks/crypto/m2.py | 2 +- shadowsocks/crypto/rc4_md5.py | 2 +- shadowsocks/crypto/salsa20_ctr.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/shadowsocks/crypto/m2.py b/shadowsocks/crypto/m2.py index 4082db0..a1d7d1c 100644 --- a/shadowsocks/crypto/m2.py +++ b/shadowsocks/crypto/m2.py @@ -96,4 +96,4 @@ def test_rc4(): if __name__ == '__main__': - test_aes_128_cfb() \ No newline at end of file + test_aes_128_cfb() diff --git a/shadowsocks/crypto/rc4_md5.py b/shadowsocks/crypto/rc4_md5.py index 952962f..b9cd72b 100644 --- a/shadowsocks/crypto/rc4_md5.py +++ b/shadowsocks/crypto/rc4_md5.py @@ -67,4 +67,4 @@ def test(): if __name__ == '__main__': - test() \ No newline at end of file + test() diff --git a/shadowsocks/crypto/salsa20_ctr.py b/shadowsocks/crypto/salsa20_ctr.py index 759aa03..0ea13b8 100644 --- a/shadowsocks/crypto/salsa20_ctr.py +++ b/shadowsocks/crypto/salsa20_ctr.py @@ -23,7 +23,6 @@ from __future__ import absolute_import, division, print_function, \ with_statement -import time import struct import logging import sys From 1205b8c50e427b2eea7d99d7954fe6e5e09f0883 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 1 Nov 2014 12:52:51 +0800 Subject: [PATCH 246/344] skip m2crypto tests on Python 3 --- .travis.yml | 2 +- shadowsocks/crypto/m2.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 79a8922..b1a64df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ before_install: script: - pep8 . - pyflakes . - - python tests/nose_plugin.py + - python tests/nose_plugin.py -v - python tests/test.py -c tests/aes.json - python tests/test.py -c tests/aes-ctr.json - python tests/test.py -c tests/aes-cfb1.json diff --git a/shadowsocks/crypto/m2.py b/shadowsocks/crypto/m2.py index a1d7d1c..3539711 100644 --- a/shadowsocks/crypto/m2.py +++ b/shadowsocks/crypto/m2.py @@ -80,18 +80,30 @@ def run_method(method): def test_aes_128_cfb(): + if bytes != str: + from nose.plugins.skip import SkipTest + raise SkipTest run_method(b'aes-128-cfb') def test_aes_256_cfb(): + if bytes != str: + from nose.plugins.skip import SkipTest + raise SkipTest run_method(b'aes-256-cfb') def test_bf_cfb(): + if bytes != str: + from nose.plugins.skip import SkipTest + raise SkipTest run_method(b'bf-cfb') def test_rc4(): + if bytes != str: + from nose.plugins.skip import SkipTest + raise SkipTest run_method(b'rc4') From 82ddafafa5d43a6714279cd44ff2a0ef1a860913 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 1 Nov 2014 13:03:13 +0800 Subject: [PATCH 247/344] skip m2crypto tests on pypy --- shadowsocks/crypto/m2.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/shadowsocks/crypto/m2.py b/shadowsocks/crypto/m2.py index 3539711..8022f81 100644 --- a/shadowsocks/crypto/m2.py +++ b/shadowsocks/crypto/m2.py @@ -79,31 +79,36 @@ def run_method(method): util.run_cipher(cipher, decipher) -def test_aes_128_cfb(): +def check_env(): + # skip this test on pypy and Python 3 + try: + import __pypy__ + from nose.plugins.skip import SkipTest + raise SkipTest + except ImportError: + __pypy__ = None if bytes != str: from nose.plugins.skip import SkipTest raise SkipTest + + +def test_aes_128_cfb(): + check_env() run_method(b'aes-128-cfb') def test_aes_256_cfb(): - if bytes != str: - from nose.plugins.skip import SkipTest - raise SkipTest + check_env() run_method(b'aes-256-cfb') def test_bf_cfb(): - if bytes != str: - from nose.plugins.skip import SkipTest - raise SkipTest + check_env() run_method(b'bf-cfb') def test_rc4(): - if bytes != str: - from nose.plugins.skip import SkipTest - raise SkipTest + check_env() run_method(b'rc4') From 13c6c393db06eb8219c8b1ede2b707beefaf124f Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 1 Nov 2014 13:10:19 +0800 Subject: [PATCH 248/344] fix pyflakes --- shadowsocks/crypto/m2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shadowsocks/crypto/m2.py b/shadowsocks/crypto/m2.py index 8022f81..4c7e148 100644 --- a/shadowsocks/crypto/m2.py +++ b/shadowsocks/crypto/m2.py @@ -83,10 +83,11 @@ def check_env(): # skip this test on pypy and Python 3 try: import __pypy__ + del __pypy__ from nose.plugins.skip import SkipTest raise SkipTest except ImportError: - __pypy__ = None + pass if bytes != str: from nose.plugins.skip import SkipTest raise SkipTest From 4affb56d36467260de6f87d8e25127601f4562f8 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 1 Nov 2014 13:24:40 +0800 Subject: [PATCH 249/344] fix test --- shadowsocks/crypto/rc4_md5.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/crypto/rc4_md5.py b/shadowsocks/crypto/rc4_md5.py index b9cd72b..ee421b6 100644 --- a/shadowsocks/crypto/rc4_md5.py +++ b/shadowsocks/crypto/rc4_md5.py @@ -60,8 +60,8 @@ ciphers = { def test(): from shadowsocks.crypto import util - cipher = create_cipher(b'salsa20-ctr', b'k' * 32, b'i' * 8, 1) - decipher = create_cipher(b'salsa20-ctr', b'k' * 32, b'i' * 8, 1) + cipher = create_cipher(b'rc4-md5', b'k' * 32, b'i' * 16, 1) + decipher = create_cipher(b'rc4-md5', b'k' * 32, b'i' * 16, 1) util.run_cipher(cipher, decipher) From 90dc03c7626a410169ac03340ecdb9770ea766f9 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 1 Nov 2014 13:52:16 +0800 Subject: [PATCH 250/344] prefer ctypes over m2crypto in rc4-md5 --- shadowsocks/crypto/rc4_md5.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/shadowsocks/crypto/rc4_md5.py b/shadowsocks/crypto/rc4_md5.py index ee421b6..7b212d3 100644 --- a/shadowsocks/crypto/rc4_md5.py +++ b/shadowsocks/crypto/rc4_md5.py @@ -28,28 +28,22 @@ import hashlib __all__ = ['ciphers'] -m2_not_found = False - def create_cipher(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, i=1, padding=1): - global m2_not_found - md5 = hashlib.md5() md5.update(key) md5.update(iv) rc4_key = md5.digest() - if not m2_not_found: - try: - import M2Crypto.EVP - return M2Crypto.EVP.Cipher(b'rc4', rc4_key, b'', op, - key_as_bytes=0, d='md5', salt=None, i=1, - padding=1) - except: - m2_not_found = True - from shadowsocks.crypto import ctypes_openssl - return ctypes_openssl.CtypesCrypto(b'rc4', rc4_key, b'', op) + try: + from shadowsocks.crypto import ctypes_openssl + return ctypes_openssl.CtypesCrypto(b'rc4', rc4_key, b'', op) + except: + import M2Crypto.EVP + return M2Crypto.EVP.Cipher(b'rc4', rc4_key, b'', op, + key_as_bytes=0, d='md5', salt=None, i=1, + padding=1) ciphers = { From e6306f7b32eda165c6b52d4f8a9e763eef128cd9 Mon Sep 17 00:00:00 2001 From: Felix Yan Date: Sat, 1 Nov 2014 22:48:22 +0800 Subject: [PATCH 251/344] setup.py: use codecs.open() so it works for non-utf8 locales --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 63ef6de..8e654dc 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,8 @@ +import codecs from setuptools import setup -with open('README.rst') as f: +with codecs.open('README.rst', encoding='utf-8') as f: long_description = f.read() setup( From 589278900576030796facb9903d4154b6aba2f14 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 1 Nov 2014 23:47:38 +0800 Subject: [PATCH 252/344] refine tests: add time limit for curl; output shadowsocks logs --- .travis.yml | 1 + tests/test.py | 66 +++++++++++++++++++++++++++++++++++---------------- 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index b1a64df..403c88a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ script: - pep8 . - pyflakes . - python tests/nose_plugin.py -v + - python setup.py sdist - python tests/test.py -c tests/aes.json - python tests/test.py -c tests/aes-ctr.json - python tests/test.py -c tests/aes-cfb1.json diff --git a/tests/test.py b/tests/test.py index b04862f..39ec374 100755 --- a/tests/test.py +++ b/tests/test.py @@ -47,6 +47,16 @@ p1 = Popen(['python', 'shadowsocks/server.py', '-c', server_config], p2 = Popen(['python', 'shadowsocks/local.py', '-c', client_config], stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) p3 = None +p4 = None +p3_fin = False +p4_fin = False + +# 1 shadowsocks started +# 2 curl started +# 3 curl finished +# 4 dig started +# 5 dig finished +stage = 1 try: local_ready = False @@ -59,6 +69,11 @@ try: for fd in r: line = fd.readline() + if not line: + if stage == 2 and fd == p3.stdout: + stage = 3 + if stage == 4 and fd == p4.stdout: + stage = 5 if bytes != str: line = str(line, 'utf8') sys.stdout.write(line) @@ -70,27 +85,38 @@ try: if local_ready and server_ready and p3 is None: time.sleep(1) + p3 = Popen(['curl', 'http://www.example.com/', '-v', '-L', + '--socks5-hostname', '127.0.0.1:1081', + '-m', '15', '--connect-timeout', '10'], + stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) + if p3 is not None: + fdset.append(p3.stdout) + fdset.append(p3.stderr) + stage = 2 + else: + sys.exit(1) + + if stage == 3 and p3 is not None: + fdset.remove(p3.stdout) + fdset.remove(p3.stderr) + r = p3.wait() + if r != 0: + sys.exit(1) + p4 = Popen(['socksify', 'dig', '@8.8.8.8', 'www.google.com'], + stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) + if p4 is not None: + fdset.append(p4.stdout) + fdset.append(p4.stderr) + stage = 4 + else: + sys.exit(1) + + if stage == 5: + r = p4.wait() + if r != 0: + sys.exit(1) + print('test passed') break - - p3 = Popen(['curl', 'http://www.example.com/', '-v', '-L', - '--socks5-hostname', '127.0.0.1:1081'], close_fds=True) - if p3 is not None: - r = p3.wait() - if r != 0: - sys.exit(r) - else: - sys.exit(1) - - p4 = Popen(['socksify', 'dig', '@8.8.8.8', 'www.google.com'], - close_fds=True) - if p4 is not None: - r = p4.wait() - if r != 0: - sys.exit(r) - else: - sys.exit(1) - print('test passed') - finally: for p in [p1, p2]: try: From 4e083440353833763ab1153d501788b4e40e6402 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 2 Nov 2014 00:10:06 +0800 Subject: [PATCH 253/344] bump --- CHANGES | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 353e18d..9be81bd 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +2.4.1 2014-11-01 +- Fix setup.py for non-utf8 locales on Python 3 + 2.4 2014-11-01 - Python 3 support - Performance improvement diff --git a/setup.py b/setup.py index 8e654dc..0add434 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with codecs.open('README.rst', encoding='utf-8') as f: setup( name="shadowsocks", - version="2.4", + version="2.4.1", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From 833c98e1149d883d0ebb5657e924c71ef58cbd63 Mon Sep 17 00:00:00 2001 From: Felix Yan Date: Sun, 2 Nov 2014 00:23:44 +0800 Subject: [PATCH 254/344] tcprelay.py: decode remote_addr to fix extra b'' in logging on python 3 --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 724c1ed..f772c93 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -261,7 +261,7 @@ class TCPRelayHandler(object): if header_result is None: raise Exception('can not parse header') addrtype, remote_addr, remote_port, header_length = header_result - logging.info('connecting %s:%d' % (remote_addr, remote_port)) + logging.info('connecting %s:%d' % (remote_addr.decode('utf-8'), remote_port)) self._remote_address = (remote_addr, remote_port) # pause reading self._update_stream(STREAM_UP, WAIT_STATUS_WRITING) From d971cb44a8810205918d88e8062e0e24536750fa Mon Sep 17 00:00:00 2001 From: Felix Yan Date: Sun, 2 Nov 2014 00:41:29 +0800 Subject: [PATCH 255/344] tcprelay.py: wrap long line for PEP8 --- shadowsocks/tcprelay.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index f772c93..4a30b42 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -261,7 +261,8 @@ class TCPRelayHandler(object): if header_result is None: raise Exception('can not parse header') addrtype, remote_addr, remote_port, header_length = header_result - logging.info('connecting %s:%d' % (remote_addr.decode('utf-8'), remote_port)) + logging.info('connecting %s:%d' % (remote_addr.decode('utf-8'), + remote_port)) self._remote_address = (remote_addr, remote_port) # pause reading self._update_stream(STREAM_UP, WAIT_STATUS_WRITING) From 9fe3cf17d54562d62269146e8eec26c536d38eee Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 2 Nov 2014 01:25:02 +0800 Subject: [PATCH 256/344] just start curl, don't wait --- tests/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test.py b/tests/test.py index 39ec374..f62485b 100755 --- a/tests/test.py +++ b/tests/test.py @@ -82,8 +82,8 @@ try: if line.find('starting server') >= 0: server_ready = True - if local_ready and server_ready and p3 is None: - time.sleep(1) + if stage == 1: + time.sleep(2) p3 = Popen(['curl', 'http://www.example.com/', '-v', '-L', '--socks5-hostname', '127.0.0.1:1081', From f02e9a313221e7b69f064c3c716718e2f46c7742 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 2 Nov 2014 17:13:56 +0800 Subject: [PATCH 257/344] fix EVP_CIPHER_CTX_new's arg types --- shadowsocks/crypto/ctypes_openssl.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/shadowsocks/crypto/ctypes_openssl.py b/shadowsocks/crypto/ctypes_openssl.py index 39a8edd..0ef8ce0 100644 --- a/shadowsocks/crypto/ctypes_openssl.py +++ b/shadowsocks/crypto/ctypes_openssl.py @@ -49,8 +49,6 @@ def load_openssl(): libcrypto = CDLL(libcrypto_path) libcrypto.EVP_get_cipherbyname.restype = c_void_p libcrypto.EVP_CIPHER_CTX_new.restype = c_void_p - libcrypto.EVP_CIPHER_CTX_new.argtypes = (c_void_p, c_void_p, c_char_p, - c_char_p) libcrypto.EVP_CipherInit_ex.argtypes = (c_void_p, c_void_p, c_char_p, c_char_p, c_char_p, c_int) @@ -90,8 +88,7 @@ class CtypesCrypto(object): raise Exception('cipher %s not found in libcrypto' % cipher_name) key_ptr = c_char_p(key) iv_ptr = c_char_p(iv) - self._ctx = libcrypto.EVP_CIPHER_CTX_new(cipher, None, - key_ptr, iv_ptr) + self._ctx = libcrypto.EVP_CIPHER_CTX_new() if not self._ctx: raise Exception('can not create cipher context') r = libcrypto.EVP_CipherInit_ex(self._ctx, cipher, None, From 7ac84349833a93b5fc821fa5ad0652bb710a2d0f Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 3 Nov 2014 18:14:25 +0800 Subject: [PATCH 258/344] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3d0ea91..7f50c02 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ A fast tunnel proxy that helps you bypass firewalls. Install ------- -You'll have a client on your local machine, and install a server on a +You'll have a client on your local side, and setup a server on a remote server. ### Client From fb27cf52a9a7b72ef8d3e3693623de43a477ebaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=BC=99?= Date: Wed, 5 Nov 2014 11:04:40 +0800 Subject: [PATCH 259/344] Fix str/byte related TypeError in Python 3.4 Traceback (most recent call last): File "/usr/bin/sslocal", line 9, in load_entry_point('shadowsocks==2.4', 'console_scripts', 'sslocal')() File "/usr/lib/python3.4/site-packages/shadowsocks/local.py", line 68, in main loop.run() File "/usr/lib/python3.4/site-packages/shadowsocks/eventloop.py", line 230, in run handler(events) File "/usr/lib/python3.4/site-packages/shadowsocks/tcprelay.py", line 630, in _handle_events handler.handle_event(sock, event) File "/usr/lib/python3.4/site-packages/shadowsocks/tcprelay.py", line 458, in handle_event self._on_local_read() File "/usr/lib/python3.4/site-packages/shadowsocks/tcprelay.py", line 377, in _on_local_read self._handle_stage_reply(data) File "/usr/lib/python3.4/site-packages/shadowsocks/tcprelay.py", line 212, in _handle_stage_reply s = remote_sock.sendto(data, MSG_FASTOPEN, self._chosen_server) TypeError: 'str' does not support the buffer interface --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 4a30b42..6ccadbe 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -207,7 +207,7 @@ class TCPRelayHandler(object): self._create_remote_socket(self._chosen_server[0], self._chosen_server[1]) self._loop.add(remote_sock, eventloop.POLL_ERR) - data = ''.join(self._data_to_write_to_local) + data = b''.join(self._data_to_write_to_local) l = len(data) s = remote_sock.sendto(data, MSG_FASTOPEN, self._chosen_server) if s < l: From aca86f86851b725c7a037839bda23e95f2f6bf28 Mon Sep 17 00:00:00 2001 From: Felix Yan Date: Wed, 5 Nov 2014 16:54:51 +0800 Subject: [PATCH 260/344] encrypt.py: port table to support python 3 --- shadowsocks/encrypt.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 336f455..c60318d 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -65,9 +65,10 @@ def get_table(key): m.update(key) s = m.digest() (a, b) = struct.unpack(' Date: Wed, 5 Nov 2014 18:27:18 +0800 Subject: [PATCH 261/344] refactor table into a single cipher plugin --- .travis.yml | 1 + shadowsocks/crypto/rc4_md5.py | 2 +- shadowsocks/crypto/table.py | 178 ++++++++++++++++++++++++++++++++++ shadowsocks/encrypt.py | 150 ++++++++++------------------ shadowsocks/local.py | 2 +- shadowsocks/server.py | 2 +- shadowsocks/utils.py | 10 +- 7 files changed, 239 insertions(+), 106 deletions(-) create mode 100644 shadowsocks/crypto/table.py diff --git a/.travis.yml b/.travis.yml index 403c88a..8864e32 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,7 @@ script: - python tests/test.py -c tests/aes-cfb8.json - python tests/test.py -c tests/rc4-md5.json - python tests/test.py -c tests/salsa20.json + - python tests/test.py -c tests/table.json - python tests/test.py -c tests/server-multi-ports.json - python tests/test.py -c tests/server-multi-passwd.json tests/server-multi-passwd-client-side.json - python tests/test.py -c tests/workers.json diff --git a/shadowsocks/crypto/rc4_md5.py b/shadowsocks/crypto/rc4_md5.py index 7b212d3..3062dcc 100644 --- a/shadowsocks/crypto/rc4_md5.py +++ b/shadowsocks/crypto/rc4_md5.py @@ -55,7 +55,7 @@ def test(): from shadowsocks.crypto import util cipher = create_cipher(b'rc4-md5', b'k' * 32, b'i' * 16, 1) - decipher = create_cipher(b'rc4-md5', b'k' * 32, b'i' * 16, 1) + decipher = create_cipher(b'rc4-md5', b'k' * 32, b'i' * 16, 0) util.run_cipher(cipher, decipher) diff --git a/shadowsocks/crypto/table.py b/shadowsocks/crypto/table.py new file mode 100644 index 0000000..2492987 --- /dev/null +++ b/shadowsocks/crypto/table.py @@ -0,0 +1,178 @@ +# !/usr/bin/env python + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import absolute_import, division, print_function, \ + with_statement + +import string +import struct +import hashlib + + +__all__ = ['ciphers'] + +cached_tables = {} + +if hasattr(string, 'maketrans'): + maketrans = string.maketrans + translate = string.translate +else: + maketrans = bytes.maketrans + translate = bytes.translate + + +def get_table(key): + m = hashlib.md5() + m.update(key) + s = m.digest() + a, b = struct.unpack(' 0: key, iv_ = EVP_BytesToKey(password, m[0], m[1]) - if iv is None: - iv = iv_ - iv = iv[:m[1]] - if op == 1: - # this iv is for cipher not decipher - self.cipher_iv = iv[:m[1]] - return m[2](method, key, iv, op) + else: + # key_length == 0 indicates we should use the key directly + key, iv = password, b'' - logging.error('method %s not supported' % method) - sys.exit(1) + iv = iv[:m[1]] + if op == 1: + # this iv is for cipher not decipher + self.cipher_iv = iv[:m[1]] + return m[2](method, key, iv, op) def encrypt(self, buf): if len(buf) == 0: return buf - if not self.method: - return translate(buf, self.encrypt_table) + if self.iv_sent: + return self.cipher.update(buf) else: - if self.iv_sent: - return self.cipher.update(buf) - else: - self.iv_sent = True - return self.cipher_iv + self.cipher.update(buf) + self.iv_sent = True + return self.cipher_iv + self.cipher.update(buf) def decrypt(self, buf): if len(buf) == 0: return buf - if not self.method: - return translate(buf, self.decrypt_table) - else: - if self.decipher is None: - decipher_iv_len = self.get_cipher_param(self.method)[1] - decipher_iv = buf[:decipher_iv_len] - self.decipher = self.get_cipher(self.key, self.method, 0, - iv=decipher_iv) - buf = buf[decipher_iv_len:] - if len(buf) == 0: - return buf - return self.decipher.update(buf) + if self.decipher is None: + decipher_iv_len = self._method_info[1] + decipher_iv = buf[:decipher_iv_len] + self.decipher = self.get_cipher(self.key, self.method, 0, + iv=decipher_iv) + buf = buf[decipher_iv_len:] + if len(buf) == 0: + return buf + return self.decipher.update(buf) def encrypt_all(password, method, op, data): - if method is not None and method.lower() == b'table': - method = None - if not method: - [encrypt_table, decrypt_table] = init_table(password) - if op: - return translate(data, encrypt_table) - else: - return translate(data, decrypt_table) + result = [] + method = method.lower() + (key_len, iv_len, m) = method_supported[method] + (key, _) = EVP_BytesToKey(password, key_len, iv_len) + if op: + iv = random_string(iv_len) + result.append(iv) else: - result = [] - method = method.lower() - (key_len, iv_len, m) = method_supported[method] - (key, _) = EVP_BytesToKey(password, key_len, iv_len) - if op: - iv = random_string(iv_len) - result.append(iv) - else: - iv = data[:iv_len] - data = data[iv_len:] - cipher = m(method, key, iv, op) - result.append(cipher.update(data)) - return b''.join(result) + iv = data[:iv_len] + data = data[iv_len:] + cipher = m(method, key, iv, op) + result.append(cipher.update(data)) + return b''.join(result) diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 0c627a5..794d929 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -46,7 +46,7 @@ def main(): utils.print_shadowsocks() - encrypt.init_table(config['password'], config['method']) + encrypt.test_cipher(config['password'], config['method']) try: logging.info("starting local at %s:%d" % diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 2ee74bd..ce36944 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -54,7 +54,7 @@ def main(): else: config['port_password'][str(server_port)] = config['password'] - encrypt.init_table(config['password'], config['method']) + encrypt.test_cipher(config['password'], config['method']) tcp_servers = [] udp_servers = [] dns_resolver = asyncdns.DNSResolver() diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 9b6cd3a..b610f40 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -68,15 +68,15 @@ def find_config(): def check_config(config): - if config.get('local_address', '') in ['0.0.0.0']: + if config.get('local_address', '') in [b'0.0.0.0']: logging.warn('warning: local set to listen 0.0.0.0, which is not safe') - if config.get('server', '') in ['127.0.0.1', 'localhost']: + if config.get('server', '') in [b'127.0.0.1', b'localhost']: logging.warn('warning: server set to listen %s:%s, are you sure?' % (config['server'], config['server_port'])) - if (config.get('method', '') or '').lower() == '': + if (config.get('method', '') or '').lower() == b'table': logging.warn('warning: table is not safe; please use a safer cipher, ' 'like AES-256-CFB') - if (config.get('method', '') or '').lower() == 'rc4': + if (config.get('method', '') or '').lower() == b'rc4': logging.warn('warning: RC4 is not safe; please use a safer cipher, ' 'like AES-256-CFB') if config.get('timeout', 300) < 100: @@ -85,7 +85,7 @@ def check_config(config): if config.get('timeout', 300) > 600: logging.warn('warning: your timeout %d seems too long' % int(config.get('timeout'))) - if config.get('password') in ['mypassword']: + if config.get('password') in [b'mypassword']: logging.error('DON\'T USE DEFAULT PASSWORD! Please change it in your ' 'config.json!') exit(1) From 039451bcfbafe657d3cc668ff0dd29aec3408041 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 5 Nov 2014 18:30:18 +0800 Subject: [PATCH 262/344] fix encrypt_all --- shadowsocks/encrypt.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 610fc29..9733bbb 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -77,7 +77,7 @@ def EVP_BytesToKey(password, key_len, iv_len): key = ms[:key_len] iv = ms[key_len:key_len + iv_len] cached_keys[password] = (key, iv) - return (key, iv) + return key, iv class Encryptor(object): @@ -148,7 +148,10 @@ def encrypt_all(password, method, op, data): result = [] method = method.lower() (key_len, iv_len, m) = method_supported[method] - (key, _) = EVP_BytesToKey(password, key_len, iv_len) + if key_len > 0: + key, _ = EVP_BytesToKey(password, key_len, iv_len) + else: + key = password if op: iv = random_string(iv_len) result.append(iv) From 710fe1419099d89b9d46002d2ea01608ebc1e4e5 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 5 Nov 2014 18:32:01 +0800 Subject: [PATCH 263/344] rename test_cipher as it will break unit tests --- shadowsocks/encrypt.py | 2 +- shadowsocks/local.py | 2 +- shadowsocks/server.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 9733bbb..2043001 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -51,7 +51,7 @@ def random_string(length): cached_keys = {} -def test_cipher(key, method=None): +def try_cipher(key, method=None): Encryptor(key, method) diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 794d929..e778ea7 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -46,7 +46,7 @@ def main(): utils.print_shadowsocks() - encrypt.test_cipher(config['password'], config['method']) + encrypt.try_cipher(config['password'], config['method']) try: logging.info("starting local at %s:%d" % diff --git a/shadowsocks/server.py b/shadowsocks/server.py index ce36944..64b656b 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -54,7 +54,7 @@ def main(): else: config['port_password'][str(server_port)] = config['password'] - encrypt.test_cipher(config['password'], config['method']) + encrypt.try_cipher(config['password'], config['method']) tcp_servers = [] udp_servers = [] dns_resolver = asyncdns.DNSResolver() From f9ba6f7390bc7819e31983762fa43e9c86a8c720 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 5 Nov 2014 18:40:51 +0800 Subject: [PATCH 264/344] fix table test --- shadowsocks/crypto/table.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/shadowsocks/crypto/table.py b/shadowsocks/crypto/table.py index 2492987..6f16d26 100644 --- a/shadowsocks/crypto/table.py +++ b/shadowsocks/crypto/table.py @@ -78,6 +78,7 @@ ciphers = { def test_table_result(): + from shadowsocks.common import ord target1 = [ [60, 53, 84, 138, 217, 94, 88, 23, 39, 242, 219, 35, 12, 157, 165, 181, 255, 143, 83, 247, 162, 16, 31, 209, 190, 171, 115, 65, 38, 41, 21, @@ -150,15 +151,15 @@ def test_table_result(): 184, 22, 104, 254, 234, 253, 187, 226, 247, 188, 156, 151, 40, 108, 51, 83, 178, 52, 3, 31, 255, 195, 53, 235, 126, 167, 120]] - encrypt_table = ''.join(get_table('foobar!')) - decrypt_table = string.maketrans(encrypt_table, string.maketrans('', '')) + encrypt_table = b''.join(get_table(b'foobar!')) + decrypt_table = maketrans(encrypt_table, maketrans(b'', b'')) for i in range(0, 256): assert (target1[0][i] == ord(encrypt_table[i])) assert (target1[1][i] == ord(decrypt_table[i])) - encrypt_table = ''.join(get_table('barfoo!')) - decrypt_table = string.maketrans(encrypt_table, string.maketrans('', '')) + encrypt_table = b''.join(get_table(b'barfoo!')) + decrypt_table = maketrans(encrypt_table, maketrans(b'', b'')) for i in range(0, 256): assert (target2[0][i] == ord(encrypt_table[i])) @@ -175,4 +176,5 @@ def test_encryption(): if __name__ == '__main__': + test_table_result() test_encryption() From 5209860db106b8956a67fb029b2e37ad85c663c6 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 5 Nov 2014 18:42:01 +0800 Subject: [PATCH 265/344] fix cipher name in test --- shadowsocks/crypto/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/crypto/table.py b/shadowsocks/crypto/table.py index 6f16d26..08c1205 100644 --- a/shadowsocks/crypto/table.py +++ b/shadowsocks/crypto/table.py @@ -170,7 +170,7 @@ def test_encryption(): from shadowsocks.crypto import util cipher = TableCipher(b'table', b'test', b'', 1) - decipher = TableCipher(b'rc4-md5', b'test', b'', 0) + decipher = TableCipher(b'table', b'test', b'', 0) util.run_cipher(cipher, decipher) From a88d47883b79912836e8b365b7db51648ff301eb Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 6 Nov 2014 12:56:00 +0800 Subject: [PATCH 266/344] add tests for command line args --- .travis.yml | 2 ++ tests/test.py | 40 ++++++++++++++++++++++++++++------------ 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8864e32..0eb865a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,3 +28,5 @@ script: - python tests/test.py -c tests/server-multi-ports.json - python tests/test.py -c tests/server-multi-passwd.json tests/server-multi-passwd-client-side.json - python tests/test.py -c tests/workers.json + - python tests/test.py -b "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388" -a "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" + - python tests/test.py -b "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388" -a "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" diff --git a/tests/test.py b/tests/test.py index f62485b..5c5241a 100755 --- a/tests/test.py +++ b/tests/test.py @@ -29,23 +29,39 @@ import os import signal import select import time +import argparse from subprocess import Popen, PIPE sys.path.insert(0, './') -if sys.argv[-3] == '-c': - client_config = sys.argv[-1] - server_config = sys.argv[-2] -elif sys.argv[-2] == '-c': - client_config = sys.argv[-1] - server_config = sys.argv[-1] -else: - raise Exception('usage: test.py -c server_conf [client_conf]') +python = 'python' -p1 = Popen(['python', 'shadowsocks/server.py', '-c', server_config], - stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) -p2 = Popen(['python', 'shadowsocks/local.py', '-c', client_config], - stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) +parser = argparse.ArgumentParser(description='test Shadowsocks') +parser.add_argument('-c', '--client-conf', type=str, default=None) +parser.add_argument('-s', '--server-conf', type=str, default=None) +parser.add_argument('-a', '--client-args', type=str, default=None) +parser.add_argument('-b', '--server-args', type=str, default=None) + +config = parser.parse_args() + +client_args = [python, 'shadowsocks/local.py'] +server_args = [python, 'shadowsocks/server.py'] + +if config.client_conf: + client_args.extend(['-c', config.client_conf]) + if config.server_conf: + server_args.extend(['-c', config.server_conf]) + else: + server_args.extend(['-c', config.client_conf]) +if config.client_args: + client_args.extend(config.client_args.split()) + if config.server_args: + server_args.extend(config.server_args.split()) + else: + server_args.extend(config.client_args.split()) + +p1 = Popen(server_args, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) +p2 = Popen(client_args, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) p3 = None p4 = None p3_fin = False From 4e3c055406559a8883c7d43804ebcb94b56a964a Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 6 Nov 2014 12:58:14 +0800 Subject: [PATCH 267/344] lint code --- tests/test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test.py b/tests/test.py index 5c5241a..cefea87 100755 --- a/tests/test.py +++ b/tests/test.py @@ -32,8 +32,6 @@ import time import argparse from subprocess import Popen, PIPE -sys.path.insert(0, './') - python = 'python' parser = argparse.ArgumentParser(description='test Shadowsocks') From b5010df575c75c7073beb5822bece5f7de1ff21f Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 6 Nov 2014 13:02:57 +0800 Subject: [PATCH 268/344] fix args on Python 3 --- .travis.yml | 2 +- shadowsocks/common.py | 4 ++++ shadowsocks/utils.py | 9 +++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0eb865a..592ac6e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ script: - python tests/test.py -c tests/salsa20.json - python tests/test.py -c tests/table.json - python tests/test.py -c tests/server-multi-ports.json - - python tests/test.py -c tests/server-multi-passwd.json tests/server-multi-passwd-client-side.json + - python tests/test.py -s tests/server-multi-passwd.json -c tests/server-multi-passwd-client-side.json - python tests/test.py -c tests/workers.json - python tests/test.py -b "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388" -a "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" - python tests/test.py -b "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388" -a "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" diff --git a/shadowsocks/common.py b/shadowsocks/common.py index fbc977b..4c599c3 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -47,6 +47,10 @@ ord = compat_ord chr = compat_chr +def to_bytes(s): + return s.encode('utf-8') + + def inet_ntop(family, ipstr): if family == socket.AF_INET: return socket.inet_ntoa(ipstr) diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index b610f40..7808d8f 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -29,6 +29,7 @@ import json import sys import getopt import logging +from shadowsocks.common import to_bytes VERBOSE_LEVEL = 5 @@ -126,15 +127,15 @@ def get_config(is_local): if key == '-p': config['server_port'] = int(value) elif key == '-k': - config['password'] = value + config['password'] = to_bytes(value) elif key == '-l': config['local_port'] = int(value) elif key == '-s': - config['server'] = value + config['server'] = to_bytes(value) elif key == '-m': - config['method'] = value + config['method'] = to_bytes(value) elif key == '-b': - config['local_address'] = value + config['local_address'] = to_bytes(value) elif key == '-v': v_count += 1 # '-vv' turns on more verbose mode From 70dae91e7c172411882d0c8d885870fffc2e0e9d Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 6 Nov 2014 14:04:28 +0800 Subject: [PATCH 269/344] add tests for common.py --- shadowsocks/common.py | 72 +++++++++++++++++++++++++++++++++-------- shadowsocks/tcprelay.py | 2 +- 2 files changed, 60 insertions(+), 14 deletions(-) diff --git a/shadowsocks/common.py b/shadowsocks/common.py index 4c599c3..e4f698c 100644 --- a/shadowsocks/common.py +++ b/shadowsocks/common.py @@ -48,31 +48,44 @@ chr = compat_chr def to_bytes(s): - return s.encode('utf-8') + if bytes != str: + if type(s) == str: + return s.encode('utf-8') + return s + + +def to_str(s): + if bytes != str: + if type(s) == bytes: + return s.decode('utf-8') + return s def inet_ntop(family, ipstr): if family == socket.AF_INET: - return socket.inet_ntoa(ipstr) + return to_bytes(socket.inet_ntoa(ipstr)) elif family == socket.AF_INET6: - v6addr = b':'.join((b'%02X%02X' % (ord(i), ord(j))) - for i, j in zip(ipstr[::2], ipstr[1::2])) - return v6addr + import re + v6addr = ':'.join(('%02X%02X' % (ord(i), ord(j))).lstrip('0') + for i, j in zip(ipstr[::2], ipstr[1::2])) + v6addr = re.sub('::+', '::', v6addr, count=1) + return to_bytes(v6addr) def inet_pton(family, addr): + addr = to_str(addr) if family == socket.AF_INET: return socket.inet_aton(addr) elif family == socket.AF_INET6: - if b'.' in addr: # a v4 addr - v4addr = addr[addr.rindex(b':') + 1:] + if '.' in addr: # a v4 addr + v4addr = addr[addr.rindex(':') + 1:] v4addr = socket.inet_aton(v4addr) - v4addr = map(lambda x: (b'%02X' % ord(x)), v4addr) - v4addr.insert(2, b':') - newaddr = addr[:addr.rindex(b':') + 1] + b''.join(v4addr) + v4addr = map(lambda x: ('%02X' % ord(x)), v4addr) + v4addr.insert(2, ':') + newaddr = addr[:addr.rindex(':') + 1] + ''.join(v4addr) return inet_pton(family, newaddr) dbyts = [0] * 8 # 8 groups - grps = addr.split(b':') + grps = addr.split(':') for i, v in enumerate(grps): if v: dbyts[i] = int(v, 16) @@ -105,9 +118,10 @@ ADDRTYPE_HOST = 3 def pack_addr(address): + address_str = to_str(address) for family in (socket.AF_INET, socket.AF_INET6): try: - r = socket.inet_pton(family, address) + r = socket.inet_pton(family, address_str) if family == socket.AF_INET6: return b'\x04' + r else: @@ -155,4 +169,36 @@ def parse_header(data): addrtype) if dest_addr is None: return None - return addrtype, dest_addr, dest_port, header_length + return addrtype, to_bytes(dest_addr), dest_port, header_length + + +def test_inet_conv(): + ipv4 = b'8.8.4.4' + b = inet_pton(socket.AF_INET, ipv4) + assert inet_ntop(socket.AF_INET, b) == ipv4 + ipv6 = b'2404:6800:4005:805::1011' + b = inet_pton(socket.AF_INET6, ipv6) + assert inet_ntop(socket.AF_INET6, b) == ipv6 + + +def test_parse_header(): + assert parse_header(b'\x03\x0ewww.google.com\x00\x50') == \ + (3, b'www.google.com', 80, 18) + assert parse_header(b'\x01\x08\x08\x08\x08\x00\x35') == \ + (1, b'8.8.8.8', 53, 7) + assert parse_header((b'\x04$\x04h\x00@\x05\x08\x05\x00\x00\x00\x00\x00' + b'\x00\x10\x11\x00\x50')) == \ + (4, b'2404:6800:4005:805::1011', 80, 19) + + +def test_pack_header(): + assert pack_addr(b'8.8.8.8') == b'\x01\x08\x08\x08\x08' + assert pack_addr(b'2404:6800:4005:805::1011') == \ + b'\x04$\x04h\x00@\x05\x08\x05\x00\x00\x00\x00\x00\x00\x10\x11' + assert pack_addr(b'www.google.com') == b'\x03\x0ewww.google.com' + + +if __name__ == '__main__': + test_inet_conv() + test_parse_header() + test_pack_header() diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 6ccadbe..39e88fe 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -261,7 +261,7 @@ class TCPRelayHandler(object): if header_result is None: raise Exception('can not parse header') addrtype, remote_addr, remote_port, header_length = header_result - logging.info('connecting %s:%d' % (remote_addr.decode('utf-8'), + logging.info('connecting %s:%d' % (common.to_str(remote_addr), remote_port)) self._remote_address = (remote_addr, remote_port) # pause reading From a8996b812b3da0b1c775d1f64678cf8dcc53e43b Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 6 Nov 2014 14:50:19 +0800 Subject: [PATCH 270/344] update CHANGES --- CHANGES | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES b/CHANGES index 9be81bd..08abeea 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,8 @@ +2.4.2 2014-11-06 +- Fix command line arguments on Python 3 +- Support table on Python 3 +- Fix TCP Fast Open on Python 3 + 2.4.1 2014-11-01 - Fix setup.py for non-utf8 locales on Python 3 From 143750f699e72c6b039c146031a18f42e720798e Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 6 Nov 2014 14:52:15 +0800 Subject: [PATCH 271/344] update README.rst --- README.rst | 71 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/README.rst b/README.rst index 983a375..19d5b2a 100644 --- a/README.rst +++ b/README.rst @@ -3,25 +3,24 @@ shadowsocks |PyPI version| |Build Status| -A fast tunnel proxy that help you get through firewalls. +A fast tunnel proxy that helps you bypass firewalls. `中文说明 `__ Install ------- -You'll have a client on your local machine, and install a server on a -remote server. +You'll have a client on your local side, and setup a server on a remote +server. Client ~~~~~~ - `Windows `__ / `OS - X `__ + X `__ - `Android `__ - / - `iOS `__ + / `iOS `__ - `OpenWRT `__ Server @@ -32,18 +31,31 @@ Debian / Ubuntu: :: - apt-get install python-pip python-m2crypto + apt-get install python-pip pip install shadowsocks +Or simply ``apt-get install shadowsocks`` if you have `Debian +sid `__ in your +source list. + CentOS: ^^^^^^^ :: - yum install m2crypto python-setuptools + yum install python-setuptools easy_install pip pip install shadowsocks +Windows: +^^^^^^^^ + +Download OpenSSL for Windows and install. Then install shadowsocks via +easy\_install and pip as Linux. If you don't know how to use them, you +can directly download `the +package `__, and use +``python shadowsocks/server.py`` instead of ``ssserver`` command below. + Configuration ------------- @@ -65,27 +77,27 @@ On your server create a config file ``/etc/shadowsocks.json``. Example: Explanation of the fields: -+------------------+-----------------------------------------------------------------------------------------------------+ -| Name | Explanation | -+==================+=====================================================================================================+ -| server | the address your server listens | -+------------------+-----------------------------------------------------------------------------------------------------+ -| server\_port | server port | -+------------------+-----------------------------------------------------------------------------------------------------+ -| local\_address | the address your local listens | -+------------------+-----------------------------------------------------------------------------------------------------+ -| local\_port | local port | -+------------------+-----------------------------------------------------------------------------------------------------+ -| password | password used for encryption | -+------------------+-----------------------------------------------------------------------------------------------------+ -| timeout | in seconds | -+------------------+-----------------------------------------------------------------------------------------------------+ -| method | encryption method, "aes-256-cfb" is recommended | -+------------------+-----------------------------------------------------------------------------------------------------+ -| fast\_open | use `TCP\_FASTOPEN `__, true / false | -+------------------+-----------------------------------------------------------------------------------------------------+ -| workers | number of workers, available on Unix/Linux | -+------------------+-----------------------------------------------------------------------------------------------------+ ++------------------+---------------------------------------------------------------------------------------------------------+ +| Name | Explanation | ++==================+=========================================================================================================+ +| server | the address your server listens | ++------------------+---------------------------------------------------------------------------------------------------------+ +| server\_port | server port | ++------------------+---------------------------------------------------------------------------------------------------------+ +| local\_address | the address your local listens | ++------------------+---------------------------------------------------------------------------------------------------------+ +| local\_port | local port | ++------------------+---------------------------------------------------------------------------------------------------------+ +| password | password used for encryption | ++------------------+---------------------------------------------------------------------------------------------------------+ +| timeout | in seconds | ++------------------+---------------------------------------------------------------------------------------------------------+ +| method | default: "aes-256-cfb", see `Encryption `__ | ++------------------+---------------------------------------------------------------------------------------------------------+ +| fast\_open | use `TCP\_FASTOPEN `__, true / false | ++------------------+---------------------------------------------------------------------------------------------------------+ +| workers | number of workers, available on Unix/Linux | ++------------------+---------------------------------------------------------------------------------------------------------+ Run ``ssserver -c /etc/shadowsocks.json`` on your server. To run it in the background, use @@ -133,6 +145,7 @@ List all available args with ``-h``. Wiki ---- +You can find all the documentation in the wiki: https://github.com/clowwindy/shadowsocks/wiki License From ea535c74b21144c769dd41e6ccba36498801db85 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 6 Nov 2014 14:52:53 +0800 Subject: [PATCH 272/344] bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0add434..38b0491 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with codecs.open('README.rst', encoding='utf-8') as f: setup( name="shadowsocks", - version="2.4.1", + version="2.4.2", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From f43906006a08712316a07392405fa70929e8e32d Mon Sep 17 00:00:00 2001 From: Bin Wang Date: Mon, 10 Nov 2014 10:37:37 +0800 Subject: [PATCH 273/344] fix str/byte issue in python 3 --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 39e88fe..e25c63f 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -407,7 +407,7 @@ class TCPRelayHandler(object): def _on_local_write(self): if self._data_to_write_to_local: - data = ''.join(self._data_to_write_to_local) + data = b''.join(self._data_to_write_to_local) self._data_to_write_to_local = [] self._write_to_sock(data, self._local_sock) else: From fa31a233b6a3d2809d766baa66b2a6f31e2aec1e Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 8 Nov 2014 02:28:37 +0800 Subject: [PATCH 274/344] add ipv6 tests --- .travis.yml | 1 + shadowsocks/tcprelay.py | 2 +- tests/ipv6-client-side.json | 10 ++++++++++ tests/ipv6.json | 10 ++++++++++ tests/test.py | 4 ++-- 5 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 tests/ipv6-client-side.json create mode 100644 tests/ipv6.json diff --git a/.travis.yml b/.travis.yml index 592ac6e..286c5df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,5 +28,6 @@ script: - python tests/test.py -c tests/server-multi-ports.json - python tests/test.py -s tests/server-multi-passwd.json -c tests/server-multi-passwd-client-side.json - python tests/test.py -c tests/workers.json + - python tests/test.py -s tests/ipv6.json -c tests/ipv6-client-side.json - python tests/test.py -b "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388" -a "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" - python tests/test.py -b "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388" -a "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index e25c63f..e55f9dd 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -241,7 +241,7 @@ class TCPRelayHandler(object): header = b'\x05\x00\x00\x04' else: header = b'\x05\x00\x00\x01' - addr, port = self._local_sock.getsockname() + addr, port = self._local_sock.getsockname()[:2] addr_to_send = socket.inet_pton(self._local_sock.family, addr) port_to_send = struct.pack('>H', port) diff --git a/tests/ipv6-client-side.json b/tests/ipv6-client-side.json new file mode 100644 index 0000000..6c3cfaf --- /dev/null +++ b/tests/ipv6-client-side.json @@ -0,0 +1,10 @@ +{ + "server":"::1", + "server_port":8388, + "local_port":1081, + "password":"aes_password", + "timeout":60, + "method":"aes-256-cfb", + "local_address":"127.0.0.1", + "fast_open":false +} diff --git a/tests/ipv6.json b/tests/ipv6.json new file mode 100644 index 0000000..d855f9c --- /dev/null +++ b/tests/ipv6.json @@ -0,0 +1,10 @@ +{ + "server":"::", + "server_port":8388, + "local_port":1081, + "password":"aes_password", + "timeout":60, + "method":"aes-256-cfb", + "local_address":"127.0.0.1", + "fast_open":false +} diff --git a/tests/test.py b/tests/test.py index cefea87..34c10fb 100755 --- a/tests/test.py +++ b/tests/test.py @@ -42,8 +42,8 @@ parser.add_argument('-b', '--server-args', type=str, default=None) config = parser.parse_args() -client_args = [python, 'shadowsocks/local.py'] -server_args = [python, 'shadowsocks/server.py'] +client_args = [python, 'shadowsocks/local.py', '-v'] +server_args = [python, 'shadowsocks/server.py', '-v'] if config.client_conf: client_args.extend(['-c', config.client_conf]) From 0a3730942fc6888b66170b123437b77cb1b9c547 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 10 Nov 2014 15:47:52 +0800 Subject: [PATCH 275/344] update CHANGES --- CHANGES | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES b/CHANGES index 08abeea..c12a1e6 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +2.4.3 2014-11-10 +- Fix an issue on Python 3 +- Fix an issue with IPv6 + 2.4.2 2014-11-06 - Fix command line arguments on Python 3 - Support table on Python 3 From ffcbf0597580fe1509d9f7a2ca402c8278693df4 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 10 Nov 2014 15:48:16 +0800 Subject: [PATCH 276/344] bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 38b0491..aa18757 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with codecs.open('README.rst', encoding='utf-8') as f: setup( name="shadowsocks", - version="2.4.2", + version="2.4.3", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From 84dcff83c196506165fbf95f1c9e8c2991e6f089 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 10 Dec 2014 23:11:22 +0800 Subject: [PATCH 277/344] add libsodium support --- shadowsocks/crypto/ctypes_libsodium.py | 135 +++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 shadowsocks/crypto/ctypes_libsodium.py diff --git a/shadowsocks/crypto/ctypes_libsodium.py b/shadowsocks/crypto/ctypes_libsodium.py new file mode 100644 index 0000000..1e81060 --- /dev/null +++ b/shadowsocks/crypto/ctypes_libsodium.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import absolute_import, division, print_function, \ + with_statement + +import logging +from ctypes import CDLL, c_char_p, c_int, c_long, c_ulonglong, byref, \ + create_string_buffer, c_void_p + +__all__ = ['ciphers'] + +libsodium = None +loaded = False + +buf_size = 2048 + +# for salsa20 and chacha20 +BLOCK_SIZE = 64 + + +def load_libsodium(): + global loaded, libsodium, buf + + from ctypes.util import find_library + for p in ('sodium',): + libsodium_path = find_library(p) + if libsodium_path: + break + else: + raise Exception('libsodium not found') + logging.info('loading libsodium from %s', libsodium_path) + libsodium = CDLL(libsodium_path) + libsodium.sodium_init.restype = c_int + libsodium.crypto_stream_salsa20_xor_ic.restype = c_int + libsodium.crypto_stream_salsa20_xor_ic.argtypes = (c_void_p, c_char_p, + c_ulonglong, + c_char_p, c_ulonglong, + c_char_p) + libsodium.crypto_stream_chacha20_xor_ic.restype = c_int + libsodium.crypto_stream_chacha20_xor_ic.argtypes = (c_void_p, c_char_p, + c_ulonglong, + c_char_p, c_ulonglong, + c_char_p) + + libsodium.sodium_init() + + buf = create_string_buffer(buf_size) + loaded = True + + +class Salsa20Crypto(object): + def __init__(self, cipher_name, key, iv, op): + if not loaded: + load_libsodium() + self.key = key + self.iv = iv + self.key_ptr = c_char_p(key) + self.iv_ptr = c_char_p(iv) + if cipher_name == b'salsa20': + self.cipher = libsodium.crypto_stream_salsa20_xor_ic + elif cipher_name == b'chacha20': + self.cipher = libsodium.crypto_stream_chacha20_xor_ic + else: + raise Exception('Unknown cipher') + # byte counter, not block counter + self.counter = 0 + + def update(self, data): + global buf_size, buf + l = len(data) + + # we can only prepend some padding to make the encryption align to + # blocks + padding = self.counter % BLOCK_SIZE + if buf_size < padding + l: + buf_size = (padding + l) * 2 + buf = create_string_buffer(buf_size) + + if padding: + data = (b'\0' * padding) + data + self.cipher(byref(buf), c_char_p(data), padding + l, + self.iv_ptr, int(self.counter / BLOCK_SIZE), self.key_ptr) + self.counter += l + # buf is copied to a str object when we access buf.raw + # strip off the padding + return buf.raw[padding:padding + l] + + +ciphers = { + b'salsa20': (32, 8, Salsa20Crypto), + b'chacha20': (32, 8, Salsa20Crypto), +} + + +def test_salsa20(): + from shadowsocks.crypto import util + + cipher = Salsa20Crypto(b'salsa20', b'k' * 32, b'i' * 16, 1) + decipher = Salsa20Crypto(b'salsa20', b'k' * 32, b'i' * 16, 0) + + util.run_cipher(cipher, decipher) + + +def test_chacha20(): + from shadowsocks.crypto import util + + cipher = Salsa20Crypto(b'chacha20', b'k' * 32, b'i' * 16, 1) + decipher = Salsa20Crypto(b'chacha20', b'k' * 32, b'i' * 16, 0) + + util.run_cipher(cipher, decipher) + + +if __name__ == '__main__': + test_chacha20() + test_salsa20() From 9d33a59b609d04c129396dba4ddb56d2cee07f97 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 10 Dec 2014 23:15:24 +0800 Subject: [PATCH 278/344] add libsodium tests --- .travis.yml | 3 +++ tests/chacha20.json | 10 ++++++++++ tests/libsodium/install.sh | 9 +++++++++ tests/salsa20-ctr.json | 10 ++++++++++ tests/salsa20.json | 2 +- 5 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 tests/chacha20.json create mode 100755 tests/libsodium/install.sh create mode 100644 tests/salsa20-ctr.json diff --git a/.travis.yml b/.travis.yml index 286c5df..4a11a47 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ before_install: - sudo apt-get install -qq build-essential libssl-dev swig python-m2crypto python-numpy dnsutils - pip install m2crypto salsa20 pep8 pyflakes nose - sudo tests/socksify/install.sh + - sudo tests/libsodium/install.sh script: - pep8 . - pyflakes . @@ -24,6 +25,8 @@ script: - python tests/test.py -c tests/aes-cfb8.json - python tests/test.py -c tests/rc4-md5.json - python tests/test.py -c tests/salsa20.json + - python tests/test.py -c tests/chacha20.json + - python tests/test.py -c tests/salsa20-ctr.json - python tests/test.py -c tests/table.json - python tests/test.py -c tests/server-multi-ports.json - python tests/test.py -s tests/server-multi-passwd.json -c tests/server-multi-passwd-client-side.json diff --git a/tests/chacha20.json b/tests/chacha20.json new file mode 100644 index 0000000..541a9be --- /dev/null +++ b/tests/chacha20.json @@ -0,0 +1,10 @@ +{ + "server":"127.0.0.1", + "server_port":8388, + "local_port":1081, + "password":"salsa20_password", + "timeout":60, + "method":"chacha20", + "local_address":"127.0.0.1", + "fast_open":false +} diff --git a/tests/libsodium/install.sh b/tests/libsodium/install.sh new file mode 100755 index 0000000..be57b90 --- /dev/null +++ b/tests/libsodium/install.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +if [ ! -d libsodium-1.0.1 ]; then + wget https://github.com/jedisct1/libsodium/releases/download/1.0.1/libsodium-1.0.1.tar.gz || exit 1 + tar xf libsodium-1.0.1.tar.gz || exit 1 +fi +pushd libsodium-1.0.1 +./configure && make && make install || exit 1 +popd diff --git a/tests/salsa20-ctr.json b/tests/salsa20-ctr.json new file mode 100644 index 0000000..5ca6c45 --- /dev/null +++ b/tests/salsa20-ctr.json @@ -0,0 +1,10 @@ +{ + "server":"127.0.0.1", + "server_port":8388, + "local_port":1081, + "password":"salsa20_password", + "timeout":60, + "method":"salsa20-ctr", + "local_address":"127.0.0.1", + "fast_open":false +} diff --git a/tests/salsa20.json b/tests/salsa20.json index 5ca6c45..7e30380 100644 --- a/tests/salsa20.json +++ b/tests/salsa20.json @@ -4,7 +4,7 @@ "local_port":1081, "password":"salsa20_password", "timeout":60, - "method":"salsa20-ctr", + "method":"salsa20", "local_address":"127.0.0.1", "fast_open":false } From e6416562fc92d81db6e47415553f47abb5ae93c9 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 10 Dec 2014 23:19:11 +0800 Subject: [PATCH 279/344] add salsa20 and chacha20 --- shadowsocks/crypto/ctypes_libsodium.py | 2 +- shadowsocks/encrypt.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/shadowsocks/crypto/ctypes_libsodium.py b/shadowsocks/crypto/ctypes_libsodium.py index 1e81060..efecfd4 100644 --- a/shadowsocks/crypto/ctypes_libsodium.py +++ b/shadowsocks/crypto/ctypes_libsodium.py @@ -24,7 +24,7 @@ from __future__ import absolute_import, division, print_function, \ with_statement import logging -from ctypes import CDLL, c_char_p, c_int, c_long, c_ulonglong, byref, \ +from ctypes import CDLL, c_char_p, c_int, c_ulonglong, byref, \ create_string_buffer, c_void_p __all__ = ['ciphers'] diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 2043001..0f6aba8 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -28,13 +28,15 @@ import sys import hashlib import logging -from shadowsocks.crypto import m2, rc4_md5, salsa20_ctr, ctypes_openssl, table +from shadowsocks.crypto import m2, rc4_md5, salsa20_ctr,\ + ctypes_openssl, ctypes_libsodium, table method_supported = {} method_supported.update(rc4_md5.ciphers) method_supported.update(salsa20_ctr.ciphers) method_supported.update(ctypes_openssl.ciphers) +method_supported.update(ctypes_libsodium.ciphers) # let M2Crypto override ctypes_openssl method_supported.update(m2.ciphers) method_supported.update(table.ciphers) From d7d9e5c1037bd591796ca8bca5ace5649597d315 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 10 Dec 2014 23:46:16 +0800 Subject: [PATCH 280/344] add ldconfig --- tests/libsodium/install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/libsodium/install.sh b/tests/libsodium/install.sh index be57b90..6cca56e 100755 --- a/tests/libsodium/install.sh +++ b/tests/libsodium/install.sh @@ -6,4 +6,5 @@ if [ ! -d libsodium-1.0.1 ]; then fi pushd libsodium-1.0.1 ./configure && make && make install || exit 1 +sudo ldconfig popd From 5fe886efadcf064b15741ca2360d6af90a47d44d Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 00:06:56 +0800 Subject: [PATCH 281/344] bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index aa18757..5103c70 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with codecs.open('README.rst', encoding='utf-8') as f: setup( name="shadowsocks", - version="2.4.3", + version="2.5", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From b4ce4b7f487439ec3585fde5ed4692fad4e0ee20 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 00:36:39 +0800 Subject: [PATCH 282/344] update CHANGES --- CHANGES | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES b/CHANGES index c12a1e6..4c07066 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +2.5 2014-12-11 +- Add salsa20 and chacha20 + 2.4.3 2014-11-10 - Fix an issue on Python 3 - Fix an issue with IPv6 From 55630f5a7e81d3f809d06df272e92bad111402be Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 12:43:16 +0800 Subject: [PATCH 283/344] add coverage --- .travis.yml | 35 ++++++++++++++++++----------------- tests/test.py | 12 ++++++++---- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4a11a47..5a885cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,26 +11,27 @@ cache: before_install: - sudo apt-get update -qq - sudo apt-get install -qq build-essential libssl-dev swig python-m2crypto python-numpy dnsutils - - pip install m2crypto salsa20 pep8 pyflakes nose + - pip install m2crypto salsa20 pep8 pyflakes nose coverage - sudo tests/socksify/install.sh - sudo tests/libsodium/install.sh script: - pep8 . - pyflakes . - - python tests/nose_plugin.py -v + - coverage run tests/nose_plugin.py -v - python setup.py sdist - - python tests/test.py -c tests/aes.json - - python tests/test.py -c tests/aes-ctr.json - - python tests/test.py -c tests/aes-cfb1.json - - python tests/test.py -c tests/aes-cfb8.json - - python tests/test.py -c tests/rc4-md5.json - - python tests/test.py -c tests/salsa20.json - - python tests/test.py -c tests/chacha20.json - - python tests/test.py -c tests/salsa20-ctr.json - - python tests/test.py -c tests/table.json - - python tests/test.py -c tests/server-multi-ports.json - - python tests/test.py -s tests/server-multi-passwd.json -c tests/server-multi-passwd-client-side.json - - python tests/test.py -c tests/workers.json - - python tests/test.py -s tests/ipv6.json -c tests/ipv6-client-side.json - - python tests/test.py -b "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388" -a "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" - - python tests/test.py -b "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388" -a "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" + - python tests/test.py --with-coverage -c tests/aes.json + - python tests/test.py --with-coverage -c tests/aes-ctr.json + - python tests/test.py --with-coverage -c tests/aes-cfb1.json + - python tests/test.py --with-coverage -c tests/aes-cfb8.json + - python tests/test.py --with-coverage -c tests/rc4-md5.json + - python tests/test.py --with-coverage -c tests/salsa20.json + - python tests/test.py --with-coverage -c tests/chacha20.json + - python tests/test.py --with-coverage -c tests/salsa20-ctr.json + - python tests/test.py --with-coverage -c tests/table.json + - python tests/test.py --with-coverage -c tests/server-multi-ports.json + - python tests/test.py --with-coverage -s tests/server-multi-passwd.json -c tests/server-multi-passwd-client-side.json + - python tests/test.py --with-coverage -c tests/workers.json + - python tests/test.py --with-coverage -s tests/ipv6.json -c tests/ipv6-client-side.json + - python tests/test.py --with-coverage -b "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388" -a "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" + - python tests/test.py --with-coverage -b "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388" -a "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" + - coverage report --include=shadowsocks/* diff --git a/tests/test.py b/tests/test.py index 34c10fb..f7cc25d 100755 --- a/tests/test.py +++ b/tests/test.py @@ -32,18 +32,22 @@ import time import argparse from subprocess import Popen, PIPE -python = 'python' +python = ['python'] parser = argparse.ArgumentParser(description='test Shadowsocks') parser.add_argument('-c', '--client-conf', type=str, default=None) parser.add_argument('-s', '--server-conf', type=str, default=None) parser.add_argument('-a', '--client-args', type=str, default=None) parser.add_argument('-b', '--server-args', type=str, default=None) +parser.add_argument('--with-coverage', action='store_true', default=None) config = parser.parse_args() -client_args = [python, 'shadowsocks/local.py', '-v'] -server_args = [python, 'shadowsocks/server.py', '-v'] +if config.with_coverage: + python = ['coverage', 'run', '-a'] + +client_args = python + ['shadowsocks/local.py', '-v'] +server_args = python + ['shadowsocks/server.py', '-v'] if config.client_conf: client_args.extend(['-c', config.client_conf]) @@ -134,6 +138,6 @@ try: finally: for p in [p1, p2]: try: - os.kill(p.pid, signal.SIGTERM) + os.kill(p.pid, signal.SIGQUIT) except OSError: pass From c214dbf7b593b130ad5fd3f970e2ccd5e73b0a24 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 12:46:52 +0800 Subject: [PATCH 284/344] parallel libsodium and socksify builds --- tests/libsodium/install.sh | 2 +- tests/socksify/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/libsodium/install.sh b/tests/libsodium/install.sh index 6cca56e..b0e35fa 100755 --- a/tests/libsodium/install.sh +++ b/tests/libsodium/install.sh @@ -5,6 +5,6 @@ if [ ! -d libsodium-1.0.1 ]; then tar xf libsodium-1.0.1.tar.gz || exit 1 fi pushd libsodium-1.0.1 -./configure && make && make install || exit 1 +./configure && make -j2 && make install || exit 1 sudo ldconfig popd diff --git a/tests/socksify/install.sh b/tests/socksify/install.sh index 1927d76..8eff72d 100755 --- a/tests/socksify/install.sh +++ b/tests/socksify/install.sh @@ -5,6 +5,6 @@ if [ ! -d dante-1.4.0 ]; then tar xf dante-1.4.0.tar.gz || exit 1 fi pushd dante-1.4.0 -./configure && make && make install || exit 1 +./configure && make -j4 && make install || exit 1 popd cp tests/socksify/socks.conf /etc/ || exit 1 From 2a149a5b3fdb6ba03b5a98724f30a163f24c0add Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 12:50:25 +0800 Subject: [PATCH 285/344] fix tests --- tests/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test.py b/tests/test.py index f7cc25d..07e9957 100755 --- a/tests/test.py +++ b/tests/test.py @@ -139,5 +139,6 @@ finally: for p in [p1, p2]: try: os.kill(p.pid, signal.SIGQUIT) + os.waitpid(p.pid, 0) except OSError: pass From d42a32fbc0462a050cf596f9cdb404a9be818a4a Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 13:00:08 +0800 Subject: [PATCH 286/344] master should wait for child to quit --- shadowsocks/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 64b656b..9abdf9c 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -104,6 +104,7 @@ def main(): for pid in children: try: os.kill(pid, signum) + os.waitpid(pid, 0) except OSError: # child may already exited pass sys.exit() From 030cdbcec0d3ec4cf5758da7701fc6de5f12514a Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 14:08:34 +0800 Subject: [PATCH 287/344] add asyncdns test --- shadowsocks/asyncdns.py | 70 +++++++++++++++++++++++++++++++++++------ 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 4354b1d..46459c9 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -307,7 +307,7 @@ class DNSResolver(object): def _parse_hosts(self): etc_path = '/etc/hosts' - if os.environ.__contains__('WINDIR'): + if 'WINDIR' in os.environ: etc_path = os.environ['WINDIR'] + '/system32/drivers/etc/hosts' try: with open(etc_path, 'rb') as f: @@ -324,7 +324,7 @@ class DNSResolver(object): except IOError: self._hosts['localhost'] = '127.0.0.1' - def add_to_loop(self, loop): + def add_to_loop(self, loop, ref=False): if self._loop: raise Exception('already add to loop') self._loop = loop @@ -333,21 +333,21 @@ class DNSResolver(object): socket.SOL_UDP) self._sock.setblocking(False) loop.add(self._sock, eventloop.POLL_IN) - loop.add_handler(self.handle_events, ref=False) + loop.add_handler(self.handle_events, ref=ref) def _call_callback(self, hostname, ip, error=None): callbacks = self._hostname_to_cb.get(hostname, []) for callback in callbacks: - if self._cb_to_hostname.__contains__(callback): + if callback in self._cb_to_hostname: del self._cb_to_hostname[callback] if ip or error: callback((hostname, ip), error) else: callback((hostname, None), Exception('unknown hostname %s' % hostname)) - if self._hostname_to_cb.__contains__(hostname): + if hostname in self._hostname_to_cb: del self._hostname_to_cb[hostname] - if self._hostname_status.__contains__(hostname): + if hostname in self._hostname_status: del self._hostname_status[hostname] def _handle_data(self, data): @@ -408,7 +408,7 @@ class DNSResolver(object): arr.remove(callback) if not arr: del self._hostname_to_cb[hostname] - if self._hostname_status.__contains__(hostname): + if hostname in self._hostname_status: del self._hostname_status[hostname] def _send_req(self, hostname, qtype): @@ -422,15 +422,17 @@ class DNSResolver(object): self._sock.sendto(req, (server, 53)) def resolve(self, hostname, callback): + if type(hostname) != bytes: + hostname = hostname.encode('utf8') if not hostname: callback(None, Exception('empty hostname')) elif is_ip(hostname): callback((hostname, hostname), None) - elif self._hosts.__contains__(hostname): + elif hostname in self._hosts: logging.debug('hit hosts: %s', hostname) ip = self._hosts[hostname] callback((hostname, ip), None) - elif self._cache.__contains__(hostname): + elif hostname in self._cache: logging.debug('hit cache: %s', hostname) ip = self._cache[hostname] callback((hostname, ip), None) @@ -453,3 +455,53 @@ class DNSResolver(object): if self._sock: self._sock.close() self._sock = None + + +def test(): + dns_resolver = DNSResolver() + loop = eventloop.EventLoop() + dns_resolver.add_to_loop(loop, ref=True) + + global counter + counter = 0 + + def make_callback(): + global counter + + def callback(result, error): + global counter + # TODO: what can we assert? + print(result, error) + counter += 1 + if counter == 10: + loop.remove_handler(dns_resolver.handle_events) + dns_resolver.close() + a_callback = callback + return a_callback + + assert(make_callback() != make_callback()) + + dns_resolver.resolve(b'google.com', make_callback()) + dns_resolver.resolve('google.com', make_callback()) + dns_resolver.resolve('example.com', make_callback()) + dns_resolver.resolve('ipv6.google.com', make_callback()) + dns_resolver.resolve('www.facebook.com', make_callback()) + dns_resolver.resolve('ns2.google.com', make_callback()) + dns_resolver.resolve('not.existed.google.com', make_callback()) + dns_resolver.resolve('invalid.@!#$%^&$@.hostname', make_callback()) + dns_resolver.resolve('toooooooooooooooooooooooooooooooooooooooooooooooooo' + 'ooooooooooooooooooooooooooooooooooooooooooooooooooo' + 'long.hostname', make_callback()) + dns_resolver.resolve('toooooooooooooooooooooooooooooooooooooooooooooooooo' + 'ooooooooooooooooooooooooooooooooooooooooooooooooooo' + 'ooooooooooooooooooooooooooooooooooooooooooooooooooo' + 'ooooooooooooooooooooooooooooooooooooooooooooooooooo' + 'ooooooooooooooooooooooooooooooooooooooooooooooooooo' + 'ooooooooooooooooooooooooooooooooooooooooooooooooooo' + 'long.hostname', make_callback()) + + loop.run() + + +if __name__ == '__main__': + test() \ No newline at end of file From e1f20462293b9337e3d65fe8554645853e4d3dcb Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 14:09:18 +0800 Subject: [PATCH 288/344] lint code --- shadowsocks/lru_cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/lru_cache.py b/shadowsocks/lru_cache.py index 6400a2c..4523399 100644 --- a/shadowsocks/lru_cache.py +++ b/shadowsocks/lru_cache.py @@ -86,13 +86,13 @@ class LRUCache(collections.MutableMapping): break if self.close_callback is not None: for key in self._time_to_keys[least]: - if self._store.__contains__(key): + if key in self._store: if now - self._keys_to_last_time[key] > self.timeout: value = self._store[key] self.close_callback(value) for key in self._time_to_keys[least]: self._last_visits.popleft() - if self._store.__contains__(key): + if key in self._store: if now - self._keys_to_last_time[key] > self.timeout: del self._store[key] del self._keys_to_last_time[key] From 134bf332a4555a6f6965915c77b8e29b8acbf395 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 14:13:58 +0800 Subject: [PATCH 289/344] remove domain not existed test(causing time out) --- shadowsocks/asyncdns.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/shadowsocks/asyncdns.py b/shadowsocks/asyncdns.py index 46459c9..18222a6 100644 --- a/shadowsocks/asyncdns.py +++ b/shadowsocks/asyncdns.py @@ -473,7 +473,7 @@ def test(): # TODO: what can we assert? print(result, error) counter += 1 - if counter == 10: + if counter == 9: loop.remove_handler(dns_resolver.handle_events) dns_resolver.close() a_callback = callback @@ -487,7 +487,6 @@ def test(): dns_resolver.resolve('ipv6.google.com', make_callback()) dns_resolver.resolve('www.facebook.com', make_callback()) dns_resolver.resolve('ns2.google.com', make_callback()) - dns_resolver.resolve('not.existed.google.com', make_callback()) dns_resolver.resolve('invalid.@!#$%^&$@.hostname', make_callback()) dns_resolver.resolve('toooooooooooooooooooooooooooooooooooooooooooooooooo' 'ooooooooooooooooooooooooooooooooooooooooooooooooooo' @@ -504,4 +503,4 @@ def test(): if __name__ == '__main__': - test() \ No newline at end of file + test() From 60c16fae697c25ada9873f25f8b4723fc1113110 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 14:41:53 +0800 Subject: [PATCH 290/344] fix coverage --- .travis.yml | 2 +- tests/test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5a885cc..90ffbaa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,4 +34,4 @@ script: - python tests/test.py --with-coverage -s tests/ipv6.json -c tests/ipv6-client-side.json - python tests/test.py --with-coverage -b "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388" -a "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" - python tests/test.py --with-coverage -b "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388" -a "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" - - coverage report --include=shadowsocks/* + - coverage combine && coverage report --include=shadowsocks/* diff --git a/tests/test.py b/tests/test.py index 07e9957..5314d6e 100755 --- a/tests/test.py +++ b/tests/test.py @@ -44,7 +44,7 @@ parser.add_argument('--with-coverage', action='store_true', default=None) config = parser.parse_args() if config.with_coverage: - python = ['coverage', 'run', '-a'] + python = ['coverage', 'run', '-p', '-a'] client_args = python + ['shadowsocks/local.py', '-v'] server_args = python + ['shadowsocks/server.py', '-v'] From 95ed8a75764109dbce0389e0f918658296fdb0ea Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 14:51:10 +0800 Subject: [PATCH 291/344] add coveralls --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 90ffbaa..7775c57 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ cache: before_install: - sudo apt-get update -qq - sudo apt-get install -qq build-essential libssl-dev swig python-m2crypto python-numpy dnsutils - - pip install m2crypto salsa20 pep8 pyflakes nose coverage + - pip install m2crypto salsa20 pep8 pyflakes nose coverage coveralls - sudo tests/socksify/install.sh - sudo tests/libsodium/install.sh script: @@ -35,3 +35,5 @@ script: - python tests/test.py --with-coverage -b "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388" -a "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" - python tests/test.py --with-coverage -b "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388" -a "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" - coverage combine && coverage report --include=shadowsocks/* +after_success: + - coveralls From 92e49099b6af26af8f4e3923cb0c6a74ff517de4 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 15:09:30 +0800 Subject: [PATCH 292/344] remove pypy as it is too slow --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7775c57..1fa8929 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ python: - 2.7 - 3.3 - 3.4 - - pypy cache: directories: - dante-1.4.0 From a377cc3fb2afe928044fd95bc9caa29c59429713 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 15:57:30 +0800 Subject: [PATCH 293/344] remove coveralls --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1fa8929..9d7a9bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ cache: before_install: - sudo apt-get update -qq - sudo apt-get install -qq build-essential libssl-dev swig python-m2crypto python-numpy dnsutils - - pip install m2crypto salsa20 pep8 pyflakes nose coverage coveralls + - pip install m2crypto salsa20 pep8 pyflakes nose coverage - sudo tests/socksify/install.sh - sudo tests/libsodium/install.sh script: @@ -34,5 +34,3 @@ script: - python tests/test.py --with-coverage -b "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388" -a "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" - python tests/test.py --with-coverage -b "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388" -a "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" - coverage combine && coverage report --include=shadowsocks/* -after_success: - - coveralls From 4d5d6c7c85cc562d9b4e4a606785edc58b4bef1d Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 17:47:12 +0800 Subject: [PATCH 294/344] add unit tests in encrypt --- shadowsocks/encrypt.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 0f6aba8..7c5329f 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -163,3 +163,29 @@ def encrypt_all(password, method, op, data): cipher = m(method, key, iv, op) result.append(cipher.update(data)) return b''.join(result) + + +def test_encryptor(): + from os import urandom + plain = urandom(10240) + for method in method_supported.keys(): + logging.warn('testing %s' % method.decode('utf-8')) + encryptor = Encryptor(b'key', method) + cipher = encryptor.encrypt(plain) + decryptor = Encryptor(b'key', method) + plain2 = decryptor.decrypt(cipher) + assert plain == plain2 + + +def test_encrypt_all(): + from os import urandom + plain = urandom(10240) + for method in method_supported.keys(): + cipher = encrypt_all(b'key', method, 1, plain) + plain2 = encrypt_all(b'key', method, 0, cipher) + assert plain == plain2 + + +if __name__ == '__main__': + test_encrypt_all() + test_encryptor() From 1459282ae643820e433767adc91ef72e3c2b72dd Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 18:02:57 +0800 Subject: [PATCH 295/344] only test some ciphers --- shadowsocks/encrypt.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 7c5329f..e08c0a2 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -165,11 +165,20 @@ def encrypt_all(password, method, op, data): return b''.join(result) +CIPHERS_TO_TEST = [ + b'aes-128-cfb', + b'aes-256-cfb', + b'rc4-md5', + b'salsa20', + b'chacha20', + b'table', +] + + def test_encryptor(): from os import urandom plain = urandom(10240) - for method in method_supported.keys(): - logging.warn('testing %s' % method.decode('utf-8')) + for method in CIPHERS_TO_TEST: encryptor = Encryptor(b'key', method) cipher = encryptor.encrypt(plain) decryptor = Encryptor(b'key', method) @@ -180,7 +189,7 @@ def test_encryptor(): def test_encrypt_all(): from os import urandom plain = urandom(10240) - for method in method_supported.keys(): + for method in CIPHERS_TO_TEST: cipher = encrypt_all(b'key', method, 1, plain) plain2 = encrypt_all(b'key', method, 0, cipher) assert plain == plain2 From 3e503bf67753c86dd1b06201326b6bfc1e166ab1 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 22:43:46 +0800 Subject: [PATCH 296/344] log method in encrypt unit test --- shadowsocks/encrypt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index e08c0a2..d5b1535 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -179,6 +179,7 @@ def test_encryptor(): from os import urandom plain = urandom(10240) for method in CIPHERS_TO_TEST: + logging.warn(method) encryptor = Encryptor(b'key', method) cipher = encryptor.encrypt(plain) decryptor = Encryptor(b'key', method) @@ -190,6 +191,7 @@ def test_encrypt_all(): from os import urandom plain = urandom(10240) for method in CIPHERS_TO_TEST: + logging.warn(method) cipher = encrypt_all(b'key', method, 1, plain) plain2 = encrypt_all(b'key', method, 0, cipher) assert plain == plain2 From 810db996cd4030a2ecfdfe45dbb8ca64af258568 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 22:54:49 +0800 Subject: [PATCH 297/344] disable M2Crypto on Python 3 --- shadowsocks/crypto/m2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shadowsocks/crypto/m2.py b/shadowsocks/crypto/m2.py index 4c7e148..5ad48a8 100644 --- a/shadowsocks/crypto/m2.py +++ b/shadowsocks/crypto/m2.py @@ -33,6 +33,8 @@ try: __import__('M2Crypto') except ImportError: has_m2 = False +if bytes != str: + has_m2 = False def create_cipher(alg, key, iv, op, key_as_bytes=0, d=None, salt=None, i=1, From cbbc880f44902e657f26ed470c18b4c3325dcfc0 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 23:22:48 +0800 Subject: [PATCH 298/344] try to find the problem --- shadowsocks/encrypt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index d5b1535..82baa1a 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -181,8 +181,8 @@ def test_encryptor(): for method in CIPHERS_TO_TEST: logging.warn(method) encryptor = Encryptor(b'key', method) - cipher = encryptor.encrypt(plain) decryptor = Encryptor(b'key', method) + cipher = encryptor.encrypt(plain) plain2 = decryptor.decrypt(cipher) assert plain == plain2 From 676bf9617b8dfa7fd95dbb3f903d3c58df42531e Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 11 Dec 2014 23:44:42 +0800 Subject: [PATCH 299/344] fix key cache when just method changes --- shadowsocks/encrypt.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 82baa1a..22c091a 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -62,7 +62,8 @@ def EVP_BytesToKey(password, key_len, iv_len): # so that we make the same key and iv as nodejs version if hasattr(password, 'encode'): password = password.encode('utf-8') - r = cached_keys.get(password, None) + cached_key = '%s-%d-%d' % (password, key_len, iv_len) + r = cached_keys.get(cached_key, None) if r: return r m = [] @@ -78,7 +79,7 @@ def EVP_BytesToKey(password, key_len, iv_len): ms = b''.join(m) key = ms[:key_len] iv = ms[key_len:key_len + iv_len] - cached_keys[password] = (key, iv) + cached_keys[cached_key] = (key, iv) return key, iv @@ -198,5 +199,5 @@ def test_encrypt_all(): if __name__ == '__main__': - test_encrypt_all() + #test_encrypt_all() test_encryptor() From 163992b98b7a2e181920e4d28660733013a4f525 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 12 Dec 2014 00:04:17 +0800 Subject: [PATCH 300/344] small fix --- shadowsocks/encrypt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index 22c091a..ba02101 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -199,5 +199,5 @@ def test_encrypt_all(): if __name__ == '__main__': - #test_encrypt_all() + test_encrypt_all() test_encryptor() From 525765344062126ceb2606e76e78e0065526cc04 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 14 Dec 2014 23:46:15 +0800 Subject: [PATCH 301/344] Update CONTRIBUTING.md --- CONTRIBUTING.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 11054e2..c164423 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,11 +13,10 @@ to read this guide. 请提交下面的信息： 1. 你是如何搭建环境的？（操作系统，Shadowsocks 版本） -2. 有无错误提示？错误是发生在哪里，客户端还是服务器？ +2. 操作步骤是什么？ 3. 浏览器里的现象是什么？一直转菊花，还是有提示错误？ -4. 发生错误时，客户端最后一页完整的日志。 -5. 发生错误时，服务器端最后一页完整的日志。 -6. 其它你认为可能和问题有关的信息。 +4. 发生错误时，客户端和服务端最后一部分日志。 +5. 其它你认为可能和问题有关的信息。 如果你不清楚其中某条的含义， 可以直接跳过那一条。 @@ -27,7 +26,7 @@ Issues Please include the following information in your submission: 1. How did you set up your environment? (OS, version of Shadowsocks) -2. Did you see any error? Where did you see this error, was it on local or on server? +2. Steps to reproduce the problem. 3. What happened in your browser? Just no response, or any error message? 4. 10 lines of log on the local side of shadowsocks when the error happened. 5. 10 lines of log on the server side of shadowsocks when the error happened. From 4b0b2529537aef45fcd9954202237d2fd0cded89 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 18 Dec 2014 13:23:59 +0800 Subject: [PATCH 302/344] improve comments --- shadowsocks/tcprelay.py | 101 ++++++++++++++++++++++++++++++---------- 1 file changed, 77 insertions(+), 24 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index e55f9dd..0c4f5de 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -35,35 +35,57 @@ import random from shadowsocks import encrypt, eventloop, utils, common from shadowsocks.common import parse_header - +# we clear at most TIMEOUTS_CLEAN_SIZE timeouts each time TIMEOUTS_CLEAN_SIZE = 512 + +# we check timeouts every TIMEOUT_PRECISION seconds TIMEOUT_PRECISION = 4 MSG_FASTOPEN = 0x20000000 +# SOCKS CMD defination CMD_CONNECT = 1 CMD_BIND = 2 CMD_UDP_ASSOCIATE = 3 +# TCP Relay can be either sslocal or ssserver +# for sslocal it is called is_local=True + +# for each opening port, we have a TCP Relay +# for each connection, we have a TCP Relay Handler to handle the connection + +# for each handler, we have 2 sockets: +# local: connected to the client +# remote: connected to remote server + +# for each handler, we have 2 streams: +# upstream: from client to server direction +# read local and write to remote +# downstream: from server to client direction +# read remote and write to local + +# for each handler, it could be at one of several stages: + # local: -# stage 0 init -# stage 1 hello received, hello sent +# stage 0 SOCKS hello received from local, send hello to local +# stage 1 addr received from local, query DNS for remote # stage 2 UDP assoc -# stage 3 DNS -# stage 4 addr received, reply sent -# stage 5 remote connected +# stage 3 DNS resolved, connect to remote +# stage 4 still connecting, more data from local received +# stage 5 remote connected, piping local and remote # remote: -# stage 0 init -# stage 3 DNS -# stage 4 addr received, reply sent -# stage 5 remote connected +# stage 0 just jump to stage 1 +# stage 1 addr received from local, query DNS for remote +# stage 3 DNS resolved, connect to remote +# stage 4 still connecting, more data from local received +# stage 5 remote connected, piping local and remote STAGE_INIT = 0 -STAGE_HELLO = 1 +STAGE_ADDR = 1 STAGE_UDP_ASSOC = 2 STAGE_DNS = 3 -STAGE_REPLY = 4 +STAGE_CONNECTING = 4 STAGE_STREAM = 5 STAGE_DESTROYED = -1 @@ -71,7 +93,7 @@ STAGE_DESTROYED = -1 STREAM_UP = 0 STREAM_DOWN = 1 -# stream wait status +# stream wait status, indicating it's waiting for reading, etc WAIT_STATUS_INIT = 0 WAIT_STATUS_READING = 1 WAIT_STATUS_WRITING = 2 @@ -128,9 +150,15 @@ class TCPRelayHandler(object): return server, server_port def _update_activity(self): + # tell the TCP Relay we have activities recently + # else it will think we are inactive and timed out self._server.update_activity(self) def _update_stream(self, stream, status): + # update a stream to a new waiting status + + # check if status is changed + # only update if dirty dirty = False if stream == STREAM_DOWN: if self._downstream_status != status: @@ -157,6 +185,9 @@ class TCPRelayHandler(object): self._loop.modify(self._remote_sock, event) def _write_to_sock(self, data, sock): + # write data to sock + # if only some of the data are written, put remaining in the buffer + # and update the stream to wait for writing if not data or not sock: return False uncomplete = False @@ -195,13 +226,16 @@ class TCPRelayHandler(object): logging.error('write_all_to_sock:unknown socket') return True - def _handle_stage_reply(self, data): + def _handle_stage_connecting(self, data): if self._is_local: data = self._encryptor.encrypt(data) self._data_to_write_to_remote.append(data) if self._is_local and not self._fastopen_connected and \ self._config['fast_open']: + # for sslocal and fastopen, we basically wait for data and use + # sendto to connect try: + # only connect once self._fastopen_connected = True remote_sock = \ self._create_remote_socket(self._chosen_server[0], @@ -231,7 +265,7 @@ class TCPRelayHandler(object): traceback.print_exc() self.destroy() - def _handle_stage_hello(self, data): + def _handle_stage_addr(self, data): try: if self._is_local: cmd = common.ord(data[1]) @@ -312,7 +346,7 @@ class TCPRelayHandler(object): ip = result[1] if ip: try: - self._stage = STAGE_REPLY + self._stage = STAGE_CONNECTING remote_addr = ip if self._is_local: remote_port = self._chosen_server[1] @@ -320,11 +354,15 @@ class TCPRelayHandler(object): remote_port = self._remote_address[1] if self._is_local and self._config['fast_open']: + # for fastopen: # wait for more data to arrive and send them in one SYN - self._stage = STAGE_REPLY + self._stage = STAGE_CONNECTING + # we don't have to wait for remote since it's not + # created self._update_stream(STREAM_UP, WAIT_STATUS_READING) # TODO when there is already data in this packet else: + # else do connect remote_sock = self._create_remote_socket(remote_addr, remote_port) try: @@ -335,7 +373,7 @@ class TCPRelayHandler(object): pass self._loop.add(remote_sock, eventloop.POLL_ERR | eventloop.POLL_OUT) - self._stage = STAGE_REPLY + self._stage = STAGE_CONNECTING self._update_stream(STREAM_UP, WAIT_STATUS_READWRITING) self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) return @@ -346,6 +384,8 @@ class TCPRelayHandler(object): self.destroy() def _on_local_read(self): + # handle all local read events and dispatch them to methods for + # each stage self._update_activity() if not self._local_sock: return @@ -372,15 +412,16 @@ class TCPRelayHandler(object): elif is_local and self._stage == STAGE_INIT: # TODO check auth method self._write_to_sock(b'\x05\00', self._local_sock) - self._stage = STAGE_HELLO + self._stage = STAGE_ADDR return - elif self._stage == STAGE_REPLY: - self._handle_stage_reply(data) - elif (is_local and self._stage == STAGE_HELLO) or \ + elif self._stage == STAGE_CONNECTING: + self._handle_stage_connecting(data) + elif (is_local and self._stage == STAGE_ADDR) or \ (not is_local and self._stage == STAGE_INIT): - self._handle_stage_hello(data) + self._handle_stage_addr(data) def _on_remote_read(self): + # handle all remote read events self._update_activity() data = None try: @@ -406,6 +447,7 @@ class TCPRelayHandler(object): self.destroy() def _on_local_write(self): + # handle local writable event if self._data_to_write_to_local: data = b''.join(self._data_to_write_to_local) self._data_to_write_to_local = [] @@ -414,6 +456,7 @@ class TCPRelayHandler(object): self._update_stream(STREAM_DOWN, WAIT_STATUS_READING) def _on_remote_write(self): + # handle remote writable event self._stage = STAGE_STREAM if self._data_to_write_to_remote: data = b''.join(self._data_to_write_to_remote) @@ -435,6 +478,7 @@ class TCPRelayHandler(object): self.destroy() def handle_event(self, sock, event): + # handle all events in this handler and dispatch them to methods if self._stage == STAGE_DESTROYED: logging.debug('ignore handle_event: destroyed') return @@ -465,7 +509,15 @@ class TCPRelayHandler(object): logging.warn('unknown socket') def destroy(self): + # destroy the handler and release any resources + # promises: + # 1. destroy won't make another destroy() call inside + # 2. destroy releases resources so it prevents future call to destroy + # 3. destroy won't raise any exceptions + # if any of the promises are broken, it indicates a bug have been + # introduced! mostly likely memory leaks, etc if self._stage == STAGE_DESTROYED: + # this couldn't happen logging.debug('already destroyed') return self._stage = STAGE_DESTROYED @@ -552,7 +604,7 @@ class TCPRelay(object): del self._handler_to_timeouts[hash(handler)] def update_activity(self, handler): - """ set handler to active """ + # set handler to active now = int(time.time()) if now - handler.last_activity < TIMEOUT_PRECISION: # thus we can lower timeout modification frequency @@ -601,6 +653,7 @@ class TCPRelay(object): self._timeout_offset = pos def _handle_events(self, events): + # handle events and dispatch to handlers for sock, fd, event in events: if sock: logging.log(utils.VERBOSE_LEVEL, 'fd %d %s', fd, From 706a137b3f225880ec15b13e5b346e8dae5c9f44 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 18 Dec 2014 13:29:00 +0800 Subject: [PATCH 303/344] fix typo --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 0c4f5de..8c4b2de 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -514,7 +514,7 @@ class TCPRelayHandler(object): # 1. destroy won't make another destroy() call inside # 2. destroy releases resources so it prevents future call to destroy # 3. destroy won't raise any exceptions - # if any of the promises are broken, it indicates a bug have been + # if any of the promises are broken, it indicates a bug has been # introduced! mostly likely memory leaks, etc if self._stage == STAGE_DESTROYED: # this couldn't happen From 7274c7ad965e6a02d9a6b9824977a6dab25f64d0 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 18 Dec 2014 13:34:41 +0800 Subject: [PATCH 304/344] fix comments --- shadowsocks/tcprelay.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 8c4b2de..146714a 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -66,7 +66,7 @@ CMD_UDP_ASSOCIATE = 3 # for each handler, it could be at one of several stages: -# local: +# sslocal: # stage 0 SOCKS hello received from local, send hello to local # stage 1 addr received from local, query DNS for remote # stage 2 UDP assoc @@ -74,7 +74,7 @@ CMD_UDP_ASSOCIATE = 3 # stage 4 still connecting, more data from local received # stage 5 remote connected, piping local and remote -# remote: +# ssserver: # stage 0 just jump to stage 1 # stage 1 addr received from local, query DNS for remote # stage 3 DNS resolved, connect to remote From 78f93a091c10293b3b5fd074bd4486e0dc74602d Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 19 Dec 2014 14:24:32 +0800 Subject: [PATCH 305/344] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7f50c02..7ee4bea 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ source list. #### Windows: -Download OpenSSL for Windows and install. Then install shadowsocks via +Download [OpenSSL for Windows] and install. Then install shadowsocks via easy_install and pip as Linux. If you don't know how to use them, you can directly download [the package], and use `python shadowsocks/server.py` instead of `ssserver` command below. @@ -134,6 +134,7 @@ Bugs and Issues [iOS]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Help [Issue Tracker]: https://github.com/clowwindy/shadowsocks/issues?state=open [Mailing list]: http://groups.google.com/group/shadowsocks +[OpenSSL for Windows]: http://slproweb.com/products/Win32OpenSSL.html [OpenWRT]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#openwrt [OS X]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Shadowsocks-for-OSX-Help [PyPI]: https://pypi.python.org/pypi/shadowsocks From 134497c24ffc4a62452cd839f50bc6e4bfab885c Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 21 Dec 2014 12:47:07 +0800 Subject: [PATCH 306/344] implement daemon --- shadowsocks/daemon.py | 173 ++++++++++++++++++++++++++++++++++++++++++ shadowsocks/local.py | 5 +- shadowsocks/server.py | 5 +- shadowsocks/utils.py | 93 ++++++++++++++--------- 4 files changed, 240 insertions(+), 36 deletions(-) create mode 100644 shadowsocks/daemon.py diff --git a/shadowsocks/daemon.py b/shadowsocks/daemon.py new file mode 100644 index 0000000..d4320e0 --- /dev/null +++ b/shadowsocks/daemon.py @@ -0,0 +1,173 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import absolute_import, division, print_function, \ + with_statement + +import os +import sys +import logging +import signal +import time +from shadowsocks import common + +# this module is ported from ShadowVPN daemon.c + + +def daemon_exec(config): + if 'daemon' in config: + if os.name != 'posix': + raise Exception('daemon mode is only supported in unix') + command = config['daemon'] + if not command: + command = 'start' + pid_file = config['pid-file'] + log_file = config['log-file'] + command = common.to_str(command) + pid_file = common.to_str(pid_file) + log_file = common.to_str(log_file) + if command == 'start': + daemon_start(pid_file, log_file) + elif command == 'stop': + daemon_stop(pid_file) + # always exit after daemon_stop + sys.exit(0) + elif command == 'restart': + daemon_stop(pid_file) + daemon_start(pid_file, log_file) + else: + raise Exception('unsupported daemon command %s' % command) + + +def write_pid_file(pid_file, pid): + import fcntl + import stat + + try: + fd = os.open(pid_file, os.O_RDWR | os.O_CREAT, + stat.S_IRUSR | stat.S_IWUSR) + except OSError as e: + logging.error(e) + return -1 + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + assert flags != -1 + flags |= fcntl.FD_CLOEXEC + r = fcntl.fcntl(fd, fcntl.F_SETFD, flags) + assert r != -1 + # There is no platform independent way to implement fcntl(fd, F_SETLK, &fl) + # via fcntl.fcntl. So use lockf instead + try: + fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB, 0, 0, os.SEEK_SET) + except IOError: + r = os.read(fd, 32) + if r: + logging.error('already started at pid %s' % common.to_str(r)) + else: + logging.error('already started') + os.close(fd) + return -1 + os.ftruncate(fd, 0) + os.write(fd, common.to_bytes(str(pid))) + return 0 + + +def freopen(f, mode, stream): + oldf = open(f, mode) + oldfd = oldf.fileno() + newfd = stream.fileno() + os.close(newfd) + os.dup2(oldfd, newfd) + + +def daemon_start(pid_file, log_file): + # fork only once because we are sure parent will exit + pid = os.fork() + assert pid != -1 + + def handle_exit(signum, _): + sys.exit(0) + + if pid > 0: + # parent waits for its child + signal.signal(signal.SIGINT, handle_exit) + time.sleep(5) + sys.exit(0) + + # child signals its parent to exit + ppid = os.getppid() + pid = os.getpid() + if write_pid_file(pid_file, pid) != 0: + os.kill(ppid, signal.SIGINT) + sys.exit(1) + + print('started') + os.kill(ppid, signal.SIGINT) + + sys.stdin.close() + freopen(log_file, 'a', sys.stdout) + freopen(log_file, 'a', sys.stderr) + + +def daemon_stop(pid_file): + import errno + try: + with open(pid_file) as f: + buf = f.read() + pid = common.to_str(buf) + if not buf: + logging.error('not running') + except IOError as e: + logging.error(e) + if e.errno == errno.ENOENT: + # always exit 0 if we are sure daemon is not running + logging.error('not running') + return + sys.exit(1) + pid = int(pid) + if pid > 0: + try: + os.kill(pid, signal.SIGTERM) + except OSError as e: + if e.errno == errno.ESRCH: + logging.error('not running') + # always exit 0 if we are sure daemon is not running + return + logging.error(e) + sys.exit(1) + else: + logging.error('pid is not positive: %d', pid) + + # sleep for maximum 10s + for i in range(0, 200): + try: + # query for the pid + os.kill(pid, 0) + except OSError as e: + if e.errno == errno.ESRCH: + break + time.sleep(0.05) + else: + logging.error('timed out when stopping pid %d', pid) + sys.exit(1) + print('stopped') + os.unlink(pid_file) diff --git a/shadowsocks/local.py b/shadowsocks/local.py index e778ea7..6a97f07 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -30,7 +30,8 @@ import logging import signal sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../')) -from shadowsocks import utils, encrypt, eventloop, tcprelay, udprelay, asyncdns +from shadowsocks import utils, daemon, encrypt, eventloop, tcprelay, udprelay,\ + asyncdns def main(): @@ -44,6 +45,8 @@ def main(): config = utils.get_config(True) + daemon.daemon_exec(config) + utils.print_shadowsocks() encrypt.try_cipher(config['password'], config['method']) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 9abdf9c..e7acc5e 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -30,7 +30,8 @@ import logging import signal sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../')) -from shadowsocks import utils, encrypt, eventloop, tcprelay, udprelay, asyncdns +from shadowsocks import utils, daemon, encrypt, eventloop, tcprelay, udprelay,\ + asyncdns def main(): @@ -38,6 +39,8 @@ def main(): config = utils.get_config(False) + daemon.daemon_exec(config) + utils.print_shadowsocks() if config['port_password']: diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 7808d8f..7caa243 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -70,9 +70,9 @@ def find_config(): def check_config(config): if config.get('local_address', '') in [b'0.0.0.0']: - logging.warn('warning: local set to listen 0.0.0.0, which is not safe') + logging.warn('warning: local set to listen on 0.0.0.0, it\'s not safe') if config.get('server', '') in [b'127.0.0.1', b'localhost']: - logging.warn('warning: server set to listen %s:%s, are you sure?' % + logging.warn('warning: server set to listen on %s:%s, are you sure?' % (config['server'], config['server_port'])) if (config.get('method', '') or '').lower() == b'table': logging.warn('warning: table is not safe; please use a safer cipher, ' @@ -96,11 +96,11 @@ def get_config(is_local): logging.basicConfig(level=logging.INFO, format='%(levelname)-s: %(message)s') if is_local: - shortopts = 'hs:b:p:k:l:m:c:t:vq' - longopts = ['fast-open'] + shortopts = 'hd:s:b:p:k:l:m:c:t:vq' + longopts = ['help', 'fast-open', 'pid-file=', 'log-file='] else: - shortopts = 'hs:p:k:m:c:t:vq' - longopts = ['fast-open', 'workers='] + shortopts = 'hd:s:p:k:m:c:t:vq' + longopts = ['help', 'fast-open', 'pid-file=', 'log-file=', 'workers='] try: config_path = find_config() optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts) @@ -146,12 +146,18 @@ def get_config(is_local): config['fast_open'] = True elif key == '--workers': config['workers'] = int(value) - elif key == '-h': + elif key in ('-h', '--help'): if is_local: print_local_help() else: print_server_help() sys.exit(0) + elif key == '-d': + config['daemon'] = value + elif key == '--pid-file': + config['pid-file'] = value + elif key == '--log-file': + config['log-file'] = value elif key == '-q': v_count -= 1 config['verbose'] = v_count @@ -171,6 +177,9 @@ def get_config(is_local): config['timeout'] = int(config.get('timeout', 300)) config['fast_open'] = config.get('fast_open', False) config['workers'] = config.get('workers', 1) + config['pid-file'] = config.get('pid-file', '/var/run/shadowsocks.pid') + config['log-file'] = config.get('log-file', '/var/log/shadowsocks.log') + config['workers'] = config.get('workers', 1) config['verbose'] = config.get('verbose', False) config['local_address'] = config.get('local_address', '127.0.0.1') config['local_port'] = config.get('local_port', 1080) @@ -231,21 +240,29 @@ def print_help(is_local): def print_local_help(): print('''usage: sslocal [-h] -s SERVER_ADDR [-p SERVER_PORT] [-b LOCAL_ADDR] [-l LOCAL_PORT] -k PASSWORD [-m METHOD] - [-t TIMEOUT] [-c CONFIG] [--fast-open] [-v] [-q] + [-t TIMEOUT] [-c CONFIG] [--fast-open] [-v] -[d] [-q] +A fast tunnel proxy that helps you bypass firewalls. -optional arguments: - -h, --help show this help message and exit - -s SERVER_ADDR server address - -p SERVER_PORT server port, default: 8388 - -b LOCAL_ADDR local binding address, default: 127.0.0.1 - -l LOCAL_PORT local port, default: 1080 - -k PASSWORD password - -m METHOD encryption method, default: aes-256-cfb - -t TIMEOUT timeout in seconds, default: 300 - -c CONFIG path to config file - --fast-open use TCP_FASTOPEN, requires Linux 3.7+ - -v, -vv verbose mode - -q, -qq quiet mode, only show warnings/errors +You can supply configurations via either config file or command line arguments. + +Proxy options: + -h, --help show this help message and exit + -c CONFIG path to config file + -s SERVER_ADDR server address + -p SERVER_PORT server port, default: 8388 + -b LOCAL_ADDR local binding address, default: 127.0.0.1 + -l LOCAL_PORT local port, default: 1080 + -k PASSWORD password + -m METHOD encryption method, default: aes-256-cfb + -t TIMEOUT timeout in seconds, default: 300 + --fast-open use TCP_FASTOPEN, requires Linux 3.7+ + +General options: + -d start/stop/restart daemon mode + --pid-file PID_FILE pid file for daemon mode + --log-file LOG_FILE log file for daemon mode + -v, -vv verbose mode + -q, -qq quiet mode, only show warnings/errors Online help: ''') @@ -254,20 +271,28 @@ Online help: def print_server_help(): print('''usage: ssserver [-h] [-s SERVER_ADDR] [-p SERVER_PORT] -k PASSWORD -m METHOD [-t TIMEOUT] [-c CONFIG] [--fast-open] - [--workers WORKERS] [-v] [-q] + [--workers WORKERS] [-v] [-d start] [-q] +A fast tunnel proxy that helps you bypass firewalls. -optional arguments: - -h, --help show this help message and exit - -s SERVER_ADDR server address, default: 0.0.0.0 - -p SERVER_PORT server port, default: 8388 - -k PASSWORD password - -m METHOD encryption method, default: aes-256-cfb - -t TIMEOUT timeout in seconds, default: 300 - -c CONFIG path to config file - --fast-open use TCP_FASTOPEN, requires Linux 3.7+ - --workers WORKERS number of workers, available on Unix/Linux - -v, -vv verbose mode - -q, -qq quiet mode, only show warnings/errors +You can supply configurations via either config file or command line arguments. + +Proxy options: + -h, --help show this help message and exit + -c CONFIG path to config file + -s SERVER_ADDR server address, default: 0.0.0.0 + -p SERVER_PORT server port, default: 8388 + -k PASSWORD password + -m METHOD encryption method, default: aes-256-cfb + -t TIMEOUT timeout in seconds, default: 300 + --fast-open use TCP_FASTOPEN, requires Linux 3.7+ + --workers WORKERS number of workers, available on Unix/Linux + +General options: + -d start/stop/restart daemon mode + --pid-file PID_FILE pid file for daemon mode + --log-file LOG_FILE log file for daemon mode + -v, -vv verbose mode + -q, -qq quiet mode, only show warnings/errors Online help: ''') From 28347b685e3e38fdf68aa88732f77e840d4e5c2d Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 21 Dec 2014 13:46:27 +0800 Subject: [PATCH 307/344] add unit test for daemon --- .travis.yml | 1 + shadowsocks/daemon.py | 16 ++++++++++++---- tests/test_daemon.sh | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 4 deletions(-) create mode 100755 tests/test_daemon.sh diff --git a/.travis.yml b/.travis.yml index 9d7a9bb..1ceac69 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,7 @@ script: - pyflakes . - coverage run tests/nose_plugin.py -v - python setup.py sdist + - tests/test_daemon.sh - python tests/test.py --with-coverage -c tests/aes.json - python tests/test.py --with-coverage -c tests/aes-ctr.json - python tests/test.py --with-coverage -c tests/aes-cfb1.json diff --git a/shadowsocks/daemon.py b/shadowsocks/daemon.py index d4320e0..2eedf03 100644 --- a/shadowsocks/daemon.py +++ b/shadowsocks/daemon.py @@ -105,11 +105,14 @@ def daemon_start(pid_file, log_file): assert pid != -1 def handle_exit(signum, _): - sys.exit(0) + if signum == signal.SIGTERM: + sys.exit(0) + sys.exit(1) if pid > 0: # parent waits for its child signal.signal(signal.SIGINT, handle_exit) + signal.signal(signal.SIGTERM, handle_exit) time.sleep(5) sys.exit(0) @@ -121,11 +124,16 @@ def daemon_start(pid_file, log_file): sys.exit(1) print('started') - os.kill(ppid, signal.SIGINT) + os.kill(ppid, signal.SIGTERM) sys.stdin.close() - freopen(log_file, 'a', sys.stdout) - freopen(log_file, 'a', sys.stderr) + try: + freopen(log_file, 'a', sys.stdout) + freopen(log_file, 'a', sys.stderr) + except IOError as e: + logging.error(e) + os.kill(ppid, signal.SIGINT) + sys.exit(1) def daemon_stop(pid_file): diff --git a/tests/test_daemon.sh b/tests/test_daemon.sh new file mode 100755 index 0000000..b05208d --- /dev/null +++ b/tests/test_daemon.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +function test { + expected=$1 + shift + echo "running test: $command $@" + $command $@ + status=$? + if [ $status -ne $expected ]; then + echo "exit $status != $expected" + exit 1 + fi + echo "exit status $status == $expected" + echo OK + return +} + +for module in local server +do + +command="coverage run -p -a shadowsocks/$module.py" + +test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log + +test 0 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log + +test 0 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +test 1 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log + +test 0 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +test 0 -c tests/aes.json -d restart --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log + +test 0 -c tests/aes.json -d restart --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log + +test 1 -c tests/aes.json -d start --pid-file /tmp/not_exist/shadowsocks.pid --log-file /tmp/shadowsocks.log +test 1 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/not_exist/shadowsocks.log + +done From 6dbabdea9d70ebbf7f662105c65198f1c4bbade9 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 21 Dec 2014 13:54:01 +0800 Subject: [PATCH 308/344] bump --- CHANGES | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 4c07066..3662f50 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +2.6 2014-12-21 +- Add daemon support + 2.5 2014-12-11 - Add salsa20 and chacha20 diff --git a/setup.py b/setup.py index 5103c70..224cb21 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with codecs.open('README.rst', encoding='utf-8') as f: setup( name="shadowsocks", - version="2.5", + version="2.6", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From 35c3b0b41c96aa1cfc53bb25f5c040562707b783 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 21 Dec 2014 14:06:28 +0800 Subject: [PATCH 309/344] refine documentation --- README.md | 43 ++++++++++++++--------------------- README.rst | 52 +++++++++++++++++-------------------------- shadowsocks/daemon.py | 2 +- 3 files changed, 39 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 7ee4bea..5fa677e 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,7 @@ Example: "password":"mypassword", "timeout":300, "method":"aes-256-cfb", - "fast_open": false, - "workers": 1 + "fast_open": false } Explanation of the fields: @@ -74,40 +73,35 @@ Explanation of the fields: | fast_open | use [TCP_FASTOPEN], true / false | | workers | number of workers, available on Unix/Linux | -Run `ssserver -c /etc/shadowsocks.json` on your server. To run it in the -background, use [Supervisor]. +On your server: -On your client machine, use the same configuration as your server, and -start your client. +To run in the foreground: -If you use Chrome, it's recommended to use [SwitchySharp]. Change the proxy -settings to + ssserver -c /etc/shadowsocks.json - protocol: socks5 - hostname: 127.0.0.1 - port: your local_port +To run in the background: -If you can't install [SwitchySharp], you can launch Chrome with the following -arguments to force Chrome to use the proxy: + sudo ssserver -c /etc/shadowsocks.json -d start + sudo ssserver -c /etc/shadowsocks.json -d stop - Chrome.exe --proxy-server="socks5://127.0.0.1:1080" --host-resolver-rules="MAP * 0.0.0.0 , EXCLUDE localhost" +On your client machine, use the same configuration as your server. Check the +README of your client for more information. -If you can't even download Chrome, find a friend to download a -[Chrome Standalone] installer for you. +Command Line Options +-------------------- -Command line args ------------------- - -You can use args to override settings from `config.json`. +Check the options via `-h`.You can use args to override settings from +`config.json`. sslocal -s server_name -p server_port -l local_port -k password -m bf-cfb ssserver -p server_port -k password -m bf-cfb --workers 2 - ssserver -c /etc/shadowsocks/config.json + ssserver -c /etc/shadowsocks/config.json -d start --pid-file=/tmp/shadowsocks.pid + ssserver -c /etc/shadowsocks/config.json -d stop --pid-file=/tmp/shadowsocks.pid List all available args with `-h`. -Wiki ----- +Documentation +------------- You can find all the documentation in the wiki: https://github.com/clowwindy/shadowsocks/wiki @@ -127,7 +121,6 @@ Bugs and Issues [Android]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#android [Build Status]: https://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat [Chinese Readme]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E -[Chrome Standalone]: https://support.google.com/installer/answer/126299 [Debian sid]: https://packages.debian.org/unstable/python/shadowsocks [the package]: https://pypi.python.org/pypi/shadowsocks [Encryption]: https://github.com/clowwindy/shadowsocks/wiki/Encryption @@ -139,9 +132,7 @@ Bugs and Issues [OS X]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Shadowsocks-for-OSX-Help [PyPI]: https://pypi.python.org/pypi/shadowsocks [PyPI version]: https://img.shields.io/pypi/v/shadowsocks.svg?style=flat -[Supervisor]: https://github.com/clowwindy/shadowsocks/wiki/Configure-Shadowsocks-with-Supervisor [TCP_FASTOPEN]: https://github.com/clowwindy/shadowsocks/wiki/TCP-Fast-Open [Travis CI]: https://travis-ci.org/clowwindy/shadowsocks [Troubleshooting]: https://github.com/clowwindy/shadowsocks/wiki/Troubleshooting -[SwitchySharp]: https://chrome.google.com/webstore/detail/proxy-switchysharp/dpplabbmogkhghncfbfdeeokoefdjegm [Windows]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#windows diff --git a/README.rst b/README.rst index 19d5b2a..fab5e1f 100644 --- a/README.rst +++ b/README.rst @@ -50,9 +50,10 @@ CentOS: Windows: ^^^^^^^^ -Download OpenSSL for Windows and install. Then install shadowsocks via -easy\_install and pip as Linux. If you don't know how to use them, you -can directly download `the +Download `OpenSSL for +Windows `__ and install. +Then install shadowsocks via easy\_install and pip as Linux. If you +don't know how to use them, you can directly download `the package `__, and use ``python shadowsocks/server.py`` instead of ``ssserver`` command below. @@ -71,8 +72,7 @@ On your server create a config file ``/etc/shadowsocks.json``. Example: "password":"mypassword", "timeout":300, "method":"aes-256-cfb", - "fast_open": false, - "workers": 1 + "fast_open": false } Explanation of the fields: @@ -99,51 +99,41 @@ Explanation of the fields: | workers | number of workers, available on Unix/Linux | +------------------+---------------------------------------------------------------------------------------------------------+ -Run ``ssserver -c /etc/shadowsocks.json`` on your server. To run it in -the background, use -`Supervisor `__. +On your server: -On your client machine, use the same configuration as your server, and -start your client. - -If you use Chrome, it's recommended to use -`SwitchySharp `__. -Change the proxy settings to +To run in the foreground: :: - protocol: socks5 - hostname: 127.0.0.1 - port: your local_port + ssserver -c /etc/shadowsocks.json -If you can't install -`SwitchySharp `__, -you can launch Chrome with the following arguments to force Chrome to -use the proxy: +To run in the background: :: - Chrome.exe --proxy-server="socks5://127.0.0.1:1080" --host-resolver-rules="MAP * 0.0.0.0 , EXCLUDE localhost" + sudo ssserver -c /etc/shadowsocks.json -d start + sudo ssserver -c /etc/shadowsocks.json -d stop -If you can't even download Chrome, find a friend to download a `Chrome -Standalone `__ -installer for you. +On your client machine, use the same configuration as your server. Check +the README of your client for more information. -Command line args ------------------ +Command Line Options +-------------------- -You can use args to override settings from ``config.json``. +Check the options via ``-h``.You can use args to override settings from +``config.json``. :: sslocal -s server_name -p server_port -l local_port -k password -m bf-cfb ssserver -p server_port -k password -m bf-cfb --workers 2 - ssserver -c /etc/shadowsocks/config.json + ssserver -c /etc/shadowsocks/config.json -d start --pid-file=/tmp/pid + ssserver -c /etc/shadowsocks/config.json -d stop --pid-file=/tmp/pid List all available args with ``-h``. -Wiki ----- +Documentation +------------- You can find all the documentation in the wiki: https://github.com/clowwindy/shadowsocks/wiki diff --git a/shadowsocks/daemon.py b/shadowsocks/daemon.py index 2eedf03..d694e94 100644 --- a/shadowsocks/daemon.py +++ b/shadowsocks/daemon.py @@ -37,7 +37,7 @@ from shadowsocks import common def daemon_exec(config): if 'daemon' in config: if os.name != 'posix': - raise Exception('daemon mode is only supported in unix') + raise Exception('daemon mode is only supported on Unix') command = config['daemon'] if not command: command = 'start' From 2733e6a4ba1f2e8c312d06103c9f425ab25c5ad1 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 21 Dec 2014 14:08:11 +0800 Subject: [PATCH 310/344] update readme --- README.md | 2 -- README.rst | 6 ++---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5fa677e..9489eed 100644 --- a/README.md +++ b/README.md @@ -98,8 +98,6 @@ Check the options via `-h`.You can use args to override settings from ssserver -c /etc/shadowsocks/config.json -d start --pid-file=/tmp/shadowsocks.pid ssserver -c /etc/shadowsocks/config.json -d stop --pid-file=/tmp/shadowsocks.pid -List all available args with `-h`. - Documentation ------------- diff --git a/README.rst b/README.rst index fab5e1f..a586f2a 100644 --- a/README.rst +++ b/README.rst @@ -127,10 +127,8 @@ Check the options via ``-h``.You can use args to override settings from sslocal -s server_name -p server_port -l local_port -k password -m bf-cfb ssserver -p server_port -k password -m bf-cfb --workers 2 - ssserver -c /etc/shadowsocks/config.json -d start --pid-file=/tmp/pid - ssserver -c /etc/shadowsocks/config.json -d stop --pid-file=/tmp/pid - -List all available args with ``-h``. + ssserver -c /etc/shadowsocks/config.json -d start --pid-file=/tmp/shadowsocks.pid + ssserver -c /etc/shadowsocks/config.json -d stop --pid-file=/tmp/shadowsocks.pid Documentation ------------- From fb789d8e9fd574ad9d48662f8b5afdc5a745b1fb Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 21 Dec 2014 14:09:07 +0800 Subject: [PATCH 311/344] update readme --- README.md | 4 ++-- README.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9489eed..7a37b26 100644 --- a/README.md +++ b/README.md @@ -81,8 +81,8 @@ To run in the foreground: To run in the background: - sudo ssserver -c /etc/shadowsocks.json -d start - sudo ssserver -c /etc/shadowsocks.json -d stop + ssserver -c /etc/shadowsocks.json -d start + ssserver -c /etc/shadowsocks.json -d stop On your client machine, use the same configuration as your server. Check the README of your client for more information. diff --git a/README.rst b/README.rst index a586f2a..003b358 100644 --- a/README.rst +++ b/README.rst @@ -111,8 +111,8 @@ To run in the background: :: - sudo ssserver -c /etc/shadowsocks.json -d start - sudo ssserver -c /etc/shadowsocks.json -d stop + ssserver -c /etc/shadowsocks.json -d start + ssserver -c /etc/shadowsocks.json -d stop On your client machine, use the same configuration as your server. Check the README of your client for more information. From dae2624b307e083bd9fbd70b7bf3a71e47ca3317 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 21 Dec 2014 14:49:51 +0800 Subject: [PATCH 312/344] add setsid --- shadowsocks/daemon.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shadowsocks/daemon.py b/shadowsocks/daemon.py index d694e94..9e252e0 100644 --- a/shadowsocks/daemon.py +++ b/shadowsocks/daemon.py @@ -126,6 +126,9 @@ def daemon_start(pid_file, log_file): print('started') os.kill(ppid, signal.SIGTERM) + os.setsid() + signal.signal(signal.SIG_IGN, signal.SIGHUP) + sys.stdin.close() try: freopen(log_file, 'a', sys.stdout) From 2b4c3619d6fd29cf80c203454eaee659b8c301ea Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 21 Dec 2014 14:52:00 +0800 Subject: [PATCH 313/344] reorder setsid and kill --- shadowsocks/daemon.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shadowsocks/daemon.py b/shadowsocks/daemon.py index 9e252e0..ec6676c 100644 --- a/shadowsocks/daemon.py +++ b/shadowsocks/daemon.py @@ -123,12 +123,12 @@ def daemon_start(pid_file, log_file): os.kill(ppid, signal.SIGINT) sys.exit(1) - print('started') - os.kill(ppid, signal.SIGTERM) - os.setsid() signal.signal(signal.SIG_IGN, signal.SIGHUP) + print('started') + os.kill(ppid, signal.SIGTERM) + sys.stdin.close() try: freopen(log_file, 'a', sys.stdout) From be2ab378ffd8c4e234b42cf7602d764ef6ef5398 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 22 Dec 2014 16:33:23 +0800 Subject: [PATCH 314/344] add jenkins --- .gitignore | 1 + .jenkins.sh | 53 ++++++++++++++++++++++++++++++++++++++++++++ .travis.yml | 22 +----------------- tests/test_daemon.sh | 28 +++++++++++------------ 4 files changed, 69 insertions(+), 35 deletions(-) create mode 100755 .jenkins.sh diff --git a/.gitignore b/.gitignore index 357232f..fb96264 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ develop-eggs pip-log.txt # Unit test / coverage reports +htmlcov .coverage .tox diff --git a/.jenkins.sh b/.jenkins.sh new file mode 100755 index 0000000..1206b00 --- /dev/null +++ b/.jenkins.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +result=0 + +function run_test { + printf '\e[0;36m' + echo "running test: $command $@" + printf '\e[0m' + + $command $@ + status=$? + if [ $status -ne 0 ]; then + printf '\e[0;31m' + echo "test failed: $command $@" + printf '\e[0m' + echo + result=1 + else + printf '\e[0;32m' + echo OK + printf '\e[0m' + echo + fi + return 0 +} + +coverage erase +run_test pep8 . +run_test pyflakes . +run_test coverage run tests/nose_plugin.py -v +run_test python setup.py sdist +run_test tests/test_daemon.sh +run_test python tests/test.py --with-coverage -c tests/aes.json +run_test python tests/test.py --with-coverage -c tests/aes-ctr.json +run_test python tests/test.py --with-coverage -c tests/aes-cfb1.json +run_test python tests/test.py --with-coverage -c tests/aes-cfb8.json +run_test python tests/test.py --with-coverage -c tests/rc4-md5.json +run_test python tests/test.py --with-coverage -c tests/salsa20.json +run_test python tests/test.py --with-coverage -c tests/chacha20.json +run_test python tests/test.py --with-coverage -c tests/salsa20-ctr.json +run_test python tests/test.py --with-coverage -c tests/table.json +run_test python tests/test.py --with-coverage -c tests/server-multi-ports.json +run_test python tests/test.py --with-coverage -s tests/server-multi-passwd.json -c tests/server-multi-passwd-client-side.json +run_test python tests/test.py --with-coverage -c tests/workers.json +run_test python tests/test.py --with-coverage -s tests/ipv6.json -c tests/ipv6-client-side.json +run_test python tests/test.py --with-coverage -b "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388" -a "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" +run_test python tests/test.py --with-coverage -b "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388" -a "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" +coverage combine && coverage report --include=shadowsocks/* +coverage combine && coverage report --include=shadowsocks/* +rm -rf htmlcov +coverage html --include=shadowsocks/* + +exit $result diff --git a/.travis.yml b/.travis.yml index 1ceac69..3b094f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,24 +14,4 @@ before_install: - sudo tests/socksify/install.sh - sudo tests/libsodium/install.sh script: - - pep8 . - - pyflakes . - - coverage run tests/nose_plugin.py -v - - python setup.py sdist - - tests/test_daemon.sh - - python tests/test.py --with-coverage -c tests/aes.json - - python tests/test.py --with-coverage -c tests/aes-ctr.json - - python tests/test.py --with-coverage -c tests/aes-cfb1.json - - python tests/test.py --with-coverage -c tests/aes-cfb8.json - - python tests/test.py --with-coverage -c tests/rc4-md5.json - - python tests/test.py --with-coverage -c tests/salsa20.json - - python tests/test.py --with-coverage -c tests/chacha20.json - - python tests/test.py --with-coverage -c tests/salsa20-ctr.json - - python tests/test.py --with-coverage -c tests/table.json - - python tests/test.py --with-coverage -c tests/server-multi-ports.json - - python tests/test.py --with-coverage -s tests/server-multi-passwd.json -c tests/server-multi-passwd-client-side.json - - python tests/test.py --with-coverage -c tests/workers.json - - python tests/test.py --with-coverage -s tests/ipv6.json -c tests/ipv6-client-side.json - - python tests/test.py --with-coverage -b "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388" -a "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" - - python tests/test.py --with-coverage -b "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388" -a "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" - - coverage combine && coverage report --include=shadowsocks/* + - ./.jenkins.sh diff --git a/tests/test_daemon.sh b/tests/test_daemon.sh index b05208d..fdfe517 100755 --- a/tests/test_daemon.sh +++ b/tests/test_daemon.sh @@ -1,6 +1,6 @@ #!/bin/bash -function test { +function run_test { expected=$1 shift echo "running test: $command $@" @@ -20,23 +20,23 @@ do command="coverage run -p -a shadowsocks/$module.py" -test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log -test 0 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log -test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log -test 0 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log -test 1 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log -test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +run_test 1 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log -test 0 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log -test 0 -c tests/aes.json -d restart --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log -test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d restart --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log -test 0 -c tests/aes.json -d restart --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log -test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d restart --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log -test 1 -c tests/aes.json -d start --pid-file /tmp/not_exist/shadowsocks.pid --log-file /tmp/shadowsocks.log -test 1 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/not_exist/shadowsocks.log +run_test 1 -c tests/aes.json -d start --pid-file /tmp/not_exist/shadowsocks.pid --log-file /tmp/shadowsocks.log +run_test 1 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/not_exist/shadowsocks.log done From c7b5a5a0110e8090a226e8af329c57bfb8af4cc2 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 22 Dec 2014 16:45:46 +0800 Subject: [PATCH 315/344] fix ci --- .jenkins.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.jenkins.sh b/.jenkins.sh index 1206b00..256fe38 100755 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -7,7 +7,7 @@ function run_test { echo "running test: $command $@" printf '\e[0m' - $command $@ + $command "$@" status=$? if [ $status -ne 0 ]; then printf '\e[0;31m' @@ -46,7 +46,6 @@ run_test python tests/test.py --with-coverage -s tests/ipv6.json -c tests/ipv6-c run_test python tests/test.py --with-coverage -b "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388" -a "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" run_test python tests/test.py --with-coverage -b "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388" -a "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" coverage combine && coverage report --include=shadowsocks/* -coverage combine && coverage report --include=shadowsocks/* rm -rf htmlcov coverage html --include=shadowsocks/* From 536b7d1ee637f9fbc78eb923656838cb3219673e Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 22 Dec 2014 17:09:37 +0800 Subject: [PATCH 316/344] use SIGINT instead in tests Conflicts: tests/test.py --- shadowsocks/local.py | 5 +++++ shadowsocks/server.py | 5 +++++ tests/test.py | 4 +++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 6a97f07..994b6d8 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -68,6 +68,11 @@ def main(): tcp_server.close(next_tick=True) udp_server.close(next_tick=True) signal.signal(getattr(signal, 'SIGQUIT', signal.SIGTERM), handler) + + def int_handler(signum, _): + sys.exit(1) + signal.signal(signal.SIGINT, int_handler) + loop.run() except (KeyboardInterrupt, IOError, OSError) as e: logging.error(e) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index e7acc5e..c5a00ca 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -77,6 +77,11 @@ def main(): tcp_servers + udp_servers)) signal.signal(getattr(signal, 'SIGQUIT', signal.SIGTERM), child_handler) + + def int_handler(signum, _): + sys.exit(1) + signal.signal(signal.SIGINT, int_handler) + try: loop = eventloop.EventLoop() dns_resolver.add_to_loop(loop) diff --git a/tests/test.py b/tests/test.py index 5314d6e..721d484 100755 --- a/tests/test.py +++ b/tests/test.py @@ -138,7 +138,9 @@ try: finally: for p in [p1, p2]: try: - os.kill(p.pid, signal.SIGQUIT) + print('kill', file=sys.stderr) + os.kill(p.pid, signal.SIGINT) + print('waitpid', file=sys.stderr) os.waitpid(p.pid, 0) except OSError: pass From be1d1d50323f0f67dc58e33dc76e24b22907e1a1 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 22 Dec 2014 17:29:58 +0800 Subject: [PATCH 317/344] add SIGINT in workers --- shadowsocks/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shadowsocks/server.py b/shadowsocks/server.py index c5a00ca..8eed4ad 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -118,6 +118,7 @@ def main(): sys.exit() signal.signal(signal.SIGTERM, handler) signal.signal(signal.SIGQUIT, handler) + signal.signal(signal.SIGINT, handler) # master for a_tcp_server in tcp_servers: From 072afd68f2a8a98fe6d6706dafb7ff1f7093a602 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 22 Dec 2014 17:39:52 +0800 Subject: [PATCH 318/344] use local tmp dir --- .jenkins.sh | 2 ++ tests/test_daemon.sh | 28 +++++++++++++++------------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.jenkins.sh b/.jenkins.sh index 256fe38..a67f606 100755 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -25,6 +25,7 @@ function run_test { } coverage erase +mkdir tmp run_test pep8 . run_test pyflakes . run_test coverage run tests/nose_plugin.py -v @@ -47,6 +48,7 @@ run_test python tests/test.py --with-coverage -b "-m rc4-md5 -k testrc4 -s 127.0 run_test python tests/test.py --with-coverage -b "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388" -a "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" coverage combine && coverage report --include=shadowsocks/* rm -rf htmlcov +rm -rf tmp coverage html --include=shadowsocks/* exit $result diff --git a/tests/test_daemon.sh b/tests/test_daemon.sh index fdfe517..02c6cf0 100755 --- a/tests/test_daemon.sh +++ b/tests/test_daemon.sh @@ -20,23 +20,25 @@ do command="coverage run -p -a shadowsocks/$module.py" -run_test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +mkdir -p tmp -run_test 0 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log -run_test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d stop --pid-file tmp/shadowsocks.pid --log-file tmp/shadowsocks.log -run_test 0 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log -run_test 1 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log -run_test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d start --pid-file tmp/shadowsocks.pid --log-file tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d stop --pid-file tmp/shadowsocks.pid --log-file tmp/shadowsocks.log -run_test 0 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log -run_test 0 -c tests/aes.json -d restart --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log -run_test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d start --pid-file tmp/shadowsocks.pid --log-file tmp/shadowsocks.log +run_test 1 -c tests/aes.json -d start --pid-file tmp/shadowsocks.pid --log-file tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d stop --pid-file tmp/shadowsocks.pid --log-file tmp/shadowsocks.log -run_test 0 -c tests/aes.json -d restart --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log -run_test 0 -c tests/aes.json -d stop --pid-file /tmp/shadowsocks.pid --log-file /tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d start --pid-file tmp/shadowsocks.pid --log-file tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d restart --pid-file tmp/shadowsocks.pid --log-file tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d stop --pid-file tmp/shadowsocks.pid --log-file tmp/shadowsocks.log -run_test 1 -c tests/aes.json -d start --pid-file /tmp/not_exist/shadowsocks.pid --log-file /tmp/shadowsocks.log -run_test 1 -c tests/aes.json -d start --pid-file /tmp/shadowsocks.pid --log-file /tmp/not_exist/shadowsocks.log +run_test 0 -c tests/aes.json -d restart --pid-file tmp/shadowsocks.pid --log-file tmp/shadowsocks.log +run_test 0 -c tests/aes.json -d stop --pid-file tmp/shadowsocks.pid --log-file tmp/shadowsocks.log + +run_test 1 -c tests/aes.json -d start --pid-file tmp/not_exist/shadowsocks.pid --log-file tmp/shadowsocks.log +run_test 1 -c tests/aes.json -d start --pid-file tmp/shadowsocks.pid --log-file tmp/not_exist/shadowsocks.log done From 5ea8403e56043ea0ad50efbd8c255d32119a1b59 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Mon, 22 Dec 2014 17:58:12 +0800 Subject: [PATCH 319/344] fix daemon and add fastopen tests --- .jenkins.sh | 7 +++++++ shadowsocks/daemon.py | 13 +++++++------ tests/test.py | 2 -- tests/test_daemon.sh | 1 - 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.jenkins.sh b/.jenkins.sh index a67f606..a8bdf15 100755 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -46,6 +46,13 @@ run_test python tests/test.py --with-coverage -c tests/workers.json run_test python tests/test.py --with-coverage -s tests/ipv6.json -c tests/ipv6-client-side.json run_test python tests/test.py --with-coverage -b "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388" -a "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" run_test python tests/test.py --with-coverage -b "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388" -a "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" + +if [ -f /proc/sys/net/ipv4/tcp_fastopen ] ; then + if [ 3 -eq `cat /proc/sys/net/ipv4/tcp_fastopen` ] ; then + run_test python tests/test.py --with-coverage -c tests/fastopen.json + fi +fi + coverage combine && coverage report --include=shadowsocks/* rm -rf htmlcov rm -rf tmp diff --git a/shadowsocks/daemon.py b/shadowsocks/daemon.py index ec6676c..d206ccf 100644 --- a/shadowsocks/daemon.py +++ b/shadowsocks/daemon.py @@ -100,19 +100,21 @@ def freopen(f, mode, stream): def daemon_start(pid_file, log_file): - # fork only once because we are sure parent will exit - pid = os.fork() - assert pid != -1 def handle_exit(signum, _): if signum == signal.SIGTERM: sys.exit(0) sys.exit(1) + signal.signal(signal.SIGINT, handle_exit) + signal.signal(signal.SIGTERM, handle_exit) + + # fork only once because we are sure parent will exit + pid = os.fork() + assert pid != -1 + if pid > 0: # parent waits for its child - signal.signal(signal.SIGINT, handle_exit) - signal.signal(signal.SIGTERM, handle_exit) time.sleep(5) sys.exit(0) @@ -135,7 +137,6 @@ def daemon_start(pid_file, log_file): freopen(log_file, 'a', sys.stderr) except IOError as e: logging.error(e) - os.kill(ppid, signal.SIGINT) sys.exit(1) diff --git a/tests/test.py b/tests/test.py index 721d484..0b63a18 100755 --- a/tests/test.py +++ b/tests/test.py @@ -138,9 +138,7 @@ try: finally: for p in [p1, p2]: try: - print('kill', file=sys.stderr) os.kill(p.pid, signal.SIGINT) - print('waitpid', file=sys.stderr) os.waitpid(p.pid, 0) except OSError: pass diff --git a/tests/test_daemon.sh b/tests/test_daemon.sh index 02c6cf0..40f35ef 100755 --- a/tests/test_daemon.sh +++ b/tests/test_daemon.sh @@ -39,6 +39,5 @@ run_test 0 -c tests/aes.json -d restart --pid-file tmp/shadowsocks.pid --log-fil run_test 0 -c tests/aes.json -d stop --pid-file tmp/shadowsocks.pid --log-file tmp/shadowsocks.log run_test 1 -c tests/aes.json -d start --pid-file tmp/not_exist/shadowsocks.pid --log-file tmp/shadowsocks.log -run_test 1 -c tests/aes.json -d start --pid-file tmp/shadowsocks.pid --log-file tmp/not_exist/shadowsocks.log done From f1b084be060f5a3129e7fff4fab2c6aee7696ebf Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 23 Dec 2014 13:09:51 +0800 Subject: [PATCH 320/344] add large file test --- .jenkins.sh | 2 ++ .travis.yml | 4 +++- tests/test_large_file.sh | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100755 tests/test_large_file.sh diff --git a/.jenkins.sh b/.jenkins.sh index a8bdf15..bae7f79 100755 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -53,6 +53,8 @@ if [ -f /proc/sys/net/ipv4/tcp_fastopen ] ; then fi fi +run_test tests/test_large_file.sh + coverage combine && coverage report --include=shadowsocks/* rm -rf htmlcov rm -rf tmp diff --git a/.travis.yml b/.travis.yml index 3b094f2..2f9bd13 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,9 @@ cache: - dante-1.4.0 before_install: - sudo apt-get update -qq - - sudo apt-get install -qq build-essential libssl-dev swig python-m2crypto python-numpy dnsutils + - sudo apt-get install -qq build-essential libssl-dev swig python-m2crypto python-numpy dnsutils iproute nginx + - sudo dd if=/dev/urandom of=/usr/share/nginx/www/file bs=1M count=10 + - sudo service nginx restart - pip install m2crypto salsa20 pep8 pyflakes nose coverage - sudo tests/socksify/install.sh - sudo tests/libsodium/install.sh diff --git a/tests/test_large_file.sh b/tests/test_large_file.sh new file mode 100755 index 0000000..3124ac9 --- /dev/null +++ b/tests/test_large_file.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +DEV=lo +PORT=8388 +DELAY=100ms + +PYTHON="coverage run -p -a" +URL=http://127.0.0.1/file + +mkdir -p tmp + +type tc > /dev/null && ( + tc qdisc add dev $DEV root handle 1: prio + tc qdisc add dev $DEV parent 1:3 handle 30: netem delay $DELAY + tc filter add dev $DEV parent 1:0 protocol ip u32 match ip dport $PORT 0xffff flowid 1:3 + tc filter add dev $DEV parent 1:0 protocol ip u32 match ip sport $PORT 0xffff flowid 1:3 + tc qdisc show dev lo +) + +$PYTHON shadowsocks/local.py -c tests/aes.json & +LOCAL=$! + +$PYTHON shadowsocks/server.py -c tests/aes.json & +SERVER=$! + +sleep 3 + +curl -o tmp/expected $URL +curl -o tmp/result --socks5-hostname 127.0.0.1:1081 $URL + +kill $LOCAL +kill $SERVER + +type tc > /dev/null && tc qdisc del dev lo root + +sleep 2 + +diff tmp/expected tmp/result || exit 1 From 9b3944c954326fd40ef4f641ea92137ae9d1feef Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 23 Dec 2014 13:18:30 +0800 Subject: [PATCH 321/344] fix large file test --- tests/test_large_file.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_large_file.sh b/tests/test_large_file.sh index 3124ac9..66cc13d 100755 --- a/tests/test_large_file.sh +++ b/tests/test_large_file.sh @@ -9,7 +9,7 @@ URL=http://127.0.0.1/file mkdir -p tmp -type tc > /dev/null && ( +type tc 2> /dev/null && ( tc qdisc add dev $DEV root handle 1: prio tc qdisc add dev $DEV parent 1:3 handle 30: netem delay $DELAY tc filter add dev $DEV parent 1:0 protocol ip u32 match ip dport $PORT 0xffff flowid 1:3 @@ -25,13 +25,13 @@ SERVER=$! sleep 3 -curl -o tmp/expected $URL -curl -o tmp/result --socks5-hostname 127.0.0.1:1081 $URL +time curl -o tmp/expected $URL +time curl -o tmp/result --socks5-hostname 127.0.0.1:1081 $URL kill $LOCAL kill $SERVER -type tc > /dev/null && tc qdisc del dev lo root +type tc 2> /dev/null && tc qdisc del dev lo root sleep 2 From 2cc7ee5053bc89f18377c754f348c5cf735d1a6f Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 23 Dec 2014 13:57:31 +0800 Subject: [PATCH 322/344] alter tc rule --- tests/test_large_file.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_large_file.sh b/tests/test_large_file.sh index 66cc13d..66e4f94 100755 --- a/tests/test_large_file.sh +++ b/tests/test_large_file.sh @@ -10,10 +10,17 @@ URL=http://127.0.0.1/file mkdir -p tmp type tc 2> /dev/null && ( - tc qdisc add dev $DEV root handle 1: prio - tc qdisc add dev $DEV parent 1:3 handle 30: netem delay $DELAY - tc filter add dev $DEV parent 1:0 protocol ip u32 match ip dport $PORT 0xffff flowid 1:3 - tc filter add dev $DEV parent 1:0 protocol ip u32 match ip sport $PORT 0xffff flowid 1:3 + tc qdisc add dev $DEV root handle 1: htb + tc class add dev $DEV parent 1: classid 1:1 htb rate 2mbps + tc class add dev $DEV parent 1:1 classid 1:6 htb rate 2mbps ceil 1mbps prio 0 + tc filter add dev $DEV parent 1:0 prio 0 protocol ip handle 6 fw flowid 1:6 + + tc filter add dev $DEV parent 1:0 protocol ip u32 match ip dport $PORT 0xffff flowid 1:6 + tc filter add dev $DEV parent 1:0 protocol ip u32 match ip sport $PORT 0xffff flowid 1:6 + +# iptables -D OUTPUT -t mangle -p tcp --sport 8388 -j MARK --set-mark 6 +# iptables -A OUTPUT -t mangle -p tcp --sport 8388 -j MARK --set-mark 6 + tc qdisc show dev lo ) From cd07001471fede6f5e3fab2fdfbc8b2d0843e51d Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 23 Dec 2014 14:05:20 +0800 Subject: [PATCH 323/344] use SIGINT instead in large file test --- tests/test_large_file.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_large_file.sh b/tests/test_large_file.sh index 66e4f94..14d3002 100755 --- a/tests/test_large_file.sh +++ b/tests/test_large_file.sh @@ -35,8 +35,8 @@ sleep 3 time curl -o tmp/expected $URL time curl -o tmp/result --socks5-hostname 127.0.0.1:1081 $URL -kill $LOCAL -kill $SERVER +kill -s SIGINT $LOCAL +kill -s SIGINT $SERVER type tc 2> /dev/null && tc qdisc del dev lo root From 9cfffa360e2194f7cc6ae1b88c79057acff1b54e Mon Sep 17 00:00:00 2001 From: clowwindy Date: Tue, 23 Dec 2014 14:20:46 +0800 Subject: [PATCH 324/344] sudo tc setup --- .travis.yml | 1 + tests/setup_tc.sh | 18 ++++++++++++++++++ tests/test_large_file.sh | 21 --------------------- 3 files changed, 19 insertions(+), 21 deletions(-) create mode 100755 tests/setup_tc.sh diff --git a/.travis.yml b/.travis.yml index 2f9bd13..4a88b77 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,5 +15,6 @@ before_install: - pip install m2crypto salsa20 pep8 pyflakes nose coverage - sudo tests/socksify/install.sh - sudo tests/libsodium/install.sh + - sudo tests/setup_tc.sh script: - ./.jenkins.sh diff --git a/tests/setup_tc.sh b/tests/setup_tc.sh new file mode 100755 index 0000000..1a5fa20 --- /dev/null +++ b/tests/setup_tc.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +DEV=lo +PORT=8388 +DELAY=100ms + +type tc 2> /dev/null && ( + tc qdisc add dev $DEV root handle 1: htb + tc class add dev $DEV parent 1: classid 1:1 htb rate 2mbps + tc class add dev $DEV parent 1:1 classid 1:6 htb rate 2mbps ceil 1mbps prio 0 + tc filter add dev $DEV parent 1:0 prio 0 protocol ip handle 6 fw flowid 1:6 + + tc filter add dev $DEV parent 1:0 protocol ip u32 match ip dport $PORT 0xffff flowid 1:6 + tc filter add dev $DEV parent 1:0 protocol ip u32 match ip sport $PORT 0xffff flowid 1:6 + + tc qdisc show dev lo +) + diff --git a/tests/test_large_file.sh b/tests/test_large_file.sh index 14d3002..e8acd79 100755 --- a/tests/test_large_file.sh +++ b/tests/test_large_file.sh @@ -1,29 +1,10 @@ #!/bin/bash -DEV=lo -PORT=8388 -DELAY=100ms - PYTHON="coverage run -p -a" URL=http://127.0.0.1/file mkdir -p tmp -type tc 2> /dev/null && ( - tc qdisc add dev $DEV root handle 1: htb - tc class add dev $DEV parent 1: classid 1:1 htb rate 2mbps - tc class add dev $DEV parent 1:1 classid 1:6 htb rate 2mbps ceil 1mbps prio 0 - tc filter add dev $DEV parent 1:0 prio 0 protocol ip handle 6 fw flowid 1:6 - - tc filter add dev $DEV parent 1:0 protocol ip u32 match ip dport $PORT 0xffff flowid 1:6 - tc filter add dev $DEV parent 1:0 protocol ip u32 match ip sport $PORT 0xffff flowid 1:6 - -# iptables -D OUTPUT -t mangle -p tcp --sport 8388 -j MARK --set-mark 6 -# iptables -A OUTPUT -t mangle -p tcp --sport 8388 -j MARK --set-mark 6 - - tc qdisc show dev lo -) - $PYTHON shadowsocks/local.py -c tests/aes.json & LOCAL=$! @@ -38,8 +19,6 @@ time curl -o tmp/result --socks5-hostname 127.0.0.1:1081 $URL kill -s SIGINT $LOCAL kill -s SIGINT $SERVER -type tc 2> /dev/null && tc qdisc del dev lo root - sleep 2 diff tmp/expected tmp/result || exit 1 From 1d0c8b1800c6775b4bad02bcbb4529848d334d1a Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 24 Dec 2014 16:47:14 +0800 Subject: [PATCH 325/344] add coverage --- .jenkins.sh | 2 ++ README.md | 6 +++++- tests/coverage_server.py | 23 +++++++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 tests/coverage_server.py diff --git a/.jenkins.sh b/.jenkins.sh index bae7f79..5630511 100755 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -60,4 +60,6 @@ rm -rf htmlcov rm -rf tmp coverage html --include=shadowsocks/* +coverage report --include=shadowsocks/* | tail -n1 | rev | cut -d' ' -f 1 | rev > /tmp/shadowsocks-coverage + exit $result diff --git a/README.md b/README.md index 7a37b26..621d2d9 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ shadowsocks =========== -[![PyPI version]][PyPI] [![Build Status]][Travis CI] +[![PyPI version]][PyPI] +[![Build Status]][Travis CI] +[![Coverage Status]][Coverage] A fast tunnel proxy that helps you bypass firewalls. @@ -119,6 +121,8 @@ Bugs and Issues [Android]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#android [Build Status]: https://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat [Chinese Readme]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E +[Coverage Status]: http://192.81.132.184/result/shadowsocks +[Coverage]: http://192.81.132.184/job/Shadowsocks/ws/htmlcov/index.html [Debian sid]: https://packages.debian.org/unstable/python/shadowsocks [the package]: https://pypi.python.org/pypi/shadowsocks [Encryption]: https://github.com/clowwindy/shadowsocks/wiki/Encryption diff --git a/tests/coverage_server.py b/tests/coverage_server.py new file mode 100644 index 0000000..4bb53df --- /dev/null +++ b/tests/coverage_server.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +import tornado.ioloop +import tornado.web +import urllib + + +class MainHandler(tornado.web.RequestHandler): + def get(self): + with open('/tmp/shadowsocks-coverage', 'rb') as f: + coverage = f.read().strip() + self.redirect(('https://img.shields.io/badge/' + 'coverage-%s-brightgreen.svg' + '?style=flat') % + urllib.quote(coverage)) + +application = tornado.web.Application([ + (r"/shadowsocks", MainHandler), +]) + +if __name__ == "__main__": + application.listen(8888, address='127.0.0.1') + tornado.ioloop.IOLoop.instance().start() From b785d95f66b08ba6308567ac5143ffdcc62b9947 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 24 Dec 2014 17:06:15 +0800 Subject: [PATCH 326/344] fix tests --- tests/coverage_server.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/coverage_server.py b/tests/coverage_server.py index 4bb53df..7a135c5 100644 --- a/tests/coverage_server.py +++ b/tests/coverage_server.py @@ -1,23 +1,23 @@ #!/usr/bin/env python -import tornado.ioloop -import tornado.web -import urllib +if __name__ == '__main__': + import tornado.ioloop + import tornado.web + import urllib + class MainHandler(tornado.web.RequestHandler): + def get(self): + with open('/tmp/shadowsocks-coverage', 'rb') as f: + coverage = f.read().strip() + self.redirect(('https://img.shields.io/badge/' + 'coverage-%s-brightgreen.svg' + '?style=flat') % + urllib.quote(coverage)) -class MainHandler(tornado.web.RequestHandler): - def get(self): - with open('/tmp/shadowsocks-coverage', 'rb') as f: - coverage = f.read().strip() - self.redirect(('https://img.shields.io/badge/' - 'coverage-%s-brightgreen.svg' - '?style=flat') % - urllib.quote(coverage)) + application = tornado.web.Application([ + (r"/shadowsocks", MainHandler), + ]) -application = tornado.web.Application([ - (r"/shadowsocks", MainHandler), -]) - -if __name__ == "__main__": - application.listen(8888, address='127.0.0.1') - tornado.ioloop.IOLoop.instance().start() + if __name__ == "__main__": + application.listen(8888, address='127.0.0.1') + tornado.ioloop.IOLoop.instance().start() From c6bc912c11df07db02e8492e10d92e1e680ac177 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 24 Dec 2014 18:02:14 +0800 Subject: [PATCH 327/344] add command line tests --- .jenkins.sh | 4 +- tests/assert.sh | 148 ++++++++++++++++++++++++++++++++++++++++++ tests/test_command.sh | 40 ++++++++++++ 3 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 tests/assert.sh create mode 100755 tests/test_command.sh diff --git a/.jenkins.sh b/.jenkins.sh index 5630511..56e2acc 100755 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -44,8 +44,8 @@ run_test python tests/test.py --with-coverage -c tests/server-multi-ports.json run_test python tests/test.py --with-coverage -s tests/server-multi-passwd.json -c tests/server-multi-passwd-client-side.json run_test python tests/test.py --with-coverage -c tests/workers.json run_test python tests/test.py --with-coverage -s tests/ipv6.json -c tests/ipv6-client-side.json -run_test python tests/test.py --with-coverage -b "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388" -a "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" -run_test python tests/test.py --with-coverage -b "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388" -a "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 -l 1081" +run_test python tests/test.py --with-coverage -b "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -q" -a "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -l 1081 -vv" +run_test python tests/test.py --with-coverage -b "-b 127.0.0.1 -m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 --workers 1" -a "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 -l 1081 -t 30 -qq" if [ -f /proc/sys/net/ipv4/tcp_fastopen ] ; then if [ 3 -eq `cat /proc/sys/net/ipv4/tcp_fastopen` ] ; then diff --git a/tests/assert.sh b/tests/assert.sh new file mode 100644 index 0000000..b0c679c --- /dev/null +++ b/tests/assert.sh @@ -0,0 +1,148 @@ +#!/bin/bash +# assert.sh 1.0 - bash unit testing framework +# Copyright (C) 2009, 2010, 2011, 2012 Robert Lehmann +# +# http://github.com/lehmannro/assert.sh +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +export DISCOVERONLY=${DISCOVERONLY:-} +export DEBUG=${DEBUG:-} +export STOP=${STOP:-} +export INVARIANT=${INVARIANT:-} +export CONTINUE=${CONTINUE:-} + +args="$(getopt -n "$0" -l \ + verbose,help,stop,discover,invariant,continue vhxdic $*)" \ +|| exit -1 +for arg in $args; do + case "$arg" in + -h) + echo "$0 [-vxidc]" \ + "[--verbose] [--stop] [--invariant] [--discover] [--continue]" + echo "`sed 's/./ /g' <<< "$0"` [-h] [--help]" + exit 0;; + --help) + cat < [stdin] + (( tests_ran++ )) || : + [[ -n "$DISCOVERONLY" ]] && return || true + # printf required for formatting + printf -v expected "x${2:-}" # x required to overwrite older results + result="$(eval 2>/dev/null $1 <<< ${3:-})" || true + # Note: $expected is already decorated + if [[ "x$result" == "$expected" ]]; then + [[ -n "$DEBUG" ]] && echo -n . || true + return + fi + result="$(sed -e :a -e '$!N;s/\n/\\n/;ta' <<< "$result")" + [[ -z "$result" ]] && result="nothing" || result="\"$result\"" + [[ -z "$2" ]] && expected="nothing" || expected="\"$2\"" + _assert_fail "expected $expected${_indent}got $result" "$1" "$3" +} + +assert_raises() { + # assert_raises [stdin] + (( tests_ran++ )) || : + [[ -n "$DISCOVERONLY" ]] && return || true + status=0 + (eval $1 <<< ${3:-}) > /dev/null 2>&1 || status=$? + expected=${2:-0} + if [[ "$status" -eq "$expected" ]]; then + [[ -n "$DEBUG" ]] && echo -n . || true + return + fi + _assert_fail "program terminated with code $status instead of $expected" "$1" "$3" +} + +_assert_fail() { + # _assert_fail + [[ -n "$DEBUG" ]] && echo -n X + report="test #$tests_ran \"$2${3:+ <<< $3}\" failed:${_indent}$1" + if [[ -n "$STOP" ]]; then + [[ -n "$DEBUG" ]] && echo + echo "$report" + exit 1 + fi + tests_errors[$tests_failed]="$report" + (( tests_failed++ )) || : +} + +_assert_reset +: ${tests_suite_status:=0} # remember if any of the tests failed so far +_assert_cleanup() { + local status=$? + # modify exit code if it's not already non-zero + [[ $status -eq 0 && -z $CONTINUE ]] && exit $tests_suite_status +} +trap _assert_cleanup EXIT diff --git a/tests/test_command.sh b/tests/test_command.sh new file mode 100755 index 0000000..fc23665 --- /dev/null +++ b/tests/test_command.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +. tests/assert.sh + +PYTHON="coverage run -a -p" +LOCAL="$PYTHON shadowsocks/local.py" +SERVER="$PYTHON shadowsocks/server.py" + +assert "$LOCAL 2>&1 | grep ERROR" "ERROR: config not specified" +assert "$LOCAL 2>&1 | grep usage | cut -d: -f1" "usage" + +assert "$SERVER 2>&1 | grep ERROR" "ERROR: config not specified" +assert "$SERVER 2>&1 | grep usage | cut -d: -f1" "usage" + +assert "$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d start | grep WARNING | awk -F\"WARNING\" '{print \$2}'" " warning: server set to listen on 127.0.0.1:8388, are you sure?" +$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop + +assert "$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 0.0.0.0 -p 8388 -t10 -d start | grep WARNING | awk -F\"WARNING\" '{print \$2}'" " warning: your timeout 10 seems too short" +$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop + +assert "$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 0.0.0.0 -p 8388 -t1000 -d start | grep WARNING | awk -F\"WARNING\" '{print \$2}'" " warning: your timeout 1000 seems too long" +$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop + +assert "$LOCAL 2>&1 -m rc4 -k testrc4 -s 0.0.0.0 -p 8388 -d start | grep WARNING | awk -F\"WARNING\" '{print \$2}'" " warning: RC4 is not safe; please use a safer cipher, like AES-256-CFB" +$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop + +assert "$LOCAL 2>&1 -m rc4-md5 -k mypassword -s 0.0.0.0 -p 8388 -d start | grep ERROR | awk -F\"ERROR\" '{print \$2}'" " DON'T USE DEFAULT PASSWORD! Please change it in your config.json!" +$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop + +assert "$LOCAL 2>&1 -m rc4-md5 -p 8388 -k testrc4 -d start | grep ERROR | awk -F\"ERROR\" '{print \$2}'" ": server addr not specified" +$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop + +assert "$LOCAL 2>&1 -m rc4-md5 -p 8388 -s 0.0.0.0 -d start | grep ERROR | awk -F\"ERROR\" '{print \$2}'" ": password not specified" +$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop + +assert "$SERVER 2>&1 -m rc4-md5 -p 8388 -s 0.0.0.0 -d start | grep ERROR | awk -F\"ERROR\" '{print \$2}'" ": password or port_password not specified" +$SERVER 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop + + +assert_end command From ab975d1753bcdc41db380b4a4399c634c0632f32 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 24 Dec 2014 18:08:16 +0800 Subject: [PATCH 328/344] fix tests --- .jenkins.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins.sh b/.jenkins.sh index 56e2acc..23ed2c7 100755 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -45,7 +45,7 @@ run_test python tests/test.py --with-coverage -s tests/server-multi-passwd.json run_test python tests/test.py --with-coverage -c tests/workers.json run_test python tests/test.py --with-coverage -s tests/ipv6.json -c tests/ipv6-client-side.json run_test python tests/test.py --with-coverage -b "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -q" -a "-m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -l 1081 -vv" -run_test python tests/test.py --with-coverage -b "-b 127.0.0.1 -m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 --workers 1" -a "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 -l 1081 -t 30 -qq" +run_test python tests/test.py --with-coverage -b "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 --workers 1" -a "-m aes-256-cfb -k testrc4 -s 127.0.0.1 -p 8388 -l 1081 -t 30 -qq -b 127.0.0.1" if [ -f /proc/sys/net/ipv4/tcp_fastopen ] ; then if [ 3 -eq `cat /proc/sys/net/ipv4/tcp_fastopen` ] ; then From d228ffbe84de78ce7492d493900ad2e6a285cc01 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 24 Dec 2014 18:10:12 +0800 Subject: [PATCH 329/344] add command line test --- .jenkins.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.jenkins.sh b/.jenkins.sh index 23ed2c7..7b381a8 100755 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -55,6 +55,8 @@ fi run_test tests/test_large_file.sh +run_test tests/test_command.sh + coverage combine && coverage report --include=shadowsocks/* rm -rf htmlcov rm -rf tmp From dd140814a8d10ccceb249c00668bd85ca6252b91 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 24 Dec 2014 18:15:52 +0800 Subject: [PATCH 330/344] suppress some warnings --- tests/test_command.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_command.sh b/tests/test_command.sh index fc23665..eba4c8c 100755 --- a/tests/test_command.sh +++ b/tests/test_command.sh @@ -13,28 +13,28 @@ assert "$SERVER 2>&1 | grep ERROR" "ERROR: config not specified" assert "$SERVER 2>&1 | grep usage | cut -d: -f1" "usage" assert "$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d start | grep WARNING | awk -F\"WARNING\" '{print \$2}'" " warning: server set to listen on 127.0.0.1:8388, are you sure?" -$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop +$LOCAL 2>/dev/null 1>/dev/null -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop assert "$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 0.0.0.0 -p 8388 -t10 -d start | grep WARNING | awk -F\"WARNING\" '{print \$2}'" " warning: your timeout 10 seems too short" -$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop +$LOCAL 2>/dev/null 1>/dev/null -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop assert "$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 0.0.0.0 -p 8388 -t1000 -d start | grep WARNING | awk -F\"WARNING\" '{print \$2}'" " warning: your timeout 1000 seems too long" -$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop +$LOCAL 2>/dev/null 1>/dev/null -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop assert "$LOCAL 2>&1 -m rc4 -k testrc4 -s 0.0.0.0 -p 8388 -d start | grep WARNING | awk -F\"WARNING\" '{print \$2}'" " warning: RC4 is not safe; please use a safer cipher, like AES-256-CFB" -$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop +$LOCAL 2>/dev/null 1>/dev/null -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop assert "$LOCAL 2>&1 -m rc4-md5 -k mypassword -s 0.0.0.0 -p 8388 -d start | grep ERROR | awk -F\"ERROR\" '{print \$2}'" " DON'T USE DEFAULT PASSWORD! Please change it in your config.json!" -$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop +$LOCAL 2>/dev/null 1>/dev/null -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop assert "$LOCAL 2>&1 -m rc4-md5 -p 8388 -k testrc4 -d start | grep ERROR | awk -F\"ERROR\" '{print \$2}'" ": server addr not specified" -$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop +$LOCAL 2>/dev/null 1>/dev/null -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop assert "$LOCAL 2>&1 -m rc4-md5 -p 8388 -s 0.0.0.0 -d start | grep ERROR | awk -F\"ERROR\" '{print \$2}'" ": password not specified" -$LOCAL 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop +$LOCAL 2>/dev/null 1>/dev/null -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop assert "$SERVER 2>&1 -m rc4-md5 -p 8388 -s 0.0.0.0 -d start | grep ERROR | awk -F\"ERROR\" '{print \$2}'" ": password or port_password not specified" -$SERVER 2>&1 -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop +$LOCAL 2>/dev/null 1>/dev/null -m rc4-md5 -k testrc4 -s 127.0.0.1 -p 8388 -d stop assert_end command From 30f4f78557e55197b8cba9d761f3cd388fc7def2 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 24 Dec 2014 18:23:35 +0800 Subject: [PATCH 331/344] fix travis --- .travis.yml | 2 +- shadowsocks/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4a88b77..4fbe78c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ cache: - dante-1.4.0 before_install: - sudo apt-get update -qq - - sudo apt-get install -qq build-essential libssl-dev swig python-m2crypto python-numpy dnsutils iproute nginx + - sudo apt-get install -qq build-essential libssl-dev swig python-m2crypto python-numpy dnsutils iproute nginx bc - sudo dd if=/dev/urandom of=/usr/share/nginx/www/file bs=1M count=10 - sudo service nginx restart - pip install m2crypto salsa20 pep8 pyflakes nose coverage diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index 7caa243..ff6801c 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -29,7 +29,7 @@ import json import sys import getopt import logging -from shadowsocks.common import to_bytes +from shadowsocks.common import to_bytes, to_str VERBOSE_LEVEL = 5 @@ -73,7 +73,7 @@ def check_config(config): logging.warn('warning: local set to listen on 0.0.0.0, it\'s not safe') if config.get('server', '') in [b'127.0.0.1', b'localhost']: logging.warn('warning: server set to listen on %s:%s, are you sure?' % - (config['server'], config['server_port'])) + (to_str(config['server']), config['server_port'])) if (config.get('method', '') or '').lower() == b'table': logging.warn('warning: table is not safe; please use a safer cipher, ' 'like AES-256-CFB') From 99c8bbafe72cdd2e7cdcf078911309be5bec9a3e Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 25 Dec 2014 16:20:15 +0800 Subject: [PATCH 332/344] fix fastopen and add a new test --- .jenkins.sh | 4 ++++ shadowsocks/tcprelay.py | 11 +++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.jenkins.sh b/.jenkins.sh index 7b381a8..4c85f1c 100755 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -49,6 +49,10 @@ run_test python tests/test.py --with-coverage -b "-m aes-256-cfb -k testrc4 -s 1 if [ -f /proc/sys/net/ipv4/tcp_fastopen ] ; then if [ 3 -eq `cat /proc/sys/net/ipv4/tcp_fastopen` ] ; then + # we have to run it twice: + # the first time there's no syn cookie + # the second time there is syn cookie + run_test python tests/test.py --with-coverage -c tests/fastopen.json run_test python tests/test.py --with-coverage -c tests/fastopen.json fi fi diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 146714a..95cbef5 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -241,19 +241,18 @@ class TCPRelayHandler(object): self._create_remote_socket(self._chosen_server[0], self._chosen_server[1]) self._loop.add(remote_sock, eventloop.POLL_ERR) - data = b''.join(self._data_to_write_to_local) + data = b''.join(self._data_to_write_to_remote) l = len(data) s = remote_sock.sendto(data, MSG_FASTOPEN, self._chosen_server) if s < l: data = data[s:] - self._data_to_write_to_local = [data] - self._update_stream(STREAM_UP, WAIT_STATUS_READWRITING) + self._data_to_write_to_remote = [data] else: - self._data_to_write_to_local = [] - self._update_stream(STREAM_UP, WAIT_STATUS_READING) - self._stage = STAGE_STREAM + self._data_to_write_to_remote = [] + self._update_stream(STREAM_UP, WAIT_STATUS_READWRITING) except (OSError, IOError) as e: if eventloop.errno_from_exception(e) == errno.EINPROGRESS: + # in this case data is not sent at all self._update_stream(STREAM_UP, WAIT_STATUS_READWRITING) elif eventloop.errno_from_exception(e) == errno.ENOTCONN: logging.error('fast open not supported on this OS') From e8501da271356af4c7fb1aefddcb8385f7b1dd77 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Thu, 25 Dec 2014 16:23:43 +0800 Subject: [PATCH 333/344] fix vim --- shadowsocks/tcprelay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/tcprelay.py b/shadowsocks/tcprelay.py index 95cbef5..567f515 100644 --- a/shadowsocks/tcprelay.py +++ b/shadowsocks/tcprelay.py @@ -249,7 +249,7 @@ class TCPRelayHandler(object): self._data_to_write_to_remote = [data] else: self._data_to_write_to_remote = [] - self._update_stream(STREAM_UP, WAIT_STATUS_READWRITING) + self._update_stream(STREAM_UP, WAIT_STATUS_READWRITING) except (OSError, IOError) as e: if eventloop.errno_from_exception(e) == errno.EINPROGRESS: # in this case data is not sent at all From 9e6e884b12f91ec54487f5851458369ae5802fb9 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Fri, 26 Dec 2014 20:58:22 +0800 Subject: [PATCH 334/344] bump --- CHANGES | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 3662f50..6822082 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +2.6.1 2014-12-26 +- Fix a problem with TCP Fast Open on local side +- Fix sometimes daemon_start returns wrong exit status + 2.6 2014-12-21 - Add daemon support diff --git a/setup.py b/setup.py index 224cb21..442a3ac 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ with codecs.open('README.rst', encoding='utf-8') as f: setup( name="shadowsocks", - version="2.6", + version="2.6.1", license='MIT', description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', From 937693519a098238d28f2efb812c99d42889a17f Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 27 Dec 2014 23:29:07 +0800 Subject: [PATCH 335/344] update url in readme --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 621d2d9..4365844 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ Documentation ------------- You can find all the documentation in the wiki: -https://github.com/clowwindy/shadowsocks/wiki +https://github.com/shadowsocks/shadowsocks/wiki License ------- @@ -118,23 +118,23 @@ Bugs and Issues * [Mailing list] -[Android]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#android -[Build Status]: https://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat -[Chinese Readme]: https://github.com/clowwindy/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E +[Android]: https://github.com/shadowsocks/shadowsocks/wiki/Ports-and-Clients#android +[Build Status]: https://img.shields.io/travis/shadowsocks/shadowsocks/master.svg?style=flat +[Chinese Readme]: https://github.com/shadowsocks/shadowsocks/wiki/Shadowsocks-%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E [Coverage Status]: http://192.81.132.184/result/shadowsocks [Coverage]: http://192.81.132.184/job/Shadowsocks/ws/htmlcov/index.html [Debian sid]: https://packages.debian.org/unstable/python/shadowsocks [the package]: https://pypi.python.org/pypi/shadowsocks -[Encryption]: https://github.com/clowwindy/shadowsocks/wiki/Encryption +[Encryption]: https://github.com/shadowsocks/shadowsocks/wiki/Encryption [iOS]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Help -[Issue Tracker]: https://github.com/clowwindy/shadowsocks/issues?state=open +[Issue Tracker]: https://github.com/shadowsocks/shadowsocks/issues?state=open [Mailing list]: http://groups.google.com/group/shadowsocks [OpenSSL for Windows]: http://slproweb.com/products/Win32OpenSSL.html -[OpenWRT]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#openwrt +[OpenWRT]: https://github.com/shadowsocks/shadowsocks/wiki/Ports-and-Clients#openwrt [OS X]: https://github.com/shadowsocks/shadowsocks-iOS/wiki/Shadowsocks-for-OSX-Help [PyPI]: https://pypi.python.org/pypi/shadowsocks [PyPI version]: https://img.shields.io/pypi/v/shadowsocks.svg?style=flat -[TCP_FASTOPEN]: https://github.com/clowwindy/shadowsocks/wiki/TCP-Fast-Open -[Travis CI]: https://travis-ci.org/clowwindy/shadowsocks -[Troubleshooting]: https://github.com/clowwindy/shadowsocks/wiki/Troubleshooting -[Windows]: https://github.com/clowwindy/shadowsocks/wiki/Ports-and-Clients#windows +[TCP_FASTOPEN]: https://github.com/shadowsocks/shadowsocks/wiki/TCP-Fast-Open +[Travis CI]: https://travis-ci.org/shadowsocks/shadowsocks +[Troubleshooting]: https://github.com/shadowsocks/shadowsocks/wiki/Troubleshooting +[Windows]: https://github.com/shadowsocks/shadowsocks/wiki/Ports-and-Clients#windows From 7f19c316405018700bf61299cfd5177d0721b9ed Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 27 Dec 2014 23:29:48 +0800 Subject: [PATCH 336/344] update project url --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 442a3ac..c2b7d88 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( description="A fast tunnel proxy that help you get through firewalls", author='clowwindy', author_email='clowwindy42@gmail.com', - url='https://github.com/clowwindy/shadowsocks', + url='https://github.com/shadowsocks/shadowsocks', packages=['shadowsocks', 'shadowsocks.crypto'], package_data={ 'shadowsocks': ['README.rst', 'LICENSE'] From 5a5158b33f24d012496060a6b6b9a061bc982ee4 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sat, 27 Dec 2014 23:36:09 +0800 Subject: [PATCH 337/344] update url --- README.rst | 64 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/README.rst b/README.rst index 003b358..41afc36 100644 --- a/README.rst +++ b/README.rst @@ -1,11 +1,11 @@ shadowsocks =========== -|PyPI version| |Build Status| +|PyPI version| |Build Status| |Coverage Status| A fast tunnel proxy that helps you bypass firewalls. -`中文说明 `__ +`中文说明 `__ Install ------- @@ -16,12 +16,12 @@ server. Client ~~~~~~ -- `Windows `__ +- `Windows `__ / `OS X `__ -- `Android `__ +- `Android `__ / `iOS `__ -- `OpenWRT `__ +- `OpenWRT `__ Server ~~~~~~ @@ -77,27 +77,27 @@ On your server create a config file ``/etc/shadowsocks.json``. Example: Explanation of the fields: -+------------------+---------------------------------------------------------------------------------------------------------+ -| Name | Explanation | -+==================+=========================================================================================================+ -| server | the address your server listens | -+------------------+---------------------------------------------------------------------------------------------------------+ -| server\_port | server port | -+------------------+---------------------------------------------------------------------------------------------------------+ -| local\_address | the address your local listens | -+------------------+---------------------------------------------------------------------------------------------------------+ -| local\_port | local port | -+------------------+---------------------------------------------------------------------------------------------------------+ -| password | password used for encryption | -+------------------+---------------------------------------------------------------------------------------------------------+ -| timeout | in seconds | -+------------------+---------------------------------------------------------------------------------------------------------+ -| method | default: "aes-256-cfb", see `Encryption `__ | -+------------------+---------------------------------------------------------------------------------------------------------+ -| fast\_open | use `TCP\_FASTOPEN `__, true / false | -+------------------+---------------------------------------------------------------------------------------------------------+ -| workers | number of workers, available on Unix/Linux | -+------------------+---------------------------------------------------------------------------------------------------------+ ++------------------+-----------------------------------------------------------------------------------------------------------+ +| Name | Explanation | ++==================+===========================================================================================================+ +| server | the address your server listens | ++------------------+-----------------------------------------------------------------------------------------------------------+ +| server\_port | server port | ++------------------+-----------------------------------------------------------------------------------------------------------+ +| local\_address | the address your local listens | ++------------------+-----------------------------------------------------------------------------------------------------------+ +| local\_port | local port | ++------------------+-----------------------------------------------------------------------------------------------------------+ +| password | password used for encryption | ++------------------+-----------------------------------------------------------------------------------------------------------+ +| timeout | in seconds | ++------------------+-----------------------------------------------------------------------------------------------------------+ +| method | default: "aes-256-cfb", see `Encryption `__ | ++------------------+-----------------------------------------------------------------------------------------------------------+ +| fast\_open | use `TCP\_FASTOPEN `__, true / false | ++------------------+-----------------------------------------------------------------------------------------------------------+ +| workers | number of workers, available on Unix/Linux | ++------------------+-----------------------------------------------------------------------------------------------------------+ On your server: @@ -134,7 +134,7 @@ Documentation ------------- You can find all the documentation in the wiki: -https://github.com/clowwindy/shadowsocks/wiki +https://github.com/shadowsocks/shadowsocks/wiki License ------- @@ -144,12 +144,14 @@ MIT Bugs and Issues --------------- -- `Troubleshooting `__ +- `Troubleshooting `__ - `Issue - Tracker `__ + Tracker `__ - `Mailing list `__ .. |PyPI version| image:: https://img.shields.io/pypi/v/shadowsocks.svg?style=flat :target: https://pypi.python.org/pypi/shadowsocks -.. |Build Status| image:: https://img.shields.io/travis/clowwindy/shadowsocks/master.svg?style=flat - :target: https://travis-ci.org/clowwindy/shadowsocks +.. |Build Status| image:: https://img.shields.io/travis/shadowsocks/shadowsocks/master.svg?style=flat + :target: https://travis-ci.org/shadowsocks/shadowsocks +.. |Coverage Status| image:: http://192.81.132.184/result/shadowsocks + :target: http://192.81.132.184/job/Shadowsocks/ws/htmlcov/index.html From d91f7d85d4adcc91cf61a2211bd59588c7421f09 Mon Sep 17 00:00:00 2001 From: v3aqb Date: Sun, 28 Dec 2014 08:30:29 +0800 Subject: [PATCH 338/344] Update ctypes_libsodium.py --- shadowsocks/crypto/ctypes_libsodium.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shadowsocks/crypto/ctypes_libsodium.py b/shadowsocks/crypto/ctypes_libsodium.py index efecfd4..69b9de9 100644 --- a/shadowsocks/crypto/ctypes_libsodium.py +++ b/shadowsocks/crypto/ctypes_libsodium.py @@ -42,7 +42,7 @@ def load_libsodium(): global loaded, libsodium, buf from ctypes.util import find_library - for p in ('sodium',): + for p in ('sodium', 'libsodium'): libsodium_path = find_library(p) if libsodium_path: break From 45ee594d587fa6606043ed08638405d2910fc0ae Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 28 Dec 2014 12:29:20 +0800 Subject: [PATCH 339/344] coverage server now supports more projects --- tests/coverage_server.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/coverage_server.py b/tests/coverage_server.py index 7a135c5..5a9ce57 100644 --- a/tests/coverage_server.py +++ b/tests/coverage_server.py @@ -6,16 +6,19 @@ if __name__ == '__main__': import urllib class MainHandler(tornado.web.RequestHandler): - def get(self): - with open('/tmp/shadowsocks-coverage', 'rb') as f: - coverage = f.read().strip() - self.redirect(('https://img.shields.io/badge/' - 'coverage-%s-brightgreen.svg' - '?style=flat') % - urllib.quote(coverage)) + def get(self, project): + try: + with open('/tmp/%s-coverage' % project, 'rb') as f: + coverage = f.read().strip() + self.redirect(('https://img.shields.io/badge/' + 'coverage-%s-brightgreen.svg' + '?style=flat') % + urllib.quote(coverage)) + except IOError: + raise tornado.web.HTTPError(404) application = tornado.web.Application([ - (r"/shadowsocks", MainHandler), + (r"/([a-zA-Z0-9\\-_]+)", MainHandler), ]) if __name__ == "__main__": From 70559a1030fe88211a119057587580153792fd78 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 28 Dec 2014 12:35:17 +0800 Subject: [PATCH 340/344] fix pep8 --- tests/coverage_server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/coverage_server.py b/tests/coverage_server.py index 5a9ce57..5cd22ae 100644 --- a/tests/coverage_server.py +++ b/tests/coverage_server.py @@ -11,9 +11,9 @@ if __name__ == '__main__': with open('/tmp/%s-coverage' % project, 'rb') as f: coverage = f.read().strip() self.redirect(('https://img.shields.io/badge/' - 'coverage-%s-brightgreen.svg' - '?style=flat') % - urllib.quote(coverage)) + 'coverage-%s-brightgreen.svg' + '?style=flat') % + urllib.quote(coverage)) except IOError: raise tornado.web.HTTPError(404) From 5a5c8b9c7e6a3bdeec3be69554df654c05ffeca5 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 28 Dec 2014 15:00:29 +0800 Subject: [PATCH 341/344] fix coverage server --- tests/coverage_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/coverage_server.py b/tests/coverage_server.py index 5cd22ae..ca3830b 100644 --- a/tests/coverage_server.py +++ b/tests/coverage_server.py @@ -18,7 +18,7 @@ if __name__ == '__main__': raise tornado.web.HTTPError(404) application = tornado.web.Application([ - (r"/([a-zA-Z0-9\\-_]+)", MainHandler), + (r"/([a-zA-Z0-9\-_]+)", MainHandler), ]) if __name__ == "__main__": From 1423fe1921536bfb830b12202762140d545d34ac Mon Sep 17 00:00:00 2001 From: clowwindy Date: Sun, 28 Dec 2014 15:06:22 +0800 Subject: [PATCH 342/344] support color --- tests/coverage_server.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/coverage_server.py b/tests/coverage_server.py index ca3830b..2df55e3 100644 --- a/tests/coverage_server.py +++ b/tests/coverage_server.py @@ -10,10 +10,15 @@ if __name__ == '__main__': try: with open('/tmp/%s-coverage' % project, 'rb') as f: coverage = f.read().strip() + n = int(coverage.strip('%')) + if n > 80: + color = 'brightgreen' + else: + color = 'yellow' self.redirect(('https://img.shields.io/badge/' - 'coverage-%s-brightgreen.svg' + 'coverage-%s-%s.svg' '?style=flat') % - urllib.quote(coverage)) + (urllib.quote(coverage), color)) except IOError: raise tornado.web.HTTPError(404) From ccd1c0b45c4c35cd7315a4f583f9f57111fd2207 Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 31 Dec 2014 09:21:10 +0800 Subject: [PATCH 343/344] implement auth in encrypt module --- shadowsocks/crypto/ctypes_libsodium.py | 34 ++++++++- shadowsocks/crypto/hmac.py | 49 ++++++++++++ shadowsocks/encrypt.py | 102 +++++++++++++++++++------ shadowsocks/local.py | 2 +- shadowsocks/server.py | 2 +- 5 files changed, 162 insertions(+), 27 deletions(-) create mode 100644 shadowsocks/crypto/hmac.py diff --git a/shadowsocks/crypto/ctypes_libsodium.py b/shadowsocks/crypto/ctypes_libsodium.py index 69b9de9..4845dd1 100644 --- a/shadowsocks/crypto/ctypes_libsodium.py +++ b/shadowsocks/crypto/ctypes_libsodium.py @@ -27,7 +27,7 @@ import logging from ctypes import CDLL, c_char_p, c_int, c_ulonglong, byref, \ create_string_buffer, c_void_p -__all__ = ['ciphers'] +__all__ = ['ciphers', 'auths'] libsodium = None loaded = False @@ -39,7 +39,7 @@ BLOCK_SIZE = 64 def load_libsodium(): - global loaded, libsodium, buf + global loaded, libsodium, buf, tag_buf from ctypes.util import find_library for p in ('sodium', 'libsodium'): @@ -62,9 +62,18 @@ def load_libsodium(): c_char_p, c_ulonglong, c_char_p) + libsodium.crypto_onetimeauth.restype = c_int + libsodium.crypto_onetimeauth.argtypes = (c_void_p, c_char_p, + c_ulonglong, c_char_p) + + libsodium.crypto_onetimeauth_verify.restype = c_int + libsodium.crypto_onetimeauth_verify.argtypes = (c_char_p, c_char_p, + c_ulonglong, c_char_p) + libsodium.sodium_init() buf = create_string_buffer(buf_size) + tag_buf = create_string_buffer(16) loaded = True @@ -106,11 +115,32 @@ class Salsa20Crypto(object): return buf.raw[padding:padding + l] +class Poly1305(object): + @staticmethod + def auth(method, key, data): + global tag_buf + if not loaded: + load_libsodium() + libsodium.crypto_onetimeauth(byref(tag_buf), data, len(data), key) + return tag_buf.raw + + @staticmethod + def verify(method, key, data, tag): + if not loaded: + load_libsodium() + r = libsodium.crypto_onetimeauth_verify(tag, data, len(data), key) + return r == 0 + + ciphers = { b'salsa20': (32, 8, Salsa20Crypto), b'chacha20': (32, 8, Salsa20Crypto), } +auths = { + b'poly1305': (32, 16, Poly1305) +} + def test_salsa20(): from shadowsocks.crypto import util diff --git a/shadowsocks/crypto/hmac.py b/shadowsocks/crypto/hmac.py new file mode 100644 index 0000000..0fbf4e9 --- /dev/null +++ b/shadowsocks/crypto/hmac.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +# Copyright (c) 2014 clowwindy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import absolute_import, division, print_function, \ + with_statement + +import hmac + +from shadowsocks import common + +__all__ = ['auths'] + + +class HMAC(object): + @staticmethod + def auth(method, key, data): + digest = common.to_str(method.replace(b'hmac-', b'')) + return hmac.new(key, data, digest).digest() + + @staticmethod + def verify(method, key, data, tag): + digest = common.to_str(method.replace(b'hmac-', b'')) + t = hmac.new(key, data, digest).digest() + return hmac.compare_digest(t, tag) + + +auths = { + b'hmac-md5': (32, 16, HMAC), + b'hmac-sha256': (32, 32, HMAC), +} diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index ba02101..b6a931d 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -29,17 +29,23 @@ import hashlib import logging from shadowsocks.crypto import m2, rc4_md5, salsa20_ctr,\ - ctypes_openssl, ctypes_libsodium, table + ctypes_openssl, ctypes_libsodium, table, hmac +from shadowsocks import common -method_supported = {} -method_supported.update(rc4_md5.ciphers) -method_supported.update(salsa20_ctr.ciphers) -method_supported.update(ctypes_openssl.ciphers) -method_supported.update(ctypes_libsodium.ciphers) +ciphers_supported = {} +ciphers_supported.update(rc4_md5.ciphers) +ciphers_supported.update(salsa20_ctr.ciphers) +ciphers_supported.update(ctypes_openssl.ciphers) +ciphers_supported.update(ctypes_libsodium.ciphers) # let M2Crypto override ctypes_openssl -method_supported.update(m2.ciphers) -method_supported.update(table.ciphers) +ciphers_supported.update(m2.ciphers) +ciphers_supported.update(table.ciphers) + + +auths_supported = {} +auths_supported.update(hmac.auths) +auths_supported.update(ctypes_libsodium.auths) def random_string(length): @@ -50,22 +56,14 @@ def random_string(length): return os.urandom(length) -cached_keys = {} - - -def try_cipher(key, method=None): +def try_cipher(key, method=None, auth=None): Encryptor(key, method) + auth_create(b'test', key, b'test', auth) def EVP_BytesToKey(password, key_len, iv_len): # equivalent to OpenSSL's EVP_BytesToKey() with count 1 # so that we make the same key and iv as nodejs version - if hasattr(password, 'encode'): - password = password.encode('utf-8') - cached_key = '%s-%d-%d' % (password, key_len, iv_len) - r = cached_keys.get(cached_key, None) - if r: - return r m = [] i = 0 while len(b''.join(m)) < (key_len + iv_len): @@ -79,7 +77,6 @@ def EVP_BytesToKey(password, key_len, iv_len): ms = b''.join(m) key = ms[:key_len] iv = ms[key_len:key_len + iv_len] - cached_keys[cached_key] = (key, iv) return key, iv @@ -102,15 +99,14 @@ class Encryptor(object): def get_method_info(self, method): method = method.lower() - m = method_supported.get(method) + m = ciphers_supported.get(method) return m def iv_len(self): return len(self.cipher_iv) def get_cipher(self, password, method, op, iv): - if hasattr(password, 'encode'): - password = password.encode('utf-8') + password = common.to_bytes(password) m = self._method_info if m[0] > 0: key, iv_ = EVP_BytesToKey(password, m[0], m[1]) @@ -150,7 +146,8 @@ class Encryptor(object): def encrypt_all(password, method, op, data): result = [] method = method.lower() - (key_len, iv_len, m) = method_supported[method] + password = common.to_bytes(password) + (key_len, iv_len, m) = ciphers_supported[method] if key_len > 0: key, _ = EVP_BytesToKey(password, key_len, iv_len) else: @@ -166,6 +163,42 @@ def encrypt_all(password, method, op, data): return b''.join(result) +def auth_create(data, password, iv, method): + if method is None: + return data + # prepend hmac to data + password = common.to_bytes(password) + method = method.lower() + method_info = auths_supported.get(method) + if not method_info: + logging.error('method %s not supported' % method) + sys.exit(1) + key_len, tag_len, m = method_info + key, _ = EVP_BytesToKey(password + iv, key_len, 0) + tag = m.auth(method, key, data) + return tag + data + + +def auth_open(data, password, iv, method): + if method is None: + return data + # verify hmac and remove the hmac or return None + password = common.to_bytes(password) + method = method.lower() + method_info = auths_supported.get(method) + if not method_info: + logging.error('method %s not supported' % method) + sys.exit(1) + key_len, tag_len, m = method_info + key, _ = EVP_BytesToKey(password + iv, key_len, 0) + if len(data) <= tag_len: + return None + result = data[tag_len:] + if not m.verify(method, key, result, data[:tag_len]): + return None + return result + + CIPHERS_TO_TEST = [ b'aes-128-cfb', b'aes-256-cfb', @@ -175,6 +208,13 @@ CIPHERS_TO_TEST = [ b'table', ] +AUTHS_TO_TEST = [ + None, + b'hmac-md5', + b'hmac-sha256', + b'poly1305', +] + def test_encryptor(): from os import urandom @@ -198,6 +238,22 @@ def test_encrypt_all(): assert plain == plain2 +def test_auth(): + from os import urandom + plain = urandom(10240) + for method in AUTHS_TO_TEST: + logging.warn(method) + boxed = auth_create(plain, b'key', b'iv', method) + unboxed = auth_open(boxed, b'key', b'iv', method) + assert plain == unboxed + if method is not None: + b = common.ord(boxed[0]) + b ^= 1 + attack = common.chr(b) + boxed[1:] + assert auth_open(attack, b'key', b'iv', method) is None + + if __name__ == '__main__': test_encrypt_all() test_encryptor() + test_auth() diff --git a/shadowsocks/local.py b/shadowsocks/local.py index 994b6d8..44e9fca 100755 --- a/shadowsocks/local.py +++ b/shadowsocks/local.py @@ -49,7 +49,7 @@ def main(): utils.print_shadowsocks() - encrypt.try_cipher(config['password'], config['method']) + encrypt.try_cipher(config['password'], config['method'], config['auth']) try: logging.info("starting local at %s:%d" % diff --git a/shadowsocks/server.py b/shadowsocks/server.py index 8eed4ad..0ddcda5 100755 --- a/shadowsocks/server.py +++ b/shadowsocks/server.py @@ -57,7 +57,7 @@ def main(): else: config['port_password'][str(server_port)] = config['password'] - encrypt.try_cipher(config['password'], config['method']) + encrypt.try_cipher(config['password'], config['method'], config['auth']) tcp_servers = [] udp_servers = [] dns_resolver = asyncdns.DNSResolver() From af5455c820308ef5da68d2111ecaf50920f8c37b Mon Sep 17 00:00:00 2001 From: clowwindy Date: Wed, 31 Dec 2014 13:41:51 +0800 Subject: [PATCH 344/344] fix test --- shadowsocks/crypto/ctypes_libsodium.py | 5 ++--- shadowsocks/crypto/hmac.py | 24 +++++++++++++++++++++--- shadowsocks/encrypt.py | 2 +- shadowsocks/utils.py | 1 + 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/shadowsocks/crypto/ctypes_libsodium.py b/shadowsocks/crypto/ctypes_libsodium.py index 4845dd1..3598e3b 100644 --- a/shadowsocks/crypto/ctypes_libsodium.py +++ b/shadowsocks/crypto/ctypes_libsodium.py @@ -39,7 +39,7 @@ BLOCK_SIZE = 64 def load_libsodium(): - global loaded, libsodium, buf, tag_buf + global loaded, libsodium, buf from ctypes.util import find_library for p in ('sodium', 'libsodium'): @@ -73,7 +73,6 @@ def load_libsodium(): libsodium.sodium_init() buf = create_string_buffer(buf_size) - tag_buf = create_string_buffer(16) loaded = True @@ -118,9 +117,9 @@ class Salsa20Crypto(object): class Poly1305(object): @staticmethod def auth(method, key, data): - global tag_buf if not loaded: load_libsodium() + tag_buf = create_string_buffer(16) libsodium.crypto_onetimeauth(byref(tag_buf), data, len(data), key) return tag_buf.raw diff --git a/shadowsocks/crypto/hmac.py b/shadowsocks/crypto/hmac.py index 0fbf4e9..ce36540 100644 --- a/shadowsocks/crypto/hmac.py +++ b/shadowsocks/crypto/hmac.py @@ -24,6 +24,7 @@ from __future__ import absolute_import, division, print_function, \ with_statement import hmac +import hashlib from shadowsocks import common @@ -34,13 +35,30 @@ class HMAC(object): @staticmethod def auth(method, key, data): digest = common.to_str(method.replace(b'hmac-', b'')) - return hmac.new(key, data, digest).digest() + return hmac.new(key, data, getattr(hashlib, digest)).digest() @staticmethod def verify(method, key, data, tag): digest = common.to_str(method.replace(b'hmac-', b'')) - t = hmac.new(key, data, digest).digest() - return hmac.compare_digest(t, tag) + t = hmac.new(key, data, getattr(hashlib, digest)).digest() + if hasattr(hmac, 'compare_digest'): + return hmac.compare_digest(t, tag) + else: + return _time_independent_equals(t, tag) + + +# from tornado +def _time_independent_equals(a, b): + if len(a) != len(b): + return False + result = 0 + if type(a[0]) is int: # python3 byte strings + for x, y in zip(a, b): + result |= x ^ y + else: # python2 + for x, y in zip(a, b): + result |= ord(x) ^ ord(y) + return result == 0 auths = { diff --git a/shadowsocks/encrypt.py b/shadowsocks/encrypt.py index b6a931d..632f147 100644 --- a/shadowsocks/encrypt.py +++ b/shadowsocks/encrypt.py @@ -180,7 +180,7 @@ def auth_create(data, password, iv, method): def auth_open(data, password, iv, method): - if method is None: + if not method: return data # verify hmac and remove the hmac or return None password = common.to_bytes(password) diff --git a/shadowsocks/utils.py b/shadowsocks/utils.py index ff6801c..07daf44 100644 --- a/shadowsocks/utils.py +++ b/shadowsocks/utils.py @@ -173,6 +173,7 @@ def get_config(is_local): config['password'] = config.get('password', '') config['method'] = config.get('method', 'aes-256-cfb') + config['auth'] = config.get('auth', None) config['port_password'] = config.get('port_password', None) config['timeout'] = int(config.get('timeout', 300)) config['fast_open'] = config.get('fast_open', False)