Merge pull request #7 from shadowsocks/master

merge code

.travis.yml

@@ -13,7 +13,7 @@ before_install:
   - sudo dd if=/dev/urandom of=/usr/share/nginx/www/file bs=1M count=10
   - sudo sh -c "echo '127.0.0.1    localhost' > /etc/hosts"
   - sudo service nginx restart
-  - pip install pep8 pyflakes nose coverage
+  - pip install pep8 pyflakes nose coverage PySocks
   - sudo tests/socksify/install.sh
   - sudo tests/libsodium/install.sh
   - sudo tests/setup_tc.sh

CHANGES

@@ -1,3 +1,13 @@
+2.6.11 2015-07-10
+- Fix a compatibility issue in UDP Relay
+
+2.6.10 2015-06-08
+- Optimize LRU cache
+- Refine logging
+
+2.6.9 2015-05-19
+- Fix a stability issue on Windows
+
 2.6.8 2015-02-10
 - Support multiple server ip on client side
 - Support --version

README.md

@@ -28,11 +28,11 @@ See [Install Server on Windows]
 
 ### Usage
 
-    ssserver -p 443 -k password -m rc4-md5
+    ssserver -p 443 -k password -m aes-256-cfb
 
 To run in the background:
 
-    sudo ssserver -p 443 -k password -m rc4-md5 --user nobody -d start
+    sudo ssserver -p 443 -k password -m aes-256-cfb --user nobody -d start
 
 To stop:
 

setup.py

@@ -7,7 +7,7 @@ with codecs.open('README.rst', encoding='utf-8') as f:
 
 setup(
     name="shadowsocks",
-    version="2.6.9",
+    version="2.6.12",
     license='http://www.apache.org/licenses/LICENSE-2.0',
     description="A fast tunnel proxy that help you get through firewalls",
     author='clowwindy',

shadowsocks/common.py

@@ -157,7 +157,7 @@ def parse_header(data):
             if len(data) >= 2 + addrlen:
                 dest_addr = data[2:2 + addrlen]
                 dest_port = struct.unpack('>H', data[2 + addrlen:4 +
-                                          addrlen])[0]
+                                                     addrlen])[0]
                 header_length = 4 + addrlen
             else:
                 logging.warn('header is too short')
@@ -171,8 +171,8 @@ def parse_header(data):
         else:
             logging.warn('header is too short')
     else:
-        logging.warn('unsupported addrtype %d, maybe wrong password' %
+        logging.warn('unsupported addrtype %d, maybe wrong password or '
-                     addrtype)
+                     'encryption method' % addrtype)
     if dest_addr is None:
         return None
     return addrtype, to_bytes(dest_addr), dest_port, header_length

shadowsocks/lru_cache.py

@@ -41,6 +41,7 @@ class LRUCache(collections.MutableMapping):
         self._time_to_keys = collections.defaultdict(list)
         self._keys_to_last_time = {}
         self._last_visits = collections.deque()
+        self._closed_values = set()
         self.update(dict(*args, **kwargs))  # use the free update to set keys
 
     def __getitem__(self, key):
@@ -83,7 +84,9 @@ class LRUCache(collections.MutableMapping):
                     if key in self._store:
                         if now - self._keys_to_last_time[key] > self.timeout:
                             value = self._store[key]
-                            self.close_callback(value)
+                            if value not in self._closed_values:
+                                self.close_callback(value)
+                                self._closed_values.add(value)
             for key in self._time_to_keys[least]:
                 self._last_visits.popleft()
                 if key in self._store:
@@ -93,6 +96,7 @@ class LRUCache(collections.MutableMapping):
                         c += 1
             del self._time_to_keys[least]
         if c:
+            self._closed_values.clear()
             logging.debug('%d keys swept' % c)
 
 
@@ -126,5 +130,21 @@ def test():
     assert 'a' not in c
     assert 'b' not in c
 
+    global close_cb_called
+    close_cb_called = False
+
+    def close_cb(t):
+        global close_cb_called
+        assert not close_cb_called
+        close_cb_called = True
+
+    c = LRUCache(timeout=0.1, close_callback=close_cb)
+    c['s'] = 1
+    c['s']
+    time.sleep(0.1)
+    c['s']
+    time.sleep(0.3)
+    c.sweep()
+
 if __name__ == '__main__':
     test()

shadowsocks/shell.py

@@ -221,7 +221,6 @@ def get_config(is_local):
     config['workers'] = config.get('workers', 1)
     config['pid-file'] = config.get('pid-file', '/var/run/shadowsocks.pid')
     config['log-file'] = config.get('log-file', '/var/log/shadowsocks.log')
-    config['workers'] = config.get('workers', 1)
     config['verbose'] = config.get('verbose', False)
     config['local_address'] = to_str(config.get('local_address', '127.0.0.1'))
     config['local_port'] = config.get('local_port', 1080)

shadowsocks/udprelay.py

@@ -76,8 +76,9 @@ from shadowsocks.common import parse_header, pack_addr
 BUF_SIZE = 65536
 
 
-def client_key(a, b, c, d):
+def client_key(source_addr, server_af):
-    return '%s:%s:%s:%s' % (a, b, c, d)
+    # notice this is server af, not dest af
+    return '%s:%s:%d' % (source_addr[0], source_addr[1], server_af)
 
 
 class UDPRelay(object):
@@ -102,6 +103,7 @@ class UDPRelay(object):
                                          close_callback=self._close_client)
         self._client_fd_to_server_addr = \
             lru_cache.LRUCache(timeout=config['timeout'])
+        self._dns_cache = lru_cache.LRUCache(timeout=300)
         self._eventloop = None
         self._closed = False
         self._last_time = time.time()
@@ -169,27 +171,33 @@ class UDPRelay(object):
         else:
             server_addr, server_port = dest_addr, dest_port
 
-        key = client_key(r_addr[0], r_addr[1], dest_addr, dest_port)
+        addrs = self._dns_cache.get(server_addr, None)
+        if addrs is None:
+            addrs = socket.getaddrinfo(server_addr, server_port, 0,
+                                       socket.SOCK_DGRAM, socket.SOL_UDP)
+            if not addrs:
+                # drop
+                return
+            else:
+                self._dns_cache[server_addr] = addrs
+
+        af, socktype, proto, canonname, sa = addrs[0]
+        key = client_key(r_addr, af)
+        logging.debug(key)
         client = self._cache.get(key, None)
         if not client:
             # TODO async getaddrinfo
-            addrs = socket.getaddrinfo(server_addr, server_port, 0,
+            if self._forbidden_iplist:
-                                       socket.SOCK_DGRAM, socket.SOL_UDP)
+                if common.to_str(sa[0]) in self._forbidden_iplist:
-            if addrs:
+                    logging.debug('IP %s is in forbidden list, drop' %
-                af, socktype, proto, canonname, sa = addrs[0]
+                                  common.to_str(sa[0]))
-                if self._forbidden_iplist:
+                    # drop
-                    if common.to_str(sa[0]) in self._forbidden_iplist:
+                    return
-                        logging.debug('IP %s is in forbidden list, drop' %
+            client = socket.socket(af, socktype, proto)
-                                      common.to_str(sa[0]))
+            client.setblocking(False)
-                        # drop
+            self._cache[key] = client
-                        return
+            self._client_fd_to_server_addr[client.fileno()] = r_addr
-                client = socket.socket(af, socktype, proto)
+
-                client.setblocking(False)
-                self._cache[key] = client
-                self._client_fd_to_server_addr[client.fileno()] = r_addr
-            else:
-                # drop
-                return
             self._sockets.add(client.fileno())
             self._eventloop.add(client, eventloop.POLL_IN)
 

tests/jenkins.sh

@@ -69,7 +69,7 @@ if [ -f /proc/sys/net/ipv4/tcp_fastopen ] ; then
 fi
 
 run_test tests/test_large_file.sh
-
+run_test tests/test_udp_src.sh
 run_test tests/test_command.sh
 
 coverage combine && coverage report --include=shadowsocks/*

tests/test_udp_src.py

@@ -0,0 +1,83 @@
+#!/usr/bin/python
+
+import socket
+import socks
+
+
+SERVER_IP = '127.0.0.1'
+SERVER_PORT = 1081
+
+
+if __name__ == '__main__':
+    # Test 1: same source port IPv4
+    sock_out = socks.socksocket(socket.AF_INET, socket.SOCK_DGRAM,
+                                socket.SOL_UDP)
+    sock_out.set_proxy(socks.SOCKS5, SERVER_IP, SERVER_PORT)
+    sock_out.bind(('127.0.0.1', 9000))
+
+    sock_in1 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM,
+                             socket.SOL_UDP)
+    sock_in2 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM,
+                             socket.SOL_UDP)
+
+    sock_in1.bind(('127.0.0.1', 9001))
+    sock_in2.bind(('127.0.0.1', 9002))
+
+    sock_out.sendto(b'data', ('127.0.0.1', 9001))
+    result1 = sock_in1.recvfrom(8)
+
+    sock_out.sendto(b'data', ('127.0.0.1', 9002))
+    result2 = sock_in2.recvfrom(8)
+
+    sock_out.close()
+    sock_in1.close()
+    sock_in2.close()
+
+    # make sure they're from the same source port
+    assert result1 == result2
+
+    # Test 2: same source port IPv6
+    # try again from the same port but IPv6
+    sock_out = socks.socksocket(socket.AF_INET, socket.SOCK_DGRAM,
+                                socket.SOL_UDP)
+    sock_out.set_proxy(socks.SOCKS5, SERVER_IP, SERVER_PORT)
+    sock_out.bind(('127.0.0.1', 9000))
+
+    sock_in1 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM,
+                             socket.SOL_UDP)
+    sock_in2 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM,
+                             socket.SOL_UDP)
+
+    sock_in1.bind(('::1', 9001))
+    sock_in2.bind(('::1', 9002))
+
+    sock_out.sendto(b'data', ('::1', 9001))
+    result1 = sock_in1.recvfrom(8)
+
+    sock_out.sendto(b'data', ('::1', 9002))
+    result2 = sock_in2.recvfrom(8)
+
+    sock_out.close()
+    sock_in1.close()
+    sock_in2.close()
+
+    # make sure they're from the same source port
+    assert result1 == result2
+
+    # Test 3: different source ports IPv6
+    sock_out = socks.socksocket(socket.AF_INET, socket.SOCK_DGRAM,
+                                socket.SOL_UDP)
+    sock_out.set_proxy(socks.SOCKS5, SERVER_IP, SERVER_PORT)
+    sock_out.bind(('127.0.0.1', 9003))
+
+    sock_in1 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM,
+                             socket.SOL_UDP)
+    sock_in1.bind(('::1', 9001))
+    sock_out.sendto(b'data', ('::1', 9001))
+    result3 = sock_in1.recvfrom(8)
+
+    # make sure they're from different source ports
+    assert result1 != result3
+
+    sock_out.close()
+    sock_in1.close()

tests/test_udp_src.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+PYTHON="coverage run -p -a"
+
+mkdir -p tmp
+
+$PYTHON shadowsocks/local.py -c tests/aes.json -v &
+LOCAL=$!
+
+$PYTHON shadowsocks/server.py -c tests/aes.json --forbidden-ip "" -v &
+SERVER=$!
+
+sleep 3
+
+python tests/test_udp_src.py
+r=$?
+
+kill -s SIGINT $LOCAL
+kill -s SIGINT $SERVER
+
+sleep 2
+
+exit $r

utils/autoban.py

@@ -42,10 +42,12 @@ if __name__ == '__main__':
             if ip not in ips:
                 ips[ip] = 1
                 print(ip)
+                sys.stdout.flush()
             else:
                 ips[ip] += 1
             if ip not in banned and ips[ip] >= config.count:
                 banned.add(ip)
                 cmd = 'iptables -A INPUT -s %s -j DROP' % ip
                 print(cmd, file=sys.stderr)
+                sys.stderr.flush()
                 os.system(cmd)

utils/fail2ban/shadowsocks.conf

@@ -0,0 +1,5 @@
+[Definition]
+
+_daemon = shadowsocks
+
+failregex = ^\s+ERROR\s+can not parse header when handling connection from <HOST>:\d+$