Merge remote-tracking branch 'upstream/phase4-11-07-2015' into python-registry-v2

test/fulldbtest.sh

@@ -36,6 +36,7 @@ down_postgres() {
 
 run_tests() {
   TEST_DATABASE_URI=$1 TEST=true python -m unittest discover -f
+  TEST_DATABASE_URI=$1 TEST=true python -m test.queue_threads -f
 }
 
 # NOTE: MySQL is currently broken on setup.
@@ -45,7 +46,7 @@ run_tests() {
 
 #echo '> Running Full Test Suite (mysql)'
 #set +e
-#run_tests "mysql+pymysql://root:password@192.168.59.103/genschema"
+#run_tests "mysql+pymysql://root:password@127.0.0.1/genschema"
 #set -e
 #down_mysql
 
@@ -55,6 +56,6 @@ up_postgres
 
 echo '> Running Full Test Suite (postgres)'
 set +e
-run_tests "postgresql://postgres@192.168.59.103/genschema"
+run_tests "postgresql://postgres@127.0.0.1/genschema"
 set -e
 down_postgres

test/queue_threads.py

@@ -0,0 +1,77 @@
+import unittest
+import json
+import time
+
+from functools import wraps
+from threading import Thread, Lock
+
+from app import app
+from data.queue import WorkQueue
+from initdb import wipe_database, initialize_database, populate_database
+
+
+QUEUE_NAME = 'testqueuename'
+
+class AutoUpdatingQueue(object):
+  def __init__(self, queue_to_wrap):
+    self._queue = queue_to_wrap
+
+  def _wrapper(self, func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+      to_return = func(*args, **kwargs)
+      self._queue.update_metrics()
+      return to_return
+    return wrapper
+
+  def __getattr__(self, attr_name):
+    method_or_attr = getattr(self._queue, attr_name)
+    if callable(method_or_attr):
+      return self._wrapper(method_or_attr)
+    else:
+      return method_or_attr
+
+
+class QueueTestCase(unittest.TestCase):
+  TEST_MESSAGE_1 = json.dumps({'data': 1})
+
+  def setUp(self):
+    self.transaction_factory = app.config['DB_TRANSACTION_FACTORY']
+    self.queue = AutoUpdatingQueue(WorkQueue(QUEUE_NAME, self.transaction_factory))
+    wipe_database()
+    initialize_database()
+    populate_database()
+
+
+class TestQueueThreads(QueueTestCase):
+  def test_queue_threads(self):
+    count = [20]
+    for i in range(count[0]):
+      self.queue.put([str(i)], self.TEST_MESSAGE_1)
+
+    lock = Lock()
+    def get(lock, count, queue):
+      item = queue.get()
+      if item is None:
+        return
+      self.assertEqual(self.TEST_MESSAGE_1, item.body)
+      with lock:
+        count[0] -= 1
+
+    threads = []
+    # The thread count needs to be a few times higher than the queue size
+    # count because some threads will get a None and thus won't decrement
+    # the counter.
+    for i in range(100):
+      t = Thread(target=get, args=(lock, count, self.queue))
+      threads.append(t)
+    for t in threads:
+      t.start()
+    for t in threads:
+      t.join()
+    self.assertEqual(count[0], 0)
+
+
+if __name__ == '__main__':
+  unittest.main()
+

test/registry_tests.py

@@ -691,6 +691,17 @@ class V1RegistryTests(V1RegistryPullMixin, V1RegistryPushMixin, RegistryTestsMix
     }]
     self.do_push('public', 'newrepo/somesubrepo', 'public', 'password', images, expected_code=400)
 
+  def test_push_unicode_metadata(self):
+    self.conduct_api_login('devtable', 'password')
+
+    images = [{
+      'id': 'onlyimagehere',
+      'comment': 'Pawe\xc5\x82 Kami\xc5\x84ski <pawel.kaminski@codewise.com>'.decode('utf-8')
+    }]
+
+    self.do_push('devtable', 'unicodetest', 'devtable', 'password', images)
+    self.do_pull('devtable', 'unicodetest', 'devtable', 'password', images=images)
+
   def test_tag_validation(self):
     image_id = 'onlyimagehere'
     images = [{

test/test_api_security.py

@@ -50,6 +50,8 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana
                                      SuperUserOrganizationManagement, SuperUserOrganizationList,
                                      SuperUserAggregateLogs)
 
+from endpoints.api.secscan import RepositoryImagePackages, RepositoryTagVulnerabilities
+
 
 try:
   app.register_blueprint(api_bp, url_prefix='/api')
@@ -4210,18 +4212,54 @@ class TestOrganizationInvoiceField(ApiTestCase):
     ApiTestCase.setUp(self)
     self._set_url(OrganizationInvoiceField, orgname='buynlarge', field_uuid='1234')
 
-  def test_get_anonymous(self):
+  def test_delete_anonymous(self):
     self._run_test('DELETE', 403, None, None)
 
-  def test_get_freshuser(self):
+  def test_delete_freshuser(self):
     self._run_test('DELETE', 403, 'freshuser', None)
 
-  def test_get_reader(self):
+  def test_delete_reader(self):
     self._run_test('DELETE', 403, 'reader', None)
 
-  def test_get_devtable(self):
+  def test_delete_devtable(self):
     self._run_test('DELETE', 201, 'devtable', None)
 
 
+class TestRepositoryTagVulnerabilities(ApiTestCase):
+  def setUp(self):
+    ApiTestCase.setUp(self)
+    self._set_url(RepositoryTagVulnerabilities, repository='devtable/simple', tag='latest')
+
+  def test_get_anonymous(self):
+    self._run_test('GET', 401, None, None)
+
+  def test_get_freshuser(self):
+    self._run_test('GET', 403, 'freshuser', None)
+
+  def test_get_reader(self):
+    self._run_test('GET', 403, 'reader', None)
+
+  def test_get_devtable(self):
+    self._run_test('GET', 200, 'devtable', None)
+
+
+class TestRepositoryImagePackages(ApiTestCase):
+  def setUp(self):
+    ApiTestCase.setUp(self)
+    self._set_url(RepositoryImagePackages, repository='devtable/simple', imageid='fake')
+
+  def test_get_anonymous(self):
+    self._run_test('GET', 401, None, None)
+
+  def test_get_freshuser(self):
+    self._run_test('GET', 403, 'freshuser', None)
+
+  def test_get_reader(self):
+    self._run_test('GET', 403, 'reader', None)
+
+  def test_get_devtable(self):
+    self._run_test('GET', 404, 'devtable', None)
+
+
 if __name__ == '__main__':
   unittest.main()

test/test_api_usage.py

@@ -1549,8 +1549,8 @@ class TestDeleteRepository(ApiTestCase):
     model.build.create_repository_build(repository, delegate_token, {}, 'someid2', 'foobar2')
 
     # Create some notifications.
-    model.notification.create_repo_notification(repository, 'repo_push', 'hipchat', {})
-    model.notification.create_repo_notification(repository, 'build_queued', 'slack', {})
+    model.notification.create_repo_notification(repository, 'repo_push', 'hipchat', {}, {})
+    model.notification.create_repo_notification(repository, 'build_queued', 'slack', {}, {})
 
     # Create some logs.
     model.log.log_action('push_repo', ADMIN_ACCESS_USER, repository=repository)
@@ -1984,7 +1984,7 @@ class TestRepositoryNotifications(ApiTestCase):
     json = self.postJsonResponse(RepositoryNotificationList,
                                  params=dict(repository=ADMIN_ACCESS_USER + '/simple'),
                                  data=dict(config={'url': 'http://example.com'}, event='repo_push',
-                                           method='webhook'),
+                                           method='webhook', eventConfig={}),
                                  expected_code=201)
 
     self.assertEquals('repo_push', json['event'])
@@ -2024,7 +2024,8 @@ class TestRepositoryNotifications(ApiTestCase):
     json = self.postJsonResponse(RepositoryNotificationList,
                                  params=dict(repository=ADMIN_ACCESS_USER + '/simple'),
                                  data=dict(config={'url': 'http://example.com'}, event='repo_push',
-                                           method='webhook', title='Some Notification'),
+                                           method='webhook', title='Some Notification',
+                                           eventConfig={}),
                                  expected_code=201)
 
     self.assertEquals('repo_push', json['event'])

test/test_buildman.py

@@ -159,6 +159,7 @@ class TestEphemeral(unittest.TestCase):
     self.etcd_client_mock.delete.assert_called_once_with(self.mock_job_key)
 
   @async_test
+  @unittest.skip('this test is flaky on Quay.io builders')
   def test_another_manager_takes_job(self):
     # Prepare a job to be taken by another manager
     test_component = yield From(self._setup_job_for_managers())

test/test_external_jwt_authn.py

@@ -12,6 +12,8 @@ from tempfile import NamedTemporaryFile
 from Crypto.PublicKey import RSA
 from datetime import datetime, timedelta
 
+_PORT_NUMBER = 5001
+
 class JWTAuthTestCase(LiveServerTestCase):
   maxDiff = None
 
@@ -30,6 +32,9 @@ class JWTAuthTestCase(LiveServerTestCase):
     JWTAuthTestCase.private_key_data = private_key_data
 
   def create_app(self):
+    global _PORT_NUMBER
+    _PORT_NUMBER = _PORT_NUMBER + 1
+
     users = [
       {'name': 'cooluser', 'email': 'user@domain.com', 'password': 'password'},
       {'name': 'some.neat.user', 'email': 'neat@domain.com', 'password': 'foobar'}
@@ -37,6 +42,7 @@ class JWTAuthTestCase(LiveServerTestCase):
 
     jwt_app = Flask('testjwt')
     private_key = JWTAuthTestCase.private_key_data
+    jwt_app.config['LIVESERVER_PORT'] = _PORT_NUMBER
 
     def _get_basic_auth():
       data = base64.b64decode(request.headers['Authorization'][len('Basic '):])

test/testconfig.py

@@ -55,3 +55,10 @@ class TestConfig(DefaultConfig):
   FEATURE_GITHUB_BUILD = True
 
   CLOUDWATCH_NAMESPACE = None
+
+  FEATURE_SECURITY_SCANNER = True
+  SECURITY_SCANNER = {
+    'ENDPOINT': 'http://localhost/some/invalid/path',
+    'ENGINE_VERSION_TARGET': 1,
+    'API_CALL_TIMEOUT': 1
+  }