Merge remote-tracking branch 'upstream/master' into python-registry-v2

data/archivedlogs.py

@@ -1,8 +1,7 @@
 import logging
 
-from gzip import GzipFile
+from util.registry.gzipinputstream import GzipInputStream
 from flask import send_file, abort
-from cStringIO import StringIO
 
 from data.userfiles import DelegateUserfiles, UserfilesHandlers
 
@@ -17,10 +16,8 @@ class LogArchiveHandlers(UserfilesHandlers):
   def get(self, file_id):
     path = self._files.get_file_id_path(file_id)
     try:
-      with self._storage.stream_read_file(self._locations, path) as gzip_stream:
-        with GzipFile(fileobj=gzip_stream) as unzipped:
-          unzipped_buffer = StringIO(unzipped.read())
-          return send_file(unzipped_buffer, mimetype=JSON_MIMETYPE)
+      data_stream = self._storage.stream_read_file(self._locations, path)
+      return send_file(GzipInputStream(data_stream), mimetype=JSON_MIMETYPE)
     except IOError:
       abort(404)
 

data/billing.py

@@ -17,6 +17,7 @@ PLANS = [
     'deprecated': True,
     'free_trial_days': 14,
     'superseded_by': None,
+    'plans_page_hidden': False,
   },
   {
     'title': 'Basic',
@@ -28,6 +29,7 @@ PLANS = [
     'deprecated': True,
     'free_trial_days': 14,
     'superseded_by': None,
+    'plans_page_hidden': False,
   },
   {
     'title': 'Yacht',
@@ -39,6 +41,7 @@ PLANS = [
     'deprecated': True,
     'free_trial_days': 180,
     'superseded_by': 'bus-small-30',
+    'plans_page_hidden': False,
   },
   {
     'title': 'Personal',
@@ -50,6 +53,7 @@ PLANS = [
     'deprecated': True,
     'free_trial_days': 14,
     'superseded_by': 'personal-30',
+    'plans_page_hidden': False,
   },
   {
     'title': 'Skiff',
@@ -61,6 +65,7 @@ PLANS = [
     'deprecated': True,
     'free_trial_days': 14,
     'superseded_by': 'bus-micro-30',
+    'plans_page_hidden': False,
   },
   {
     'title': 'Yacht',
@@ -72,6 +77,7 @@ PLANS = [
     'deprecated': True,
     'free_trial_days': 14,
     'superseded_by': 'bus-small-30',
+    'plans_page_hidden': False,
   },
   {
     'title': 'Freighter',
@@ -83,6 +89,7 @@ PLANS = [
     'deprecated': True,
     'free_trial_days': 14,
     'superseded_by': 'bus-medium-30',
+    'plans_page_hidden': False,
   },
   {
     'title': 'Tanker',
@@ -94,6 +101,7 @@ PLANS = [
     'deprecated': True,
     'free_trial_days': 14,
     'superseded_by': 'bus-large-30',
+    'plans_page_hidden': False,
   },
 
   # Active plans
@@ -107,6 +115,7 @@ PLANS = [
     'deprecated': False,
     'free_trial_days': 30,
     'superseded_by': None,
+    'plans_page_hidden': False,
   },
   {
     'title': 'Personal',
@@ -118,6 +127,7 @@ PLANS = [
     'deprecated': False,
     'free_trial_days': 30,
     'superseded_by': None,
+    'plans_page_hidden': False,
   },
   {
     'title': 'Skiff',
@@ -129,6 +139,7 @@ PLANS = [
     'deprecated': False,
     'free_trial_days': 30,
     'superseded_by': None,
+    'plans_page_hidden': False,
   },
   {
     'title': 'Yacht',
@@ -140,6 +151,7 @@ PLANS = [
     'deprecated': False,
     'free_trial_days': 30,
     'superseded_by': None,
+    'plans_page_hidden': False,
   },
   {
     'title': 'Freighter',
@@ -151,6 +163,7 @@ PLANS = [
     'deprecated': False,
     'free_trial_days': 30,
     'superseded_by': None,
+    'plans_page_hidden': False,
   },
   {
     'title': 'Tanker',
@@ -162,6 +175,19 @@ PLANS = [
     'deprecated': False,
     'free_trial_days': 30,
     'superseded_by': None,
+    'plans_page_hidden': False,
+  },
+  {
+    'title': 'Carrier',
+    'price': 35000,
+    'privateRepos': 250,
+    'stripeId': 'bus-xlarge-30',
+    'audience': 'For extra large businesses',
+    'bus_features': True,
+    'deprecated': False,
+    'free_trial_days': 30,
+    'superseded_by': None,
+    'plans_page_hidden': True,
   },
 ]
 

data/database.py

@@ -544,6 +544,15 @@ class ImageStoragePlacement(BaseModel):
     )
 
 
+class UserRegion(BaseModel):
+  user = QuayUserField(index=True, allows_robots=False)
+  location = ForeignKeyField(ImageStorageLocation)
+
+  indexes = (
+    (('user', 'location'), True),
+  )
+
+
 class Image(BaseModel):
   # This class is intentionally denormalized. Even though images are supposed
   # to be globally unique we can't treat them as such for permissions and
@@ -733,6 +742,7 @@ class RepositoryNotification(BaseModel):
   repository = ForeignKeyField(Repository, index=True)
   event = ForeignKeyField(ExternalNotificationEvent)
   method = ForeignKeyField(ExternalNotificationMethod)
+  title = CharField(null=True)
   config_json = TextField()
 
 
@@ -777,4 +787,4 @@ all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission,
               ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification,
               RepositoryAuthorizedEmail, ImageStorageTransformation, DerivedImageStorage,
               TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind,
-              AccessTokenKind, Star, RepositoryActionCount, TagManifest, BlobUpload]
+              AccessTokenKind, Star, RepositoryActionCount, TagManifest, BlobUpload, UserRegion]

data/migrations/versions/499f6f08de3_add_title_field_to_notification.py

@@ -0,0 +1,26 @@
+"""Add title field to notification
+
+Revision ID: 499f6f08de3
+Revises: 246df01a6d51
+Create Date: 2015-08-21 14:18:07.287743
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '499f6f08de3'
+down_revision = '246df01a6d51'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('repositorynotification', sa.Column('title', sa.String(length=255), nullable=True))
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('repositorynotification', 'title')
+    ### end Alembic commands ###

data/migrations/versions/9512773a4a2_add_userregion_table.py

@@ -0,0 +1,35 @@
+"""Add UserRegion table
+
+Revision ID: 9512773a4a2
+Revises: 499f6f08de3
+Create Date: 2015-09-01 14:17:08.628052
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '9512773a4a2'
+down_revision = '499f6f08de3'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('userregion',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('user_id', sa.Integer(), nullable=False),
+    sa.Column('location_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['location_id'], ['imagestoragelocation.id'], name=op.f('fk_userregion_location_id_imagestoragelocation')),
+    sa.ForeignKeyConstraint(['user_id'], ['user.id'], name=op.f('fk_userregion_user_id_user')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_userregion'))
+    )
+    op.create_index('userregion_location_id', 'userregion', ['location_id'], unique=False)
+    op.create_index('userregion_user_id', 'userregion', ['user_id'], unique=False)
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('userregion')
+    ### end Alembic commands ###

data/model/notification.py

@@ -113,12 +113,12 @@ def delete_matching_notifications(target, kind_name, **kwargs):
     notification.delete_instance()
 
 
-def create_repo_notification(repo, event_name, method_name, config):
+def create_repo_notification(repo, event_name, method_name, config, title=None):
   event = ExternalNotificationEvent.get(ExternalNotificationEvent.name == event_name)
   method = ExternalNotificationMethod.get(ExternalNotificationMethod.name == method_name)
 
   return RepositoryNotification.create(repository=repo, event=event, method=method,
-                                       config_json=json.dumps(config))
+                                       config_json=json.dumps(config), title=title)
 
 
 def get_repo_notification(uuid):

data/model/oauth.py

@@ -8,8 +8,9 @@ from oauth2lib import utils
 
 from data.database import (OAuthApplication, OAuthAuthorizationCode, OAuthAccessToken, User,
                            AccessToken, random_string_generator)
-from data.model import user
+from data.model import user, config
 from auth import scopes
+from util import get_app_url
 
 
 logger = logging.getLogger(__name__)
@@ -45,7 +46,10 @@ class DatabaseAuthorizationProvider(AuthorizationProvider):
       return False
 
   def validate_redirect_uri(self, client_id, redirect_uri):
-    if redirect_uri == url_for('web.oauth_local_handler', _external=True):
+    internal_redirect_url = '%s%s' % (get_app_url(config.app_config),
+                                      url_for('web.oauth_local_handler'))
+
+    if redirect_uri == internal_redirect_url:
       return True
 
     try:

data/model/permission.py

@@ -17,14 +17,19 @@ def list_robot_permissions(robot_name):
           .where(User.username == robot_name, User.robot == True))
 
 
-def list_organization_member_permissions(organization):
+def list_organization_member_permissions(organization, limit_to_user=None):
   query = (RepositoryPermission
            .select(RepositoryPermission, Repository, User)
            .join(Repository)
            .switch(RepositoryPermission)
            .join(User)
-           .where(Repository.namespace_user == organization)
-           .where(User.robot == False))
+           .where(Repository.namespace_user == organization))
+
+  if limit_to_user is not None:
+    query = query.where(RepositoryPermission.user == limit_to_user)
+  else:
+    query = query.where(User.robot == False)
+
   return query
 
 

data/model/storage.py

@@ -11,6 +11,12 @@ from data.database import (ImageStorage, Image, DerivedImageStorage, ImageStorag
 logger = logging.getLogger(__name__)
 
 
+def add_storage_placement(storage, location_name):
+  """ Adds a storage placement for the given storage at the given location. """
+  location = ImageStorageLocation.get(name=location_name)
+  ImageStoragePlacement.create(location=location, storage=storage)
+
+
 def find_or_create_derived_storage(source, transformation_name, preferred_location):
   existing = find_derived_storage(source, transformation_name)
   if existing is not None:

data/model/user.py

@@ -8,7 +8,8 @@ from datetime import datetime, timedelta
 
 from data.database import (User, LoginService, FederatedLogin, RepositoryPermission, TeamMember,
                            Team, Repository, TupleSelector, TeamRole, Namespace, Visibility,
-                           EmailConfirmation, Role, db_for_update, random_string_generator)
+                           EmailConfirmation, Role, db_for_update, random_string_generator,
+                           UserRegion, ImageStorageLocation)
 from data.model import (DataModelException, InvalidPasswordException, InvalidRobotException,
                         InvalidUsernameException, InvalidEmailAddressException,
                         TooManyUsersException, TooManyLoginAttemptsException, db_transaction,
@@ -463,6 +464,13 @@ def get_user_by_id(user_db_id):
     return None
 
 
+def get_namespace_user_by_user_id(namespace_user_db_id):
+  try:
+    return User.get(User.id == namespace_user_db_id, User.robot == False)
+  except User.DoesNotExist:
+    raise InvalidUsernameException('User with id does not exist: %s' % namespace_user_db_id)
+
+
 def get_namespace_by_user_id(namespace_user_db_id):
   try:
     return User.get(User.id == namespace_user_db_id, User.robot == False).username
@@ -664,3 +672,8 @@ def get_pull_credentials(robotname):
       'registry': '%s://%s/v1/' % (config.app_config['PREFERRED_URL_SCHEME'],
                                    config.app_config['SERVER_HOSTNAME']),
   }
+
+def get_region_locations(user):
+  """ Returns the locations defined as preferred storage for the given user. """
+  query = UserRegion.select().join(ImageStorageLocation).where(UserRegion.user == user)
+  return set([region.location.name for region in query])

data/queue.py

@@ -13,6 +13,17 @@ class NoopWith:
   def __exit__(self, type, value, traceback):
     pass
 
+class MetricQueueReporter(object):
+  def __init__(self, metric_queue):
+    self._metric_queue = metric_queue
+
+  def __call__(self, currently_processing, running_count, total_count):
+    need_capacity_count = total_count - running_count
+    self._metric_queue.put('BuildCapacityShortage', need_capacity_count, unit='Count')
+
+    building_percent = 100 if currently_processing else 0
+    self._metric_queue.put('PercentBuilding', building_percent, unit='Percent')
+
 class WorkQueue(object):
   def __init__(self, queue_name, transaction_factory,
                canonical_name_match_list=None, reporter=None):

data/users/externaljwt.py

@@ -1,13 +1,15 @@
 import logging
 import json
 import os
-import jwt
 
 from datetime import datetime, timedelta
 from data.users.federated import FederatedUsers, VerifiedCredentials
+from util.security import strictjwt
+
 
 logger = logging.getLogger(__name__)
 
+
 class ExternalJWTAuthN(FederatedUsers):
   """ Delegates authentication to a REST endpoint that returns JWTs. """
   PUBLIC_KEY_FILENAME = 'jwt-authn.cert'
@@ -45,9 +47,9 @@ class ExternalJWTAuthN(FederatedUsers):
     # Load the JWT returned.
     encoded = result_data.get('token', '')
     try:
-      payload = jwt.decode(encoded, self.public_key, algorithms=['RS256'],
-                           audience='quay.io/jwtauthn', issuer=self.issuer)
-    except jwt.InvalidTokenError:
+      payload = strictjwt.decode(encoded, self.public_key, algorithms=['RS256'],
+                                 audience='quay.io/jwtauthn', issuer=self.issuer)
+    except strictjwt.InvalidTokenError:
       logger.exception('Exception when decoding returned JWT')
       return (None, 'Invalid username or password')
 

data/users/externalldap.py

@@ -9,6 +9,16 @@ from data.users.federated import FederatedUsers, VerifiedCredentials
 logger = logging.getLogger(__name__)
 
 
+class LDAPConnectionBuilder(object):
+  def __init__(self, ldap_uri, user_dn, user_pw):
+    self._ldap_uri = ldap_uri
+    self._user_dn = user_dn
+    self._user_pw = user_pw
+
+  def get_connection(self):
+    return LDAPConnection(self._ldap_uri, self._user_dn, self._user_pw)
+
+
 class LDAPConnection(object):
   def __init__(self, ldap_uri, user_dn, user_pw):
     self._ldap_uri = ldap_uri
@@ -20,13 +30,7 @@ class LDAPConnection(object):
     trace_level = 2 if os.environ.get('USERS_DEBUG') == '1' else 0
     self._conn = ldap.initialize(self._ldap_uri, trace_level=trace_level)
     self._conn.set_option(ldap.OPT_REFERRALS, 1)
-
-    try:
-      self._conn.simple_bind_s(self._user_dn, self._user_pw)
-    except ldap.INVALID_CREDENTIALS:
-      logger.exception('LDAP admin dn or password are invalid')
-      return None
-
+    self._conn.simple_bind_s(self._user_dn, self._user_pw)
     return self._conn
 
   def __exit__(self, exc_type, value, tb):
@@ -38,7 +42,7 @@ class LDAPUsers(FederatedUsers):
 
   def __init__(self, ldap_uri, base_dn, admin_dn, admin_passwd, user_rdn, uid_attr, email_attr):
     super(LDAPUsers, self).__init__('ldap')
-    self._ldap_conn = LDAPConnection(ldap_uri, admin_dn, admin_passwd)
+    self._ldap = LDAPConnectionBuilder(ldap_uri, admin_dn, admin_passwd)
     self._ldap_uri = ldap_uri
     self._base_dn = base_dn
     self._user_rdn = user_rdn
@@ -65,10 +69,15 @@ class LDAPUsers(FederatedUsers):
     return referral_dn
 
   def _ldap_user_search(self, username_or_email):
-    with self._ldap_conn as conn:
-      if conn is None:
-        return (None, 'LDAP Admin dn or password is invalid')
+    # Verify the admin connection works first. We do this here to avoid wrapping
+    # the entire block in the INVALID CREDENTIALS check.
+    try:
+      with self._ldap.get_connection():
+        pass
+    except ldap.INVALID_CREDENTIALS:
+      return (None, 'LDAP Admin dn or password is invalid')
 
+    with self._ldap.get_connection() as conn:
       logger.debug('Incoming username or email param: %s', username_or_email.__repr__())
       user_search_dn = ','.join(self._user_rdn + self._base_dn)
       query = u'(|({0}={2})({1}={2}))'.format(self._uid_attr, self._email_attr,