Merge branch 'master' of https://bitbucket.org/yackob03/quay

app.py

@@ -3,15 +3,19 @@ import logging
 from flask import Flask
 from flask.ext.principal import Principal
 from flask.ext.login import LoginManager
-
+from flask.ext.mail import Mail
+from config import ProductionConfig
 
 app = Flask(__name__)
+app.config.from_object(ProductionConfig())
+
 logger = logging.getLogger(__name__)
 
 Principal(app, use_sessions=True)
 
-app.secret_key = '1cb18882-6d12-440d-a4cc-b7430fb5f884'
-
 login_manager = LoginManager()
 login_manager.init_app(app)
 login_manager.login_view = 'signin'
+
+mail = Mail()
+mail.init_app(app)

config.py

@@ -0,0 +1,17 @@
+class FlaskConfig(object):
+  SECRET_KEY = '1cb18882-6d12-440d-a4cc-b7430fb5f884'
+
+
+class MailConfig(object):
+  MAIL_SERVER = 'email-smtp.us-east-1.amazonaws.com'
+  MAIL_USE_TLS = True
+  MAIL_PORT = 587
+  MAIL_USERNAME = 'AKIAIXV5SDGCPVMU3N4Q'
+  MAIL_PASSWORD = 'AhmX/vWE91uQ2RtcEKTkfNrzZehEjPNXOXeOXgQNfLao'
+  DEFAULT_MAIL_SENDER = 'support@fluxmonkey.io'
+  MAIL_FAIL_SILENTLY = False
+  TESTING = False
+
+
+class ProductionConfig(FlaskConfig, MailConfig):
+  pass

data/database.py

@@ -15,16 +15,14 @@ class BaseModel(Model):
 
 
 class User(BaseModel):
-  username = CharField(unique=True)
+  username = CharField(unique=True, index=True)
   password_hash = CharField()
-  email = CharField(unique=True)
+  email = CharField(unique=True, index=True)
-
+  verified = BooleanField(default=False)
-  # TODO move this to False and require email verification
-  verified = BooleanField(default=True)
 
 
 class Visibility(BaseModel):
-  name = CharField()
+  name = CharField(index=True)
 
 
 class Repository(BaseModel):
@@ -42,14 +40,20 @@ class Repository(BaseModel):
 
 
 class Role(BaseModel):
-  name = CharField()
+  name = CharField(index=True)
 
 
 class RepositoryPermission(BaseModel):
-  user = ForeignKeyField(User)
+  user = ForeignKeyField(User, index=True)
-  repository = ForeignKeyField(Repository)
+  repository = ForeignKeyField(Repository, index=True)
   role = ForeignKeyField(Role)
 
+  class Meta:
+    database = db
+    indexes = (
+      (('user', 'repository'), True),
+    )
+
 
 def random_string_generator(length=16):
   def random_string():
@@ -60,12 +64,20 @@ def random_string_generator(length=16):
 
 
 class AccessToken(BaseModel):
-  code = CharField(default=random_string_generator(), unique=True)
+  code = CharField(default=random_string_generator(), unique=True, index=True)
   user = ForeignKeyField(User)
   repository = ForeignKeyField(Repository)
   created = DateTimeField(default=datetime.now)
 
 
+class EmailConfirmation(BaseModel):
+  code = CharField(default=random_string_generator(), unique=True, index=True)
+  user = ForeignKeyField(User)
+  pw_reset = BooleanField(default=False)
+  email_confirm = BooleanField(default=False)
+  created = DateTimeField(default=datetime.now)
+
+
 class Image(BaseModel):
   # This class is intentionally denormalized. Even though images are supposed
   # to be globally unique we can't treat them as such for permissions and
@@ -90,10 +102,17 @@ class RepositoryTag(BaseModel):
   image = ForeignKeyField(Image)
   repository = ForeignKeyField(Repository)
 
+  class Meta:
+    database = db
+    indexes = (
+      (('repository', 'name'), True),
+    )
+
 
 def initialize_db():
   create_model_tables([User, Repository, Image, AccessToken, Role,
-                       RepositoryPermission, Visibility, RepositoryTag])
+                       RepositoryPermission, Visibility, RepositoryTag,
+                       EmailConfirmation])
   Role.create(name='admin')
   Role.create(name='write')
   Role.create(name='read')

data/model.py

@@ -1,6 +1,7 @@
 import bcrypt
 import logging
 import dateutil.parser
+import operator
 
 from database import *
 from util.validation import (validate_email, validate_username,
@@ -28,9 +29,27 @@ def create_user(username, password, email):
   try:
     new_user = User.create(username=username, password_hash=pw_hash,
                            email=email)
+    return new_user
   except Exception as ex:
     raise DataModelException(ex.message)
-  return new_user
+
+
+def create_confirm_email_code(user):
+  code = EmailConfirmation.create(user=user, email_confirm=True)
+  return code
+
+
+def confirm_user_email(code):
+  code = EmailConfirmation.get(EmailConfirmation.code == code,
+                               EmailConfirmation.email_confirm == True)
+
+  user = code.user
+  user.verified = True
+  user.save()
+
+  code.delete_instance()
+
+  return user
 
 
 def get_user(username):
@@ -41,7 +60,9 @@ def get_user(username):
 
 
 def get_matching_users(username_prefix):
-  return list(User.select().where(User.username ** (username_prefix + '%')).limit(10))
+  query = User.select().where(User.username ** (username_prefix + '%'))
+  return list(query.limit(10))
+
 
 def verify_user(username, password):
   try:
@@ -75,8 +96,28 @@ def get_token(code):
   return AccessToken.get(AccessToken.code == code)
 
 
-def get_matching_repositories(repo_term):
+def get_visible_repositories(username=None):
-  return list(Repository.select().where(Repository.name ** ('%' + repo_term + '%') | Repository.namespace ** ('%' + repo_term + '%') | Repository.description ** ('%' + repo_term + '%')).limit(10))
+  query = Repository.select().distinct().join(Visibility)
+  or_clauses = [(Visibility.name == 'public')]
+
+  if username:
+    with_perms = query.switch(Repository).join(RepositoryPermission,
+                                               JOIN_LEFT_OUTER)
+    query = with_perms.join(User)
+    or_clauses.append(User.username == username)
+
+  return query.where(reduce(operator.or_, or_clauses))
+
+
+def get_matching_repositories(repo_term, username=None):
+  visible = get_visible_repositories(username)
+  search_clauses = (Repository.name ** ('%' + repo_term + '%') |
+                    Repository.namespace ** ('%' + repo_term + '%') |
+                    Repository.description ** ('%' + repo_term + '%'))
+
+
+  final = visible.where(search_clauses).limit(10)
+  return list(final)
 
 
 def change_password(user, new_password):
@@ -108,20 +149,20 @@ def get_all_repo_users(namespace_name, repository_name):
                          Repository.name == repository_name)
 
 
-def get_repository(namespace, name):
+def get_repository(namespace_name, repository_name):
   try:
-    return Repository.get(Repository.name == name,
+    return Repository.get(Repository.name == repository_name,
-                          Repository.namespace == namespace)
+                          Repository.namespace == namespace_name)
   except Repository.DoesNotExist:
     return None
 
 
-def get_user_repositories(user):
+def repository_is_public(namespace_name, repository_name):
-  select = RepositoryPermission.select(RepositoryPermission, Repository, Role)
+  joined = Repository.select().join(Visibility)
-  with_user = select.join(User).where(User.username == user.username)
+  query = joined.where(Repository.namespace == namespace_name,
-  with_role = with_user.switch(RepositoryPermission).join(Role)
+                       Repository.name == repository_name,
-  with_repo = with_role.switch(RepositoryPermission).join(Repository)
+                       Visibility.name == 'public')
-  return with_repo
+  return len(list(query)) > 0
 
 
 def create_repository(namespace, name, owner):
@@ -151,8 +192,8 @@ def set_image_metadata(image_id, namespace_name, repository_name,
                        created_date_str, comment):
   joined = Image.select().join(Repository)
   image_list = list(joined.where(Repository.name == repository_name,
-                            Repository.namespace == namespace_name,
+                                 Repository.namespace == namespace_name,
-                            Image.image_id == image_id))
+                                 Image.image_id == image_id))
 
   if not image_list:
     raise DataModelException('No image with specified id and repository')
@@ -169,6 +210,7 @@ def get_repository_images(namespace_name, repository_name):
   return joined.where(Repository.name == repository_name,
                       Repository.namespace == namespace_name)
 
+
 def get_tag_images(namespace_name, repository_name, tag_name):
   joined = Image.select().join(RepositoryTag).join(Repository)
   fetched = list(joined.where(Repository.name == repository_name,
@@ -177,6 +219,7 @@ def get_tag_images(namespace_name, repository_name, tag_name):
 
   return fetched
 
+
 def list_repository_tags(namespace_name, repository_name):
   select = RepositoryTag.select(RepositoryTag, Image)
   with_repo = select.join(Repository)
@@ -272,6 +315,7 @@ def set_user_repo_permission(username, namespace_name, repository_name,
                                            role=new_role)
     return new_perm
 
+
 def delete_user_permission(username, namespace_name, repository_name):
   if username == namespace_name:
     raise DataModelException('Namespace owner must always be admin.')

endpoints/api.py

@@ -64,10 +64,10 @@ def match_repos_api(prefix):
       'description': repo.description,
     }
 
-  repos = [repo_view(repo) for repo in model.get_matching_repositories(prefix) if
+  username = current_user.db_user.username
-           ReadRepositoryPermission(repo.namespace, repo.name).can()]
+  matching = model.get_matching_repositories(prefix, username)
   response = {
-    'repositories': repos
+    'repositories': [repo_view(repo) for repo in matching]
   }
 
   return jsonify(response)
@@ -76,16 +76,16 @@ def match_repos_api(prefix):
 @app.route('/api/repository/', methods=['GET'])
 @login_required
 def list_repos_api():
-  def repo_view(repo_perm):
+  def repo_view(repo_obj):
     return {
-      'namespace': repo_perm.repository.namespace,
+      'namespace': repo_obj.namespace,
-      'name': repo_perm.repository.name,
+      'name': repo_obj.name,
-      'role': repo_perm.role.name,
+      'description': repo_obj.description,
-      'description': repo_perm.repository.description,
     }
 
+  username = current_user.db_user.username
   repos = [repo_view(repo)
-           for repo in model.get_user_repositories(current_user.db_user)]
+           for repo in model.get_visible_repositories(username)]
   response = {
     'repositories': repos
   }
@@ -136,7 +136,7 @@ def get_repo_api(namespace, repository):
     }
 
   permission = ReadRepositoryPermission(namespace, repository)
-  if permission.can(): 
+  if permission.can() or model.repository_is_public(namespace, repository):
     repo = model.get_repository(namespace, repository)
     if repo:
       tags = model.list_repository_tags(namespace, repository)
@@ -162,12 +162,13 @@ def role_view(repo_perm_obj):
   }
 
 
-@app.route('/api/repository/<path:repository>/tag/<tag>/images', methods=['GET'])
+@app.route('/api/repository/<path:repository>/tag/<tag>/images',
+           methods=['GET'])
 @login_required
 @parse_repository_name
 def list_tag_images(namespace, repository, tag):
   permission = ReadRepositoryPermission(namespace, repository)
-  if permission.can():
+  if permission.can() or model.repository_is_public(namespace, repository):
     images = model.get_tag_images(namespace, repository, tag)
 
     return jsonify({
@@ -234,6 +235,7 @@ def change_permissions(namespace, repository, username):
 
   abort(403)  # Permission denied
 
+
 @app.route('/api/repository/<path:repository>/permissions/<username>',
            methods=['DELETE'])
 @login_required

endpoints/index.py

@@ -11,6 +11,7 @@ from app import app
 from auth.auth import (process_auth, get_authenticated_user,
                        get_validated_token)
 from util.names import parse_namespace_repository, parse_repository_name
+from util.email import send_confirmation_email
 from auth.permissions import (ModifyRepositoryPermission,
                               ReadRepositoryPermission, UserPermission)
 
@@ -46,8 +47,10 @@ def generate_headers(f):
 @app.route('/v1/users/', methods=['POST'])
 def create_user():
   user_data = request.get_json()
-  model.create_user(user_data['username'], user_data['password'],
+  new_user = model.create_user(user_data['username'], user_data['password'],
-                    user_data['email'])
+                               user_data['email'])
+  code = model.create_confirm_email_code(new_user)
+  send_confirmation_email(new_user.username, new_user.email, code.code)
   return make_response('Created', 201)
 
 
@@ -154,7 +157,7 @@ def get_repository_images(namespace, repository):
 
   # TODO invalidate token?
 
-  if permission.can():
+  if permission.can() or model.repository_is_public(namespace, repository):
     all_images = []
     for image in model.get_repository_images(namespace, repository):
       new_image_view = {

endpoints/registry.py

@@ -80,14 +80,14 @@ def set_cache_headers(f):
 @set_cache_headers
 def get_image_layer(namespace, repository, image_id, headers):
   permission = ReadRepositoryPermission(namespace, repository)
-  if not permission.can():
+  if permission.can() or model.repository_is_public(namespace, repository):
-    abort(403)
+    try:
+      return Response(store.stream_read(store.image_layer_path(
+        namespace, repository, image_id)), headers=headers)
+    except IOError:
+      abort(404)  #'Image not found', 404)
 
-  try:
+  abort(403)
-    return Response(store.stream_read(store.image_layer_path(
-      namespace, repository, image_id)), headers=headers)
-  except IOError:
-    abort(404)  #'Image not found', 404)
 
 
 @app.route('/v1/images/<image_id>/layer', methods=['PUT'])
@@ -182,7 +182,8 @@ def put_image_checksum(namespace, repository, image_id):
 @set_cache_headers
 def get_image_json(namespace, repository, image_id, headers):
   permission = ReadRepositoryPermission(namespace, repository)
-  if not permission.can():
+  if (not permission.can() and not
+      model.repository_is_public(namespace, repository)):
     abort(403)
 
   try:
@@ -211,7 +212,8 @@ def get_image_json(namespace, repository, image_id, headers):
 @set_cache_headers
 def get_image_ancestry(namespace, repository, image_id, headers):
   permission = ReadRepositoryPermission(namespace, repository)
-  if not permission.can():
+  if (not permission.can() and not
+      model.repository_is_public(namespace, repository)):
     abort(403)
 
   try:

endpoints/tags.py

@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
 def get_tags(namespace, repository):
   permission = ReadRepositoryPermission(namespace, repository)
 
-  if permission.can():
+  if permission.can() or model.repository_is_public(namespace, repository):
     tags = model.list_repository_tags(namespace, repository)
     tag_map = {tag.name: tag.image.image_id for tag in tags}
     return jsonify(tag_map)
@@ -40,7 +40,7 @@ def get_tags(namespace, repository):
 def get_tag(namespace, repository, tag):
   permission = ReadRepositoryPermission(namespace, repository)
 
-  if permission.can():
+  if permission.can() or model.repository_is_public(namespace, repository):
     tag_image = model.get_tag_image(namespace, repository, tag)
     response = make_response(tag_image.image_id, 200)
 

endpoints/web.py

@@ -36,6 +36,12 @@ def index():
   return send_file('templates/index.html')
 
 
+def common_login(db_user):
+  logger.debug('Successfully signed in as: %s' % db_user.username)
+  login_user(_LoginWrappedDBUser(db_user))
+  identity_changed.send(app, identity=Identity(db_user.username, 'username'))
+
+
 @app.route('/signin', methods=['POST'])
 def signin():
   username = request.form['username']
@@ -44,18 +50,28 @@ def signin():
   #TODO Allow email login
   verified = model.verify_user(username, password)
   if verified:
-    logger.debug('Successfully signed in as: %s' % username)
+    common_login(verified)
-
-    login_user(_LoginWrappedDBUser(verified))
-
-    identity_changed.send(app, identity=Identity(verified.username,
-                                                 'username'))
 
     return redirect(request.args.get('next') or url_for('index'))
 
   abort(403)
 
 
+@app.route('/confirm', methods=['GET'])
+def confirm_email():
+  code = request.values['code']
+  user = model.confirm_user_email(code)
+
+  common_login(user)
+
+  return redirect(url_for('index'))
+
+
+@app.route('/reset', methods=['GET'])
+def password_reset():
+  pass
+
+
 @app.route('/signin', methods=['GET'])
 def render_signin_page():
   return send_file('templates/signin.html')

requirements.txt

@@ -3,3 +3,4 @@ flask
 py-bcrypt
 Flask-Principal
 Flask-Login
+Flask-Mail

util/email.py

@@ -0,0 +1,20 @@
+from flask.ext.mail import Message
+
+from app import mail, app
+
+
+CONFIRM_MESSAGE = """
+This email address was recently used to register the username '%s'
+at <a href="http://quay.io">Quay.io</a>.<br>
+<br>
+To confirm this email address, please click the following link:<br>
+<a href="http://quay.io/confirm?code=%s">http://quay.io/confirm?code=%s</a>
+"""
+
+
+def send_confirmation_email(username, email, token):
+  msg = Message('Welcome to Quay! Please confirm your email.',
+                sender='support@fluxmonkey.io',  # Why do I need this?
+                recipients=[email])
+  msg.html = CONFIRM_MESSAGE % (username, token, token)
+  mail.send(msg)

util/validation.py

@@ -14,6 +14,7 @@ def validate_username(username):
           len(username) > 1 and
           len(username) < 256)
 
+
 def validate_password(password):
   # No whitespace and minimum length of 8
   if re.search(r'\s', password):