Merge branch 'master' of github.com:coreos-inc/quay

auth/auth.py

@@ -6,6 +6,7 @@ from datetime import datetime
 from flask import request, session
 from flask.ext.principal import identity_changed, Identity
 from flask.ext.login import current_user
+from flask.sessions import SecureCookieSessionInterface, BadSignature
 from base64 import b64decode
 
 import scopes
@@ -22,6 +23,9 @@ from util.http import abort
 logger = logging.getLogger(__name__)
 
 
+SIGNATURE_PREFIX = 'signature='
+
+
 def _load_user_from_cookie():
   if not current_user.is_anonymous():
     try:
@@ -69,7 +73,7 @@ def _validate_and_apply_oauth_token(token):
   identity_changed.send(app, identity=new_identity)
 
 
-def process_basic_auth(auth):
+def _process_basic_auth(auth):
   normalized = [part.strip() for part in auth.split(' ') if part]
   if normalized[0].lower() != 'basic' or len(normalized) != 2:
     logger.debug('Invalid basic auth format.')
@@ -127,44 +131,41 @@ def process_basic_auth(auth):
   logger.debug('Basic auth present but could not be validated.')
 
 
-def process_token(auth):
+def generate_signed_token(grants):
+  ser = SecureCookieSessionInterface().get_signing_serializer(app)
+  data_to_sign = {
+    'grants': grants,
+  }
+
+  encrypted = ser.dumps(data_to_sign)
+  return '{0}{1}'.format(SIGNATURE_PREFIX, encrypted)
+
+
+def _process_signed_grant(auth):
   normalized = [part.strip() for part in auth.split(' ') if part]
   if normalized[0].lower() != 'token' or len(normalized) != 2:
-    logger.debug('Not an auth token: %s' % auth)
+    logger.debug('Not a token: %s', auth)
     return
 
-  token_details = normalized[1].split(',')
+  if not normalized[1].startswith(SIGNATURE_PREFIX):
+    logger.debug('Not a signed grant token: %s', auth)
+    return
 
-  if len(token_details) != 1:
+  encrypted = normalized[1][len(SIGNATURE_PREFIX):]
-    logger.warning('Invalid token format: %s' % auth)
+  ser = SecureCookieSessionInterface().get_signing_serializer(app)
-    abort(401, message='Invalid token format: %(auth)s', issue='invalid-auth-token', auth=auth)
-
-  def safe_get(lst, index, default_value):
-    try:
-      return lst[index]
-    except IndexError:
-      return default_value
-
-  token_vals = {val[0]: safe_get(val, 1, '') for val in
-                (detail.split('=') for detail in token_details)}
-
-  if 'signature' not in token_vals:
-    logger.warning('Token does not contain signature: %s' % auth)
-    abort(401, message='Token does not contain a valid signature: %(auth)s',
-          issue='invalid-auth-token', auth=auth)
 
   try:
-    token_data = model.load_token_data(token_vals['signature'])
+    token_data = ser.loads(encrypted, max_age=app.config['SIGNED_GRANT_EXPIRATION_SEC'])
-
+  except BadSignature:
-  except model.InvalidTokenException:
+    logger.warning('Signed grant could not be validated: %s', encrypted)
-    logger.warning('Token could not be validated: %s', token_vals['signature'])
+    abort(401, message='Signed grant could not be validated: %(auth)s', issue='invalid-auth-token',
-    abort(401, message='Token could not be validated: %(auth)s', issue='invalid-auth-token',
           auth=auth)
 
-  logger.debug('Successfully validated token: %s', token_data.code)
+  logger.debug('Successfully validated signed grant with data: %s', token_data)
-  set_validated_token(token_data)
 
-  identity_changed.send(app, identity=Identity(token_data.code, 'token'))
+  loaded_identity = Identity(None, 'signed_grant')
+  loaded_identity.provides.update(token_data['grants'])
+  identity_changed.send(app, identity=loaded_identity)
 
 
 def process_oauth(func):
@@ -192,8 +193,8 @@ def process_auth(func):
 
     if auth:
       logger.debug('Validating auth header: %s' % auth)
-      process_token(auth)
+      _process_signed_grant(auth)
-      process_basic_auth(auth)
+      _process_basic_auth(auth)
     else:
       logger.debug('No auth header.')
 

auth/permissions.py

@@ -57,6 +57,14 @@ SCOPE_MAX_USER_ROLES.update({
 })
 
 
+def repository_read_grant(namespace, repository):
+  return _RepositoryNeed(namespace, repository, 'read')
+
+
+def repository_write_grant(namespace, repository):
+  return _RepositoryNeed(namespace, repository, 'write')
+
+
 class QuayDeferredPermissionUser(Identity):
   def __init__(self, uuid, auth_type, scopes):
     super(QuayDeferredPermissionUser, self).__init__(uuid, auth_type)
@@ -226,6 +234,11 @@ class ViewTeamPermission(Permission):
                                              team_member, admin_org)
 
 
+class AlwaysFailPermission(Permission):
+  def can(self):
+    return False
+
+
 @identity_loaded.connect_via(app)
 def on_identity_loaded(sender, identity):
   logger.debug('Identity loaded: %s' % identity)
@@ -249,5 +262,8 @@ def on_identity_loaded(sender, identity):
     logger.debug('Delegate token added permission: {0}'.format(repo_grant))
     identity.provides.add(repo_grant)
 
+  elif identity.auth_type == 'signed_grant':
+    logger.debug('Loaded signed grants identity')
+
   else:
     logger.error('Unknown identity auth type: %s', identity.auth_type)

conf/http-base.conf

@@ -16,6 +16,11 @@ gzip_types text/plain text/xml text/css
            text/javascript application/x-javascript
            application/octet-stream;
 
+map $proxy_protocol_addr $proper_forwarded_for {
+  ""      $proxy_add_x_forwarded_for;
+  default $proxy_protocol_addr;
+}
+
 upstream web_app_server {
     server unix:/tmp/gunicorn_web.sock fail_timeout=0;
 }
@@ -33,3 +38,4 @@ upstream build_manager_controller_server {
 upstream build_manager_websocket_server {
     server localhost:8787;
 }
+

conf/nginx-nossl.conf

@@ -4,7 +4,6 @@ include root-base.conf;
 
 http {
     include http-base.conf;
-
     include rate-limiting.conf;
 
     server {

conf/nginx.conf

@@ -4,9 +4,7 @@ include root-base.conf;
 
 http {
     include http-base.conf;
-
     include hosted-http-base.conf;
-
     include rate-limiting.conf;
 
     server {
@@ -25,8 +23,7 @@ http {
 
     server {
         include proxy-protocol.conf;
-
+        include server-base.conf;
-        include proxy-server-base.conf;
 
         listen 8443 default proxy_protocol;
 

conf/proxy-server-base.conf

@@ -1,87 +0,0 @@
-# vim: ft=nginx
-
-client_body_temp_path /var/log/nginx/client_body 1 2;
-server_name _;
-
-keepalive_timeout 5;
-
-if ($args ~ "_escaped_fragment_") {
-    rewrite ^ /snapshot$uri;
-}
-
-proxy_set_header X-Forwarded-For $proxy_protocol_addr;
-proxy_set_header X-Forwarded-Proto $scheme;
-proxy_set_header Host $http_host;
-proxy_redirect off;
-
-proxy_set_header Transfer-Encoding $http_transfer_encoding;
-
-location / {
-    proxy_pass   http://web_app_server;
-
-    limit_req zone=webapp burst=25 nodelay;
-}
-
-location /realtime {
-    proxy_pass   http://web_app_server;
-    proxy_buffering off;
-    proxy_request_buffering off;
-}
-
-location /v1/repositories/ {
-    proxy_buffering off;
-
-    proxy_request_buffering off;
-
-    proxy_pass http://registry_app_server;
-    proxy_temp_path /var/log/nginx/proxy_temp 1 2;
-
-    client_max_body_size 20G;
-
-    limit_req zone=repositories burst=5 nodelay;
-}
-
-location /v1/ {
-    proxy_buffering off;
-
-    proxy_request_buffering off;
-
-    proxy_pass   http://registry_app_server;
-    proxy_temp_path /var/log/nginx/proxy_temp 1 2;
-
-    client_max_body_size 20G;
-}
-
-location /c1/ {
-    proxy_buffering off;
-
-    proxy_request_buffering off;
-
-    proxy_pass   http://verbs_app_server;
-    proxy_temp_path /var/log/nginx/proxy_temp 1 2;
-
-    limit_req zone=api burst=5 nodelay;
-}
-
-location /static/ {
-    # checks for static file, if not found proxy to app
-    alias /static/;
-}
-
-location /v1/_ping {
-    add_header Content-Type text/plain;
-    add_header X-Docker-Registry-Version 0.6.0;
-    add_header X-Docker-Registry-Standalone 0;
-    return 200 'true';
-}
-
-location ~ ^/b1/controller(/?)(.*) {
-    proxy_pass http://build_manager_controller_server/$2;
-}
-
-location ~ ^/b1/socket(/?)(.*) {
-    proxy_pass http://build_manager_websocket_server/$2;
-    proxy_http_version 1.1;
-    proxy_set_header Upgrade $http_upgrade;
-    proxy_set_header Connection "upgrade";
-}

conf/rate-limiting.conf

@@ -1,7 +1,16 @@
 # vim: ft=nginx
 
+# Check the Authorization header and, if it is empty, use their proxy protocol
+# IP, else use the header as their unique identifier for rate limiting.
+# Enterprise users will never be using proxy protocol, thus the value will be
+# empty string. This means they will not get rate limited.
+map $http_authorization $registry_bucket {
+  ""      $proxy_protocol_addr;
+  default $http_authorization;
+}
+
 limit_req_zone $proxy_protocol_addr zone=webapp:10m rate=25r/s;
-limit_req_zone $proxy_protocol_addr zone=repositories:10m rate=1r/s;
 limit_req_zone $proxy_protocol_addr zone=api:10m rate=1r/s;
+limit_req_zone $registry_bucket zone=repositories:10m rate=1r/s;
 limit_req_status 429;
 limit_req_log_level warn;

conf/server-base.conf

@@ -3,16 +3,13 @@
 client_body_temp_path /var/log/nginx/client_body 1 2;
 server_name _;
 
-set_real_ip_from 172.17.0.0/16;
-real_ip_header X-Forwarded-For;
-
 keepalive_timeout 5;
 
 if ($args ~ "_escaped_fragment_") {
     rewrite ^ /snapshot$uri;
 }
 
-proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+proxy_set_header X-Forwarded-For $proper_forwarded_for;
 proxy_set_header X-Forwarded-Proto $scheme;
 proxy_set_header Host $http_host;
 proxy_redirect off;
@@ -21,6 +18,8 @@ proxy_set_header Transfer-Encoding $http_transfer_encoding;
 
 location / {
     proxy_pass   http://web_app_server;
+
+    limit_req zone=webapp;
 }
 
 location /realtime {
@@ -29,6 +28,18 @@ location /realtime {
     proxy_request_buffering off;
 }
 
+location /v1/repositories/ {
+    proxy_buffering off;
+
+    proxy_request_buffering off;
+
+    proxy_pass http://registry_app_server;
+    proxy_read_timeout 2000;
+    proxy_temp_path /var/log/nginx/proxy_temp 1 2;
+
+    limit_req zone=repositories;
+}
+
 location /v1/ {
     proxy_buffering off;
 
@@ -47,6 +58,8 @@ location /c1/ {
 
     proxy_pass   http://verbs_app_server;
     proxy_temp_path /var/log/nginx/proxy_temp 1 2;
+
+    limit_req zone=api;
 }
 
 location /static/ {

config.py

@@ -197,4 +197,7 @@ class DefaultConfig(object):
   SYSTEM_SERVICE_BLACKLIST = []
 
   # Temporary tag expiration in seconds, this may actually be longer based on GC policy
-  PUSH_TEMP_TAG_EXPIRATION_SEC = 60 * 60
+  PUSH_TEMP_TAG_EXPIRATION_SEC = 60 * 60  # One hour per layer
+
+  # Signed registry grant token expiration in seconds
+  SIGNED_GRANT_EXPIRATION_SEC = 60 * 60 * 24  # One day to complete a push/pull

endpoints/index.py

@@ -9,12 +9,13 @@ from collections import OrderedDict
 from data import model
 from data.model import oauth
 from app import app, authentication, userevents, storage
-from auth.auth import process_auth
+from auth.auth import process_auth, generate_signed_token
 from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token
 from util.names import parse_repository_name
 from util.useremails import send_confirmation_email
 from auth.permissions import (ModifyRepositoryPermission, UserAdminPermission,
-                              ReadRepositoryPermission, CreateRepositoryPermission)
+                              ReadRepositoryPermission, CreateRepositoryPermission,
+                              AlwaysFailPermission, repository_read_grant, repository_write_grant)
 
 from util.http import abort
 from endpoints.trackhelper import track_and_log
@@ -26,7 +27,13 @@ logger = logging.getLogger(__name__)
 
 index = Blueprint('index', __name__)
 
-def generate_headers(role='read'):
+
+class GrantType(object):
+  READ_REPOSITORY = 'read'
+  WRITE_REPOSITORY = 'write'
+
+
+def generate_headers(scope=GrantType.READ_REPOSITORY):
   def decorator_method(f):
     @wraps(f)
     def wrapper(namespace, repository, *args, **kwargs):
@@ -35,12 +42,6 @@ def generate_headers(role='read'):
       # Setting session namespace and repository
       session['namespace'] = namespace
       session['repository'] = repository
-
-      if get_authenticated_user():
-        session['username'] = get_authenticated_user().username
-      else:
-        session.pop('username', None)
-
       # We run our index and registry on the same hosts for now
       registry_server = urlparse.urlparse(request.url).netloc
       response.headers['X-Docker-Endpoints'] = registry_server
@@ -48,16 +49,23 @@ def generate_headers(role='read'):
       has_token_request = request.headers.get('X-Docker-Token', '')
 
       if has_token_request:
-        repo = model.get_repository(namespace, repository)
+        permission = AlwaysFailPermission()
-        if repo:
+        grants = []
-          token = model.create_access_token(repo, role, 'pushpull-token')
+        if scope == GrantType.READ_REPOSITORY:
-          token_str = 'signature=%s' % token.code
+          permission = ReadRepositoryPermission(namespace, repository)
-          response.headers['WWW-Authenticate'] = token_str
+          grants.append(repository_read_grant(namespace, repository))
-          response.headers['X-Docker-Token'] = token_str
+        elif scope == GrantType.WRITE_REPOSITORY:
-        else:
+          permission = ModifyRepositoryPermission(namespace, repository)
-          logger.info('Token request in non-existing repo: %s/%s' %
+          grants.append(repository_write_grant(namespace, repository))
-                      (namespace, repository))
 
+        if permission.can():
+          # Generate a signed grant which expires here
+          signature = generate_signed_token(grants)
+          response.headers['WWW-Authenticate'] = signature
+          response.headers['X-Docker-Token'] = signature
+        else:
+          logger.warning('Registry request with invalid credentials on repository: %s/%s',
+                         namespace, repository)
       return response
     return wrapper
   return decorator_method
@@ -186,7 +194,7 @@ def update_user(username):
 @index.route('/repositories/<path:repository>', methods=['PUT'])
 @process_auth
 @parse_repository_name
-@generate_headers(role='write')
+@generate_headers(scope=GrantType.WRITE_REPOSITORY)
 def create_repository(namespace, repository):
   logger.debug('Parsing image descriptions')
   image_descriptions = json.loads(request.data.decode('utf8'))
@@ -228,7 +236,7 @@ def create_repository(namespace, repository):
 @index.route('/repositories/<path:repository>/images', methods=['PUT'])
 @process_auth
 @parse_repository_name
-@generate_headers(role='write')
+@generate_headers(scope=GrantType.WRITE_REPOSITORY)
 def update_images(namespace, repository):
   permission = ModifyRepositoryPermission(namespace, repository)
 
@@ -273,7 +281,7 @@ def update_images(namespace, repository):
 @index.route('/repositories/<path:repository>/images', methods=['GET'])
 @process_auth
 @parse_repository_name
-@generate_headers(role='read')
+@generate_headers(scope=GrantType.READ_REPOSITORY)
 def get_repository_images(namespace, repository):
   permission = ReadRepositoryPermission(namespace, repository)
 
@@ -307,7 +315,7 @@ def get_repository_images(namespace, repository):
 @index.route('/repositories/<path:repository>/images', methods=['DELETE'])
 @process_auth
 @parse_repository_name
-@generate_headers(role='write')
+@generate_headers(scope=GrantType.WRITE_REPOSITORY)
 def delete_repository_images(namespace, repository):
   abort(501, 'Not Implemented', issue='not-implemented')
 

endpoints/registry.py

@@ -455,14 +455,15 @@ def put_image_json(namespace, repository, image_id):
           issue='invalid-request', image_id=image_id)
 
   logger.debug('Looking up repo image')
-  repo_image = model.get_repo_image_extended(namespace, repository, image_id)
+
-  if not repo_image:
-    logger.debug('Image not found, creating image')
   repo = model.get_repository(namespace, repository)
   if repo is None:
     abort(404, 'Repository does not exist: %(namespace)s/%(repository)s', issue='no-repo',
           namespace=namespace, repository=repository)
 
+  repo_image = model.get_repo_image_extended(namespace, repository, image_id)
+  if not repo_image:
+    logger.debug('Image not found, creating image')
     username = get_authenticated_user() and get_authenticated_user().username
     repo_image = model.find_create_or_link_image(image_id, repo, username, {},
                                                  store.preferred_locations[0])