quay/endpoints/common.py

import logging
import urlparse
import json
import string
import datetime
import os

# Register the various exceptions via decorators.
import endpoints.decorated

from flask import make_response, render_template, request, abort, session
from flask.ext.login import login_user
from flask.ext.principal import identity_changed
from random import SystemRandom

from data import model
from data.database import db
from app import app, oauth_apps, dockerfile_build_queue, LoginWrappedDBUser

from auth.permissions import QuayDeferredPermissionUser
from auth import scopes
from auth.auth_context import get_authenticated_user
from endpoints.api.discovery import swagger_route_data
from werkzeug.routing import BaseConverter
from functools import wraps
from config import getFrontendVisibleConfig
from external_libraries import get_external_javascript, get_external_css
from endpoints.notificationhelper import spawn_notification

import features

logger = logging.getLogger(__name__)

route_data = None

CACHE_BUSTERS_JSON = 'static/dist/cachebusters.json'
CACHE_BUSTERS = None

def get_cache_busters():
  """ Retrieves the cache busters hashes. """
  global CACHE_BUSTERS
  if CACHE_BUSTERS is not None:
    return CACHE_BUSTERS

  if not os.path.exists(CACHE_BUSTERS_JSON):
    return {}

  with open(CACHE_BUSTERS_JSON, 'r') as f:
    CACHE_BUSTERS = json.loads(f.read())
    return CACHE_BUSTERS


class RepoPathConverter(BaseConverter):
  regex = '[\.a-zA-Z0-9_\-]+/[\.a-zA-Z0-9_\-]+'
  weight = 200

app.url_map.converters['repopath'] = RepoPathConverter

def route_show_if(value):
  def decorator(f):
    @wraps(f)
    def decorated_function(*args, **kwargs):
      if not value:
        abort(404)

      return f(*args, **kwargs)
    return decorated_function
  return decorator


def route_hide_if(value):
  def decorator(f):
    @wraps(f)
    def decorated_function(*args, **kwargs):
      if value:
        abort(404)

      return f(*args, **kwargs)
    return decorated_function
  return decorator


def get_route_data():
  global route_data
  if route_data:
    return route_data

  route_data = swagger_route_data(include_internal=True, compact=True)
  return route_data


def truthy_param(param):
  return param not in {False, 'false', 'False', '0', 'FALSE', '', 'null'}


def param_required(param_name):
  def wrapper(wrapped):
    @wraps(wrapped)
    def decorated(*args, **kwargs):
      if param_name not in request.args:
        abort(make_response('Required param: %s' % param_name, 400))
      return wrapped(*args, **kwargs)
    return decorated
  return wrapper


def common_login(db_user):
  if login_user(LoginWrappedDBUser(db_user.uuid, db_user)):
    logger.debug('Successfully signed in as: %s (%s)' % (db_user.username, db_user.uuid))
    new_identity = QuayDeferredPermissionUser(db_user.uuid, 'user_uuid', {scopes.DIRECT_LOGIN})
    identity_changed.send(app, identity=new_identity)
    session['login_time'] = datetime.datetime.now()
    return True
  else:
    logger.debug('User could not be logged in, inactive?.')
    return False

def random_string():
  random = SystemRandom()
  return ''.join([random.choice(string.ascii_uppercase + string.digits) for _ in range(8)])

def list_files(path, extension):
  import os
  def matches(f):
    return os.path.splitext(f)[1] == '.' + extension

  def join_path(dp, f):
    # Remove the static/ prefix. It is added in the template.
    return os.path.join(dp, f)[len('static/'):]

  filepath = 'static/' + path
  return [join_path(dp, f) for dp, dn, files in os.walk(filepath) for f in files if matches(f)]

def render_page_template(name, **kwargs):
  debugging = app.config.get('DEBUGGING', False)
  if debugging:
    # If DEBUGGING is enabled, then we load the full set of individual JS and CSS files
    # from the file system.
    library_styles = list_files('lib', 'css')
    main_styles = list_files('css', 'css')
    library_scripts = list_files('lib', 'js')
    main_scripts = list_files('js', 'js')

    file_lists = [library_styles, main_styles, library_scripts, main_scripts]
    for file_list in file_lists:
      file_list.sort()
  else:
    library_styles = []
    main_styles = ['dist/quay-frontend.css']
    library_scripts = []
    main_scripts = ['dist/quay-frontend.min.js']

  use_cdn = app.config.get('USE_CDN', True)
  if request.args.get('use_cdn') is not None:
    use_cdn = request.args.get('use_cdn') == 'true'

  external_styles = get_external_css(local=not use_cdn)
  external_scripts = get_external_javascript(local=not use_cdn)

  def add_cachebusters(filenames):
    cachebusters = get_cache_busters()
    for filename in filenames:
      cache_buster = cachebusters.get(filename, random_string()) if not debugging else 'debugging'
      yield (filename, cache_buster)

  def get_oauth_config():
    oauth_config = {}
    for oauth_app in oauth_apps:
      oauth_config[oauth_app.key_name] = oauth_app.get_public_config()

    return oauth_config

  contact_href = None
  if len(app.config.get('CONTACT_INFO', [])) == 1:
    contact_href = app.config['CONTACT_INFO'][0]

  resp = make_response(render_template(name,
                                       route_data=json.dumps(get_route_data()),
                                       external_styles=external_styles,
                                       external_scripts=external_scripts,
                                       main_styles=add_cachebusters(main_styles),
                                       library_styles=add_cachebusters(library_styles),
                                       main_scripts=add_cachebusters(main_scripts),
                                       library_scripts=add_cachebusters(library_scripts),
                                       feature_set=json.dumps(features.get_features()),
                                       config_set=json.dumps(getFrontendVisibleConfig(app.config)),
                                       oauth_set=json.dumps(get_oauth_config()),
                                       scope_set=json.dumps(scopes.ALL_SCOPES),
                                       mixpanel_key=app.config.get('MIXPANEL_KEY', ''),
                                       google_analytics_key=app.config.get('GOOGLE_ANALYTICS_KEY', ''),
                                       sentry_public_dsn=app.config.get('SENTRY_PUBLIC_DSN', ''),
                                       is_debug=str(app.config.get('DEBUGGING', False)).lower(),
                                       show_chat=features.OLARK_CHAT,
                                       has_billing=features.BILLING,
                                       contact_href=contact_href,
                                       hostname=app.config['SERVER_HOSTNAME'],
                                       preferred_scheme=app.config['PREFERRED_URL_SCHEME'],
                                       **kwargs))

  resp.headers['X-FRAME-OPTIONS'] = 'DENY'
  return resp


def check_repository_usage(user_or_org, plan_found):
  private_repos = model.get_private_repo_count(user_or_org.username)
  repos_allowed = plan_found['privateRepos']

  if private_repos > repos_allowed:
    model.create_notification('over_private_usage', user_or_org, {'namespace': user_or_org.username})
  else:
    model.delete_notifications_by_kind(user_or_org, 'over_private_usage')


def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
                trigger=None, pull_robot_name=None, trigger_metadata=None):
  host = urlparse.urlparse(request.url).netloc
  repo_path = '%s/%s/%s' % (host, repository.namespace_user.username, repository.name)

  token = model.create_access_token(repository, 'write', kind='build-worker',
                                    friendly_name='Repository Build Token')
  logger.debug('Creating build %s with repo %s tags %s and dockerfile_id %s',
               build_name, repo_path, tags, dockerfile_id)

  job_config = {
    'docker_tags': tags,
    'registry': host,
    'build_subdir': subdir,
    'trigger_metadata': trigger_metadata or {},
    'is_manual': manual,
    'manual_user': get_authenticated_user().username if get_authenticated_user() else None
  }

  with app.config['DB_TRANSACTION_FACTORY'](db):
    build_request = model.create_repository_build(repository, token, job_config,
                                                  dockerfile_id, build_name,
                                                  trigger, pull_robot_name=pull_robot_name)

    json_data = json.dumps({
      'build_uuid': build_request.uuid,
      'pull_credentials': model.get_pull_credentials(pull_robot_name) if pull_robot_name else None
    })

    queue_id = dockerfile_build_queue.put([repository.namespace_user.username, repository.name],
                                          json_data,
                                          retries_remaining=3)

    build_request.queue_id = queue_id
    build_request.save()

  # Add the build to the repo's log.
  metadata = {
    'repo': repository.name,
    'namespace': repository.namespace_user.username,
    'fileid': dockerfile_id,
    'is_manual': manual,
    'manual_user': get_authenticated_user().username if get_authenticated_user() else None
  }

  if trigger:
    metadata['trigger_id'] = trigger.uuid
    metadata['config'] = json.loads(trigger.config)
    metadata['service'] = trigger.service.name

  model.log_action('build_dockerfile', repository.namespace_user.username, ip=request.remote_addr,
                   metadata=metadata, repository=repository)

  # Add notifications for the build queue.
  logger.debug('Adding notifications for repository')
  event_data = {
    'build_id': build_request.uuid,
    'build_name': build_name,
    'docker_tags': tags,
    'is_manual': manual,
    'manual_user': get_authenticated_user().username if get_authenticated_user() else None
  }

  if trigger:
    event_data['trigger_id'] = trigger.uuid
    event_data['trigger_kind'] = trigger.service.name

  spawn_notification(repository, 'build_queued', event_data,
                     subpage='build?current=%s' % build_request.uuid,
                     pathargs=['build', build_request.uuid])
  return build_request