vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'master' of https://bitbucket.org/yackob03/quay

Browse source
This commit is contained in:
Joseph Schorr 2013-10-01 00:41:52 -04:00
commit b7d43e1593
10 changed files with 184 additions and 115 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.ebextensions/python27.config
View file

@ -36,3 +36,5 @@ container_commands:
option_settings: option_settings:
"aws:elasticbeanstalk:container:python:staticfiles": "aws:elasticbeanstalk:container:python:staticfiles":
"/static": "static/" "/static": "static/"
"aws:elasticbeanstalk:application:environment":
"STACK": "prod"

8
app.py
View file

@ -1,13 +1,17 @@
import logging import logging
import os
from flask import Flask from flask import Flask
from flask.ext.principal import Principal from flask.ext.principal import Principal
from flask.ext.login import LoginManager from flask.ext.login import LoginManager
from flask.ext.mail import Mail from flask.ext.mail import Mail
from config import ProductionConfig from config import ProductionConfig, DebugConfig
app = Flask(__name__) app = Flask(__name__)
app.config.from_object(ProductionConfig())
is_prod = os.environ.get('STACK', '').strip().lower().startswith('prod')
config_object = ProductionConfig() if is_prod else DebugConfig()
app.config.from_object(config_object)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)

6
application.py
View file

@ -11,9 +11,7 @@ import endpoints.registry
# Remove this for prod config # Remove this for prod config
application.debug = True application.debug = True
if __name__ == '__main__': logging.basicConfig(**application.config['LOGGING_CONFIG'])
FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - ' + \
'%(funcName)s - %(message)s'
logging.basicConfig(format=FORMAT, level=logging.DEBUG)
if __name__ == '__main__':
application.run(port=5001, debug=True) application.run(port=5001, debug=True)

56
config.py
View file

@ -1,3 +1,13 @@
import logging
import sys
from peewee import MySQLDatabase, SqliteDatabase
LOG_FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - ' + \
'%(funcName)s - %(message)s'
class FlaskConfig(object): class FlaskConfig(object):
SECRET_KEY = '1cb18882-6d12-440d-a4cc-b7430fb5f884' SECRET_KEY = '1cb18882-6d12-440d-a4cc-b7430fb5f884'
@ -13,5 +23,49 @@ class MailConfig(object):
TESTING = False TESTING = False
class ProductionConfig(FlaskConfig, MailConfig): class SQLiteDB(object):
DB_NAME = 'test.db'
DB_CONNECTION_ARGS = {
'threadlocals': True
}
DB_DRIVER = SqliteDatabase
class RDSMySQL(object):
DB_NAME = 'quay'
DB_CONNECTION_ARGS = {
'host': 'fluxmonkeylogin.cb0vumcygprn.us-east-1.rds.amazonaws.com',
'user': 'fluxmonkey',
'passwd': '8eifM#uoZ85xqC^',
'threadlocals': True,
}
DB_DRIVER = MySQLDatabase
class S3Storage(object):
AWS_ACCESS_KEY = 'AKIAJWZWUIS24TWSMWRA'
AWS_SECRET_KEY = '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'
REGISTRY_S3_BUCKET = 'quay-registry'
STORAGE_KIND = 's3'
class LocalStorage(object):
STORAGE_KIND = 'local'
LOCAL_STORAGE_DIR = '/tmp/registry'
class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB):
REGISTRY_SERVER = 'localhost:5000' REGISTRY_SERVER = 'localhost:5000'
LOGGING_CONFIG = {
'level': logging.DEBUG,
'format': LOG_FORMAT
}
class ProductionConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL):
REGISTRY_SERVER = 'quay.io'
LOGGING_CONFIG = {
'stream': sys.stderr,
'level': logging.DEBUG,
'format': LOG_FORMAT,
}

15
data/database.py
View file

@ -1,12 +1,15 @@
import string import string
from random import SystemRandom
from random import SystemRandom
from datetime import datetime
from peewee import * from peewee import *
from peewee import create_model_tables from peewee import create_model_tables
from datetime import datetime from app import app
db = SqliteDatabase('test.db', threadlocals=True)
db = app.config['DB_DRIVER'](app.config['DB_NAME'],
**app.config['DB_CONNECTION_ARGS'])
class BaseModel(Model): class BaseModel(Model):
@ -29,7 +32,7 @@ class Repository(BaseModel):
namespace = CharField() namespace = CharField()
name = CharField() name = CharField()
visibility = ForeignKeyField(Visibility) visibility = ForeignKeyField(Visibility)
description = CharField(null=True) description = TextField(null=True)
class Meta: class Meta:
database = db database = db
@ -86,11 +89,11 @@ class Image(BaseModel):
image_id = CharField() image_id = CharField()
checksum = CharField(null=True) checksum = CharField(null=True)
created = DateTimeField(null=True) created = DateTimeField(null=True)
comment = CharField(null=True) comment = TextField(null=True)
repository = ForeignKeyField(Repository) repository = ForeignKeyField(Repository)
# '/' separated list of ancestory ids, e.g. /1/2/6/7/10/ # '/' separated list of ancestory ids, e.g. /1/2/6/7/10/
ancestors = CharField(index=True, default='/', max_length=65535) ancestors = CharField(index=True, default='/', max_length=64535)
class Meta: class Meta:
database = db database = db

4
data/model.py
View file

@ -112,9 +112,7 @@ def get_visible_repositories(username=None):
def get_matching_repositories(repo_term, username=None): def get_matching_repositories(repo_term, username=None):
visible = get_visible_repositories(username) visible = get_visible_repositories(username)
search_clauses = (Repository.name ** ('%' + repo_term + '%') | search_clauses = (Repository.name ** ('%' + repo_term + '%') |
Repository.namespace ** ('%' + repo_term + '%') | Repository.namespace ** ('%' + repo_term + '%'))
Repository.description ** ('%' + repo_term + '%'))
final = visible.where(search_clauses).limit(10) final = visible.where(search_clauses).limit(10)
return list(final) return list(final)

1
requirements-nover.txt
View file

@ -6,3 +6,4 @@ Flask-Login
Flask-Mail Flask-Mail
python-dateutil python-dateutil
boto boto
pymysql

2
requirements.txt
View file

@ -4,11 +4,11 @@ Flask-Mail==0.9.0
Flask-Principal==0.4.0 Flask-Principal==0.4.0
Jinja2==2.7.1 Jinja2==2.7.1
MarkupSafe==0.18 MarkupSafe==0.18
PyMySQL==0.5
Werkzeug==0.9.4 Werkzeug==0.9.4
argparse==1.2.1 argparse==1.2.1
blinker==1.3 blinker==1.3
boto==2.13.3 boto==2.13.3
distribute==0.6.34
itsdangerous==0.23 itsdangerous==0.23
peewee==2.1.4 peewee==2.1.4
py-bcrypt==0.4 py-bcrypt==0.4

200
storage/__init__.py
View file

@ -1,119 +1,123 @@
import logging
import contextlib import contextlib
import tempfile import tempfile
from app import app
__all__ = ['load'] __all__ = ['load']
logger = logging.getLogger(__name__)
class Storage(object): class Storage(object):
"""Storage is organized as follow: """Storage is organized as follow:
$ROOT/images/<image_id>/json $ROOT/images/<image_id>/json
$ROOT/images/<image_id>/layer $ROOT/images/<image_id>/layer
$ROOT/repositories/<namespace>/<repository_name>/<tag_name> $ROOT/repositories/<namespace>/<repository_name>/<tag_name>
""" """
# Useful if we want to change those locations later without rewriting # Useful if we want to change those locations later without rewriting
# the code which uses Storage # the code which uses Storage
repositories = 'repositories' repositories = 'repositories'
images = 'images' images = 'images'
# Set the IO buffer to 64kB # Set the IO buffer to 64kB
buffer_size = 64 * 1024 buffer_size = 64 * 1024
#FIXME(samalba): Move all path resolver in each module (out of the base) #FIXME(samalba): Move all path resolver in each module (out of the base)
def images_list_path(self, namespace, repository): def images_list_path(self, namespace, repository):
return '{0}/{1}/{2}/_images_list'.format(self.repositories, return '{0}/{1}/{2}/_images_list'.format(self.repositories,
namespace, namespace,
repository) repository)
def image_json_path(self, namespace, repository, image_id): def image_json_path(self, namespace, repository, image_id):
return '{0}/{1}/{2}/{3}/json'.format(self.images, namespace, return '{0}/{1}/{2}/{3}/json'.format(self.images, namespace,
repository, image_id) repository, image_id)
def image_mark_path(self, namespace, repository, image_id): def image_mark_path(self, namespace, repository, image_id):
return '{0}/{1}/{2}/{3}/_inprogress'.format(self.images, namespace, return '{0}/{1}/{2}/{3}/_inprogress'.format(self.images, namespace,
repository, image_id) repository, image_id)
def image_checksum_path(self, namespace, repository, image_id): def image_checksum_path(self, namespace, repository, image_id):
return '{0}/{1}/{2}/{3}/_checksum'.format(self.images, namespace, return '{0}/{1}/{2}/{3}/_checksum'.format(self.images, namespace,
repository, image_id) repository, image_id)
def image_layer_path(self, namespace, repository, image_id): def image_layer_path(self, namespace, repository, image_id):
return '{0}/{1}/{2}/{3}/layer'.format(self.images, namespace, return '{0}/{1}/{2}/{3}/layer'.format(self.images, namespace,
repository, image_id) repository, image_id)
def image_ancestry_path(self, namespace, repository, image_id): def image_ancestry_path(self, namespace, repository, image_id):
return '{0}/{1}/{2}/{3}/ancestry'.format(self.images, namespace, return '{0}/{1}/{2}/{3}/ancestry'.format(self.images, namespace,
repository, image_id) repository, image_id)
def tag_path(self, namespace, repository, tagname=None): def tag_path(self, namespace, repository, tagname=None):
if not tagname: if not tagname:
return '{0}/{1}/{2}'.format(self.repositories, return '{0}/{1}/{2}'.format(self.repositories,
namespace, namespace,
repository) repository)
return '{0}/{1}/{2}/tag_{3}'.format(self.repositories, return '{0}/{1}/{2}/tag_{3}'.format(self.repositories,
namespace, namespace,
repository, repository,
tagname) tagname)
def index_images_path(self, namespace, repository): def index_images_path(self, namespace, repository):
return '{0}/{1}/{2}/_index_images'.format(self.repositories, return '{0}/{1}/{2}/_index_images'.format(self.repositories,
namespace, namespace,
repository) repository)
def get_content(self, path): def get_content(self, path):
raise NotImplementedError raise NotImplementedError
def put_content(self, path, content): def put_content(self, path, content):
raise NotImplementedError raise NotImplementedError
def stream_read(self, path): def stream_read(self, path):
raise NotImplementedError raise NotImplementedError
def stream_write(self, path, fp): def stream_write(self, path, fp):
raise NotImplementedError raise NotImplementedError
def list_directory(self, path=None): def list_directory(self, path=None):
raise NotImplementedError raise NotImplementedError
def exists(self, path): def exists(self, path):
raise NotImplementedError raise NotImplementedError
def remove(self, path): def remove(self, path):
raise NotImplementedError raise NotImplementedError
def get_size(self, path): def get_size(self, path):
raise NotImplementedError raise NotImplementedError
@contextlib.contextmanager @contextlib.contextmanager
def store_stream(stream): def store_stream(stream):
"""Stores the entire stream to a temporary file.""" """Stores the entire stream to a temporary file."""
tmpf = tempfile.TemporaryFile() tmpf = tempfile.TemporaryFile()
while True: while True:
try: try:
buf = stream.read(4096) buf = stream.read(4096)
if not buf: if not buf:
break break
tmpf.write(buf) tmpf.write(buf)
except IOError: except IOError:
break break
tmpf.seek(0) tmpf.seek(0)
yield tmpf yield tmpf
tmpf.close() tmpf.close()
def temp_store_handler(): def temp_store_handler():
tmpf = tempfile.TemporaryFile() tmpf = tempfile.TemporaryFile()
def fn(buf): def fn(buf):
try: try:
tmpf.write(buf) tmpf.write(buf)
except IOError: except IOError:
pass pass
return tmpf, fn return tmpf, fn
from local import LocalStorage from local import LocalStorage
@ -124,20 +128,24 @@ _storage = {}
def load(kind=None): def load(kind=None):
"""Returns the right storage class according to the configuration.""" """Returns the right storage class according to the configuration."""
global _storage global _storage
# TODO hard code to local for now # TODO hard code to local for now
kind = 'local' kind = app.config['STORAGE_KIND']
# if not kind: # if not kind:
# kind = cfg.storage.lower() # kind = cfg.storage.lower()
if kind in _storage: if kind in _storage:
return _storage[kind] return _storage[kind]
if kind == 's3': if kind == 's3':
store = S3Storage('/registry', 'access_key', 'secret_key', 'bucket') logger.debug('Using s3 storage.')
elif kind == 'local': store = S3Storage('', app.config['AWS_ACCESS_KEY'],
store = LocalStorage('/tmp/registry') app.config['AWS_SECRET_KEY'],
else: app.config['REGISTRY_S3_BUCKET'])
raise ValueError('Not supported storage \'{0}\''.format(kind)) elif kind == 'local':
_storage[kind] = store logger.debug('Using local storage.')
return store store = LocalStorage(app.config['LOCAL_STORAGE_DIR'])
else:
raise ValueError('Not supported storage \'{0}\''.format(kind))
_storage[kind] = store
return store

1
wsgi.conf
View file

@ -3,6 +3,7 @@ LoadModule wsgi_module modules/mod_wsgi.so
WSGIPythonHome /opt/python/run/venv WSGIPythonHome /opt/python/run/venv
WSGISocketPrefix run/wsgi WSGISocketPrefix run/wsgi
WSGIRestrictEmbedded On WSGIRestrictEmbedded On
WSGIPassAuthorization On
<VirtualHost *:80> <VirtualHost *:80>