Merge branch 'master' into star
This commit is contained in:
Jimmy Zelinskie
commit
917dd6b674
229 changed files with 10807 additions and 3003 deletions
|
|
@ -2,8 +2,10 @@ import bcrypt
|
|||
import logging
|
||||
import dateutil.parser
|
||||
import json
|
||||
import time
|
||||
|
||||
from datetime import datetime, timedelta, date
|
||||
from uuid import uuid4
|
||||
|
||||
from data.database import (User, Repository, Image, AccessToken, Role, RepositoryPermission,
|
||||
Visibility, RepositoryTag, EmailConfirmation, FederatedLogin,
|
||||
|
|
@ -14,7 +16,9 @@ from data.database import (User, Repository, Image, AccessToken, Role, Repositor
|
|||
ExternalNotificationEvent, ExternalNotificationMethod,
|
||||
RepositoryNotification, RepositoryAuthorizedEmail, TeamMemberInvite,
|
||||
DerivedImageStorage, ImageStorageTransformation, random_string_generator,
|
||||
db, BUILD_PHASE, QuayUserField, Star)
|
||||
db, BUILD_PHASE, QuayUserField, ImageStorageSignature, QueueItem,
|
||||
ImageStorageSignatureKind, validate_database_url, db_for_update,
|
||||
AccessTokenKind, Star)
|
||||
from peewee import JOIN_LEFT_OUTER, fn
|
||||
from util.validation import (validate_username, validate_email, validate_password,
|
||||
INVALID_PASSWORD_MESSAGE)
|
||||
|
|
@ -105,12 +109,15 @@ class TooManyLoginAttemptsException(Exception):
|
|||
self.retry_after = retry_after
|
||||
|
||||
|
||||
def _get_repository(namespace_name, repository_name):
|
||||
return (Repository
|
||||
.select(Repository, Namespace)
|
||||
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
|
||||
.where(Namespace.username == namespace_name, Repository.name == repository_name)
|
||||
.get())
|
||||
def _get_repository(namespace_name, repository_name, for_update=False):
|
||||
query = (Repository
|
||||
.select(Repository, Namespace)
|
||||
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
|
||||
.where(Namespace.username == namespace_name, Repository.name == repository_name))
|
||||
if for_update:
|
||||
query = db_for_update(query)
|
||||
|
||||
return query.get()
|
||||
|
||||
|
||||
def hash_password(password, salt=None):
|
||||
|
|
@ -164,8 +171,7 @@ def _create_user(username, email):
|
|||
pass
|
||||
|
||||
try:
|
||||
new_user = User.create(username=username, email=email)
|
||||
return new_user
|
||||
return User.create(username=username, email=email)
|
||||
except Exception as ex:
|
||||
raise DataModelException(ex.message)
|
||||
|
||||
|
|
@ -295,6 +301,9 @@ def delete_robot(robot_username):
|
|||
|
||||
|
||||
def _list_entity_robots(entity_name):
|
||||
""" Return the list of robots for the specified entity. This MUST return a query, not a
|
||||
materialized list so that callers can use db_for_update.
|
||||
"""
|
||||
return (User
|
||||
.select()
|
||||
.join(FederatedLogin)
|
||||
|
|
@ -901,14 +910,17 @@ def change_password(user, new_password):
|
|||
delete_notifications_by_kind(user, 'password_required')
|
||||
|
||||
|
||||
def change_username(user, new_username):
|
||||
def change_username(user_id, new_username):
|
||||
(username_valid, username_issue) = validate_username(new_username)
|
||||
if not username_valid:
|
||||
raise InvalidUsernameException('Invalid username %s: %s' % (new_username, username_issue))
|
||||
|
||||
with config.app_config['DB_TRANSACTION_FACTORY'](db):
|
||||
# Reload the user for update
|
||||
user = db_for_update(User.select().where(User.id == user_id)).get()
|
||||
|
||||
# Rename the robots
|
||||
for robot in _list_entity_robots(user.username):
|
||||
for robot in db_for_update(_list_entity_robots(user.username)):
|
||||
_, robot_shortname = parse_robot_username(robot.username)
|
||||
new_robot_name = format_robot_username(new_username, robot_shortname)
|
||||
robot.username = new_robot_name
|
||||
|
|
@ -924,6 +936,11 @@ def change_invoice_email(user, invoice_email):
|
|||
user.save()
|
||||
|
||||
|
||||
def change_user_tag_expiration(user, tag_expiration_s):
|
||||
user.removed_tag_expiration_s = tag_expiration_s
|
||||
user.save()
|
||||
|
||||
|
||||
def update_email(user, new_email, auto_verify=False):
|
||||
user.email = new_email
|
||||
user.verified = auto_verify
|
||||
|
|
@ -1087,6 +1104,26 @@ def get_repository(namespace_name, repository_name):
|
|||
return None
|
||||
|
||||
|
||||
def get_image(repo, dockerfile_id):
|
||||
try:
|
||||
return Image.get(Image.docker_image_id == dockerfile_id, Image.repository == repo)
|
||||
except Image.DoesNotExist:
|
||||
return None
|
||||
|
||||
|
||||
def find_child_image(repo, parent_image, command):
|
||||
try:
|
||||
return (Image.select()
|
||||
.join(ImageStorage)
|
||||
.switch(Image)
|
||||
.where(Image.ancestors % '%/' + parent_image.id + '/%',
|
||||
ImageStorage.command == command)
|
||||
.order_by(ImageStorage.created.desc())
|
||||
.get())
|
||||
except Image.DoesNotExist:
|
||||
return None
|
||||
|
||||
|
||||
def get_repo_image(namespace_name, repository_name, docker_image_id):
|
||||
def limit_to_image_id(query):
|
||||
return query.where(Image.docker_image_id == docker_image_id).limit(1)
|
||||
|
|
@ -1249,9 +1286,9 @@ def _find_or_link_image(existing_image, repository, username, translations, pref
|
|||
storage.locations = {placement.location.name
|
||||
for placement in storage.imagestorageplacement_set}
|
||||
|
||||
new_image = Image.create(docker_image_id=existing_image.docker_image_id,
|
||||
repository=repository, storage=storage,
|
||||
ancestors=new_image_ancestry)
|
||||
new_image = Image.create(docker_image_id=existing_image.docker_image_id,
|
||||
repository=repository, storage=storage,
|
||||
ancestors=new_image_ancestry)
|
||||
|
||||
logger.debug('Storing translation %s -> %s', existing_image.id, new_image.id)
|
||||
translations[existing_image.id] = new_image.id
|
||||
|
|
@ -1315,7 +1352,28 @@ def find_create_or_link_image(docker_image_id, repository, username, translation
|
|||
ancestors='/')
|
||||
|
||||
|
||||
def find_or_create_derived_storage(source, transformation_name, preferred_location):
|
||||
def find_or_create_storage_signature(storage, signature_kind):
|
||||
found = lookup_storage_signature(storage, signature_kind)
|
||||
if found is None:
|
||||
kind = ImageStorageSignatureKind.get(name=signature_kind)
|
||||
found = ImageStorageSignature.create(storage=storage, kind=kind)
|
||||
|
||||
return found
|
||||
|
||||
|
||||
def lookup_storage_signature(storage, signature_kind):
|
||||
kind = ImageStorageSignatureKind.get(name=signature_kind)
|
||||
try:
|
||||
return (ImageStorageSignature
|
||||
.select()
|
||||
.where(ImageStorageSignature.storage == storage,
|
||||
ImageStorageSignature.kind == kind)
|
||||
.get())
|
||||
except ImageStorageSignature.DoesNotExist:
|
||||
return None
|
||||
|
||||
|
||||
def find_derived_storage(source, transformation_name):
|
||||
try:
|
||||
found = (ImageStorage
|
||||
.select(ImageStorage, DerivedImageStorage)
|
||||
|
|
@ -1328,11 +1386,19 @@ def find_or_create_derived_storage(source, transformation_name, preferred_locati
|
|||
found.locations = {placement.location.name for placement in found.imagestorageplacement_set}
|
||||
return found
|
||||
except ImageStorage.DoesNotExist:
|
||||
logger.debug('Creating storage dervied from source: %s', source.uuid)
|
||||
trans = ImageStorageTransformation.get(name=transformation_name)
|
||||
new_storage = _create_storage(preferred_location)
|
||||
DerivedImageStorage.create(source=source, derivative=new_storage, transformation=trans)
|
||||
return new_storage
|
||||
return None
|
||||
|
||||
|
||||
def find_or_create_derived_storage(source, transformation_name, preferred_location):
|
||||
existing = find_derived_storage(source, transformation_name)
|
||||
if existing is not None:
|
||||
return existing
|
||||
|
||||
logger.debug('Creating storage dervied from source: %s', source.uuid)
|
||||
trans = ImageStorageTransformation.get(name=transformation_name)
|
||||
new_storage = _create_storage(preferred_location)
|
||||
DerivedImageStorage.create(source=source, derivative=new_storage, transformation=trans)
|
||||
return new_storage
|
||||
|
||||
|
||||
def delete_derived_storage_by_uuid(storage_uuid):
|
||||
|
|
@ -1401,7 +1467,7 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created
|
|||
Image.docker_image_id == docker_image_id))
|
||||
|
||||
try:
|
||||
fetched = query.get()
|
||||
fetched = db_for_update(query).get()
|
||||
except Image.DoesNotExist:
|
||||
raise DataModelException('No image with specified id and repository')
|
||||
|
||||
|
|
@ -1489,19 +1555,48 @@ def get_repository_images(namespace_name, repository_name):
|
|||
return _get_repository_images_base(namespace_name, repository_name, lambda q: q)
|
||||
|
||||
|
||||
def list_repository_tags(namespace_name, repository_name):
|
||||
return (RepositoryTag
|
||||
.select(RepositoryTag, Image)
|
||||
.join(Repository)
|
||||
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
|
||||
.switch(RepositoryTag)
|
||||
.join(Image)
|
||||
.where(Repository.name == repository_name, Namespace.username == namespace_name))
|
||||
def _tag_alive(query):
|
||||
return query.where((RepositoryTag.lifetime_end_ts >> None) |
|
||||
(RepositoryTag.lifetime_end_ts > int(time.time())))
|
||||
|
||||
|
||||
def list_repository_tags(namespace_name, repository_name, include_hidden=False):
|
||||
query = _tag_alive(RepositoryTag
|
||||
.select(RepositoryTag, Image)
|
||||
.join(Repository)
|
||||
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
|
||||
.switch(RepositoryTag)
|
||||
.join(Image)
|
||||
.where(Repository.name == repository_name,
|
||||
Namespace.username == namespace_name))
|
||||
|
||||
if not include_hidden:
|
||||
query = query.where(RepositoryTag.hidden == False)
|
||||
|
||||
return query
|
||||
|
||||
|
||||
def _garbage_collect_tags(namespace_name, repository_name):
|
||||
to_delete = (RepositoryTag
|
||||
.select(RepositoryTag.id)
|
||||
.join(Repository)
|
||||
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
|
||||
.where(Repository.name == repository_name, Namespace.username == namespace_name,
|
||||
~(RepositoryTag.lifetime_end_ts >> None),
|
||||
(RepositoryTag.lifetime_end_ts + Namespace.removed_tag_expiration_s) <=
|
||||
int(time.time())))
|
||||
|
||||
(RepositoryTag
|
||||
.delete()
|
||||
.where(RepositoryTag.id << to_delete)
|
||||
.execute())
|
||||
|
||||
|
||||
def garbage_collect_repository(namespace_name, repository_name):
|
||||
storage_id_whitelist = {}
|
||||
|
||||
_garbage_collect_tags(namespace_name, repository_name)
|
||||
|
||||
with config.app_config['DB_TRANSACTION_FACTORY'](db):
|
||||
# TODO (jake): We could probably select this and all the images in a single query using
|
||||
# a different kind of join.
|
||||
|
|
@ -1535,12 +1630,10 @@ def garbage_collect_repository(namespace_name, repository_name):
|
|||
|
||||
if len(to_remove) > 0:
|
||||
logger.info('Garbage collecting storage for images: %s', to_remove)
|
||||
garbage_collect_storage(storage_id_whitelist)
|
||||
|
||||
return len(to_remove)
|
||||
_garbage_collect_storage(storage_id_whitelist)
|
||||
|
||||
|
||||
def garbage_collect_storage(storage_id_whitelist):
|
||||
def _garbage_collect_storage(storage_id_whitelist):
|
||||
if len(storage_id_whitelist) == 0:
|
||||
return
|
||||
|
||||
|
|
@ -1632,10 +1725,10 @@ def garbage_collect_storage(storage_id_whitelist):
|
|||
|
||||
def get_tag_image(namespace_name, repository_name, tag_name):
|
||||
def limit_to_tag(query):
|
||||
return (query
|
||||
.switch(Image)
|
||||
.join(RepositoryTag)
|
||||
.where(RepositoryTag.name == tag_name))
|
||||
return _tag_alive(query
|
||||
.switch(Image)
|
||||
.join(RepositoryTag)
|
||||
.where(RepositoryTag.name == tag_name))
|
||||
|
||||
images = _get_repository_images_base(namespace_name, repository_name, limit_to_tag)
|
||||
if not images:
|
||||
|
|
@ -1643,7 +1736,6 @@ def get_tag_image(namespace_name, repository_name, tag_name):
|
|||
else:
|
||||
return images[0]
|
||||
|
||||
|
||||
def get_image_by_id(namespace_name, repository_name, docker_image_id):
|
||||
image = get_repo_image_extended(namespace_name, repository_name, docker_image_id)
|
||||
if not image:
|
||||
|
|
@ -1672,45 +1764,69 @@ def get_parent_images(namespace_name, repository_name, image_obj):
|
|||
|
||||
def create_or_update_tag(namespace_name, repository_name, tag_name,
|
||||
tag_docker_image_id):
|
||||
try:
|
||||
repo = _get_repository(namespace_name, repository_name)
|
||||
except Repository.DoesNotExist:
|
||||
raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name))
|
||||
|
||||
try:
|
||||
image = Image.get(Image.docker_image_id == tag_docker_image_id, Image.repository == repo)
|
||||
except Image.DoesNotExist:
|
||||
raise DataModelException('Invalid image with id: %s' % tag_docker_image_id)
|
||||
with config.app_config['DB_TRANSACTION_FACTORY'](db):
|
||||
try:
|
||||
repo = _get_repository(namespace_name, repository_name)
|
||||
except Repository.DoesNotExist:
|
||||
raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name))
|
||||
|
||||
try:
|
||||
tag = RepositoryTag.get(RepositoryTag.repository == repo, RepositoryTag.name == tag_name)
|
||||
tag.image = image
|
||||
tag.save()
|
||||
except RepositoryTag.DoesNotExist:
|
||||
tag = RepositoryTag.create(repository=repo, image=image, name=tag_name)
|
||||
try:
|
||||
image = Image.get(Image.docker_image_id == tag_docker_image_id, Image.repository == repo)
|
||||
except Image.DoesNotExist:
|
||||
raise DataModelException('Invalid image with id: %s' % tag_docker_image_id)
|
||||
|
||||
return tag
|
||||
now_ts = int(time.time())
|
||||
|
||||
try:
|
||||
# When we move a tag, we really end the timeline of the old one and create a new one
|
||||
query = _tag_alive(RepositoryTag
|
||||
.select()
|
||||
.where(RepositoryTag.repository == repo, RepositoryTag.name == tag_name))
|
||||
tag = query.get()
|
||||
tag.lifetime_end_ts = now_ts
|
||||
tag.save()
|
||||
except RepositoryTag.DoesNotExist:
|
||||
# No tag that needs to be ended
|
||||
pass
|
||||
|
||||
return RepositoryTag.create(repository=repo, image=image, name=tag_name,
|
||||
lifetime_start_ts=now_ts)
|
||||
|
||||
|
||||
def delete_tag(namespace_name, repository_name, tag_name):
|
||||
try:
|
||||
found = (RepositoryTag
|
||||
.select()
|
||||
.join(Repository)
|
||||
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
|
||||
.where(Repository.name == repository_name, Namespace.username == namespace_name,
|
||||
RepositoryTag.name == tag_name)
|
||||
.get())
|
||||
with config.app_config['DB_TRANSACTION_FACTORY'](db):
|
||||
try:
|
||||
query = _tag_alive(RepositoryTag
|
||||
.select(RepositoryTag, Repository)
|
||||
.join(Repository)
|
||||
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
|
||||
.where(Repository.name == repository_name,
|
||||
Namespace.username == namespace_name,
|
||||
RepositoryTag.name == tag_name))
|
||||
found = db_for_update(query).get()
|
||||
except RepositoryTag.DoesNotExist:
|
||||
msg = ('Invalid repository tag \'%s\' on repository \'%s/%s\'' %
|
||||
(tag_name, namespace_name, repository_name))
|
||||
raise DataModelException(msg)
|
||||
|
||||
except RepositoryTag.DoesNotExist:
|
||||
msg = ('Invalid repository tag \'%s\' on repository \'%s/%s\'' %
|
||||
(tag_name, namespace_name, repository_name))
|
||||
raise DataModelException(msg)
|
||||
|
||||
found.delete_instance()
|
||||
found.lifetime_end_ts = int(time.time())
|
||||
found.save()
|
||||
|
||||
|
||||
def delete_all_repository_tags(namespace_name, repository_name):
|
||||
def create_temporary_hidden_tag(repo, image, expiration_s):
|
||||
""" Create a tag with a defined timeline, that will not appear in the UI or CLI. Returns the name
|
||||
of the temporary tag. """
|
||||
now_ts = int(time.time())
|
||||
expire_ts = now_ts + expiration_s
|
||||
tag_name = str(uuid4())
|
||||
RepositoryTag.create(repository=repo, image=image, name=tag_name, lifetime_start_ts=now_ts,
|
||||
lifetime_end_ts=expire_ts, hidden=True)
|
||||
return tag_name
|
||||
|
||||
|
||||
def purge_all_repository_tags(namespace_name, repository_name):
|
||||
""" Immediately purge all repository tags without respecting the lifeline procedure """
|
||||
try:
|
||||
repo = _get_repository(namespace_name, repository_name)
|
||||
except Repository.DoesNotExist:
|
||||
|
|
@ -1825,7 +1941,7 @@ def set_team_repo_permission(team_name, namespace_name, repository_name,
|
|||
|
||||
def purge_repository(namespace_name, repository_name):
|
||||
# Delete all tags to allow gc to reclaim storage
|
||||
delete_all_repository_tags(namespace_name, repository_name)
|
||||
purge_all_repository_tags(namespace_name, repository_name)
|
||||
|
||||
# Gc to remove the images and storage
|
||||
garbage_collect_repository(namespace_name, repository_name)
|
||||
|
|
@ -1845,10 +1961,14 @@ def get_private_repo_count(username):
|
|||
.count())
|
||||
|
||||
|
||||
def create_access_token(repository, role):
|
||||
def create_access_token(repository, role, kind=None, friendly_name=None):
|
||||
role = Role.get(Role.name == role)
|
||||
kind_ref = None
|
||||
if kind is not None:
|
||||
kind_ref = AccessTokenKind.get(AccessTokenKind.name == kind)
|
||||
|
||||
new_token = AccessToken.create(repository=repository, temporary=True,
|
||||
role=role)
|
||||
role=role, kind=kind_ref, friendly_name=friendly_name)
|
||||
return new_token
|
||||
|
||||
|
||||
|
|
@ -1967,10 +2087,10 @@ def create_repository_build(repo, access_token, job_config_obj, dockerfile_id,
|
|||
pull_robot = lookup_robot(pull_robot_name)
|
||||
|
||||
return RepositoryBuild.create(repository=repo, access_token=access_token,
|
||||
job_config=json.dumps(job_config_obj),
|
||||
display_name=display_name, trigger=trigger,
|
||||
resource_key=dockerfile_id,
|
||||
pull_robot=pull_robot)
|
||||
job_config=json.dumps(job_config_obj),
|
||||
display_name=display_name, trigger=trigger,
|
||||
resource_key=dockerfile_id,
|
||||
pull_robot=pull_robot)
|
||||
|
||||
|
||||
def get_pull_robot_name(trigger):
|
||||
|
|
@ -2255,11 +2375,20 @@ def delete_user(user):
|
|||
# TODO: also delete any repository data associated
|
||||
|
||||
|
||||
def check_health():
|
||||
def check_health(app_config):
|
||||
# Attempt to connect to the database first. If the DB is not responding,
|
||||
# using the validate_database_url will timeout quickly, as opposed to
|
||||
# making a normal connect which will just hang (thus breaking the health
|
||||
# check).
|
||||
try:
|
||||
validate_database_url(app_config['DB_URI'], connect_timeout=3)
|
||||
except Exception:
|
||||
logger.exception('Could not connect to the database')
|
||||
return False
|
||||
|
||||
# We will connect to the db, check that it contains some log entry kinds
|
||||
try:
|
||||
found_count = LogEntryKind.select().count()
|
||||
return found_count > 0
|
||||
return bool(list(LogEntryKind.select().limit(1)))
|
||||
except:
|
||||
return False
|
||||
|
||||
|
|
@ -2365,6 +2494,32 @@ def confirm_team_invite(code, user):
|
|||
found.delete_instance()
|
||||
return (team, inviter)
|
||||
|
||||
def cancel_repository_build(build):
|
||||
with config.app_config['DB_TRANSACTION_FACTORY'](db):
|
||||
# Reload the build for update.
|
||||
try:
|
||||
build = db_for_update(RepositoryBuild.select().where(RepositoryBuild.id == build.id)).get()
|
||||
except RepositoryBuild.DoesNotExist:
|
||||
return False
|
||||
|
||||
if build.phase != BUILD_PHASE.WAITING or not build.queue_item:
|
||||
return False
|
||||
|
||||
# Load the build queue item for update.
|
||||
try:
|
||||
queue_item = db_for_update(QueueItem.select()
|
||||
.where(QueueItem.id == build.queue_item.id)).get()
|
||||
except QueueItem.DoesNotExist:
|
||||
return False
|
||||
|
||||
# Check the queue item.
|
||||
if not queue_item.available or queue_item.retries_remaining == 0:
|
||||
return False
|
||||
|
||||
# Delete the queue item and build.
|
||||
queue_item.delete_instance(recursive=True)
|
||||
build.delete_instance()
|
||||
return True
|
||||
|
||||
def get_repository_usage():
|
||||
one_month_ago = date.today() - timedelta(weeks=4)
|
||||
|
|
|
|||
Reference in a new issue