vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'master' into pagesnew

Browse source
This commit is contained in:
Joseph Schorr 2015-03-05 14:22:10 -05:00
commit 86447c0a99
52 changed files with 553 additions and 211 deletions
Download patch file Download diff file Expand all files Collapse all files

2
data/database.py
View file

@ -527,7 +527,7 @@ class RepositoryBuild(BaseModel):
trigger = ForeignKeyField(RepositoryBuildTrigger, null=True, index=True)
pull_robot = QuayUserField(null=True, related_name='buildpullrobot')
logs_archived = BooleanField(default=False)
queue_item = ForeignKeyField(QueueItem, null=True, index=True)
queue_id = CharField(null=True, index=True)
class LogEntryKind(BaseModel):

34
data/migrations/versions/707d5191eda_change_build_queue_reference_from_.py Normal file
View file

@ -0,0 +1,34 @@
"""Change build queue reference from foreign key to an id.
Revision ID: 707d5191eda
Revises: 4ef04c61fcf9
Create Date: 2015-02-23 12:36:33.814528
"""
# revision identifiers, used by Alembic.
revision = '707d5191eda'
down_revision = '4ef04c61fcf9'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
def upgrade(tables):
### commands auto generated by Alembic - please adjust! ###
op.add_column('repositorybuild', sa.Column('queue_id', sa.String(length=255), nullable=True))
op.create_index('repositorybuild_queue_id', 'repositorybuild', ['queue_id'], unique=False)
op.drop_constraint(u'fk_repositorybuild_queue_item_id_queueitem', 'repositorybuild', type_='foreignkey')
op.drop_index('repositorybuild_queue_item_id', table_name='repositorybuild')
op.drop_column('repositorybuild', 'queue_item_id')
### end Alembic commands ###
def downgrade(tables):
### commands auto generated by Alembic - please adjust! ###
op.add_column('repositorybuild', sa.Column('queue_item_id', mysql.INTEGER(display_width=11), autoincrement=False, nullable=True))
op.create_foreign_key(u'fk_repositorybuild_queue_item_id_queueitem', 'repositorybuild', 'queueitem', ['queue_item_id'], ['id'])
op.create_index('repositorybuild_queue_item_id', 'repositorybuild', ['queue_item_id'], unique=False)
op.drop_index('repositorybuild_queue_id', table_name='repositorybuild')
op.drop_column('repositorybuild', 'queue_id')
### end Alembic commands ###

92
data/model/legacy.py
View file

@ -541,7 +541,7 @@ def confirm_user_email(code):
old_email = None
new_email = code.new_email
if new_email:
if new_email and new_email != old_email:
if find_user_by_email(new_email):
raise DataModelException('E-mail address already used.')
@ -903,6 +903,7 @@ def change_password(user, new_password):
raise InvalidPasswordException(INVALID_PASSWORD_MESSAGE)
pw_hash = hash_password(new_password)
user.invalid_login_attempts = 0
user.password_hash = pw_hash
user.save()
@ -1577,19 +1578,16 @@ def list_repository_tags(namespace_name, repository_name, include_hidden=False):
def _garbage_collect_tags(namespace_name, repository_name):
to_delete = (RepositoryTag
.select(RepositoryTag.id)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Repository.name == repository_name, Namespace.username == namespace_name,
~(RepositoryTag.lifetime_end_ts >> None),
(RepositoryTag.lifetime_end_ts + Namespace.removed_tag_expiration_s) <=
int(time.time())))
# We do this without using a join to prevent holding read locks on the repository table
repo = _get_repository(namespace_name, repository_name)
now = int(time.time())
(RepositoryTag
.delete()
.where(RepositoryTag.id << to_delete)
.execute())
(RepositoryTag
.delete()
.where(RepositoryTag.repository == repo,
~(RepositoryTag.lifetime_end_ts >> None),
(RepositoryTag.lifetime_end_ts + repo.namespace_user.removed_tag_expiration_s) <= now)
.execute())
def garbage_collect_repository(namespace_name, repository_name):
@ -1659,6 +1657,7 @@ def _garbage_collect_storage(storage_id_whitelist):
logger.debug('Garbage collecting derived storage from candidates: %s', storage_id_whitelist)
with config.app_config['DB_TRANSACTION_FACTORY'](db):
# Find out which derived storages will be removed, and add them to the whitelist
# The comma after ImageStorage.id is VERY important, it makes it a tuple, which is a sequence
orphaned_from_candidates = list(orphaned_storage_query(ImageStorage.select(ImageStorage.id),
storage_id_whitelist,
(ImageStorage.id,)))
@ -1698,22 +1697,32 @@ def _garbage_collect_storage(storage_id_whitelist):
paths_to_remove = placements_query_to_paths_set(placements_to_remove.clone())
# Remove the placements for orphaned storages
placements_subquery = list(placements_to_remove.clone().select(ImageStoragePlacement.id))
if len(placements_subquery) > 0:
(ImageStoragePlacement
.delete()
.where(ImageStoragePlacement.id << list(placements_subquery))
.execute())
placements_subquery = (placements_to_remove
.clone()
.select(ImageStoragePlacement.id)
.alias('ps'))
inner = (ImageStoragePlacement
.select(placements_subquery.c.id)
.from_(placements_subquery))
placements_removed = (ImageStoragePlacement
.delete()
.where(ImageStoragePlacement.id << inner)
.execute())
logger.debug('Removed %s image storage placements', placements_removed)
# Remove the all orphaned storages
orphaned_storages = list(orphaned_storage_query(ImageStorage.select(ImageStorage.id),
storage_id_whitelist,
(ImageStorage.id,)))
if len(orphaned_storages) > 0:
(ImageStorage
.delete()
.where(ImageStorage.id << orphaned_storages)
.execute())
# Remove all orphaned storages
# The comma after ImageStorage.id is VERY important, it makes it a tuple, which is a sequence
orphaned_storages = orphaned_storage_query(ImageStorage.select(ImageStorage.id),
storage_id_whitelist,
(ImageStorage.id,)).alias('osq')
orphaned_storage_inner = (ImageStorage
.select(orphaned_storages.c.id)
.from_(orphaned_storages))
storages_removed = (ImageStorage
.delete()
.where(ImageStorage.id << orphaned_storage_inner)
.execute())
logger.debug('Removed %s image storage records', storages_removed)
# We are going to make the conscious decision to not delete image storage blobs inside
# transactions.
@ -1778,11 +1787,15 @@ def create_or_update_tag(namespace_name, repository_name, tag_name,
now_ts = int(time.time())
created = RepositoryTag.create(repository=repo, image=image, name=tag_name,
lifetime_start_ts=now_ts)
try:
# When we move a tag, we really end the timeline of the old one and create a new one
query = _tag_alive(RepositoryTag
.select()
.where(RepositoryTag.repository == repo, RepositoryTag.name == tag_name))
.where(RepositoryTag.repository == repo, RepositoryTag.name == tag_name,
RepositoryTag.id != created.id))
tag = query.get()
tag.lifetime_end_ts = now_ts
tag.save()
@ -1790,8 +1803,7 @@ def create_or_update_tag(namespace_name, repository_name, tag_name,
# No tag that needs to be ended
pass
return RepositoryTag.create(repository=repo, image=image, name=tag_name,
lifetime_start_ts=now_ts)
return created
def delete_tag(namespace_name, repository_name, tag_name):
@ -2494,7 +2506,7 @@ def confirm_team_invite(code, user):
found.delete_instance()
return (team, inviter)
def cancel_repository_build(build):
def cancel_repository_build(build, work_queue):
with config.app_config['DB_TRANSACTION_FACTORY'](db):
# Reload the build for update.
try:
@ -2502,22 +2514,14 @@ def cancel_repository_build(build):
except RepositoryBuild.DoesNotExist:
return False
if build.phase != BUILD_PHASE.WAITING or not build.queue_item:
if build.phase != BUILD_PHASE.WAITING or not build.queue_id:
return False
# Load the build queue item for update.
try:
queue_item = db_for_update(QueueItem.select()
.where(QueueItem.id == build.queue_item.id)).get()
except QueueItem.DoesNotExist:
# Try to cancel the queue item.
if not work_queue.cancel(build.queue_id):
return False
# Check the queue item.
if not queue_item.available or queue_item.retries_remaining == 0:
return False
# Delete the queue item and build.
queue_item.delete_instance(recursive=True)
# Delete the build row.
build.delete_instance()
return True

37
data/queue.py
View file

@ -82,10 +82,19 @@ class WorkQueue(object):
self._reporter(self._currently_processing, running_count,
running_count + available_not_running_count)
def has_retries_remaining(self, item_id):
""" Returns whether the queue item with the given id has any retries remaining. If the
queue item does not exist, returns False. """
with self._transaction_factory(db):
try:
return QueueItem.get(id=item_id).retries_remaining > 0
except QueueItem.DoesNotExist:
return False
def put(self, canonical_name_list, message, available_after=0, retries_remaining=5):
"""
Put an item, if it shouldn't be processed for some number of seconds,
specify that amount as available_after.
specify that amount as available_after. Returns the ID of the queue item added.
"""
params = {
@ -98,7 +107,7 @@ class WorkQueue(object):
params['available_after'] = available_date
with self._transaction_factory(db):
return QueueItem.create(**params)
return str(QueueItem.create(**params).id)
def get(self, processing_time=300):
"""
@ -141,10 +150,32 @@ class WorkQueue(object):
# Return a view of the queue item rather than an active db object
return item
def cancel(self, item_id):
""" Attempts to cancel the queue item with the given ID from the queue. Returns true on success
and false if the queue item could not be canceled. A queue item can only be canceled if
if is available and has retries remaining.
"""
with self._transaction_factory(db):
# Load the build queue item for update.
try:
queue_item = db_for_update(QueueItem.select()
.where(QueueItem.id == item_id)).get()
except QueueItem.DoesNotExist:
return False
# Check the queue item.
if not queue_item.available or queue_item.retries_remaining == 0:
return False
# Delete the queue item.
queue_item.delete_instance(recursive=True)
return True
def complete(self, completed_item):
with self._transaction_factory(db):
completed_item_obj = self._item_by_id_for_update(completed_item.id)
completed_item_obj.delete_instance()
completed_item_obj.delete_instance(recursive=True)
self._currently_processing = False
def incomplete(self, incomplete_item, retry_after=300, restore_retry=False):

2
data/users.py
View file

@ -24,7 +24,7 @@ class LDAPConnection(object):
self._conn = None
def __enter__(self):
self._conn = ldap.initialize(self._ldap_uri)
self._conn = ldap.initialize(self._ldap_uri, trace_level=1)
self._conn.simple_bind_s(self._user_dn, self._user_pw)
return self._conn