vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #3329 from quay/tag-backfill-improvements

Browse source 
A few small improvements to the tag backfill worker
This commit is contained in:
Joseph Schorr 2019-01-15 12:05:44 -05:00 committed by GitHub
commit 6d5ec4eddf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 13 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
workers/tagbackfillworker.py
View file

@ -25,7 +25,7 @@ from util.migrate.allocator import yield_random_entries
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
WORKER_TIMEOUT = 600 WORKER_TIMEOUT = 6000
class BrokenManifest(ManifestInterface): class BrokenManifest(ManifestInterface):
@ -120,11 +120,7 @@ class TagBackfillWorker(Worker):
.join(TagToRepositoryTag, JOIN.LEFT_OUTER) .join(TagToRepositoryTag, JOIN.LEFT_OUTER)
.where(TagToRepositoryTag.id >> None, RepositoryTag.hidden == False)) .where(TagToRepositoryTag.id >> None, RepositoryTag.hidden == False))
min_id = (RepositoryTag min_id = (RepositoryTag.select(fn.Min(RepositoryTag.id)).scalar())
.select(fn.Min(RepositoryTag.id))
.join(TagToRepositoryTag, JOIN.LEFT_OUTER)
.where(TagToRepositoryTag.id >> None, RepositoryTag.hidden == False)
.scalar())
max_id = RepositoryTag.select(fn.Max(RepositoryTag.id)).scalar() max_id = RepositoryTag.select(fn.Max(RepositoryTag.id)).scalar()
iterator = yield_random_entries( iterator = yield_random_entries(
@ -227,7 +223,17 @@ def _get_manifest_id(repositorytag):
return None return None
try: try:
return TagManifestToManifest.get(tag_manifest=tag_manifest).manifest_id found = TagManifestToManifest.get(tag_manifest=tag_manifest).manifest
# Verify that the new-style manifest has the same contents as the old-style manifest.
# If not, update and then return. This is an extra check put in place to ensure unicode
# manifests have been correctly copied.
if found.manifest_bytes != tag_manifest.json_data:
logger.warning('Fixing manifest `%s`', found.id)
found.manifest_bytes = tag_manifest.json_data
found.save()
return found.id
except TagManifestToManifest.DoesNotExist: except TagManifestToManifest.DoesNotExist:
# Could not find the new style manifest, so backfill. # Could not find the new style manifest, so backfill.
_backfill_manifest(tag_manifest) _backfill_manifest(tag_manifest)