vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Allow filtering of the backfill by namespace

Browse source
This commit is contained in:
Joseph Schorr 2019-01-16 11:36:09 -05:00
parent 2a4646b1ca
commit 0f18b3a70a
1 changed files with 17 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
workers/tagbackfillworker.py
View file

@ -8,11 +8,13 @@ from peewee import JOIN, fn, IntegrityError
from app import app from app import app
from data.database import (UseThenDisconnect, TagToRepositoryTag, RepositoryTag, from data.database import (UseThenDisconnect, TagToRepositoryTag, RepositoryTag,
TagManifestToManifest, Tag, TagManifest, TagManifestToManifest, Image, TagManifestToManifest, Tag, TagManifest, TagManifestToManifest, Image,
Manifest, TagManifestLabel, ManifestLabel, TagManifestLabelMap, db_transaction) Manifest, TagManifestLabel, ManifestLabel, TagManifestLabelMap,
Repository, db_transaction)
from data.model import DataModelException from data.model import DataModelException
from data.model.image import get_parent_images from data.model.image import get_parent_images
from data.model.tag import populate_manifest from data.model.tag import populate_manifest
from data.model.blob import get_repo_blob_by_digest, BlobDoesNotExist from data.model.blob import get_repo_blob_by_digest, BlobDoesNotExist
from data.model.user import get_namespace_user
from data.registry_model import pre_oci_model from data.registry_model import pre_oci_model
from data.registry_model.datatypes import Tag as TagDataType from data.registry_model.datatypes import Tag as TagDataType
from image.docker.schema1 import (DockerSchema1Manifest, ManifestException, ManifestInterface, from image.docker.schema1 import (DockerSchema1Manifest, ManifestException, ManifestInterface,
@ -109,19 +111,27 @@ class BrokenManifest(ManifestInterface):
class TagBackfillWorker(Worker): class TagBackfillWorker(Worker):
def __init__(self): def __init__(self, namespace_filter=None):
super(TagBackfillWorker, self).__init__() super(TagBackfillWorker, self).__init__()
self._namespace_filter = namespace_filter
self.add_operation(self._backfill_tags, WORKER_TIMEOUT) self.add_operation(self._backfill_tags, WORKER_TIMEOUT)
def _filter(self, query):
if self._namespace_filter:
namespace_user = get_namespace_user(self._namespace_filter)
query = query.join(Repository).where(Repository.namespace_user == namespace_user)
return query
def _candidates_to_backfill(self): def _candidates_to_backfill(self):
def missing_tmt_query(): def missing_tmt_query():
return (RepositoryTag return (self._filter(RepositoryTag.select())
.select()
.join(TagToRepositoryTag, JOIN.LEFT_OUTER) .join(TagToRepositoryTag, JOIN.LEFT_OUTER)
.where(TagToRepositoryTag.id >> None, RepositoryTag.hidden == False)) .where(TagToRepositoryTag.id >> None, RepositoryTag.hidden == False))
min_id = (RepositoryTag.select(fn.Min(RepositoryTag.id)).scalar()) min_id = (self._filter(RepositoryTag.select(fn.Min(RepositoryTag.id))).scalar())
max_id = RepositoryTag.select(fn.Max(RepositoryTag.id)).scalar() max_id = self._filter(RepositoryTag.select(fn.Max(RepositoryTag.id))).scalar()
iterator = yield_random_entries( iterator = yield_random_entries(
missing_tmt_query, missing_tmt_query,
@ -368,5 +378,5 @@ if __name__ == "__main__":
while True: while True:
time.sleep(100000) time.sleep(100000)
worker = TagBackfillWorker() worker = TagBackfillWorker(app.config.get('BACKFILL_TAGS_NAMESPACE'))
worker.start() worker.start()