Load images and storage references in bulk during V1 synthesize
Currently, we perform multiple queries for each layer, making it much slower (especially cross-region) Fixes #413
This commit is contained in:
Joseph Schorr
parent
78e8aefd45
commit
35c35d9913
4 changed files with 173 additions and 157 deletions
|
|
@ -205,6 +205,9 @@ class BaseRegistryMixin(object):
|
|||
self.assertEquals(response.status_code, expected_code)
|
||||
return response
|
||||
|
||||
def _get_default_images(self):
|
||||
return [{'id': 'someid', 'contents': 'somecontent'}]
|
||||
|
||||
|
||||
class V1RegistryMixin(BaseRegistryMixin):
|
||||
def v1_ping(self):
|
||||
|
|
@ -212,20 +215,21 @@ class V1RegistryMixin(BaseRegistryMixin):
|
|||
|
||||
|
||||
class V1RegistryPushMixin(V1RegistryMixin):
|
||||
def do_push(self, namespace, repository, username, password, images):
|
||||
def do_push(self, namespace, repository, username, password, images=None):
|
||||
images = images or self._get_default_images()
|
||||
auth = (username, password)
|
||||
|
||||
# Ping!
|
||||
self.v1_ping()
|
||||
|
||||
# PUT /v1/repositories/{namespace}/{repository}/
|
||||
data = [{"id": image_id} for image_id, _ in images.iteritems()]
|
||||
self.conduct('PUT', '/v1/repositories/%s/%s' % (namespace, repository),
|
||||
data=json.dumps(data), auth=auth,
|
||||
data=json.dumps(images), auth=auth,
|
||||
expected_code=201)
|
||||
|
||||
last_image_id = None
|
||||
for image_id, _ in images.iteritems():
|
||||
for image_data in images:
|
||||
image_id = image_data['id']
|
||||
last_image_id = image_id
|
||||
|
||||
# PUT /v1/images/{imageID}/json
|
||||
|
|
@ -364,8 +368,10 @@ class V2RegistryMixin(BaseRegistryMixin):
|
|||
|
||||
|
||||
class V2RegistryPushMixin(V2RegistryMixin):
|
||||
def do_push(self, namespace, repository, username, password, images, tag_name=None,
|
||||
cancel=False, invalid=False):
|
||||
def do_push(self, namespace, repository, username, password, images=None, tag_name=None,
|
||||
cancel=False, invalid=False, expected_manifest_code=202):
|
||||
images = images or self._get_default_images()
|
||||
|
||||
# Ping!
|
||||
self.v2_ping()
|
||||
|
||||
|
|
@ -375,30 +381,22 @@ class V2RegistryPushMixin(V2RegistryMixin):
|
|||
# Build a fake manifest.
|
||||
tag_name = tag_name or 'latest'
|
||||
builder = SignedManifestBuilder(namespace, repository, tag_name)
|
||||
for image_id, contents in images.iteritems():
|
||||
if isinstance(contents, dict):
|
||||
full_contents = contents['contents']
|
||||
else:
|
||||
full_contents = contents
|
||||
|
||||
checksum = 'sha256:' + hashlib.sha256(full_contents).hexdigest()
|
||||
for image_data in images:
|
||||
checksum = 'sha256:' + hashlib.sha256(image_data['contents']).hexdigest()
|
||||
if invalid:
|
||||
checksum = 'sha256:' + hashlib.sha256('foobarbaz').hexdigest()
|
||||
|
||||
builder.add_layer(checksum, json.dumps({'id': image_id, 'data': contents}))
|
||||
builder.add_layer(checksum, json.dumps(image_data))
|
||||
|
||||
# Build the manifest.
|
||||
manifest = builder.build(_JWK)
|
||||
|
||||
# Push the image's layers.
|
||||
checksums = {}
|
||||
for image_id, contents in images.iteritems():
|
||||
chunks = None
|
||||
if isinstance(contents, dict):
|
||||
full_contents = contents['contents']
|
||||
chunks = contents['chunks']
|
||||
else:
|
||||
full_contents = contents
|
||||
for image_data in images:
|
||||
image_id = image_data['id']
|
||||
full_contents = image_data['contents']
|
||||
chunks = image_data.get('chunks')
|
||||
|
||||
# Layer data should not yet exist.
|
||||
checksum = 'sha256:' + hashlib.sha256(full_contents).hexdigest()
|
||||
|
|
@ -414,7 +412,7 @@ class V2RegistryPushMixin(V2RegistryMixin):
|
|||
|
||||
# PATCH the image data into the layer.
|
||||
if chunks is None:
|
||||
self.conduct('PATCH', location, data=contents, expected_code=204, auth='jwt')
|
||||
self.conduct('PATCH', location, data=full_contents, expected_code=204, auth='jwt')
|
||||
else:
|
||||
for chunk in chunks:
|
||||
if len(chunk) == 3:
|
||||
|
|
@ -461,7 +459,7 @@ class V2RegistryPushMixin(V2RegistryMixin):
|
|||
self.assertEquals(response.headers['Content-Length'], str(len(full_contents)))
|
||||
|
||||
# Write the manifest.
|
||||
put_code = 404 if invalid else 202
|
||||
put_code = 404 if invalid else expected_manifest_code
|
||||
self.conduct('PUT', '/v2/%s/%s/manifests/%s' % (namespace, repository, tag_name),
|
||||
data=manifest.bytes, expected_code=put_code,
|
||||
headers={'Content-Type': 'application/json'}, auth='jwt')
|
||||
|
|
@ -508,11 +506,7 @@ class V2RegistryPullMixin(V2RegistryMixin):
|
|||
class RegistryTestsMixin(object):
|
||||
def test_pull_publicrepo_anonymous(self):
|
||||
# Add a new repository under the public user, so we have a real repository to pull.
|
||||
images = {
|
||||
'someid': 'onlyimagehere'
|
||||
}
|
||||
|
||||
self.do_push('public', 'newrepo', 'public', 'password', images)
|
||||
self.do_push('public', 'newrepo', 'public', 'password')
|
||||
self.clearSession()
|
||||
|
||||
# First try to pull the (currently private) repo anonymously, which should fail (since it is
|
||||
|
|
@ -530,11 +524,7 @@ class RegistryTestsMixin(object):
|
|||
|
||||
def test_pull_publicrepo_devtable(self):
|
||||
# Add a new repository under the public user, so we have a real repository to pull.
|
||||
images = {
|
||||
'someid': 'onlyimagehere'
|
||||
}
|
||||
|
||||
self.do_push('public', 'newrepo', 'public', 'password', images)
|
||||
self.do_push('public', 'newrepo', 'public', 'password')
|
||||
self.clearSession()
|
||||
|
||||
# First try to pull the (currently private) repo as devtable, which should fail as it belongs
|
||||
|
|
@ -552,11 +542,7 @@ class RegistryTestsMixin(object):
|
|||
|
||||
def test_pull_private_repo(self):
|
||||
# Add a new repository under the devtable user, so we have a real repository to pull.
|
||||
images = {
|
||||
'someid': 'onlyimagehere'
|
||||
}
|
||||
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', images)
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password')
|
||||
self.clearSession()
|
||||
|
||||
# First try to pull the (currently private) repo as public, which should fail as it belongs
|
||||
|
|
@ -572,11 +558,7 @@ class RegistryTestsMixin(object):
|
|||
# Turn off anonymous access.
|
||||
with TestFeature(self, 'ANONYMOUS_ACCESS', False):
|
||||
# Add a new repository under the public user, so we have a real repository to pull.
|
||||
images = {
|
||||
'someid': 'onlyimagehere'
|
||||
}
|
||||
|
||||
self.do_push('public', 'newrepo', 'public', 'password', images)
|
||||
self.do_push('public', 'newrepo', 'public', 'password')
|
||||
self.clearSession()
|
||||
|
||||
# First try to pull the (currently private) repo as devtable, which should fail as it belongs
|
||||
|
|
@ -596,11 +578,7 @@ class RegistryTestsMixin(object):
|
|||
# Turn off anonymous access.
|
||||
with TestFeature(self, 'ANONYMOUS_ACCESS', False):
|
||||
# Add a new repository under the public user, so we have a real repository to pull.
|
||||
images = {
|
||||
'someid': 'onlyimagehere'
|
||||
}
|
||||
|
||||
self.do_push('public', 'newrepo', 'public', 'password', images)
|
||||
self.do_push('public', 'newrepo', 'public', 'password')
|
||||
self.clearSession()
|
||||
|
||||
# First try to pull the (currently private) repo as devtable, which should fail as it belongs
|
||||
|
|
@ -615,11 +593,7 @@ class RegistryTestsMixin(object):
|
|||
# Turn off anonymous access.
|
||||
with TestFeature(self, 'ANONYMOUS_ACCESS', False):
|
||||
# Add a new repository under the public user, so we have a real repository to pull.
|
||||
images = {
|
||||
'someid': 'onlyimagehere'
|
||||
}
|
||||
|
||||
self.do_push('public', 'newrepo', 'public', 'password', images)
|
||||
self.do_push('public', 'newrepo', 'public', 'password')
|
||||
self.clearSession()
|
||||
|
||||
# First try to pull the (currently private) repo as anonymous, which should fail as it
|
||||
|
|
@ -643,11 +617,7 @@ class RegistryTestsMixin(object):
|
|||
|
||||
|
||||
def test_create_repo_creator_user(self):
|
||||
images = {
|
||||
'someid': 'onlyimagehere'
|
||||
}
|
||||
|
||||
self.do_push('buynlarge', 'newrepo', 'creator', 'password', images)
|
||||
self.do_push('buynlarge', 'newrepo', 'creator', 'password')
|
||||
|
||||
# Pull the repository as devtable, which should succeed because the repository is owned by the
|
||||
# org.
|
||||
|
|
@ -660,11 +630,7 @@ class RegistryTestsMixin(object):
|
|||
resp = self.conduct('GET', '/api/v1/organization/buynlarge/robots/ownerbot')
|
||||
robot_token = json.loads(resp.text)['token']
|
||||
|
||||
images = {
|
||||
'someid': 'onlyimagehere'
|
||||
}
|
||||
|
||||
self.do_push('buynlarge', 'newrepo', 'buynlarge+ownerbot', robot_token, images)
|
||||
self.do_push('buynlarge', 'newrepo', 'buynlarge+ownerbot', robot_token)
|
||||
|
||||
# Pull the repository as devtable, which should succeed because the repository is owned by the
|
||||
# org.
|
||||
|
|
@ -677,11 +643,7 @@ class RegistryTestsMixin(object):
|
|||
resp = self.conduct('GET', '/api/v1/organization/buynlarge/robots/creatorbot')
|
||||
robot_token = json.loads(resp.text)['token']
|
||||
|
||||
images = {
|
||||
'someid': 'onlyimagehere'
|
||||
}
|
||||
|
||||
self.do_push('buynlarge', 'newrepo', 'buynlarge+creatorbot', robot_token, images)
|
||||
self.do_push('buynlarge', 'newrepo', 'buynlarge+creatorbot', robot_token)
|
||||
|
||||
# Pull the repository as devtable, which should succeed because the repository is owned by the
|
||||
# org.
|
||||
|
|
@ -697,27 +659,15 @@ class V2RegistryTests(V2RegistryPullMixin, V2RegistryPushMixin, RegistryTestsMix
|
|||
""" Tests for V2 registry. """
|
||||
|
||||
def test_invalid_push(self):
|
||||
images = {
|
||||
'someid': 'onlyimagehere'
|
||||
}
|
||||
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', images, invalid=True)
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', invalid=True)
|
||||
|
||||
def test_cancel_push(self):
|
||||
images = {
|
||||
'someid': 'onlyimagehere'
|
||||
}
|
||||
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', images, cancel=True)
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', cancel=True)
|
||||
|
||||
|
||||
def test_pull_by_checksum(self):
|
||||
# Add a new repository under the user, so we have a real repository to pull.
|
||||
images = {
|
||||
'someid': 'onlyimagehere'
|
||||
}
|
||||
|
||||
_, digest = self.do_push('devtable', 'newrepo', 'devtable', 'password', images)
|
||||
_, digest = self.do_push('devtable', 'newrepo', 'devtable', 'password')
|
||||
|
||||
# Attempt to pull by digest.
|
||||
self.do_pull('devtable', 'newrepo', 'devtable', 'password', manifest_id=digest)
|
||||
|
|
@ -725,11 +675,7 @@ class V2RegistryTests(V2RegistryPullMixin, V2RegistryPushMixin, RegistryTestsMix
|
|||
|
||||
def test_pull_invalid_image_tag(self):
|
||||
# Add a new repository under the user, so we have a real repository to pull.
|
||||
images = {
|
||||
'someid': 'onlyimagehere'
|
||||
}
|
||||
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', images)
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password')
|
||||
self.clearSession()
|
||||
|
||||
# Attempt to pull the invalid tag.
|
||||
|
|
@ -745,15 +691,16 @@ class V2RegistryTests(V2RegistryPullMixin, V2RegistryPushMixin, RegistryTestsMix
|
|||
chunk_count = int(math.ceil((len(contents) * 1.0) / chunksize))
|
||||
chunks = [(index * chunksize, (index + 1)*chunksize) for index in range(chunk_count)]
|
||||
|
||||
images = {
|
||||
'someid': {
|
||||
images = [
|
||||
{
|
||||
'id':'someid',
|
||||
'contents': contents,
|
||||
'chunks': chunks
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
# Push the chunked upload.
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', images)
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', images=images)
|
||||
|
||||
# Pull the image back and verify the contents.
|
||||
blobs = self.do_pull('devtable', 'newrepo', 'devtable', 'password')
|
||||
|
|
@ -765,15 +712,16 @@ class V2RegistryTests(V2RegistryPullMixin, V2RegistryPushMixin, RegistryTestsMix
|
|||
contents = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(size))
|
||||
chunks = [(0, 100), (100, size)]
|
||||
|
||||
images = {
|
||||
'someid': {
|
||||
images = [
|
||||
{
|
||||
'id':'someid',
|
||||
'contents': contents,
|
||||
'chunks': chunks
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
# Push the chunked upload.
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', images)
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', images=images)
|
||||
|
||||
# Pull the image back and verify the contents.
|
||||
blobs = self.do_pull('devtable', 'newrepo', 'devtable', 'password')
|
||||
|
|
@ -786,15 +734,16 @@ class V2RegistryTests(V2RegistryPullMixin, V2RegistryPushMixin, RegistryTestsMix
|
|||
|
||||
chunks = [(0, 100), (10, size)]
|
||||
|
||||
images = {
|
||||
'someid': {
|
||||
images = [
|
||||
{
|
||||
'id':'someid',
|
||||
'contents': contents,
|
||||
'chunks': chunks
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
# Push the chunked upload.
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', images)
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', images=images)
|
||||
|
||||
# Pull the image back and verify the contents.
|
||||
blobs = self.do_pull('devtable', 'newrepo', 'devtable', 'password')
|
||||
|
|
@ -807,22 +756,66 @@ class V2RegistryTests(V2RegistryPullMixin, V2RegistryPushMixin, RegistryTestsMix
|
|||
|
||||
chunks = [(0, 100), (101, size, 416)]
|
||||
|
||||
images = {
|
||||
'someid': {
|
||||
images = [
|
||||
{
|
||||
'id':'someid',
|
||||
'contents': contents,
|
||||
'chunks': chunks
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
# Attempt to push the chunked upload, which should fail.
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', images)
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', images=images)
|
||||
|
||||
def test_multiple_layers_invalid(self):
|
||||
# Attempt to push a manifest with an image depending on an unknown base layer.
|
||||
images = [
|
||||
{
|
||||
'id': 'latestid',
|
||||
'contents': 'the latest image',
|
||||
'parent': 'baseid',
|
||||
}
|
||||
]
|
||||
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', images=images,
|
||||
expected_manifest_code=400)
|
||||
|
||||
def test_multiple_layers(self):
|
||||
# Push a manifest with multiple layers.
|
||||
images = [
|
||||
{
|
||||
'id': 'latestid',
|
||||
'contents': 'the latest image',
|
||||
'parent': 'baseid',
|
||||
},
|
||||
{
|
||||
'id': 'baseid',
|
||||
'contents': 'The base image',
|
||||
}
|
||||
]
|
||||
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', images=images)
|
||||
|
||||
def test_multiple_tags(self):
|
||||
latest_images = [
|
||||
{
|
||||
'id': 'latestid',
|
||||
'contents': 'the latest image'
|
||||
}
|
||||
]
|
||||
|
||||
foobar_images = [
|
||||
{
|
||||
'id': 'foobarid',
|
||||
'contents': 'the foobar image',
|
||||
}
|
||||
]
|
||||
|
||||
# Create the repo.
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', {'someid': 'onlyimagehere'},
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', images=latest_images,
|
||||
tag_name='latest')
|
||||
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', {'anotherid': 'anotherimage'},
|
||||
self.do_push('devtable', 'newrepo', 'devtable', 'password', images=foobar_images,
|
||||
tag_name='foobar')
|
||||
|
||||
# Retrieve the tags.
|
||||
|
|
@ -846,6 +839,7 @@ class V2RegistryTests(V2RegistryPullMixin, V2RegistryPushMixin, RegistryTestsMix
|
|||
self.conduct('GET', '/v2/devtable/doesnotexist/tags/list', auth='jwt', expected_code=401)
|
||||
|
||||
|
||||
|
||||
class V1PushV2PullRegistryTests(V2RegistryPullMixin, V1RegistryPushMixin, RegistryTestsMixin,
|
||||
RegistryTestCaseMixin, LiveServerTestCase):
|
||||
""" Tests for V1 push, V2 pull registry. """
|
||||
|
|
|
|||
Reference in a new issue