Add support for creating schema 2 manifests and manifest lists via the OCI model
This commit is contained in:
Joseph Schorr
parent
e344d4a5cf
commit
30f072aeff
16 changed files with 398 additions and 110 deletions
|
|
@ -1,13 +1,22 @@
|
|||
import json
|
||||
|
||||
from playhouse.test_utils import assert_query_count
|
||||
|
||||
from app import docker_v2_signing_key
|
||||
from app import docker_v2_signing_key, storage
|
||||
|
||||
from data.database import Tag, ManifestBlob, get_epoch_timestamp_ms
|
||||
from digest.digest_tools import sha256_digest
|
||||
from data.database import Tag, ManifestBlob, ImageStorageLocation, ManifestChild, get_epoch_timestamp_ms
|
||||
from data.model.oci.manifest import lookup_manifest, get_or_create_manifest
|
||||
from data.model.oci.tag import filter_to_alive_tags, get_tag
|
||||
from data.model.oci.shared import get_legacy_image_for_manifest
|
||||
from data.model.repository import get_repository
|
||||
from data.model.oci.label import list_manifest_labels
|
||||
from data.model.repository import get_repository, create_repository
|
||||
from data.model.image import find_create_or_link_image
|
||||
from data.model.blob import store_blob_record_and_temp_link
|
||||
from data.model.storage import get_layer_path
|
||||
from image.docker.schema1 import DockerSchema1ManifestBuilder, DockerSchema1Manifest
|
||||
from image.docker.schema2.manifest import DockerSchema2ManifestBuilder
|
||||
from image.docker.schema2.list import DockerSchema2ManifestListBuilder
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
|
|
@ -38,35 +47,104 @@ def test_lookup_manifest_dead_tag(initialized_db):
|
|||
dead_tag.manifest)
|
||||
|
||||
|
||||
def test_get_or_create_manifest(initialized_db):
|
||||
repository = get_repository('devtable', 'simple')
|
||||
def _populate_blob(content):
|
||||
digest = str(sha256_digest(content))
|
||||
location = ImageStorageLocation.get(name='local_us')
|
||||
blob = store_blob_record_and_temp_link('devtable', 'newrepo', digest, location, len(content), 120)
|
||||
storage.put_content(['local_us'], get_layer_path(blob), content)
|
||||
return blob, digest
|
||||
|
||||
latest_tag = get_tag(repository, 'latest')
|
||||
legacy_image = get_legacy_image_for_manifest(latest_tag.manifest)
|
||||
parsed = DockerSchema1Manifest(latest_tag.manifest.manifest_bytes, validate=False)
|
||||
|
||||
builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'anothertag')
|
||||
builder.add_layer(parsed.blob_digests[0], '{"id": "%s"}' % legacy_image.docker_image_id)
|
||||
sample_manifest_instance = builder.build(docker_v2_signing_key)
|
||||
@pytest.mark.parametrize('schema_version', [
|
||||
1,
|
||||
2,
|
||||
])
|
||||
def test_get_or_create_manifest(schema_version, initialized_db):
|
||||
repository = create_repository('devtable', 'newrepo', None)
|
||||
|
||||
expected_labels = {
|
||||
'Foo': 'Bar',
|
||||
'Baz': 'Meh',
|
||||
}
|
||||
|
||||
layer_json = json.dumps({
|
||||
'id': 'somelegacyid',
|
||||
'config': {
|
||||
'Labels': expected_labels,
|
||||
},
|
||||
"rootfs": {
|
||||
"type": "layers",
|
||||
"diff_ids": []
|
||||
},
|
||||
"history": [
|
||||
{
|
||||
"created": "2018-04-03T18:37:09.284840891Z",
|
||||
"created_by": "do something",
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
# Create a legacy image.
|
||||
find_create_or_link_image('somelegacyid', repository, 'devtable', {}, 'local_us')
|
||||
|
||||
# Add a blob containing the config.
|
||||
_, config_digest = _populate_blob(layer_json)
|
||||
|
||||
# Add a blob of random data.
|
||||
random_data = 'hello world'
|
||||
_, random_digest = _populate_blob(random_data)
|
||||
|
||||
# Build the manifest.
|
||||
if schema_version == 1:
|
||||
builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'anothertag')
|
||||
builder.add_layer(random_digest, layer_json)
|
||||
sample_manifest_instance = builder.build(docker_v2_signing_key)
|
||||
elif schema_version == 2:
|
||||
builder = DockerSchema2ManifestBuilder()
|
||||
builder.set_config_digest(config_digest, len(layer_json))
|
||||
builder.add_layer(random_digest, len(random_data))
|
||||
sample_manifest_instance = builder.build()
|
||||
|
||||
# Create a new manifest.
|
||||
created, newly_created = get_or_create_manifest(repository, sample_manifest_instance)
|
||||
created_manifest = get_or_create_manifest(repository, sample_manifest_instance, storage)
|
||||
created = created_manifest.manifest
|
||||
newly_created = created_manifest.newly_created
|
||||
|
||||
assert newly_created
|
||||
assert created is not None
|
||||
assert created.media_type.name == sample_manifest_instance.media_type
|
||||
assert created.digest == sample_manifest_instance.digest
|
||||
assert created.manifest_bytes == sample_manifest_instance.bytes
|
||||
assert created_manifest.labels_to_apply == expected_labels
|
||||
|
||||
assert get_legacy_image_for_manifest(created) is not None
|
||||
# Verify the legacy image.
|
||||
legacy_image = get_legacy_image_for_manifest(created)
|
||||
assert legacy_image is not None
|
||||
assert legacy_image.storage.content_checksum == random_digest
|
||||
|
||||
# Verify the linked blobs.
|
||||
blob_digests = [mb.blob.content_checksum for mb
|
||||
in ManifestBlob.select().where(ManifestBlob.manifest == created)]
|
||||
assert parsed.blob_digests[0] in blob_digests
|
||||
|
||||
assert random_digest in blob_digests
|
||||
if schema_version == 2:
|
||||
assert config_digest in blob_digests
|
||||
|
||||
# Retrieve it again and ensure it is the same manifest.
|
||||
created2, newly_created2 = get_or_create_manifest(repository, sample_manifest_instance)
|
||||
created_manifest2 = get_or_create_manifest(repository, sample_manifest_instance, storage)
|
||||
created2 = created_manifest2.manifest
|
||||
newly_created2 = created_manifest2.newly_created
|
||||
|
||||
assert not newly_created2
|
||||
assert created2 == created
|
||||
|
||||
# Ensure the labels were added.
|
||||
labels = list(list_manifest_labels(created))
|
||||
assert len(labels) == 2
|
||||
|
||||
labels_dict = {label.key: label.value for label in labels}
|
||||
assert labels_dict == expected_labels
|
||||
|
||||
|
||||
def test_get_or_create_manifest_invalid_image(initialized_db):
|
||||
repository = get_repository('devtable', 'simple')
|
||||
|
|
@ -78,6 +156,86 @@ def test_get_or_create_manifest_invalid_image(initialized_db):
|
|||
builder.add_layer(parsed.blob_digests[0], '{"id": "foo", "parent": "someinvalidimageid"}')
|
||||
sample_manifest_instance = builder.build(docker_v2_signing_key)
|
||||
|
||||
created, newly_created = get_or_create_manifest(repository, sample_manifest_instance)
|
||||
assert created is None
|
||||
assert newly_created is None
|
||||
created_manifest = get_or_create_manifest(repository, sample_manifest_instance, storage)
|
||||
assert created_manifest is None
|
||||
|
||||
|
||||
def test_get_or_create_manifest_list(initialized_db):
|
||||
repository = create_repository('devtable', 'newrepo', None)
|
||||
|
||||
expected_labels = {
|
||||
'Foo': 'Bar',
|
||||
'Baz': 'Meh',
|
||||
}
|
||||
|
||||
layer_json = json.dumps({
|
||||
'id': 'somelegacyid',
|
||||
'config': {
|
||||
'Labels': expected_labels,
|
||||
},
|
||||
"rootfs": {
|
||||
"type": "layers",
|
||||
"diff_ids": []
|
||||
},
|
||||
"history": [
|
||||
{
|
||||
"created": "2018-04-03T18:37:09.284840891Z",
|
||||
"created_by": "do something",
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
# Create a legacy image.
|
||||
find_create_or_link_image('somelegacyid', repository, 'devtable', {}, 'local_us')
|
||||
|
||||
# Add a blob containing the config.
|
||||
_, config_digest = _populate_blob(layer_json)
|
||||
|
||||
# Add a blob of random data.
|
||||
random_data = 'hello world'
|
||||
_, random_digest = _populate_blob(random_data)
|
||||
|
||||
# Build the manifests.
|
||||
v1_builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'anothertag')
|
||||
v1_builder.add_layer(random_digest, layer_json)
|
||||
v1_manifest = v1_builder.build(docker_v2_signing_key).unsigned()
|
||||
|
||||
v2_builder = DockerSchema2ManifestBuilder()
|
||||
v2_builder.set_config_digest(config_digest, len(layer_json))
|
||||
v2_builder.add_layer(random_digest, len(random_data))
|
||||
v2_manifest = v2_builder.build()
|
||||
|
||||
# Write the manifests as blobs.
|
||||
location = ImageStorageLocation.get(name='local_us')
|
||||
blob = store_blob_record_and_temp_link('devtable', 'newrepo', v1_manifest.digest, location,
|
||||
len(v1_manifest.bytes), 120)
|
||||
storage.put_content(['local_us'], get_layer_path(blob), v1_manifest.bytes)
|
||||
|
||||
blob = store_blob_record_and_temp_link('devtable', 'newrepo', v2_manifest.digest, location,
|
||||
len(v2_manifest.bytes), 120)
|
||||
storage.put_content(['local_us'], get_layer_path(blob), v2_manifest.bytes)
|
||||
|
||||
# Build the manifest list.
|
||||
list_builder = DockerSchema2ManifestListBuilder()
|
||||
list_builder.add_manifest(v1_manifest, 'amd64', 'linux')
|
||||
list_builder.add_manifest(v2_manifest, 'amd32', 'linux')
|
||||
manifest_list = list_builder.build()
|
||||
|
||||
# Write the manifest list, which should also write the manifests themselves.
|
||||
created_tuple = get_or_create_manifest(repository, manifest_list, storage)
|
||||
assert created_tuple is not None
|
||||
|
||||
created_list = created_tuple.manifest
|
||||
assert created_list
|
||||
assert created_list.media_type.name == manifest_list.media_type
|
||||
assert created_list.digest == manifest_list.digest
|
||||
|
||||
# Ensure the child manifest links exist.
|
||||
child_manifests = {cm.child_manifest.digest: cm.child_manifest
|
||||
for cm in ManifestChild.select().where(ManifestChild.manifest == created_list)}
|
||||
assert len(child_manifests) == 2
|
||||
assert v1_manifest.digest in child_manifests
|
||||
assert v2_manifest.digest in child_manifests
|
||||
|
||||
assert child_manifests[v1_manifest.digest].media_type.name == v1_manifest.media_type
|
||||
assert child_manifests[v2_manifest.digest].media_type.name == v2_manifest.media_type
|
||||
|
|
|
|||
Reference in a new issue