import unittest import hashlib from app import app, storage, docker_v2_signing_key from initdb import setup_database_for_testing, finished_database_for_testing from data import model, database from endpoints.v2.manifest import _write_manifest from image.docker.schema1 import DockerSchema1ManifestBuilder ADMIN_ACCESS_USER = 'devtable' REPO = 'simple' FIRST_TAG = 'first' SECOND_TAG = 'second' THIRD_TAG = 'third' class TestManifests(unittest.TestCase): @staticmethod def _set_tag_expiration_policy(namespace, expiration_s): namespace_user = model.user.get_user(namespace) model.user.change_user_tag_expiration(namespace_user, expiration_s) def setUp(self): setup_database_for_testing(self) self._set_tag_expiration_policy(ADMIN_ACCESS_USER, 0) self.app = app.test_client() self.ctx = app.test_request_context() self.ctx.__enter__() def tearDown(self): finished_database_for_testing(self) self.ctx.__exit__(True, None, None) def _perform_cleanup(self): database.RepositoryTag.delete().where(database.RepositoryTag.hidden == True).execute() repo_object = model.repository.get_repository(ADMIN_ACCESS_USER, REPO) model.repository.garbage_collect_repo(repo_object) def test_missing_link(self): """ Tests for a corner case that could result in missing a link to a blob referenced by a manifest. The test exercises the case as follows: 1) Push a manifest of a single layer with a Docker ID `FIRST_ID`, pointing to blob `FIRST_BLOB`. The database should contain the tag referencing the layer, with no changed ID and the blob not being GCed. 2) Push a manifest of two layers: Layer 1: `FIRST_ID` with blob `SECOND_BLOB`: Will result in a new synthesized ID Layer 2: `SECOND_ID` with blob `THIRD_BLOB`: Will result in `SECOND_ID` pointing to the `THIRD_BLOB`, with a parent pointing to the new synthesized ID's layer. 3) Push a manifest of two layers: Layer 1: `THIRD_ID` with blob `FOURTH_BLOB`: Will result in a new `THIRD_ID` layer Layer 2: `FIRST_ID` with blob `THIRD_BLOB`: Since `FIRST_ID` already points to `SECOND_BLOB`, this will synthesize a new ID. With the current bug, the synthesized ID will match that of `SECOND_ID`, leaving `THIRD_ID` unlinked and therefore, after a GC, missing `FOURTH_BLOB`. """ location_name = storage.preferred_locations[0] location = database.ImageStorageLocation.get(name=location_name) # Create first blob. first_blob_sha = 'sha256:' + hashlib.sha256("FIRST").hexdigest() model.blob.store_blob_record_and_temp_link(ADMIN_ACCESS_USER, REPO, first_blob_sha, location, 0, 0, 0) # Push the first manifest. first_manifest = (DockerSchema1ManifestBuilder(ADMIN_ACCESS_USER, REPO, FIRST_TAG) .add_layer(first_blob_sha, '{"id": "first"}') .build(docker_v2_signing_key)) _write_manifest(ADMIN_ACCESS_USER, REPO, first_manifest) # Delete all temp tags and perform GC. self._perform_cleanup() # Ensure that the first blob still exists, along with the first tag. self.assertIsNotNone(model.blob.get_repo_blob_by_digest(ADMIN_ACCESS_USER, REPO, first_blob_sha)) self.assertIsNotNone(model.tag.load_tag_manifest(ADMIN_ACCESS_USER, REPO, FIRST_TAG)) self.assertEquals("first", model.tag.get_tag_image(ADMIN_ACCESS_USER, REPO, FIRST_TAG).docker_image_id) # Create the second and third blobs. second_blob_sha = 'sha256:' + hashlib.sha256("SECOND").hexdigest() third_blob_sha = 'sha256:' + hashlib.sha256("THIRD").hexdigest() model.blob.store_blob_record_and_temp_link(ADMIN_ACCESS_USER, REPO, second_blob_sha, location, 0, 0, 0) model.blob.store_blob_record_and_temp_link(ADMIN_ACCESS_USER, REPO, third_blob_sha, location, 0, 0, 0) # Push the second manifest. second_manifest = (DockerSchema1ManifestBuilder(ADMIN_ACCESS_USER, REPO, SECOND_TAG) .add_layer(third_blob_sha, '{"id": "second", "parent": "first"}') .add_layer(second_blob_sha, '{"id": "first"}') .build(docker_v2_signing_key)) _write_manifest(ADMIN_ACCESS_USER, REPO, second_manifest) # Delete all temp tags and perform GC. self._perform_cleanup() # Ensure that the first and second blobs still exists, along with the second tag. self.assertIsNotNone(model.blob.get_repo_blob_by_digest(ADMIN_ACCESS_USER, REPO, first_blob_sha)) self.assertIsNotNone(model.blob.get_repo_blob_by_digest(ADMIN_ACCESS_USER, REPO, second_blob_sha)) self.assertIsNotNone(model.blob.get_repo_blob_by_digest(ADMIN_ACCESS_USER, REPO, third_blob_sha)) self.assertIsNotNone(model.tag.load_tag_manifest(ADMIN_ACCESS_USER, REPO, FIRST_TAG)) self.assertIsNotNone(model.tag.load_tag_manifest(ADMIN_ACCESS_USER, REPO, SECOND_TAG)) self.assertEquals("first", model.tag.get_tag_image(ADMIN_ACCESS_USER, REPO, FIRST_TAG).docker_image_id) # Ensure the IDs have changed. self.assertNotEquals("first", model.tag.get_tag_image(ADMIN_ACCESS_USER, REPO, SECOND_TAG).parent.docker_image_id) self.assertNotEquals("second", model.tag.get_tag_image(ADMIN_ACCESS_USER, REPO, SECOND_TAG).docker_image_id) # Create the fourth blob. fourth_blob_sha = 'sha256:' + hashlib.sha256("FOURTH").hexdigest() model.blob.store_blob_record_and_temp_link(ADMIN_ACCESS_USER, REPO, fourth_blob_sha, location, 0, 0, 0) # Push the third manifest. third_manifest = (DockerSchema1ManifestBuilder(ADMIN_ACCESS_USER, REPO, THIRD_TAG) .add_layer(third_blob_sha, '{"id": "second", "parent": "first"}') .add_layer(fourth_blob_sha, '{"id": "first"}') # Note the change in BLOB from the second manifest. .build(docker_v2_signing_key)) _write_manifest(ADMIN_ACCESS_USER, REPO, third_manifest) # Delete all temp tags and perform GC. self._perform_cleanup() # Ensure all blobs are present. self.assertIsNotNone(model.blob.get_repo_blob_by_digest(ADMIN_ACCESS_USER, REPO, first_blob_sha)) self.assertIsNotNone(model.blob.get_repo_blob_by_digest(ADMIN_ACCESS_USER, REPO, second_blob_sha)) self.assertIsNotNone(model.blob.get_repo_blob_by_digest(ADMIN_ACCESS_USER, REPO, third_blob_sha)) self.assertIsNotNone(model.blob.get_repo_blob_by_digest(ADMIN_ACCESS_USER, REPO, fourth_blob_sha)) # Ensure new synthesized IDs were created. self.assertNotEquals( model.tag.get_tag_image(ADMIN_ACCESS_USER, REPO, SECOND_TAG).docker_image_id, model.tag.get_tag_image(ADMIN_ACCESS_USER, REPO, THIRD_TAG).docker_image_id) if __name__ == '__main__': unittest.main()