145 lines
4.9 KiB
Python
145 lines
4.9 KiB
Python
import hashlib
|
|
import os
|
|
import tarfile
|
|
|
|
from io import BytesIO
|
|
from contextlib import closing
|
|
|
|
import pytest
|
|
|
|
from data.registry_model.blobuploader import (retrieve_blob_upload_manager,
|
|
upload_blob, BlobUploadException,
|
|
BlobDigestMismatchException, BlobTooLargeException,
|
|
BlobUploadSettings)
|
|
from data.registry_model.registry_pre_oci_model import PreOCIModel
|
|
|
|
from storage.distributedstorage import DistributedStorage
|
|
from storage.fakestorage import FakeStorage
|
|
from test.fixtures import *
|
|
|
|
@pytest.fixture()
|
|
def pre_oci_model(initialized_db):
|
|
return PreOCIModel()
|
|
|
|
@pytest.mark.parametrize('chunk_count', [
|
|
0,
|
|
1,
|
|
2,
|
|
10,
|
|
])
|
|
@pytest.mark.parametrize('subchunk', [
|
|
True,
|
|
False,
|
|
])
|
|
def test_basic_upload_blob(chunk_count, subchunk, pre_oci_model):
|
|
repository_ref = pre_oci_model.lookup_repository('devtable', 'complex')
|
|
storage = DistributedStorage({'local_us': FakeStorage(None)}, ['local_us'])
|
|
settings = BlobUploadSettings('2M', 512 * 1024, 3600)
|
|
app_config = {'TESTING': True}
|
|
|
|
data = ''
|
|
with upload_blob(repository_ref, storage, settings) as manager:
|
|
assert manager
|
|
assert manager.blob_upload_id
|
|
|
|
for index in range(0, chunk_count):
|
|
chunk_data = os.urandom(100)
|
|
data += chunk_data
|
|
|
|
if subchunk:
|
|
manager.upload_chunk(app_config, BytesIO(chunk_data))
|
|
manager.upload_chunk(app_config, BytesIO(chunk_data), (index * 100) + 50)
|
|
else:
|
|
manager.upload_chunk(app_config, BytesIO(chunk_data))
|
|
|
|
blob = manager.commit_to_blob(app_config)
|
|
|
|
# Check the blob.
|
|
assert blob.compressed_size == len(data)
|
|
assert not blob.uploading
|
|
assert blob.digest == 'sha256:' + hashlib.sha256(data).hexdigest()
|
|
|
|
# Ensure the blob exists in storage and has the expected data.
|
|
assert storage.get_content(['local_us'], blob.storage_path) == data
|
|
|
|
|
|
def test_cancel_upload(pre_oci_model):
|
|
repository_ref = pre_oci_model.lookup_repository('devtable', 'complex')
|
|
storage = DistributedStorage({'local_us': FakeStorage(None)}, ['local_us'])
|
|
settings = BlobUploadSettings('2M', 512 * 1024, 3600)
|
|
app_config = {'TESTING': True}
|
|
|
|
blob_upload_id = None
|
|
with upload_blob(repository_ref, storage, settings) as manager:
|
|
blob_upload_id = manager.blob_upload_id
|
|
assert pre_oci_model.lookup_blob_upload(repository_ref, blob_upload_id) is not None
|
|
|
|
manager.upload_chunk(app_config, BytesIO('hello world'))
|
|
|
|
# Since the blob was not comitted, the upload should be deleted.
|
|
assert blob_upload_id
|
|
assert pre_oci_model.lookup_blob_upload(repository_ref, blob_upload_id) is None
|
|
|
|
|
|
def test_too_large(pre_oci_model):
|
|
repository_ref = pre_oci_model.lookup_repository('devtable', 'complex')
|
|
storage = DistributedStorage({'local_us': FakeStorage(None)}, ['local_us'])
|
|
settings = BlobUploadSettings('1K', 512 * 1024, 3600)
|
|
app_config = {'TESTING': True}
|
|
|
|
with upload_blob(repository_ref, storage, settings) as manager:
|
|
with pytest.raises(BlobTooLargeException):
|
|
manager.upload_chunk(app_config, BytesIO(os.urandom(1024 * 1024 * 2)))
|
|
|
|
|
|
def test_extra_blob_stream_handlers(pre_oci_model):
|
|
handler1_result = []
|
|
handler2_result = []
|
|
|
|
def handler1(bytes):
|
|
handler1_result.append(bytes)
|
|
|
|
def handler2(bytes):
|
|
handler2_result.append(bytes)
|
|
|
|
repository_ref = pre_oci_model.lookup_repository('devtable', 'complex')
|
|
storage = DistributedStorage({'local_us': FakeStorage(None)}, ['local_us'])
|
|
settings = BlobUploadSettings('1K', 512 * 1024, 3600)
|
|
app_config = {'TESTING': True}
|
|
|
|
with upload_blob(repository_ref, storage, settings,
|
|
extra_blob_stream_handlers=[handler1, handler2]) as manager:
|
|
manager.upload_chunk(app_config, BytesIO('hello '))
|
|
manager.upload_chunk(app_config, BytesIO('world'))
|
|
|
|
assert ''.join(handler1_result) == 'hello world'
|
|
assert ''.join(handler2_result) == 'hello world'
|
|
|
|
|
|
def valid_tar_gz(contents):
|
|
with closing(BytesIO()) as layer_data:
|
|
with closing(tarfile.open(fileobj=layer_data, mode='w|gz')) as tar_file:
|
|
tar_file_info = tarfile.TarInfo(name='somefile')
|
|
tar_file_info.type = tarfile.REGTYPE
|
|
tar_file_info.size = len(contents)
|
|
tar_file_info.mtime = 1
|
|
tar_file.addfile(tar_file_info, BytesIO(contents))
|
|
|
|
layer_bytes = layer_data.getvalue()
|
|
return layer_bytes
|
|
|
|
|
|
def test_uncompressed_size(pre_oci_model):
|
|
repository_ref = pre_oci_model.lookup_repository('devtable', 'complex')
|
|
storage = DistributedStorage({'local_us': FakeStorage(None)}, ['local_us'])
|
|
settings = BlobUploadSettings('1K', 512 * 1024, 3600)
|
|
app_config = {'TESTING': True}
|
|
|
|
with upload_blob(repository_ref, storage, settings) as manager:
|
|
manager.upload_chunk(app_config, BytesIO(valid_tar_gz('hello world')))
|
|
|
|
blob = manager.commit_to_blob(app_config)
|
|
|
|
assert blob.compressed_size is not None
|
|
assert blob.uncompressed_size is not None
|
|
|