From 6ed28930b293018625838059a88bc24e1cb773db Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 13 Jan 2015 17:46:11 -0500 Subject: [PATCH 01/37] Work in progress: Docker -> ACI conversion --- application.py | 2 +- conf/gunicorn_local.py | 2 +- endpoints/verbs.py | 95 +++++++++++++----- endpoints/web.py | 5 + formats/__init__.py | 0 formats/aci.py | 185 +++++++++++++++++++++++++++++++++++ formats/squashed.py | 102 +++++++++++++++++++ formats/tarimageformatter.py | 46 +++++++++ initdb.py | 1 + templates/index.html | 2 + util/dockerloadformat.py | 132 ------------------------- util/streamlayerformat.py | 4 +- util/tarlayerformat.py | 10 +- 13 files changed, 424 insertions(+), 162 deletions(-) create mode 100644 formats/__init__.py create mode 100644 formats/aci.py create mode 100644 formats/squashed.py create mode 100644 formats/tarimageformatter.py delete mode 100644 util/dockerloadformat.py diff --git a/application.py b/application.py index a9bd0df6e..d8b6b2838 100644 --- a/application.py +++ b/application.py @@ -12,4 +12,4 @@ import registry if __name__ == '__main__': logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False) - application.run(port=5000, debug=True, threaded=True, host='0.0.0.0') + application.run(port=80, debug=True, threaded=True, host='0.0.0.0') diff --git a/conf/gunicorn_local.py b/conf/gunicorn_local.py index aa16e63ec..1389c0472 100644 --- a/conf/gunicorn_local.py +++ b/conf/gunicorn_local.py @@ -1,4 +1,4 @@ -bind = '0.0.0.0:5000' +bind = '0.0.0.0:80' workers = 2 worker_class = 'gevent' timeout = 2000 diff --git a/endpoints/verbs.py b/endpoints/verbs.py index f0aef83c4..f316b256c 100644 --- a/endpoints/verbs.py +++ b/endpoints/verbs.py @@ -16,12 +16,15 @@ from storage import Storage from util.queuefile import QueueFile from util.queueprocess import QueueProcess from util.gzipwrap import GzipWrap -from util.dockerloadformat import build_docker_load_stream +from formats.squashed import SquashedDockerImage +from formats.aci import ACIImage + verbs = Blueprint('verbs', __name__) logger = logging.getLogger(__name__) -def _open_stream(namespace, repository, tag, synthetic_image_id, image_json, image_id_list): +def _open_stream(formatter, namespace, repository, tag, synthetic_image_id, image_json, + image_id_list): store = Storage(app) # For performance reasons, we load the full image list here, cache it, then disconnect from @@ -45,17 +48,17 @@ def _open_stream(namespace, repository, tag, synthetic_image_id, image_json, ima logger.debug('Returning image layer %s: %s' % (current_image_id, current_image_path)) yield current_image_stream - stream = build_docker_load_stream(namespace, repository, tag, synthetic_image_id, image_json, + stream = formatter.build_stream(namespace, repository, tag, synthetic_image_id, image_json, get_next_image, get_next_layer) return stream.read -def _write_synthetic_image_to_storage(linked_storage_uuid, linked_locations, queue_file): +def _write_synthetic_image_to_storage(verb, linked_storage_uuid, linked_locations, queue_file): store = Storage(app) def handle_exception(ex): - logger.debug('Exception when building squashed image %s: %s', linked_storage_uuid, ex) + logger.debug('Exception when building %s image %s: %s', verb, linked_storage_uuid, ex) with database.UseThenDisconnect(app.config): model.delete_derived_storage_by_uuid(linked_storage_uuid) @@ -73,11 +76,13 @@ def _write_synthetic_image_to_storage(linked_storage_uuid, linked_locations, que done_uploading.save() -@verbs.route('/squash///', methods=['GET']) -@process_auth -def get_squashed_tag(namespace, repository, tag): +def _repo_verb(namespace, repository, tag, verb, formatter, checker=None, **kwargs): permission = ReadRepositoryPermission(namespace, repository) - if permission.can() or model.repository_is_public(namespace, repository): + + # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + # TODO: renable auth! + # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + if True or permission.can() or model.repository_is_public(namespace, repository): # Lookup the requested tag. try: tag_image = model.get_tag_image(namespace, repository, tag) @@ -89,38 +94,54 @@ def get_squashed_tag(namespace, repository, tag): if not repo_image: abort(404) - # Log the action. - track_and_log('repo_verb', repo_image.repository, tag=tag, verb='squash') - + # If there is a data checker, call it first. store = Storage(app) - derived = model.find_or_create_derived_storage(repo_image.storage, 'squash', + uuid = repo_image.storage.uuid + image_json = None + + if checker is not None: + image_json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid)) + image_json = json.loads(image_json_data) + + if not checker(image_json): + logger.debug('Check mismatch on %s/%s:%s, verb %s', namespace, repository, tag, verb) + abort(404) + + # Log the action. + track_and_log('repo_verb', repo_image.repository, tag=tag, verb=verb, **kwargs) + + derived = model.find_or_create_derived_storage(repo_image.storage, verb, store.preferred_locations[0]) - if not derived.uploading: - logger.debug('Derived image %s exists in storage', derived.uuid) + + # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + # TODO: renable caching! + # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + if False and not derived.uploading: + logger.debug('Derived %s image %s exists in storage', verb, derived.uuid) derived_layer_path = store.image_layer_path(derived.uuid) download_url = store.get_direct_download_url(derived.locations, derived_layer_path) if download_url: - logger.debug('Redirecting to download URL for derived image %s', derived.uuid) + logger.debug('Redirecting to download URL for derived %s image %s', verb, derived.uuid) return redirect(download_url) # Close the database handle here for this process before we send the long download. database.close_db_filter(None) - logger.debug('Sending cached derived image %s', derived.uuid) + logger.debug('Sending cached derived %s image %s', verb, derived.uuid) return send_file(store.stream_read_file(derived.locations, derived_layer_path)) # Load the ancestry for the image. - logger.debug('Building and returning derived image %s', derived.uuid) - uuid = repo_image.storage.uuid + logger.debug('Building and returning derived %s image %s', verb, derived.uuid) ancestry_data = store.get_content(repo_image.storage.locations, store.image_ancestry_path(uuid)) full_image_list = json.loads(ancestry_data) # Load the image's JSON layer. - image_json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid)) - image_json = json.loads(image_json_data) + if not image_json: + image_json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid)) + image_json = json.loads(image_json_data) # Calculate a synthetic image ID. - synthetic_image_id = hashlib.sha256(tag_image.docker_image_id + ':squash').hexdigest() + synthetic_image_id = hashlib.sha256(tag_image.docker_image_id + ':' + verb).hexdigest() # Create a queue process to generate the data. The queue files will read from the process # and send the results to the client and storage. @@ -128,7 +149,7 @@ def get_squashed_tag(namespace, repository, tag): # Close any existing DB connection once the process has exited. database.close_db_filter(None) - args = (namespace, repository, tag, synthetic_image_id, image_json, full_image_list) + args = (formatter, namespace, repository, tag, synthetic_image_id, image_json, full_image_list) queue_process = QueueProcess(_open_stream, 8 * 1024, 10 * 1024 * 1024, # 8K/10M chunk/max args, finished=_cleanup) @@ -140,7 +161,7 @@ def get_squashed_tag(namespace, repository, tag): queue_process.run() # Start the storage saving. - storage_args = (derived.uuid, derived.locations, storage_queue_file) + storage_args = (verb, derived.uuid, derived.locations, storage_queue_file) QueueProcess.run_process(_write_synthetic_image_to_storage, storage_args, finished=_cleanup) # Close the database handle here for this process before we send the long download. @@ -150,3 +171,29 @@ def get_squashed_tag(namespace, repository, tag): return send_file(client_queue_file) abort(403) + + +@verbs.route('/aci/////aci///', methods=['GET']) +@process_auth +def get_rocket_image(server, namespace, repository, tag, os, arch): + def checker(image_json): + # Verify the architecture and os. + operating_system = image_json.get('os', 'linux') + if operating_system != os: + return False + + architecture = image_json.get('architecture', 'amd64') + if architecture != arch: + return False + + return True + + return _repo_verb(namespace, repository, tag, 'aci', ACIImage(), + checker=checker, os=os, arch=arch) + + +@verbs.route('/squash///', methods=['GET']) +@process_auth +def get_squashed_tag(namespace, repository, tag): + return _repo_verb(namespace, repository, tag, 'squash', SquashedDockerImage()) + diff --git a/endpoints/web.py b/endpoints/web.py index 519fc5c5e..8239e2938 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -140,6 +140,11 @@ def repository(path): return index('') +@web.route('//', methods=['GET']) +@no_cache +def repository_test(namespace, repository): + return index('') + @web.route('/security/') @no_cache def security(): diff --git a/formats/__init__.py b/formats/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/formats/aci.py b/formats/aci.py new file mode 100644 index 000000000..c02cac857 --- /dev/null +++ b/formats/aci.py @@ -0,0 +1,185 @@ +from app import app +from util.streamlayerformat import StreamLayerMerger +from formats.tarimageformatter import TarImageFormatter + +import json + +class ACIImage(TarImageFormatter): + """ Image formatter which produces an ACI-compatible TAR. + """ + + def stream_generator(self, namespace, repository, tag, synthetic_image_id, + layer_json, get_image_iterator, get_layer_iterator): + # ACI Format (.tar): + # manifest - The JSON manifest + # rootfs - The root file system + + # Yield the manifest. + yield self.tar_file('manifest', self._build_manifest(namespace, repository, tag, layer_json, + synthetic_image_id)) + + # Yield the merged layer dtaa. + yield self.tar_folder('rootfs') + + layer_merger = StreamLayerMerger(get_layer_iterator, path_prefix='rootfs/') + for entry in layer_merger.get_generator(): + yield entry + + def _build_isolators(self, docker_config): + """ Builds ACI isolator config from the docker config. """ + + def _isolate_memory(memory): + return { + "name": "memory/limit", + "value": str(memory) + 'B' + } + + def _isolate_swap(memory): + return { + "name": "memory/swap", + "value": str(memory) + 'B' + } + + def _isolate_cpu(cpu): + return { + "name": "cpu/shares", + "value": str(cpu) + } + + def _isolate_capabilities(capabilities_set): + return { + "name": "capabilities/bounding-set", + "value": str(capabilities_set) + } + + mappers = { + 'Memory': _isolate_memory, + 'MemorySwap': _isolate_swap, + 'CpuShares': _isolate_cpu, + 'Cpuset': _isolate_capabilities + } + + isolators = [] + + for config_key in mappers: + value = docker_config.get(config_key) + if value: + isolators.append(mappers[config_key](value)) + + return isolators + + def _build_ports(self, docker_config): + """ Builds the ports definitions for the ACI. """ + ports = [] + + for docker_port_definition in docker_config.get('ports', {}): + # Formats: + # port/tcp + # port/udp + # port + + protocol = 'tcp' + port_number = -1 + + if '/' in docker_port_definition: + (port_number, protocol) = docker_port_definition.split('/') + else: + port_number = docker_port_definition + + try: + port_number = int(port_number) + ports.append({ + "name": "port-%s" % port_number, + "port": port_number, + "protocol": protocol + }) + except ValueError: + pass + + return ports + + def _build_volumes(self, docker_config): + """ Builds the volumes definitions for the ACI. """ + volumes = [] + names = set() + + def get_name(docker_volume_path): + parts = docker_volume_path.split('/') + name = '' + + while True: + name = name + parts[-1] + parts = parts[0:-1] + if names.add(name): + break + + name = '/' + name + + return name + + for docker_volume_path in docker_config.get('volumes', {}): + volumes.append({ + "name": get_name(docker_volume_path), + "path": docker_volume_path, + "readOnly": False + }) + return volumes + + + def _build_manifest(self, namespace, repository, tag, docker_layer_data, synthetic_image_id): + """ Builds an ACI manifest from the docker layer data. """ + + config = docker_layer_data.get('config', {}) + config.update(docker_layer_data.get('container_config', {})) + + source_url = "%s://%s/%s/%s:%s" % (app.config['PREFERRED_URL_SCHEME'], + app.config['SERVER_HOSTNAME'], + namespace, repository, tag) + + exec_path = config.get('Cmd', []) + if exec_path: + if not exec_path[0].startswith('/'): + exec_path[0] = '/bin/' + exec_path[0] + + # TODO: ACI doesn't support : in the name, so remove any ports. + hostname = app.config['SERVER_HOSTNAME'] + hostname = hostname.split(':', 1)[0] + + manifest = { + "acKind": "ImageManifest", + "acVersion": "0.1.1", + "name": '%s/%s/%s/%s' % (hostname, namespace, repository, tag), + "labels": [ + { + "name": "version", + "value": "1.0.0" + }, + { + "name": "arch", + "value": docker_layer_data.get('architecture', 'amd64') + }, + { + "name": "os", + "value": docker_layer_data.get('os', 'linux') + } + ], + "app": { + "exec": exec_path, + "user": config.get('User', '') or 'root', + "group": config.get('Group', '') or 'root', + "eventHandlers": [], + "workingDirectory": config.get('WorkingDir', ''), + "environment": {key:value for (key, value) in [e.split('=') for e in config.get('Env')]}, + "isolators": self._build_isolators(config), + "mountPoints": self._build_volumes(config), + "ports": self._build_ports(config), + "annotations": [ + {"name": "created", "value": docker_layer_data.get('created', '')}, + {"name": "homepage", "value": source_url}, + {"name": "quay.io/derived_image", "value": synthetic_image_id}, + ] + }, + } + + return json.dumps(manifest) + diff --git a/formats/squashed.py b/formats/squashed.py new file mode 100644 index 000000000..187d1e74f --- /dev/null +++ b/formats/squashed.py @@ -0,0 +1,102 @@ +from app import app +from util.gzipwrap import GZIP_BUFFER_SIZE +from util.streamlayerformat import StreamLayerMerger +from formats.tarimageformatter import TarImageFormatter + +import copy +import json +import tarfile + +class FileEstimationException(Exception): + """ Exception raised by build_docker_load_stream if the estimated size of the layer TAR + was lower than the actual size. This means the sent TAR header is wrong, and we have + to fail. + """ + pass + + +class SquashedDockerImage(TarImageFormatter): + """ Image formatter which produces a squashed image compatible with the `docker load` + command. + """ + + def stream_generator(self, namespace, repository, tag, synthetic_image_id, + layer_json, get_image_iterator, get_layer_iterator): + # Docker import V1 Format (.tar): + # repositories - JSON file containing a repo -> tag -> image map + # {image ID folder}: + # json - The layer JSON + # layer.tar - The TARed contents of the layer + # VERSION - The docker import version: '1.0' + layer_merger = StreamLayerMerger(get_layer_iterator) + + # Yield the repositories file: + synthetic_layer_info = {} + synthetic_layer_info[tag + '.squash'] = synthetic_image_id + + hostname = app.config['SERVER_HOSTNAME'] + repositories = {} + repositories[hostname + '/' + namespace + '/' + repository] = synthetic_layer_info + + yield self.tar_file('repositories', json.dumps(repositories)) + + # Yield the image ID folder. + yield self.tar_folder(synthetic_image_id) + + # Yield the JSON layer data. + layer_json = self._build_layer_json(layer_json, synthetic_image_id) + yield self.tar_file(synthetic_image_id + '/json', json.dumps(layer_json)) + + # Yield the VERSION file. + yield self.tar_file(synthetic_image_id + '/VERSION', '1.0') + + # Yield the merged layer data's header. + estimated_file_size = 0 + for image in get_image_iterator(): + estimated_file_size += image.storage.uncompressed_size + + yield self.tar_file_header(synthetic_image_id + '/layer.tar', estimated_file_size) + + # Yield the contents of the merged layer. + yielded_size = 0 + for entry in layer_merger.get_generator(): + yield entry + yielded_size += len(entry) + + # If the yielded size is more than the estimated size (which is unlikely but possible), then + # raise an exception since the tar header will be wrong. + if yielded_size > estimated_file_size: + raise FileEstimationException() + + # If the yielded size is less than the estimated size (which is likely), fill the rest with + # zeros. + if yielded_size < estimated_file_size: + to_yield = estimated_file_size - yielded_size + while to_yield > 0: + yielded = min(to_yield, GZIP_BUFFER_SIZE) + yield '\0' * yielded + to_yield -= yielded + + # Yield any file padding to 512 bytes that is necessary. + yield self.tar_file_padding(estimated_file_size) + + # Last two records are empty in TAR spec. + yield '\0' * 512 + yield '\0' * 512 + + + def _build_layer_json(self, layer_json, synthetic_image_id): + updated_json = copy.deepcopy(layer_json) + updated_json['id'] = synthetic_image_id + + if 'parent' in updated_json: + del updated_json['parent'] + + if 'config' in updated_json and 'Image' in updated_json['config']: + updated_json['config']['Image'] = synthetic_image_id + + if 'container_config' in updated_json and 'Image' in updated_json['container_config']: + updated_json['container_config']['Image'] = synthetic_image_id + + return updated_json + diff --git a/formats/tarimageformatter.py b/formats/tarimageformatter.py new file mode 100644 index 000000000..162c89b90 --- /dev/null +++ b/formats/tarimageformatter.py @@ -0,0 +1,46 @@ +import tarfile +from util.gzipwrap import GzipWrap + +class TarImageFormatter(object): + """ Base class for classes which produce a TAR containing image and layer data. """ + + def build_stream(self, namespace, repository, tag, synthetic_image_id, layer_json, + get_image_iterator, get_layer_iterator): + """ Builds and streams a synthetic .tar.gz that represents the formatted TAR created by this + class's implementation. + """ + return GzipWrap(self.stream_generator(namespace, repository, tag, + synthetic_image_id, layer_json, + get_image_iterator, get_layer_iterator)) + + def stream_generator(self, namespace, repository, tag, synthetic_image_id, + layer_json, get_image_iterator, get_layer_iterator): + raise NotImplementedError + + def tar_file(self, name, contents): + """ Returns the TAR binary representation for a file with the given name and file contents. """ + length = len(contents) + tar_data = self.tar_file_header(name, length) + tar_data += contents + tar_data += self.tar_file_padding(length) + return tar_data + + def tar_file_padding(self, length): + """ Returns TAR file padding for file data of the given length. """ + if length % 512 != 0: + return '\0' * (512 - (length % 512)) + + return '' + + def tar_file_header(self, name, file_size): + """ Returns TAR file header data for a file with the given name and size. """ + info = tarfile.TarInfo(name=name) + info.type = tarfile.REGTYPE + info.size = file_size + return info.tobuf() + + def tar_folder(self, name): + """ Returns TAR file header data for a folder with the given name. """ + info = tarfile.TarInfo(name=name) + info.type = tarfile.DIRTYPE + return info.tobuf() \ No newline at end of file diff --git a/initdb.py b/initdb.py index 74199d024..8b50431a1 100644 --- a/initdb.py +++ b/initdb.py @@ -255,6 +255,7 @@ def initialize_database(): ImageStorageLocation.create(name='local_us') ImageStorageTransformation.create(name='squash') + ImageStorageTransformation.create(name='aci') # NOTE: These MUST be copied over to NotificationKind, since every external # notification can also generate a Quay.io notification. diff --git a/templates/index.html b/templates/index.html index e6e698bac..a0bf60469 100644 --- a/templates/index.html +++ b/templates/index.html @@ -10,6 +10,8 @@ + + {% endblock %} {% block body_content %} diff --git a/util/dockerloadformat.py b/util/dockerloadformat.py deleted file mode 100644 index b4a8393c3..000000000 --- a/util/dockerloadformat.py +++ /dev/null @@ -1,132 +0,0 @@ -from util.gzipwrap import GzipWrap, GZIP_BUFFER_SIZE -from util.streamlayerformat import StreamLayerMerger -from app import app - -import copy -import json -import tarfile - -class FileEstimationException(Exception): - """ Exception raised by build_docker_load_stream if the estimated size of the layer TAR - was lower than the actual size. This means the sent TAR header is wrong, and we have - to fail. - """ - pass - - -def build_docker_load_stream(namespace, repository, tag, synthetic_image_id, - layer_json, get_image_iterator, get_layer_iterator): - """ Builds and streams a synthetic .tar.gz that represents a squashed version - of the given layers, in `docker load` V1 format. - """ - return GzipWrap(_import_format_generator(namespace, repository, tag, - synthetic_image_id, layer_json, - get_image_iterator, get_layer_iterator)) - - -def _import_format_generator(namespace, repository, tag, synthetic_image_id, - layer_json, get_image_iterator, get_layer_iterator): - # Docker import V1 Format (.tar): - # repositories - JSON file containing a repo -> tag -> image map - # {image ID folder}: - # json - The layer JSON - # layer.tar - The TARed contents of the layer - # VERSION - The docker import version: '1.0' - layer_merger = StreamLayerMerger(get_layer_iterator) - - # Yield the repositories file: - synthetic_layer_info = {} - synthetic_layer_info[tag + '.squash'] = synthetic_image_id - - hostname = app.config['SERVER_HOSTNAME'] - repositories = {} - repositories[hostname + '/' + namespace + '/' + repository] = synthetic_layer_info - - yield _tar_file('repositories', json.dumps(repositories)) - - # Yield the image ID folder. - yield _tar_folder(synthetic_image_id) - - # Yield the JSON layer data. - layer_json = _build_layer_json(layer_json, synthetic_image_id) - yield _tar_file(synthetic_image_id + '/json', json.dumps(layer_json)) - - # Yield the VERSION file. - yield _tar_file(synthetic_image_id + '/VERSION', '1.0') - - # Yield the merged layer data's header. - estimated_file_size = 0 - for image in get_image_iterator(): - estimated_file_size += image.storage.uncompressed_size - - yield _tar_file_header(synthetic_image_id + '/layer.tar', estimated_file_size) - - # Yield the contents of the merged layer. - yielded_size = 0 - for entry in layer_merger.get_generator(): - yield entry - yielded_size += len(entry) - - # If the yielded size is more than the estimated size (which is unlikely but possible), then - # raise an exception since the tar header will be wrong. - if yielded_size > estimated_file_size: - raise FileEstimationException() - - # If the yielded size is less than the estimated size (which is likely), fill the rest with - # zeros. - if yielded_size < estimated_file_size: - to_yield = estimated_file_size - yielded_size - while to_yield > 0: - yielded = min(to_yield, GZIP_BUFFER_SIZE) - yield '\0' * yielded - to_yield -= yielded - - # Yield any file padding to 512 bytes that is necessary. - yield _tar_file_padding(estimated_file_size) - - # Last two records are empty in TAR spec. - yield '\0' * 512 - yield '\0' * 512 - - -def _build_layer_json(layer_json, synthetic_image_id): - updated_json = copy.deepcopy(layer_json) - updated_json['id'] = synthetic_image_id - - if 'parent' in updated_json: - del updated_json['parent'] - - if 'config' in updated_json and 'Image' in updated_json['config']: - updated_json['config']['Image'] = synthetic_image_id - - if 'container_config' in updated_json and 'Image' in updated_json['container_config']: - updated_json['container_config']['Image'] = synthetic_image_id - - return updated_json - - -def _tar_file(name, contents): - length = len(contents) - tar_data = _tar_file_header(name, length) - tar_data += contents - tar_data += _tar_file_padding(length) - return tar_data - - -def _tar_file_padding(length): - if length % 512 != 0: - return '\0' * (512 - (length % 512)) - - return '' - -def _tar_file_header(name, file_size): - info = tarfile.TarInfo(name=name) - info.type = tarfile.REGTYPE - info.size = file_size - return info.tobuf() - - -def _tar_folder(name): - info = tarfile.TarInfo(name=name) - info.type = tarfile.DIRTYPE - return info.tobuf() diff --git a/util/streamlayerformat.py b/util/streamlayerformat.py index 914dea4a2..686c16204 100644 --- a/util/streamlayerformat.py +++ b/util/streamlayerformat.py @@ -11,8 +11,8 @@ AUFS_WHITEOUT_PREFIX_LENGTH = len(AUFS_WHITEOUT) class StreamLayerMerger(TarLayerFormat): """ Class which creates a generator of the combined TAR data for a set of Docker layers. """ - def __init__(self, layer_iterator): - super(StreamLayerMerger, self).__init__(layer_iterator) + def __init__(self, layer_iterator, path_prefix=None): + super(StreamLayerMerger, self).__init__(layer_iterator, path_prefix) self.path_trie = marisa_trie.Trie() self.path_encountered = [] diff --git a/util/tarlayerformat.py b/util/tarlayerformat.py index 3468678c5..2d7a6b52d 100644 --- a/util/tarlayerformat.py +++ b/util/tarlayerformat.py @@ -8,8 +8,9 @@ class TarLayerReadException(Exception): class TarLayerFormat(object): """ Class which creates a generator of the combined TAR data. """ - def __init__(self, tar_iterator): + def __init__(self, tar_iterator, path_prefix=None): self.tar_iterator = tar_iterator + self.path_prefix = path_prefix def get_generator(self): for current_tar in self.tar_iterator(): @@ -36,7 +37,12 @@ class TarLayerFormat(object): continue # Yield the tar header. - yield tar_info.tobuf() + if self.path_prefix: + clone = tarfile.TarInfo.frombuf(tar_info.tobuf()) + clone.name = os.path.join(self.path_prefix, clone.name) + yield clone.tobuf() + else: + yield tar_info.tobuf() # Try to extract any file contents for the tar. If found, we yield them as well. if tar_info.isreg(): From d8817a2965a054abe7ae0023ca1bbf997d45e1a9 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 13 Jan 2015 17:55:27 -0500 Subject: [PATCH 02/37] Fix ACI conversion handling of relative exec commands --- formats/aci.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/formats/aci.py b/formats/aci.py index c02cac857..05f1dce3e 100644 --- a/formats/aci.py +++ b/formats/aci.py @@ -136,10 +136,11 @@ class ACIImage(TarImageFormatter): app.config['SERVER_HOSTNAME'], namespace, repository, tag) + # ACI requires that the execution command be absolutely referenced. Therefore, if we find + # a relative command, we give it as an argument to /bin/sh to resolve and execute for us. exec_path = config.get('Cmd', []) - if exec_path: - if not exec_path[0].startswith('/'): - exec_path[0] = '/bin/' + exec_path[0] + if exec_path and not exec_path[0].startswith('/'): + exec_path = ['/bin/sh', '-c', '""%s""' % ' '.join(exec_path)] # TODO: ACI doesn't support : in the name, so remove any ports. hostname = app.config['SERVER_HOSTNAME'] From 5bbf1d0c14c678952ac82f6ffc140205fc68eb61 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 13 Jan 2015 18:00:01 -0500 Subject: [PATCH 03/37] Make sure the ac-discovery URL is generated properly from config values --- endpoints/common.py | 2 ++ templates/index.html | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/endpoints/common.py b/endpoints/common.py index e81b4facf..bc57c1beb 100644 --- a/endpoints/common.py +++ b/endpoints/common.py @@ -207,6 +207,8 @@ def render_page_template(name, **kwargs): cache_buster=cache_buster, has_billing=features.BILLING, contact_href=contact_href, + hostname=app.config['SERVER_HOSTNAME'], + preferred_scheme=app.config['PREFERRED_URL_SCHEME'], **kwargs)) resp.headers['X-FRAME-OPTIONS'] = 'DENY' diff --git a/templates/index.html b/templates/index.html index a0bf60469..120b7865c 100644 --- a/templates/index.html +++ b/templates/index.html @@ -10,7 +10,7 @@ - + {% endblock %} From e902cd62fdbde268f87690f5ca7fa9f816d7dac2 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 16 Jan 2015 17:29:59 -0500 Subject: [PATCH 04/37] Handle capabilities strings with both spaces and commas (since Docker doesn't have this documented yet) --- formats/aci.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/formats/aci.py b/formats/aci.py index 05f1dce3e..ea108830f 100644 --- a/formats/aci.py +++ b/formats/aci.py @@ -3,6 +3,7 @@ from util.streamlayerformat import StreamLayerMerger from formats.tarimageformatter import TarImageFormatter import json +import re class ACIImage(TarImageFormatter): """ Image formatter which produces an ACI-compatible TAR. @@ -46,10 +47,11 @@ class ACIImage(TarImageFormatter): "value": str(cpu) } - def _isolate_capabilities(capabilities_set): + def _isolate_capabilities(capabilities_set_value): + capabilities_set = re.split(r'[\s,]', capabilities_set_value) return { "name": "capabilities/bounding-set", - "value": str(capabilities_set) + "value": ' '.join(capabilities_set) } mappers = { From fa55169c35afba5921f522a1f352da48a24a057f Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 16 Jan 2015 18:23:40 -0500 Subject: [PATCH 05/37] - Fix bug in tarlayerformat when dealing with larger headers - Fix bug around entry points and config in the ACI converter --- formats/aci.py | 4 ++-- util/tarlayerformat.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/formats/aci.py b/formats/aci.py index ea108830f..156858ed2 100644 --- a/formats/aci.py +++ b/formats/aci.py @@ -132,7 +132,6 @@ class ACIImage(TarImageFormatter): """ Builds an ACI manifest from the docker layer data. """ config = docker_layer_data.get('config', {}) - config.update(docker_layer_data.get('container_config', {})) source_url = "%s://%s/%s/%s:%s" % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'], @@ -140,7 +139,8 @@ class ACIImage(TarImageFormatter): # ACI requires that the execution command be absolutely referenced. Therefore, if we find # a relative command, we give it as an argument to /bin/sh to resolve and execute for us. - exec_path = config.get('Cmd', []) + entrypoint = config.get('Entrypoint') or [] + exec_path = entrypoint + config.get('Cmd') or [] if exec_path and not exec_path[0].startswith('/'): exec_path = ['/bin/sh', '-c', '""%s""' % ' '.join(exec_path)] diff --git a/util/tarlayerformat.py b/util/tarlayerformat.py index 2d7a6b52d..3d6ddd94b 100644 --- a/util/tarlayerformat.py +++ b/util/tarlayerformat.py @@ -1,5 +1,6 @@ import os import tarfile +import copy class TarLayerReadException(Exception): """ Exception raised when reading a layer has failed. """ @@ -38,7 +39,7 @@ class TarLayerFormat(object): # Yield the tar header. if self.path_prefix: - clone = tarfile.TarInfo.frombuf(tar_info.tobuf()) + clone = copy.deepcopy(tar_info) clone.name = os.path.join(self.path_prefix, clone.name) yield clone.tobuf() else: From 9583023749aa110feca0cb44f3662c9ddefbb5f1 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 29 Jan 2015 14:25:42 -0500 Subject: [PATCH 06/37] Start cleanup in prep for merge --- endpoints/verbs.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/endpoints/verbs.py b/endpoints/verbs.py index f316b256c..6e9de7d90 100644 --- a/endpoints/verbs.py +++ b/endpoints/verbs.py @@ -78,11 +78,7 @@ def _write_synthetic_image_to_storage(verb, linked_storage_uuid, linked_location def _repo_verb(namespace, repository, tag, verb, formatter, checker=None, **kwargs): permission = ReadRepositoryPermission(namespace, repository) - - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - # TODO: renable auth! - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - if True or permission.can() or model.repository_is_public(namespace, repository): + if permission.can() or model.repository_is_public(namespace, repository): # Lookup the requested tag. try: tag_image = model.get_tag_image(namespace, repository, tag) @@ -113,10 +109,7 @@ def _repo_verb(namespace, repository, tag, verb, formatter, checker=None, **kwar derived = model.find_or_create_derived_storage(repo_image.storage, verb, store.preferred_locations[0]) - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - # TODO: renable caching! - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - if False and not derived.uploading: + if not derived.uploading: logger.debug('Derived %s image %s exists in storage', verb, derived.uuid) derived_layer_path = store.image_layer_path(derived.uuid) download_url = store.get_direct_download_url(derived.locations, derived_layer_path) From 15397d270ae338ef41fa12873163d52589a59644 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 29 Jan 2015 14:57:42 -0500 Subject: [PATCH 07/37] Add tests for path prefixing and super long filenames --- test/test_streamlayerformat.py | 56 ++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/test/test_streamlayerformat.py b/test/test_streamlayerformat.py index 093bbaee4..6e65e2b34 100644 --- a/test/test_streamlayerformat.py +++ b/test/test_streamlayerformat.py @@ -34,11 +34,11 @@ class TestStreamLayerMerger(unittest.TestCase): def create_empty_layer(self): return '' - def squash_layers(self, layers): + def squash_layers(self, layers, path_prefix=None): def get_layers(): return [StringIO(layer) for layer in layers] - merger = StreamLayerMerger(get_layers) + merger = StreamLayerMerger(get_layers, path_prefix=path_prefix) merged_data = ''.join(merger.get_generator()) return merged_data @@ -395,5 +395,57 @@ class TestStreamLayerMerger(unittest.TestCase): except TarLayerReadException as ex: self.assertEquals('Could not read layer', ex.message) + def test_single_layer_with_prefix(self): + tar_layer = self.create_layer( + foo = 'some_file', + bar = 'another_file', + meh = 'third_file') + + squashed = self.squash_layers([tar_layer], path_prefix='foo/') + + self.assertHasFile(squashed, 'foo/some_file', 'foo') + self.assertHasFile(squashed, 'foo/another_file', 'bar') + self.assertHasFile(squashed, 'foo/third_file', 'meh') + + def test_multiple_layers_overwrite_with_prefix(self): + second_layer = self.create_layer( + foo = 'some_file', + bar = 'another_file', + meh = 'third_file') + + first_layer = self.create_layer( + top = 'another_file') + + squashed = self.squash_layers([first_layer, second_layer], path_prefix='foo/') + + self.assertHasFile(squashed, 'foo/some_file', 'foo') + self.assertHasFile(squashed, 'foo/third_file', 'meh') + self.assertHasFile(squashed, 'foo/another_file', 'top') + + + def test_superlong_filename(self): + tar_layer = self.create_layer( + meh = 'this_is_the_filename_that_never_ends_it_goes_on_and_on_my_friend_some_people_started') + + squashed = self.squash_layers([tar_layer], + path_prefix='foo/') + + self.assertHasFile(squashed, 'foo/this_is_the_filename_that_never_ends_it_goes_on_and_on_my_friend_some_people_started', 'meh') + + + def test_superlong_prefix(self): + tar_layer = self.create_layer( + foo = 'some_file', + bar = 'another_file', + meh = 'third_file') + + squashed = self.squash_layers([tar_layer], + path_prefix='foo/bar/baz/something/foo/bar/baz/anotherthing/whatever/this/is/a/really/long/filename/that/goes/here/') + + self.assertHasFile(squashed, 'foo/bar/baz/something/foo/bar/baz/anotherthing/whatever/this/is/a/really/long/filename/that/goes/here/some_file', 'foo') + self.assertHasFile(squashed, 'foo/bar/baz/something/foo/bar/baz/anotherthing/whatever/this/is/a/really/long/filename/that/goes/here/another_file', 'bar') + self.assertHasFile(squashed, 'foo/bar/baz/something/foo/bar/baz/anotherthing/whatever/this/is/a/really/long/filename/that/goes/here/third_file', 'meh') + + if __name__ == '__main__': unittest.main() From ae85ea247e703604a1332f6754de3d41cba4e75f Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 29 Jan 2015 14:59:49 -0500 Subject: [PATCH 08/37] Revert changes accidentally checked in --- application.py | 2 +- conf/gunicorn_local.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application.py b/application.py index d8b6b2838..a9bd0df6e 100644 --- a/application.py +++ b/application.py @@ -12,4 +12,4 @@ import registry if __name__ == '__main__': logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False) - application.run(port=80, debug=True, threaded=True, host='0.0.0.0') + application.run(port=5000, debug=True, threaded=True, host='0.0.0.0') diff --git a/conf/gunicorn_local.py b/conf/gunicorn_local.py index 1389c0472..aa16e63ec 100644 --- a/conf/gunicorn_local.py +++ b/conf/gunicorn_local.py @@ -1,4 +1,4 @@ -bind = '0.0.0.0:80' +bind = '0.0.0.0:5000' workers = 2 worker_class = 'gevent' timeout = 2000 From 1022355bb132373c47436af05bb48adde660a4ef Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 29 Jan 2015 15:00:44 -0500 Subject: [PATCH 09/37] Revert changes accidentally checked in --- endpoints/web.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/endpoints/web.py b/endpoints/web.py index 8239e2938..519fc5c5e 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -140,11 +140,6 @@ def repository(path): return index('') -@web.route('//', methods=['GET']) -@no_cache -def repository_test(namespace, repository): - return index('') - @web.route('/security/') @no_cache def security(): From 84e5c0644eb20b34da33aee8921c8f8b84e75f4d Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 2 Feb 2015 14:07:32 -0500 Subject: [PATCH 10/37] Address comments --- endpoints/verbs.py | 2 +- formats/aci.py | 5 +++-- test/data/test.db | Bin 251904 -> 684032 bytes util/tarlayerformat.py | 3 +++ 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/endpoints/verbs.py b/endpoints/verbs.py index 6e9de7d90..8eb46bcc3 100644 --- a/endpoints/verbs.py +++ b/endpoints/verbs.py @@ -168,7 +168,7 @@ def _repo_verb(namespace, repository, tag, verb, formatter, checker=None, **kwar @verbs.route('/aci/////aci///', methods=['GET']) @process_auth -def get_rocket_image(server, namespace, repository, tag, os, arch): +def get_aci_image(server, namespace, repository, tag, os, arch): def checker(image_json): # Verify the architecture and os. operating_system = image_json.get('os', 'linux') diff --git a/formats/aci.py b/formats/aci.py index 156858ed2..1d82d8c6a 100644 --- a/formats/aci.py +++ b/formats/aci.py @@ -150,7 +150,7 @@ class ACIImage(TarImageFormatter): manifest = { "acKind": "ImageManifest", - "acVersion": "0.1.1", + "acVersion": "0.2.0", "name": '%s/%s/%s/%s' % (hostname, namespace, repository, tag), "labels": [ { @@ -172,7 +172,8 @@ class ACIImage(TarImageFormatter): "group": config.get('Group', '') or 'root', "eventHandlers": [], "workingDirectory": config.get('WorkingDir', ''), - "environment": {key:value for (key, value) in [e.split('=') for e in config.get('Env')]}, + "environment": [(key, value) + for (key, value) in [e.split('=') for e in config.get('Env')]], "isolators": self._build_isolators(config), "mountPoints": self._build_volumes(config), "ports": self._build_ports(config), diff --git a/test/data/test.db b/test/data/test.db index cdaa66e57f768e5e4463ddc644aca0f4add444b4..c37aec119f0b2d599ca2fc5db9c1e1660280afa0 100644 GIT binary patch delta 24289 zcmeHv2Y6IP*YM7r-Mw4xy+aQz5JEyi*=%nVNg)N&6Oxcnw*>;E0HK!{nu5|?WdNlJ zHc*-oS3qfsg(kv_sHg#vE+VL?H2-sV0|Y{N{l546zvua%=g*^;yEA9boS8ZI%xN=E zZX!>f>==@rnNn2l2`MZcUF;|iF-jUp5RycPgh(WKh(scJeNjkH^AL&Vhj1DEV#&Ag zlu&Y$Tqjq_Pvji=fqbi+q3FrJ%u3`Y*(~Yz*vc$nWT?Q~j{DfFawMXrUf+;Vug1~P zo9qnsu68u>P8jk8Y67Kwz0UMl)YxDDNxI#;E!B(~RsE7W8hM+fCZdKkb2Zb{yQ~}M zotoOi>rRP74ZJ(TT6$Ac1@GaMUdrAA8|1N~>Dc?XJ>0tu;7YqSMdWU?La(=Lyj+OO zijV2*UDU0u_jXsC_qsh!&fhKS=pELzvv;%2?)@OFmA8jY=@snfy`v+#c^7pE^?n|q z_s$NpA{BHx-aDZ~3#7b@n)6n5Xy;80k3b4PMGL}@d-L1A?p@brQ`J0l3`vVUp&h-J z*5jgJx@J^jc#awik3k-I^v;Hd=?QqW=mL+R#ymU``CUSOuWC5p3a$znaI)6JtK$ZE ze`gy=$#n_wHavg!q53KvHxzrXs|iQeOUN~HihMzKlB$N>OSO>r+W@8^-u0v(M>a^v z23WcyWETK7SpG=B4UO{%*(@O!$w~4r@(I~aHb08IMHzy3b5LJG-j+bupOW{q{gqm;s!}yk?o@Cc&%R6wpIu73++Ye6CX=90zZ-p_foFbWYWoTLPr#>j=&tBU)M$+o90B6)4*^!uiiVig9 zYrxrK`^we1#f>)7#a{!?Hp^9&olcynq~>n`XOptrtk%opU!_&w08ZJT*-l0O*RRlm z6M(Z*pWA!7nLk_Wof=Z0P<`EFEYXF2>8%b)Rp6IiiIxc!&w5Sms6w(dI!5y4ya?~n zcH?N@Krx9`lCOxGlc;BL*VQV%t@=3gu51Y^VdhCA z_)+Q&($$JJ?BCLfSi!haYd%R9%yi>_;wCHG;5%GL;RiBM5h-ky`xKd~1>8Z^dqPwE z7Tcf9W(UtyE+r8CyRU=59#jl*Pv zB5@9)yWy3dWU!15`4D$};V;~gUYdZzPysd00LZroLqWwYrX#gtc=Iyw2u1L?JQ7f3 z`%%m-2KX#m$N>E25_r#`Wdq@Th}iaw1s>(QaYyfl&fzo>LHU_IP!z-9PPD8)ZjTbZ z8J(fiN~jUUK4PZB!z(_nWP;B>n42DC?5| zU3a6G%z)?%k>X!&GvVH;fLmq7TI#go2paPQU^E2=4zI{an}}4Gn}L|0!hjeZU3Qz% z&@18)+us(44YNW|U>r`D?gq%q&7oCCy1@$XUU3|oTS7qx+D0UQjyT{n4hq`S9v0~H z1q<|P6-%vu1Fq%-dT-)zU|Cf~ zO zNhXp}q=-03KFK7#Ner_?N5^@W4lV4!h`jLDOvf&6hKt6_TZX08s6N0}JEkh6#~G%AZ!3W<_IGjx*yS|& zjvNPu*-!R@im;Vbk@e(tV44@n3uFPX%?vV?j0eUUK|EwIuue8fC5ga1-H4fJfqmMO zU=jojq$V=qZ(yNch0DS@V4@SkG2wGyqYs4-1RpTco5HKYN?;|gFt0**3b<*qFjgo9 zesT!|g*@P>WFcPY2|Q&KG(v>XPG}`G6$r2u=5OJ7}x zX!^@gB&!OG`IcF{3rQ;`bDbpQ1}IJ!K_mJG7-k>Y1MKoPSqIFrj64UdGM!8UMj1|= zz$RIw4=_nrVgMEiB`twLRD=QcxF%c#=J-Z90<5t|*anQTPFO806P^?10#8g5MgvDU zg#zG*K0+*TgF)y7ywFmpXaual@Hc=FF7iJB8yw;H0TXQF-v$;~%`XE6n9EOxwQxA1;>b${AqX;siAvaH(d3blgBpKYbam$AwH`)i=Fb;^qh;hGAq< zB~mg0L3SC`RC4(tcDt)yX0?$sSklzJl;_e-R~J7 zwTzQ0!QL4?XuOm?Xx=FKx2vfbk3N2#|+$a(l4 z-i7aA5qXl70sr?Uy+9U3!Zw10v%-F1lR)9)b_%KRF^3B({ycw}e~W(_KFbVX(U!oL z-*6vtZ*tFbQ@KK5I}O)VeN+9t`a|`WT1%^IJ{{Ba{mf=x;YZ90+`R?*T7tf2X5u{j zt+bEyb6F4BA$ba0!hWTQSL{|MDyviy)dbZcZZda_Zv$fSpb$zbgd?PnWCPd*L!}j6 z*q}!mpKkA!R0Zv~ni0IMovr^ph!-{YZgM`2S_Uk?G*@5L%Dcuj(EG=f&Zu>u&^C1d zXb>+sQA_W}sgqEfKxxluxu|W`;%S`_3i0l)=;@scE!x3bb8p(xd=v_A?Yy5Y9faEZ z%c7U1qcHEnWu+)wY~Re=?8UCAgE#ZV$tc2m@I_PJ&fn?vzy}-AE{opxb;f8W2NW3sbT`e5o-r|~p ziqVq0NjmahfGq34Ul7!=44Hg4emy^j5z;Wn4&#QM`>%^ z1JSAbndbC|511rrdz_5usr^h_6j?R2Y$j%mN_zPdCWw=Qz7G=TBKejat&QlejSLdq z3;;Euk}Lsz&QQ072YU@q9sS>zO^=m_t7J-YS^{(P6FL1b$u-(Sg7@^AP$l_M zLjC~FsXqCh=>75ac1lt$;<`wx>%kT4t$wqElAIKAT_Pvzfr|5<+!zufN*Yd4C?Sc$ z8~opV7Pp>LsfVgQRfQ|xQmPb%_Z!1PkEa7h;D%u?gWK(PIjvyl>x~+d-3E5P-DcLf z9S)DhV$(a_MjKr;0ynM5%*-^KExpaD{W3BP35hvzy<-c^y?Ptdk_;(HSt-U|eUkEH z;uBMiu~vJplw?z?&JdTa>zkPw(>q;n$jdV5^k$9D0RM&PtzAvJu6j$9-JrJ{OqFfb zf+WW%spJ(Bg;Jr2F&Z2meWAypad`|TjmcrPYMc%94wuGiF8|pGk@?#AyyEncriwATa=F~VJ}!;Z;;_L&!-r;f8{Hb6 z$5vQq)7x}TlYX?L#9dk}?m{&*#*8YRkeu0jXnvMqLSAf1UPg>1C2_1SFE2AMDK>YQ zBdNe;Nu8LOKS5vCZ=!beq?ojkNwChQ=={emv?&T!*{m~EhRT3O`~6SOVbXa#E|X2; zvbaGGz-((A@VRSjI-Nu3avLpHgWJ1td$Wq(dFiozV)K$rx#@{9aXBfO8L`&%tdwMZ zMq*5EOk%P*FEb}Ur(bqLe3~&yXUXbgP41nQptBbk?73#UDWxv+0Zk^m!Dy`PBn6t< zR?2$Td>}fP#qKe9oEozZX3JzZ88yxvr`w`2=nEY>qu%Z^8y$tCJ!6K|nz!VvN$z2N z%7PJY$!M9^)=haW?4Nt*Vw);XQHmy;f@+#Q5J9Pm0sk` z8|o=?7da*lceqNOQN^WWhIvXxltvFNb`*_#^w(>QG8^qWqoJ}j1C(m+FKVIFY|y!F zK$=IS)D2VTaNBJfBM2>>%OK7!e7fWON>Jwtclqd2XKA@DAv-h1sMBZniH++AYnh#! zpJU56#>e){&Fp0}rls^v>SfKh8v6B00l^xZXa`m>Wt#Ki@@;V@Q+|5g1s3Tufvl^v zAfPX}Qf5}_{a>%ss@K~cM!m-9aTNB46?Qma_i-3BW``ZpcQ%CNthq)~l~O z&zWG+_sK1CWlkDt(w9VK*hvvKV)}iL~UgL^JTP)5#BTVU*31hvxsybR9 zlFP;@gVAiVTSTpmBSs0>>8ZpBUZZMoa=b~#fb#PaSgZ5y@0x*NfnSp`Za>nB>=hj( z*T`k?1$+@S1+!Mif*TOsV!}jt2 zDbq1Q*sO$AZt~nmvYLhY!@o4=?=2223;F!vxxKqE-o2khw<0e~$Qg*xoFHFAMy!lW&U&u)2OYD7h{}8osKmxv>LefGw@DNayj^nZSz#;X2tU67env7Wp zCRXwd)%d@@d=(GsJb;a8Kud--^C)jtP5(OfN@l7O$SBN-)}s=HdDm=0XXu>bLLNfTwTROr94QvQXi zxTl@s8btr;A~6XfLCX5p=b`24RKIKwtPad!jm!MuIjs4dV!fwMhdst_8zPNvgr-T* zG<2PL3uj2z$uzRJ*>UV`#bVVab%fdnp6V<7bYTxE6Fw%}B+pC6OU3|W)cW%tIbuLM zFuQ}jSB|jWjz>L6A28F?F0}OSI2y}Uw$rjf=^asJUQx-vh4$V!{{%AJMNo;1RfjJO zLdYB_>2;}0ZV3dsyIpGKJ$9+HxAk9IWUWg1>mWil`m!2)JyAC&Ap6}{wfCu;#Cz;! zXWB!7w5Y4ESb>5Wh~xlhTe?n#hM?}^dkDRvLTYMJqv5EBSTLC8w?KVX6f}tuu7`aijZ$~gnLnwxBZVMid^KFnIwkbogVnH_Z-}U_c zCSwH%wagIegj#eQL>bRsZyMPz*N~TE?U$5g$%x6yPBY~i?B@KmoW#uZn1mE_dLF2T zz4caOqIc&X;ohV_(j!y*CFJ+kC#9q&rRA8jEWPbEuNwqJsW~5ho+7Vb^^W*rE*OumK1{s7htKra>`0 zZ?k#K7SZMb1IPk8I~cAuv(BUOxIk$K6VUE1G>-O^m3DQwi;GIiJ!8ttOGkK0YAHVk zT~~vps~L=3z13tiii1JqD+&4PQT>BK-qR3)bCmz2-vFvVCFIX~Pz|PkHb%UR+yVuq z9uOD36@--k10d~@@-bifV_hkatpx5P%Oqq6Syp!h)tD|bqQV*m`6r9t4RLZY=i^pA zZgX%oiA-{%CsNDFMbRZg8@`8@KD_aSWW0vy#W>W86@!q!)gOeUXA@D|$IQzRsi~>t zH;LpoCW=|ioWY&(Te4W$9(k<%0Q;FjuUM)K14$U7Ud+k4VqqObO8QAkCEqTJY0`YU zwWr^CIWyC*x0iby#p3^Qbch)(thd$&wMhu|qH8T^5Q?V?D@sNQw67KAqC~%xqr0pq znx0Gnah2q2kcwVHs1Mzo4&v%S8i=dpzv3_jB)}6Cntj>Eq)f6QLb;zvnZ88dGzXh zv=;Rf{|TiZEI^6>Z8`W@HeDrCi2NePyz1eYSAb&%qh^Bm)nf8XO;EKiQ~Lvr*0uxp_d7#3jYeD;WcPqy&7lLkOLUzW zT;K$>cuX4b|AAM_Xf?ZBCVFx-GFO1}F2Umr5yjkHFP=F>ql3TDE+L`irGwp$a!+X2 zkkHi9k}e?+K->I2zq`;=Ye`DIAy5@%wCIdFbEOn1B{@i0DT`6qb#|x61kMPLTL-yF zO>Iu_0eH+{ahpAMyB;i9z0F3~jzQ7w@0oY^u$gUUvnjBNhzVij=EE~kqptWjh{j4r zijh(sfd4~Wra$U|AK*9R=*} z27j9d9GPy7T@QX_kIQBS=JD9AE;nYhj6#BPlwiB40NM=hTr>DL?V`KWXw%qrIar&Jw0Ukd-!z?YNR9pdgU={OO_}KRP?Xc5$FHVZ#@1(e-pKoi>A=;q&}OegrOHU zq9CKV!i^XtVGy$x8}TmbY?)LxTz-PhVZT*OQ5jVqsdLnSa&!1Jp|P+R0#I%V?YSIv zhx94GFkP`6nQ&!zU%Gxi#L7N=3AIG|VohhNT#iQnm!XEC{sFulSD-0qz`Z`?v|$R)OFOSf%@O)uggvLd&?747i@%<=aJP;cM9gD4vzkJzv|U3Chy@}GZz zJt*mOREP@im2&jx=SZWf3~{hQnPya(NT2)xBARcW0&RTb52y(mdJl9a8g}npO$VO_ zWa&Ska#R!`NIS+9vXrk}aIq#sx zVMepgW^op}GDtv)wvBG+I9uX1m>P0- z$)s_bTzZWGqDXd~4T399GmU%~g@$=7CY#v=kXEY>aC&q`jm_b3g4}ZGT~?jNm=OciHSZjll^+gTR$XV+T_~W7fM&Iv8!C!(yd!ABqk$Lg2?}bQv`|6U>&$=5TAA z4)7$~Y4UTAT6425*K52)4|+zy>M`DT*^AkBp)aLOB;8jr(bFxiX-kItZ{n|!EC zEI9VyE^x&eln+rV}m&@QZfFIv&u=>pqHQ6PBz~tRbrdp|85Anj?RFuYa z*G?4QLUecra@9o-*y4n=Zz5#D4Tb z2XRjz1adB56Mx7J__^qxg(S8uWLur~unW*FM^Q#Q0vc1G=_!bnABV(^-SF`ofSB)D zAY?nKop2#45xx)|^BtydA4l2k$QxqY$K=h`={DnN!1{RW1j-K)^U16H(|i&}y#o^M z&OqSz8%RvNNj?N0Gi~=H)4i@;vL1_*Eu&S{D0T5iNE%AW5^*c~4cyab!KM8bcr^C` zg7;zI8(`*_kd<}m_z=@2=TLDwvPj~O$^S%-0R^u^#>%&_Xa`}@Zb0L`k81ipri(6t zaQk0RBzmlNK1AA@Q#&QFW>ZK6WM&Q#c*uV%;%;+GxHz@5)(k+7AEs@O<3>RbDQi{; zQtQkn8%;WnpG(tQ&ALJl#G@ft20Dqp5W-}7r$ghgc#NRI7%V!EIdp2aTGV(XC`K|= zsd|KlcMoT}8myux6-?)Pdwp8Pbf%XTOlZ9jcX7yQw36voFBl+-q}!BCmkldL7I*9dGP$M>81B16XP58LESC>l*6o)|>;$w3YITh6p$i$}6S zR;UeQnJ_daQ0risE^Jvb8)S!y=UFBkl?N)0uuK#?utXlDfeJ@!n2@nzg`IVjvO!uX z+ufQ8LE{3zH(E2{;cOWj)V@8MmE(77I9x8zm@$5zTCLadzW87`e83JH#Rh5GBf6;# z5IP~C5MF4*bY`cFW`lY{MXR<@BW5+&sD?uUf# zN5oHeFn+W6e!x9ovELOlTpJvN>YBxOtJROMRz1EN4y)WnUJ=Meda!fF{rX2$v-hhX zB_ZVxRM!!TvInXw>eNdgsIMa#@#Cv&xf6i4I;};s50Rx3_%HBR)qKGDTIt+=W~56a z7;pr4zd1J%=y zlit)aDj9h^h*1QYFCp{mV97X!>3l20h5ZwqfbOs|v8sPe)8icXaGpq_{3n=_MbaOK z=^upvvKgh4kCu=oU^7NPJPA5(2BOh9%qQZOT>E>0<+dBG1{-aY%N%=DrOs$KQB@vu zOk205&1wNx(%7PrZcT}!*dvY=li3n7n{0eU^d#dmOlJ>fCiM^*T#!XeWYMyp?(F=i%;M;9X20gTuNkq>FEO#hqK;yT;u_t3cCJI%G zR(i0nq95}rqpIvYjt$BK1kQ~NW6(ac%}3;Hv)m6YYf88v-<6(i8L!a0%utL`xD?rnIE7KsUeQ>A*{0MsULfBoCMt>)`HJ2O zyP~6_g+j^x#a>{)W9<C<7qpJTTIS@|$1{(%Ot#n@MXkYi$;-&8qF!5Jx;%nCBaOCx z)?X4L7WMKs4nV~D8^e37zi|xRP{FjrkaR{jS1>K5J;f=6w;)Kh*r>(LEBw=EHfzlm zt=XzI+q7o8w!6Q_+J)WxJ>Dzn>hCb{)9&vO-fjL4t$s+oO{=$S%?7RI!JlS-!$7SG z))3JqTHGYTPee_rRco?oO?IupPekoPeJxxjt5#PFmsZU1qcPK%mNY|yVV9V~DCyxS ztfHAwIF2@q!qK=Jo_rs|2M}spp%xE~Y3*8-pLS(nSXx+3t-{aQFzH&>4`bHK{V=jX zuUdoFsFen|AKtMaMz4jC2vKVN^5UPIBdz$qbCC2OpCirv-#$p%{;q%VGTi%d3Nj9l zf$;ha{Bs|`(V@*Cf?g#n;ry`|e5Ox>>>k69-Kl>Ch?>voI^Ge4T*r0 z$gN0ILSU!D!foLXIE{QsI3rZUdE}$QLE#hdduftekwm6A`2sU4?h^rD`)emd?K7!?#7#WEu2|y z@85dr9wZ$n+LOMBIm~xV1w%%PP_KczailO%*;$dV`kZ}A-BfW#{jy>uE@pDn{kbv9 z+tQxOQ9_V#fq9Ytf;lDMqM9SW$!$eFh!)L|H5AtH@v`NdMis{pHb&l2@`vm+_cs~K z9+d9qpHZJvG?vanpULu+KKz%w)K6C6;qAgj$^413v?2z_&^uFbGCiipNz`S)y(pf~ z#7O;LS$a1dBR2*hL1x#K7KPv#ncfeu8(!h?eb6-~nc{{xfmU2#VrbMefUngy_^EG# z;fNWHs(^x>>bnR=HHB6Y^l)z|T`*Myn*xu)(Rb@#4ubkndPgU=N)`$H4e%4N4e0`m=3sr|5IN4`-mYIg=MhEUdiCB1k8xT`*2Hv(AVE|Fv;pZg-|z@*a?&%v8ciPP!1HUN?*KE%bOWo@A>LWJ8D3~#WC{x_Ds9*q-e zt_r5(f(jZw+gcZU(IOac6 z7yyU+y9-_5{JIL1z4IVuKIR{evsCH0O~|+91tx>ZBtbQy_W!BF_z#-`;93*oGBVow zMdm^#X%P_Fb-Zh0I0y{WwPX!h4ksWNk!QdMRsokrj3LD^a0djta!5L8l5wO5vBK36 zok=*Txh?8;*CY@I3SP;a&K0-L2tg{w^Dn{af}_yGHn!fq&xWAE4`Wj>cuw<2Z zZgtheREhS`q6`^b^(NE4Mc?G)JZqoCG;5ZjK$n?igCo=uWY#DdG4sB4-~61^l(_h$ z{OtI+oTOBnI0S|qmjE~X^eD98>*;P5HJ*pjGL5lD`Dkpv``5f4Yz6+?2=x#Vy2+^qUqn1&X~nCrt!1EDt~@9GXO2{U7gJkL>s)!7-^-AX+g_YGg3O_WyXmX z1q@m!Xjsd6ykZs8H{f@84S56M{RV58U+BTDOwz;tg=w`s5U-)dwVV$*+kuR`!Gd6V zemm1VR`iDi!}U&N1zazY1y^F2xs(63o1x(R_bC4VU-k6=&Q(tZDP8Bv$}@i}QHYD* zMlwJGs$_3ag>LcFxN5k(B2qnFbwV{zxkFh5$6B52cD4sx3)Wk9P8K6w2It79;z;IW z@E=W-+>{Jh6w{n`Pry{s^7y3#;m>cHNlzr;aSC=@0jK(6b1#~oh*v6_HH;YD=EuWF zsJ%O0p$Imfu8#bnr;Jv2#|4T8*SX_CJHG#z#`M6c3Nr7*woN}CzlbjG0XRGKG-a8e z+4veYKLI!!eR=&{%OM@c(W)l^XUHo3`M+0=tjzf&^0ha6Q&TwLL^+|i(06nQJsJ);Tex5Sxc#%|o6^hv->w7NO;(WS@NF+G0KMbK$2ppRxLBU9U2_Z^|PT0kE{(S@g)XimY)7F+-dw+h-Le~j^6W!=Lk@@J`wUi?`wXn%Iizfsd zWGBR>?+k7p(4UJ)OSnqrf7E*fG9klvCloXFIE>!KgEW%qV#4VaxXYpegxg)b00DO2u_J%B5dQ-YlIt0eM6&K?+ z%mHRR1oOT{bCFr{on$%?5aP=lhu0#X-i<%UzL*re6Z>*)cpdgRLhy3Ne}7RYsaYa5 zOD9YB$ds}S*%H}Fd8E8d{vIn~lh}oD`$dRi82EMm1joL=Bo zat(moRor=A$4}sQ2?`-YSR$N+pjH`qul^dC_P`(Q{39dy2L8yb!7{(P6nMXUP>5KKpUnIzvT1%}rUc%`da+XrEfwEewv|AKLx;ifYUhR7{m<2zksL%R9vyEoxzW~IyDq0+Z2<-#tH!$rkHe6MPPts3E;b{Mm zv^YW{?J1oFm$?vr#U4;JQuI@-Q2eOuq%6Z#%6FBw zzX5pLxel-bl_s zHNP-l)5;4>Pv0|FnL&uo-3m&$^ePxij_dH)xd3$S2VllB0aI_V90u$Ajadgd0&(zq z{yNhf`Aw|5hE)!f-f4j0&bfeTBidBHX4~M_AOFZZOP`QP_ep2U8pv{GFUfuYUM!Kn z6BzY_V}A6ArB6I+%;NYSJsS^>Tz}W*6YY{HBR)ct<}(StE=kzJP*XDa7msZPOJsNm zj;33;?#h2ik7TzlnyVz_gX42&Flu=w5`c-N-Z#EZ5}{%nS#eLz5c=W z!~T=)$21YdBR4_2y5U$dg&v&<;!(Qc?FI7rnkbq%3B+Tg^^xaVhOb;p|C|KkF{DXU z+b$RQL3G7r5RbuEq9Z$=pC_erM}o*z(&F)a@zQ8oJrZz+uaiC_2@omLDu^=_jn zKKxV?y;Tf28{dASeXCpT_t7;afKwhju8rcu(Vb}TQoyP1yMHYCuJxDn%~HUL+pEq> z-LYe7bU(mp)T`BrZ<4XJx}PY<7n0%Ox~9+5f%$l46UI5|HJtnAmdemR_bzIr4GQpc z?4Uk>NMCP;2b`Y(zJqS27crm$UYJTKjMzB8?UA10>3b5U1=jKWMh z@B^5b4(+=1H9KAzOQkztVoZN6KG&y{eJx$L112U+c;QFGq%Xdr+)kM6&?eFn?Ut1X zsbeSLl>WB-ru>%%gQ?>!z}e8ye^K+sDR*e~TY$6cmwbBGpz0Djdp+Q68{Fh%=LNxb z+GPXaz}Zff_*_&BEu^P608Vq(jFEj_^WCP?Hv&%cOJ`ik!Z#mK z+Y-R3Sn+aSu3=IvtzH5+gK`H92zjgJIXY`8;Oy4u*{`#z7Tu!fmI6-wnl`rhseiPh z-IoDQ)o%+zNv`7-J+TaMN|R5VkmG|Mn*So;WcofS!Er5?Qu|E6smN+KUHw|r1zJ55 zaJD@1_P@gC4cS9uW&zHIdyhWV{M7Cb=;B#`Qxp5zH$4ZY-=^l-fYZ3~qgF`+CS0Rc zvjHcGy0M~Z%-%R!@DyIjw*AqU=H(@H>I6U;zHhw8bUu0)t)75WD>~iq%DZ=7_OfVM zc9YMSC^8k36l>td(F5#y_HV`SY=31NWuhWdS*l#ZDwVsH7gdc_-PxHck7~aBB!q)b zs+DSky1#m+dcFEf_1|11T!=V{Tf-gTeuulS64@{LGPrbck93jZrLCsKDx)bgU9p%3b zwD>KkFQZA~1*P*h!uxPK>NXQaWA8w^BHYnBm4@F2-_(lROecEvHk0Z7DFp5czQbtg z**nau^qs$%6saGPc(3Y~-|%9FR$-j#UDvr|pj^Cplm?q|rFTcCNZ%DR?iJXPd&(|I z6cgpuiq(n;R#n`^wGVbyZ zU&bm#{<+IN!UB0Ck$dFb{QNUO6t;e6U{ZjJ+PO>87if-#M@bi%z)Xux&aDC&`aWVw2PL)`%t=6ys0&25LCYHzuMIb zSGyKY7aLe{XZjA@66-rW0EZy@c1ySqwgX)-27uFu4W#Z5EGjO*5O53vCQ2x^dDC)ikYw7cQAedzP5NFYJ4!6&0FMgTf zvLzBxyJnL_{ThP_|P1Y26}Lj0TUbb__iJSj2$d_AD+%zH{?& z3q)U8#3a4GYYS%R>al>XfMy)R>ArJ^pe{iiMq75a*cxFOBc=ooz3_Ro=2r~@UwJCG>&c{ZoX0;@_lM7{`)ShK(@G! z>%q`lpMks$j$9BGdBn-nHTyt@^B1|a#*@C6)86|*;5H90bhbPA`}_3G{UF0pS@T~Q IoZI^U0OrgtQUCw| delta 25833 zcmeIa2Xs}%+CRK!=A4|pPZB~2gq{!rNeCyWH(C^pXreZf9qAF(6s5ZlY%WUsJY>>2hH+sYnc_p`g%2FBUVYzhL=e-yTyHsD4x&=S5II z8A0_V1XD{9c*i0rFGNr}3cRlBdo}w>U_asUEDQXkj|gLr3G91zf*run zw|6jXV1dU{#)q-T1@<_m<{*0!xv6)uz~coa5p0{l&amU`9rikVmTkjJP_V6hTm*YU zz$@Nh&#^Dr6YO0SY-E8as`IIZe6W|a z=Z_szYXY0v4GpXb-`q6cONvOcyj~sHBM5~b{{4CdI>ba9l0~ghBsUNQ$!2j`?M|Cl z=XBdFI*Z@o(>c6Oo6c?X`HVK3-|aKI0*e=xiT2e?X6C(g`at0L!aOnTj&F)PJkaCg zfOQd%>~YV{oBMC|f3{F1zt-HRT%{5NuC;Li)#5Bs<2;+C?e&>EFlOxrIg-#(a3-{7tYzw`K+%Ak+m)292z=}xVcwOA~`8V80RcODL8|~Ue zIdvlN)-ok=gijb8%z7#C$5JKhMSi@$HZe>dIb|mW_5(W!(EES|PHu|UurCDmBl`jw zM_J&DE!`Sqcp~gPLg7om(+T!Ifc{PP65GkPv4^nRZ)5A&D%OZj&tw(M#Y)*|mdnzZ zlNnhe>%+RS4oqT%{z|{2r|Aj$K0QF+q%YB(bQ^t$-c4_#>*y-lNaxX+w1T>5DIHC7 zX&QA>BTb}beP}n@fl8E+U&(jmG&w=uCkM!z4Fur$B+QO%S$`b2Pv# zLafkRsS^w$+G!ed+i9YFcCv)={S~AizrGT%ky=SQla8%Rdb1cQ=SzVU(Mj4V0KxB^ ziszd&s=!_1XZ-aFk{;4NeN1NBxWKxQMDCnMYC<}+JV*1r)5vvKzAvjmMoSAwjKD4c z|9%T(@)`RW*!LjY!(Icbd4@d+wDTal2blL(b`x98mI3e1WwmTN^RvmUjE!UYEQbwe zLzs;j*Z>yCdb6&qBMW08OrXEfAL-Zh6#ayLNZ+IT=|AZ!^aZ+uK0zO)_tU#*!|n7I zdLvy)m(YcDHm#x4sFz+t$I~%%6wRWkbP%=BWST%@X;0dPwx?REB7cxy$oJ$dIZ2L_ z56C-YFL{IPCeM+l$yV|(xtDAt>&eaJ2C{-&OXib$Qbnc`HhhSG9N1kRE}kNEE{k+m|2+~klB~@Vo`w2P^P9A0i8e4ujm(m&!hBRx(^WgGJT#l z0Y)F8_t8y&(g0o4K(7O!E}*k$H9*xvC(#lB>qweOQvj}Js-y9=5A8ua(+EH;BfpcM z0kdC{&&e@B?IH3uc^z>3EP0A-0pxBbcM=ZRy`C&54S?P{GJ{M3{7wKGF8~CmlcB&s zMj(@Z7`OZjbcyN7zdfc>Z;$)vL)su?4#IwvOHO0ce!@NgNc@xS20%Q)9tJqvj$?Q= zTf*i86s9pZ0AUOp0WcWEOaOvd)*T?AWfTDLJw1cv|9~FE;_s%D(G6x@jV=|B{nTlXr* zb36XabmAplnkLO4PZAQv`z#@`fj_8Ip##XJBfq`Gx=EM5C8&TNe`B}C5H-0nIhm+B zh6p&Q=3r<~(UmNmEf7l4rwrCr6U9GVO1kqI)g*;ss@Irp^5y_Ye^Q$(1|BW z52d60`9{){Z&`vdbYF^|_T(qjnl5}5(S-85mI7FM1*`R1M#hoeymlFx6e8n_D?8X0 zjc>;mx(7)6T2=#?7{@ZOArgS52|Y>QqFX5kgu7`TAi5vbl2c?qd6cXKY!+hKx?`z6 z5MCA@5pEF{34ZxoZGWBPnPBw#ru*uB<*u5V=@lMVeMNOu=l)F%3&}zz6Mj#`2f_AB zx!pEUKAY)!fM7oM-ZtSMqz88F&CDiD5stDtAnBR|o4cB`(%Sh!2rfFUhq{X(Mn+|}5w1Nj}RiTT#mB(lvgXOjMM*ZXk3l3YU) z_)jYVn2G$FRiv5>2oChqRamVg9=)0@Xnx0KU}~UhiG2$C8l1(a=>EHmf0{yLCH9%X ze#Tk%74qI={IdeuM`Fk1dS}=%RFWGVA9c27?HI0ds$rg$ojNeh2 z3>F>}asnMq+vhMkbyln2sk69z7Mj4DeXUo?EIDb1FhF+M9heU#BODmS+< zBYR|7R^HWmWpO%_)HIW%t+m>ZZzrk2AUAjHbAm`Z04e3yX@z6=Y7$tk;*MXBx8$W{s_!Q(~Q2T#{?A z^SEbL`$o8&(`%;~3VkE$+?G+3cQ>V;u zRZPF?x*D9xc7xGuvNZOj=&A0~y87)Vr_<#%>l}WQOK0(z%{rF}6KJz~T^^&?Z}3~q zUf=9`mwUP|sli*X;F^&IS;cwjnPt}8qS3{+yy7wDLZ^9TcD~VCG`zI1)Mg!T8B;j6 zFvmIEFtXHOvYXQi?PIJ3Sq58H={Q^F@GDjr{j)i(2BWdjM$o@rjcTi8RKVL{z#7{u zKAqFxwNB7kjOd%gX>sUW4!7TFa@h@TlcA-9&68=*7@t35Tvly%Sz-3nDYHfzO7lyn z`zl;@UVnXVa`C(j&%ElgQb+cbyu4ZIMYD>M$JHBs{KW>EsK0E1vGtyknfxc!GExf> z9utNGdVNwQ`8DHA~IMtpTNEQd;x-){yKrZ;FwR!!TwRKm_J7GpM9vAekRwbwK7D zKpCw8$^b?+ucZfjuvcXQ`+N2kNXw7G=Dfr90j-<@MfECb-N*RW@(>k&a}yoXgPH|8 zkM*Hu9Hh(H6m|;K^Si7aJx_PZFQk0yaY@~fomc8CD=Zx~!kpn8XD=RKm@y_T$2Pja zSz;e$v6bYd7mh5*E-B2gj$V)3JF@|NT8pDYrV_eqoF=lIl zJ!|Zk5r%YAzH_W48(>gaG9V~ao2LhGdip1#*djz!_8NYiU^@f6I05{-g-RK_VK1cbU zk7?+Cy&4M^jbs`~$r{5ugtGs9Q5u?8rFl{CHro6)HT!SZ1}`JcFQe=)7AECyqn8DX zf&?`fiyy(>90bcSQ5Y5I5HqlSqd8dW?$shZ#OpRR60>z->~UDo`f|8`+%mOL=&uhz6> zNsa&Z$Ouj~rtMeAZ%?xm^0YC2BAUgu5Q$)0+NnU@3%`+#^cS!!KVii`XJ(UzKWbvb zh*MrOJ@0O2Q^E%+B8$wJacRRxW%Bk`7R9HnQuS?m!_2x8GK8P7u;n2`l}h|83+uu2 ztxUsjw6eI%=)N|@=wznWP{^W|QbiU;`N=BQIbmdRsVUEBwB?L<=H%p;{@BsR5gFO$5%#h& z`)I>h8*p*vIHPS;iQFWuy$tA#^b|->dYnxmT`?`-D{WQ{e*!{+@L`vYv)aMB1jiX~ zV=Dl(K$mN|;QC07&tFN!F^i{Q`56pwC__HQkEC!q19^BYN1z{*87kk+?SYjQ&)xq>+ zc++bR){Byi;6U09VWAwNlF1~qH6x5K8_Z^rtk#T?&3y6^fgM5b_cBvFE0X?Br{&op z>=%Lkf_7gs{>v^Fu4Wf-{^I$2#xLw+(q+#VHDY_U0HJ@W@C{i)ce5Iy2%R6!Qif@G z=@7tYc5Ao7`1~O(oTsL+Fn-5i7Qv4XVWUWHYk4S-7>e@Jq0CE0G`%pC=2$r?>Dgw2n3Rrs*=c;utrr5AqBi>iRv0s$e&!I8c9aqx>Qx%_HE68cm>#{sKPX|fCCamboERS z7=J5Jg=*Ltg~LMb$BdsTSM|`yLgF8rg~a@1wJN%07JxI_tAyi%a2#U6pV(}nT$q4y z<*=M=4IjQjHHeh(MJrTU{L!Tvf@r>exoR>g4yL=VP>d1dwsVV_U3f8@QlYf-)5pHW*FRyhT&}b9=e%#^yf5WF{{#CH^*&l^cZ!P&Tggeew)z5Q){HUKunCDjmG{6xdHpb%|~Ca>cR7FP{r_D zuU91#S1{xF^{W0n=6aQZOqMe;dE6>hd~mIz_}0~^culb4Z>zB{lsa{7+Kz7f;^wXL zv%IQ5qnlqgQ~c-kYORL7EI`2eGP0W(UvrH{GyE!Jm1mCjVrkOVC-%}*Y!TQN1!rz% z<+2v%?<_u*k?Wt<;;Kub9BOQa0QO}gn}HseGd`nS9ok^=n$bfGSX7?{G6=H;$BD;p z0Qc{+I*eY3JX}^r^;8r{kpxOt>oplY7KG12o?0R%cb*sVVba;IfhlkRD zKzwBP;z%<2T{^GHE^80n&?DNNKAp?#GkK$YE~m-ku`s6eP?#YIGoaa&xxjKh;|7(L zc$zlfpvt4f$G^K#^$_vPQ7nIcttwl~*mv@}e5atRPZ|HNS{*3`^L}REVUV8&QP{Z# zHKetuD=VsHU_)J*Z!<6z6NmzF%I8A_4txM9?3=Wh^#sxmC>WwS7+<2{FCJ1CkV?4= zy;VXP0FfM2XOSs#MiSQ@RBL$30d*y*2zK)A18NH&a!73<)A-Co>W#scvHa6R>TZ1B zT2(Lp?Ey5Yen;&j)A>X1sF#o#K@^krt~#+9p?uR!3ZvU6g2VY?le(kCkpE3H*>FNZ zG+O@a7Im0{S$=DSS$3(rYhvj5UK8F>E!D|7fV(=PuZkNfSgC^N)G&s#RvzbT#u1=0KnEf`3*XYxk z{7zZ4<8kSn5IpG&4wK1dGMSx_xbd04tF3WHtI=bGbWZ2=T73{snGN_j1a;6uTWk&= zw9-}^|L5=OL2(|Z(_;2|{5q=zodaB(bPjwJw35f>wp;98;3=N)hdMUS?m(w4E}PC| z11)87y3kF7*{d^o{YIC~V|LpOUS9QwddNht-EQ_nCaANzJ$R?f?gGxU`*k*-#cj2D z-DaQNVpC}ZsSgo~rnqXSxT@-BxvHqq;PzPUHXujH(9q6`Ui%@xa)3Q|IXnit*<|G* z7uBh8HlM+S;hA&>OdU3t*{^fif%WZ1$Yrbsx6$D-@@W^*Ym;A=OS*hIP;geA#b<{& z%w{y`oPJP&Zio>*9y{N9Q9aNGagWIe(H&+&5u6!pI+x30)EQkqgWqiPdYne5f^UVd z1>tL2PTyyfg)><2^}Oh?+8Qylnc&UM&k(Meq~X)vRre)Tyx?PX*QWd5Rr?63;T^Y8R;uLiE5R>cZYK&JSfT3XtEnXzbePECHVPF#cjhNM<)# z>=Lvpk-7R;*-6wt&i1o^qRs;?LYL@}A>G&n0Y)WI=0KM6SY$lkWs-OK(-I>4k z5zt-HQFSJn(`FW4II5lp0f7QJn5-3KeruC0e{P<4YkQ3|G4#0jRb<1c1B^0Qn8IUsdW>GQMt8 zNPQd1HB2SIK*Rwh*HdhAphHXo@AR>HFj?3HXx~OSggvQbaj?((s3u&dJ+%B;rl}we zZE{DbG($DLrfQ<{oRr^)2P$0Krg#|DSpEhv+lH^MgNhfiK4>NCTI+g63x^~O_aG8 zFST2{OYi7>SL^+Xphp6dCg%$iIQIpwQ zR;e^wg8anJ!-8Wn4TmzjLik-+FQj7z`z4GGmFWh7qzO|b?Nz8Cd$P>|-qpwpE{1gG z*Io!oiuw~7)mmfaA0bKns|z8qe_5mD9crHXd&uCwtcDjQ@MnGx>GZE_g#8iHvAKpJ zrkQ?i?aXDLQt`2Wg!B&9H9D`Ti|>v4%fc`J2+3@a&lu7QT*lR#ELBu0l z_?^W*dXK$@s&_N~l3tV1gFPEWOwWS2`Vxra7{ug+- zr%bd24K#LP69x@7cFMg$)!Tv?>G=Xpdb71#nIM!2&ye2qAr{F3LUEu&%&_o9wL-ln z>YA#MC>@|NT+;NW!gWF5O_wyW8h5oON{^E1k|u@N}7Heu|^Zsw=d=UB~33X ztZw>R(s+rcVx}fahuqRoc^h0G#6s&rHA(Fj)oP+r`qIX}3eU}d57l%FcX>R%x;jOq zBHX~g3Du0$RMuf1;RR`7^2S&rXZGMr!ZdL~MFU-QXa)RG*d{Ouh81tYjz(5A z%%BSA(5eRe6O@JZW@+R~&Y=YeL;rQL_ForEmx>z{jpd&{E?z3XTCwJ`;wv^tNZaBo z_k{Yg;)W~sjOw!TE5IAZN|z?)O6b3gjO{jmYH9tMa zFg7=Be2LL!%E`ABn(XOz`xs|_R=Qzarpa1ngRaG4&9s=zR{QV~8D-hU>A7P|N(>nV zg=W)8@yg}0CYwQ<}ikQ?ftLpA8kE{_?uo(9nDKK{ZM&G2^e zTv6d(LAaNV6YPNwF^Ss7#95lCBAhw}yiY$Exb2S>m7lMw=JihiZ{u^d^_gHB(^=Fijc!SP}~AKpjoCigL%Z;2Nj+QxyiHBluP z%-A-A+0svp;Pw4PoTKZ4=cv+PYd1&dtgwmj)Nt|j;GqFY zPxGb*3_7evlZp3kC%${NLL+Q^xi&(4w^d_%sioav1QRo>V!BsX<(i?GMl!)H2xjPQ zYcRY0ctNa~Ol}P>@{oa|)U+v9yqkvBSJ%!{+6n7h2e0N~N#bO3TbqphBurn+LaX^> zNn$6Nh`+YN=jH&o6c$y`E&zA^gOj4jj zEzE=MIEf&eXn*=7yH9w&83_pd#UybUxxKYxsy}C_`Nv5B#DN1vKiSY)P0hCs6syUd zZ8C=H#OY*Xn~WQDfVX#a;&8I5DJoesw8#rD@9p>HVyCN(fd5n=Mo4Tmgz8{gwqR%r z8DHHX&hH({-jsX)JT^l5r8 zT}$WCiF6q41-smX@GGN)?AIS{?BD4tVDh@ergs$ z^aBU81>CqnbTmDE3c{WkmdP7$7rTer%=%Phyfu(blItWkC_L2vo(XINOd{}Ych&F39H*zWqtwTtdFKiX7YdxiSfNBG%`?4QM5rJXews7 zrufQ1%hbwog_mdR#lGY&Ua1!+kh}TodeO-fr%DNAGaoZmsv!6B=cY;v$$bii1S@tm zkF1b-^J7z`2!3salt=ClZvT@N($CH2$$eXR*010masn#)cL4@(f@yo6HNjN;5q2Nj z1Y_|4c=+o85)0TYFx?f*!zKYVMzfJDlchj8VrDuP53}(etTT&%;W(qe)1P5F{w4jK z9)t1tA^J9b9n8nG^eMUp2IQM*!<{f$Tt}~`@GODtS{%VwQ;1`?ARgpZutlrncbmZC-O)%94=! zMzICq)o(~?a*N}R;=?zh7@qDmFLAEvxv@Tpbi0mUF{0L+5 zsjMf=yq8f2?6dDe_E%&P1~K^7CW-U-9a0V-@{!n8{><0BFkhP^r#|EDM+H+K@zh*x zj%?WTUQo-e<=5ULrK-mu=Kyd2vZU01v5+bT*XP;V_ZJ`T@^&!~0*B z->>4mBBW0{2I-vk0%=cH!Q%Q!mc1^m8!xVuoa8}yxA)|mVsPXrhw7V^(m3*Pkh6gx zPHck|7XRDYT0V5TbRBslxVb-?E`3`pQ=IYeE`d<_OPEC+!f>8}&(bP*7~Bdzrw+h0 z31;P)3|emx>YcDg=iwZ%8+Kj=G!^?U7CTM_+xlZTI_|?xT?O-ESu624X;SsZQLSNj z33%;Y$hw8`yYy1uZkf56g=0(83e8rK(YEa2 z1vz8oE>d<_0Ks)I&W-NkdR7nlwu^>I5WPDjAaq{?|tBtFcB7;kY zN}c$rYN@MC2YI@Z*92JCrvj|&=yl{%IAC;!GsfHW6l1X19|Uup36Od=!qs6Lc@50> zDfk)uXQt6jQ#JsV@nd%oZ$oG3Z@}nQ&#I~~_rSCj$Aia1d$7TO&aH9P)y=7{^|CI^ zEd8lP&2(2)Czf=nNp;n1Uv0grE3*=p*IOBPm3V%11Bq!L!TPouileVuzHlS#@b}=W{Lu}htD@WN zOo_Z?PFOek>0&rd+)v-2pVB{Yh#9fL=dtVH5JKRWvJ;|QSv9kd@Tcz}i4k%~nr+n4 zk#K$quU5=Cd@~SeDqq{ zso_60wfwWCWg7upvkwMr-ONUl(=xE_J8bF9Zk@|u zYiHKMzTRVWL(grsoA|MLRNpF}z4StS#pG;^kub_Wjq1c^HwBB!X)}9dH8%9H&``rh z!Uat-By3ir!|m`|VY^p7tG=$n>jM&QwyvK)cQ}r(O8xlk`D0DR*@hZ_mS?0ddz9I0 zo#&i0b^a86!<=ldv$}NDl-ad2GP7)YV{u+dRki6~I-VHe0AM!T&5b=p^bxKUEq(O5 zojyNiUFUJbS;b;E>DPWTUer z)9lBEXb(KHI`*(7-~GTvNLAn z`9@lbE2kFcq*;Bn_Uufovn6%V-|;Y#_o47_znBsOk@LA2wfrGigpQ30Y@a_{>~cnPAtrL>Edk?#O=900 zB5p9g()pFZkp&wQtQzgX5kn*T%O4FB@&!;86|6hFb{A1SoNd?O<39-Z2>JMYa#Q{W z>1oz46b47&#KtofeGG(JIC+6gCsD#KWcLw9bkzWRkqBP3QcQ1pW~F!w>m#!j;Xrrz z+bo=cBko-CA{A*NT@Pd71UOG}sLF>lt=TGV4pt6>#~l7n5yKT#cPN$ZJZ6Vf&;%B{ zlqqU24OGyq6b$VU@&#Fr`EsDcv4L;uH;IGiF5W(N_=w*Ex6axq>WZHol`?Pooq-{< z%S3kH`_t4tJhuck&dw9rxtqtRuNT4sKg{NlNpIhA^U$$j7Z<{`zUAfxUY?|l3+$eg zC3fzVe`zr$N=WZHkoHOag%+#mrt>)?Q8a!|{6}T!v2tn?Uo}{No#sNF$pL z9G1=yzT=RT#rN%&(wZ*3FLfndgCOG)*b-37H`0M*1*X6y7{T2dl6zgcl)`IpdHgOv z)E0gjiTROoT77Q#?3vsapUGo~uscvVbGF!XQ2mhj4hoLIF&qX41b zSo#OFpCurv?X>9T}C?d->uu(rhvAg@x12Gb1^_ zutwS>CZ_Fq-MM>t7ryyMX`|Tf{q-SnhP&_Kxof2|G3A!NrKOaA${$%P<#jge-VDvY zzv08ip1qo{F6pIG1zxOKZP7=QSJHm5J}NI)81~0sZV~LP<~<_tKR%S!HI*Ng(wRbK zi9$W}l}ryY${3)t3gKI1B(&5H#v=x_OyQ7sM0pAyZxnB2&3g*kgGX2{-Apc!#Tfo- zp$ro++>o4fX;Wd}xC~Z5NCAVx3+e@wf!k(){juNfu()yLIc+Xpu}aDl2i#LzIe4q{ z4gSI^%%4hH5vRWY>x+ESYRq5CYWm9fAE#yVqpLB0Y~!lY>JI(yUXdE9GF}l#gEa!u=8&(*WVk@?6kHf@u2^hT<|v+T+AXFvJ^Gk5jy64htw>u? z&!5jvArH;0hGcN{Ef?Z|BJdfVE|YBi3`0OGjAq>)BTQlqRx6AkJx06V%fDET0b0C| zmgSbd5XEy>VAA?MI96-fQJ=;)uE3-Xdidn=F|o6!^P$&a(qe6U9@U>5{XBp6I!s!J zoyons?hl{D-7BSyk-g`aWDR&E>-=P8smy)kt8e(7I&By%7#FCECkc#5KvbFFU^6#dMO0gYazhC1A^=aAkf|p!S*f)xJ?jqAA-R9H2H!2 z4l#HK2u=IJ;lCY4^w2o!7bwb|`OVJ;{k&kgtnpDRMeyrDx}y@>l`u^Svy^b8 z5*8@oSS6gKgyl-;RYJcKPF2Eb4N5d!39FQ_MhR<`uwDsgE8$!X>x4e)X|Qj4lO9)p z^pJB*db3^?%rzMFzUGukKczWk(N{F5Z2HRPltVv5Ni{%Os;>@a8cq6{%_)n%t~q7X z&uUIN^m9bk86R}$I|aY&&>Nll$TsPyHt8O1()u=Oi!3yfUxKzK1J!rt;hSK+Vb*u$ z2S6!ww(EO#VDYUr?0SA3 zr(#rn`?Jw; zGoE=N;+}^)z-|Y$ybf!k&)LsVI=l}x&X4qWD0aF+$)m%KH2F{gEx?Xl4|(!N5NJ=r zr}GdLNZ$x+g%!elqIa0}4vR7a-StkZ-f7djlzBvYaxh`mU&HsV7gJP}Y2(4vII?F70FVybGms!<~s4HhjHy{_eTgB$Q< zvFelCOufafH?)~&JX)|MviP!jZGt%#ERrnj{Bx$&pf|U9o7JSZUjArPz}2b`k)bC? zZ`IrM8U+gElwB_>Bez=hj$o!$Z?);2$}5nGwAHR3r1Y=#(dy6-=KF4Mh6CtXBUM9X zI6x|j4C9M$7kev|T{~f*fWN<_?SjV-)Qu;0izA!9_*`1W{&U(yCXq&iHsNb*T3sNp zI*B{=+IW65Zl>bj=(P9ogB`VfNnO+0WbI56yyvK8hye^b^Q~{p!d_+kk&75W^eYd%IdERn zc0Tc13}9H)y3xg7y)~1czZL_~COxETH#GVnpVugDve=`=NAr?uSIe9I?*c8FuZwB79v4;JLxnWBII=YQ5j@2PkfS)BEW(GvO+p%$r*G45 z_er~2NSD802eL06$zEJkp%ABVX$0ls^H{+cz_@v)3uKqipue8~R zxcSlFq)lByKAQBgw4u<}*f*o~x;b1;*JpVgfAYLET{NX+@5~y{H}HfDn7#P;n$EWU z`+M=#7Ze&Oso&r&?+N_a1f#J=n6aS0z(7{)A6B>Zer|NUpc}(4R4o!P=x-$2{=fM*i&2(x%9iZ_dsv zT^e{*FkCptuS3wZ&!LSWyxa~Z;HK-PICX!8 zN7&O%8wY0~C$b&P<-ew+d!&5~vP0Q=Z=%2^ZtoB?hH?K~+M&U2a=>^QG_uEG04?bC z;W72N#YAcFfeX3eXb4u23|Qk4(tvMGwj>A!VM4@x-)w!LabcIkzfQE}(c zfWW^xy_&H?z;TLw(koEv-%&;1GXzU`5@14gZSaFD#-e*+C3-wS&G)L!Wbcf10b!3+DO z;e5JG`N!mFIfqR{4sXiQ-o!PqZfM5;xDRO?E_&1A?y?e?0jvjtc^m8|uFZ^WIvTF+ zN2Sv73OAqwl#bG-HobM1H2L2#0uwLIp8@x|xIN&DeejT!o#9-#^u^}XnH@Gb1Nq#L zBKvr5w6;tfmT>XMk`Qw&e&6xl4LzH;S>F-pVx0D|nqJ^u*Lf zt{$_d=S%m$%2)Tq)C~#z@Xfe=vHAR1Pc0Y2``HI~oRT+&FYcwCErvfa@yAXBPp#w^ zdTBR_j+w8WD^PFk%QyGdZc6GrHa&k?+N9RGs-<1Xeql!tvyj!YhzclYAjyDiYjKJ8 z7&n(|*KqeW+DyJFAK2xOiNGm0P0}9Z7he`L`B&Fyv-zyo#MD^%jiJIRTuXOV8Q%_J zJm$&9YqGQj{4lI!n%3rOrxLy^3-HPTv`s&c&~}50SJ?!k1kk!Z)(OT;(`gK@Sz88> zJq2G1<*p5J#IzuP&nJHjfmw}KTgeLL@5hm|0_=9qQG%NmV`0{0r&+j*(FL=og)o6y z2m0qB;JH_DmE)JN80ZZWdlEiV1BVnZqnx}ty&-LX8nj@E31A~ zu$5JBw^i!%g3aUyz1^X>K7!|N5~B;0UOB;!1QXy;lf7#1jU9o9GufMrd`@}tsT+Kk114ecXA8b9@kx%N5O%rAskk+A-=0|?CyLMAU-!0h>8P9nY`x!HxCg6_8 z8akI=i<>@gf{^>INJS-?Vx0Iq#c#G6YZqLku*|?JJZfe+=X^g;wT!W#9e7uCGJMM zDRFn&U5R_p9!lJk_Eh3tw3ib1roELonno*e42@|}5`AbNC61-BO5B(BRpK}rr^Nke zKPB!@`zvugjaT9XnxMpqG*O8M&;d%EM3a-KiXJ`hzsqk1E5B*IJjH?{ffD3D5XjdRnkqWr1_FGsE9w$eD zhu;8NG?Bo7vyU}csa5H#joOshPVGwUpbjN=Ql}CRqJxxpFdeMKL+B7C9!iHQ@i00} ziBo8b5~tEsB~GJhN<5qnSK@S&uW;LlUQqep;G-vD0S0sQ(P@a%fv+m*n(bAf-S0uPU8 zg#h86ERguz)_<*Jn|X=8wZJUHg=++L9;XLhLKZmPyWr!lGdTTl6!ICIu$$mLB5{g1 zBW{T<_+zbz6cQ}6FS7@RbQpXI5YaWW15;PuzWzV+DxezG%8ro^XK%^KugGnD6a z!O7J9m-!R=?aug|-`NExQ``~NLh~($BKeRg?d*i8VHwL}R+pa*l1Ag@s|mbtb9n>j z(LnMcxd~n`KSKK^(^rnBh7j$8L1mGa7=){mVlDbvJ5P{_u?I$hS$I*SP36%!T9MBI zLDmE&;8(&=WolEq%Xmm4s|8t!NmjS0G0A?#E4zj$*ZIj*Tv|}VSgp~f!AU&j6>%gt zACuk=<{r3FOykM>L@=hi#cY1~Z85c(`n@WbKcPJ)VTfT%^|*|Lz5KWKnArZG?Ad9n z*QJ&b8#W~JhBfLDm6DBq>{Dncz zd7IDhg`KqA)XARIz5B^`zmo^mfn*E+N>50#Wo0FRr5D8cR`BR)!$8?TQ3cuCe7<9t z76g+GRDQb@|gE2pM&+pHX3>!(dWtF$YL z;%u)XaBXTxYKi6gX*4iuGH2MFa>+)avHGg$6BnpmxxY#`!FwvQD_xV;w>{f@i79UuC{rQla M`~Mw(#@I#w55Zu_YXATM diff --git a/util/tarlayerformat.py b/util/tarlayerformat.py index 3d6ddd94b..37aee16a7 100644 --- a/util/tarlayerformat.py +++ b/util/tarlayerformat.py @@ -39,6 +39,9 @@ class TarLayerFormat(object): # Yield the tar header. if self.path_prefix: + # Note: We use a copy here because we need to make sure we copy over all the internal + # data of the tar header. We cannot use frombuf(tobuf()), however, because it doesn't + # properly handle large filenames. clone = copy.deepcopy(tar_info) clone.name = os.path.join(self.path_prefix, clone.name) yield clone.tobuf() From 8d6cda44b5579524a314e30c748cd1c80d08231c Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 2 Feb 2015 14:25:03 -0500 Subject: [PATCH 11/37] Switch to a name, value dict, rather than a tuple --- formats/aci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/formats/aci.py b/formats/aci.py index 1d82d8c6a..e4091bbac 100644 --- a/formats/aci.py +++ b/formats/aci.py @@ -172,7 +172,7 @@ class ACIImage(TarImageFormatter): "group": config.get('Group', '') or 'root', "eventHandlers": [], "workingDirectory": config.get('WorkingDir', ''), - "environment": [(key, value) + "environment": [{"name": key, "value": value} for (key, value) in [e.split('=') for e in config.get('Env')]], "isolators": self._build_isolators(config), "mountPoints": self._build_volumes(config), From e95b4f43dd85116b8ea8b9a772bef041e20f4d36 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Mon, 2 Feb 2015 16:12:23 -0500 Subject: [PATCH 12/37] aci format: fix indentation --- formats/aci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/formats/aci.py b/formats/aci.py index e4091bbac..048a479b7 100644 --- a/formats/aci.py +++ b/formats/aci.py @@ -24,7 +24,7 @@ class ACIImage(TarImageFormatter): layer_merger = StreamLayerMerger(get_layer_iterator, path_prefix='rootfs/') for entry in layer_merger.get_generator(): - yield entry + yield entry def _build_isolators(self, docker_config): """ Builds ACI isolator config from the docker config. """ From 7f6b42f8f32dd5d5214ef7e2e4627af309aa364b Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Mon, 2 Feb 2015 16:13:37 -0500 Subject: [PATCH 13/37] aci format: change methods to staticmethods --- formats/aci.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/formats/aci.py b/formats/aci.py index 048a479b7..7983d6f08 100644 --- a/formats/aci.py +++ b/formats/aci.py @@ -26,7 +26,8 @@ class ACIImage(TarImageFormatter): for entry in layer_merger.get_generator(): yield entry - def _build_isolators(self, docker_config): + @staticmethod + def _build_isolators(docker_config): """ Builds ACI isolator config from the docker config. """ def _isolate_memory(memory): @@ -70,7 +71,8 @@ class ACIImage(TarImageFormatter): return isolators - def _build_ports(self, docker_config): + @staticmethod + def _build_ports(docker_config): """ Builds the ports definitions for the ACI. """ ports = [] @@ -100,7 +102,8 @@ class ACIImage(TarImageFormatter): return ports - def _build_volumes(self, docker_config): + @staticmethod + def _build_volumes(docker_config): """ Builds the volumes definitions for the ACI. """ volumes = [] names = set() @@ -128,7 +131,8 @@ class ACIImage(TarImageFormatter): return volumes - def _build_manifest(self, namespace, repository, tag, docker_layer_data, synthetic_image_id): + @staticmethod + def _build_manifest(namespace, repository, tag, docker_layer_data, synthetic_image_id): """ Builds an ACI manifest from the docker layer data. """ config = docker_layer_data.get('config', {}) @@ -174,9 +178,9 @@ class ACIImage(TarImageFormatter): "workingDirectory": config.get('WorkingDir', ''), "environment": [{"name": key, "value": value} for (key, value) in [e.split('=') for e in config.get('Env')]], - "isolators": self._build_isolators(config), - "mountPoints": self._build_volumes(config), - "ports": self._build_ports(config), + "isolators": ACIImage._build_isolators(config), + "mountPoints": ACIImage._build_volumes(config), + "ports": ACIImage._build_ports(config), "annotations": [ {"name": "created", "value": docker_layer_data.get('created', '')}, {"name": "homepage", "value": source_url}, @@ -186,4 +190,3 @@ class ACIImage(TarImageFormatter): } return json.dumps(manifest) - From d4ab4a2ea1f0ffb0e19ed53d029fbc423f3a664b Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Mon, 2 Feb 2015 16:15:11 -0500 Subject: [PATCH 14/37] aci format: fix name values to follow AC Name spec This means no underscores or dashes. --- formats/aci.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/formats/aci.py b/formats/aci.py index 7983d6f08..b5289bd44 100644 --- a/formats/aci.py +++ b/formats/aci.py @@ -93,7 +93,7 @@ class ACIImage(TarImageFormatter): try: port_number = int(port_number) ports.append({ - "name": "port-%s" % port_number, + "name": "port%s" % port_number, "port": port_number, "protocol": protocol }) @@ -184,7 +184,7 @@ class ACIImage(TarImageFormatter): "annotations": [ {"name": "created", "value": docker_layer_data.get('created', '')}, {"name": "homepage", "value": source_url}, - {"name": "quay.io/derived_image", "value": synthetic_image_id}, + {"name": "quay.io/derivedimage", "value": synthetic_image_id}, ] }, } From a3afedfbbdc2870cffa0ef566c8afb32243e92db Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Mon, 2 Feb 2015 16:16:10 -0500 Subject: [PATCH 15/37] aci format: default workdir / due to ACI spec --- formats/aci.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/formats/aci.py b/formats/aci.py index b5289bd44..23ddfd43e 100644 --- a/formats/aci.py +++ b/formats/aci.py @@ -148,7 +148,7 @@ class ACIImage(TarImageFormatter): if exec_path and not exec_path[0].startswith('/'): exec_path = ['/bin/sh', '-c', '""%s""' % ' '.join(exec_path)] - # TODO: ACI doesn't support : in the name, so remove any ports. + # TODO(jschorr): ACI doesn't support : in the name, so remove any ports. hostname = app.config['SERVER_HOSTNAME'] hostname = hostname.split(':', 1)[0] @@ -172,10 +172,11 @@ class ACIImage(TarImageFormatter): ], "app": { "exec": exec_path, + # Below, `or 'root'` is required to replace empty string from Dockerfiles. "user": config.get('User', '') or 'root', "group": config.get('Group', '') or 'root', "eventHandlers": [], - "workingDirectory": config.get('WorkingDir', ''), + "workingDirectory": config.get('WorkingDir', '') or '/', "environment": [{"name": key, "value": value} for (key, value) in [e.split('=') for e in config.get('Env')]], "isolators": ACIImage._build_isolators(config), From 844a960608435c5980d0ce8323565e2662362c0b Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Mon, 2 Feb 2015 16:38:58 -0500 Subject: [PATCH 16/37] linter: rm unused imports, shut the linter up --- endpoints/verbs.py | 10 ++++++---- formats/aci.py | 4 ++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/endpoints/verbs.py b/endpoints/verbs.py index 8eb46bcc3..c14d8dcf0 100644 --- a/endpoints/verbs.py +++ b/endpoints/verbs.py @@ -2,11 +2,10 @@ import logging import json import hashlib -from flask import redirect, Blueprint, abort, send_file, request +from flask import redirect, Blueprint, abort, send_file from app import app from auth.auth import process_auth -from auth.auth_context import get_authenticated_user from auth.permissions import ReadRepositoryPermission from data import model from data import database @@ -15,11 +14,11 @@ from storage import Storage from util.queuefile import QueueFile from util.queueprocess import QueueProcess -from util.gzipwrap import GzipWrap from formats.squashed import SquashedDockerImage from formats.aci import ACIImage +# pylint: disable=invalid-name verbs = Blueprint('verbs', __name__) logger = logging.getLogger(__name__) @@ -45,7 +44,7 @@ def _open_stream(formatter, namespace, repository, tag, synthetic_image_id, imag current_image_path) current_image_id = current_image_entry.id - logger.debug('Returning image layer %s: %s' % (current_image_id, current_image_path)) + logger.debug('Returning image layer %s: %s', current_image_id, current_image_path) yield current_image_stream stream = formatter.build_stream(namespace, repository, tag, synthetic_image_id, image_json, @@ -76,8 +75,10 @@ def _write_synthetic_image_to_storage(verb, linked_storage_uuid, linked_location done_uploading.save() +# pylint: disable=too-many-locals def _repo_verb(namespace, repository, tag, verb, formatter, checker=None, **kwargs): permission = ReadRepositoryPermission(namespace, repository) + # pylint: disable=no-member if permission.can() or model.repository_is_public(namespace, repository): # Lookup the requested tag. try: @@ -168,6 +169,7 @@ def _repo_verb(namespace, repository, tag, verb, formatter, checker=None, **kwar @verbs.route('/aci/////aci///', methods=['GET']) @process_auth +# pylint: disable=unused-argument def get_aci_image(server, namespace, repository, tag, os, arch): def checker(image_json): # Verify the architecture and os. diff --git a/formats/aci.py b/formats/aci.py index 23ddfd43e..d425c4497 100644 --- a/formats/aci.py +++ b/formats/aci.py @@ -1,3 +1,5 @@ +# pylint: disable=missing-docstring,bad-continuation + from app import app from util.streamlayerformat import StreamLayerMerger from formats.tarimageformatter import TarImageFormatter @@ -5,10 +7,12 @@ from formats.tarimageformatter import TarImageFormatter import json import re + class ACIImage(TarImageFormatter): """ Image formatter which produces an ACI-compatible TAR. """ + # pylint: disable=too-many-arguments def stream_generator(self, namespace, repository, tag, synthetic_image_id, layer_json, get_image_iterator, get_layer_iterator): # ACI Format (.tar): From 75fa007c5417288e7c1fae36b13113ca1a574ff7 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Mon, 2 Feb 2015 16:53:39 -0500 Subject: [PATCH 17/37] squashed format: _build_layer_json now static --- formats/squashed.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/formats/squashed.py b/formats/squashed.py index 187d1e74f..ef66d39f0 100644 --- a/formats/squashed.py +++ b/formats/squashed.py @@ -44,7 +44,7 @@ class SquashedDockerImage(TarImageFormatter): yield self.tar_folder(synthetic_image_id) # Yield the JSON layer data. - layer_json = self._build_layer_json(layer_json, synthetic_image_id) + layer_json = SquashedDockerImage._build_layer_json(layer_json, synthetic_image_id) yield self.tar_file(synthetic_image_id + '/json', json.dumps(layer_json)) # Yield the VERSION file. @@ -85,7 +85,8 @@ class SquashedDockerImage(TarImageFormatter): yield '\0' * 512 - def _build_layer_json(self, layer_json, synthetic_image_id): + @staticmethod + def _build_layer_json(layer_json, synthetic_image_id): updated_json = copy.deepcopy(layer_json) updated_json['id'] = synthetic_image_id @@ -99,4 +100,3 @@ class SquashedDockerImage(TarImageFormatter): updated_json['container_config']['Image'] = synthetic_image_id return updated_json - From 90a37829334adb631494c153545c764c3ed8bc26 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Mon, 2 Feb 2015 16:54:23 -0500 Subject: [PATCH 18/37] lint: pylint comments and unused imports --- app.py | 2 ++ formats/aci.py | 3 +-- formats/squashed.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index cdac98a27..417722752 100644 --- a/app.py +++ b/app.py @@ -27,6 +27,8 @@ from data.queue import WorkQueue from data.userevent import UserEventsBuilderModule from avatars.avatars import Avatar +# pylint: disable=invalid-name,too-many-public-methods,too-few-public-methods,too-many-ancestors + class Config(BaseConfig): """ Flask config enhanced with a `from_yamlfile` method """ diff --git a/formats/aci.py b/formats/aci.py index d425c4497..f0fdf6cb6 100644 --- a/formats/aci.py +++ b/formats/aci.py @@ -1,5 +1,3 @@ -# pylint: disable=missing-docstring,bad-continuation - from app import app from util.streamlayerformat import StreamLayerMerger from formats.tarimageformatter import TarImageFormatter @@ -7,6 +5,7 @@ from formats.tarimageformatter import TarImageFormatter import json import re +# pylint: disable=bad-continuation class ACIImage(TarImageFormatter): """ Image formatter which produces an ACI-compatible TAR. diff --git a/formats/squashed.py b/formats/squashed.py index ef66d39f0..10580a9d8 100644 --- a/formats/squashed.py +++ b/formats/squashed.py @@ -5,7 +5,6 @@ from formats.tarimageformatter import TarImageFormatter import copy import json -import tarfile class FileEstimationException(Exception): """ Exception raised by build_docker_load_stream if the estimated size of the layer TAR @@ -20,6 +19,7 @@ class SquashedDockerImage(TarImageFormatter): command. """ + # pylint: disable=too-many-arguments,too-many-locals def stream_generator(self, namespace, repository, tag, synthetic_image_id, layer_json, get_image_iterator, get_layer_iterator): # Docker import V1 Format (.tar): From 81b5b8d1dc438e8f10f0d8cf325337f677542265 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 3 Feb 2015 12:15:08 -0500 Subject: [PATCH 19/37] aci format: - allowed in AC Name spec --- formats/aci.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/formats/aci.py b/formats/aci.py index f0fdf6cb6..ff7753652 100644 --- a/formats/aci.py +++ b/formats/aci.py @@ -96,7 +96,7 @@ class ACIImage(TarImageFormatter): try: port_number = int(port_number) ports.append({ - "name": "port%s" % port_number, + "name": "port-%s" % port_number, "port": port_number, "protocol": protocol }) @@ -188,7 +188,7 @@ class ACIImage(TarImageFormatter): "annotations": [ {"name": "created", "value": docker_layer_data.get('created', '')}, {"name": "homepage", "value": source_url}, - {"name": "quay.io/derivedimage", "value": synthetic_image_id}, + {"name": "quay.io/derived-image", "value": synthetic_image_id}, ] }, } From 6102f905bc49b9af8a4ef0dcf26fba4e324a17e1 Mon Sep 17 00:00:00 2001 From: Alex Malinovich Date: Tue, 3 Feb 2015 12:09:21 -0800 Subject: [PATCH 20/37] Updating DMCA contact info --- templates/tos.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/tos.html b/templates/tos.html index 038ebdfd8..65c97a069 100644 --- a/templates/tos.html +++ b/templates/tos.html @@ -105,11 +105,11 @@

Also, in accordance with the Digital Millennium Copyright Act (DMCA) and other applicable law, CoreOS has adopted a policy of terminating, in appropriate circumstances and at our discretion, account holders who are deemed to be repeat infringers. CoreOS also may, at its discretion, limit access to the Services and terminate the accounts of any users who infringe any intellectual property rights of others, whether or not there is any repeat infringement.

If you think that anything on the Services infringes upon any copyright that you own or control, you may file a notification with CoreOS’ Designated Agent as set forth below:

- - - - - + + + + +
Designated Agent:[insert name]
Address of Designated Agent:[insert address]
Telephone Number of Designated Agent:[insert telephone]
Fax Number of Designated Agent:[insert telephone number]
Email Address of Designated Agent:[insert email address]
Designated Agent:DMCA Agent
Address of Designated Agent:3043 Mission Street, San Francisco, CA 94110
Telephone Number of Designated Agent:(800) 774-3507
Fax Number of Designated Agent:(415) 580-7362
Email Address of Designated Agent:support@quay.io

Please see 17 U.S.C. § 512(c)(3) for the requirements of a proper notification. If you knowingly misrepresent that any material or activity is infringing, you may be liable for any damages, including costs and attorneys’ fees, CoreOS or the alleged infringer incurs because we relied on the misrepresentation when removing or disabling access to the material or activity.

From ec4f77fa7e3e46d0e6e8991aa425e3977637a0d9 Mon Sep 17 00:00:00 2001 From: Alex Malinovich Date: Tue, 3 Feb 2015 13:42:22 -0800 Subject: [PATCH 21/37] Fix date in ToS --- templates/tos.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/tos.html b/templates/tos.html index 65c97a069..35d13b81e 100644 --- a/templates/tos.html +++ b/templates/tos.html @@ -28,7 +28,7 @@ {% block body_content %}

CoreOS Terms of Service

-

Last Revised: February 2, 2015

+

Last Revised: February 3, 2015

These Quay.io Terms of Service (these “Terms”) apply to the features and functions provided by CoreOS, Inc. (“CoreOS,” “our,” or “we”) via quay.io (the “Site”) (collectively, the “Services”). By accessing or using the Services, you agree to be bound by these Terms. If you do not agree to these Terms, do not use any of the Services. The “Effective Date” of these Terms is the date you first access any of the Services.

If you are accessing the Services in your capacity as an employee, consultant or agent of a company (or other entity), you represent that you are an employee, consultant or agent of such company (or other entity) and you have the authority to agree (and be legally bound) on behalf of such company (or other entity) to all of the terms and conditions of these Terms.

From 4355e07f9f553f8ac59c95494e7971ad482c2b23 Mon Sep 17 00:00:00 2001 From: Alex Malinovich Date: Tue, 3 Feb 2015 17:40:58 -0800 Subject: [PATCH 22/37] Fix date in ToS again --- templates/tos.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/tos.html b/templates/tos.html index 35d13b81e..ede476da9 100644 --- a/templates/tos.html +++ b/templates/tos.html @@ -28,7 +28,7 @@ {% block body_content %}

CoreOS Terms of Service

-

Last Revised: February 3, 2015

+

Last Revised: February 4, 2015

These Quay.io Terms of Service (these “Terms”) apply to the features and functions provided by CoreOS, Inc. (“CoreOS,” “our,” or “we”) via quay.io (the “Site”) (collectively, the “Services”). By accessing or using the Services, you agree to be bound by these Terms. If you do not agree to these Terms, do not use any of the Services. The “Effective Date” of these Terms is the date you first access any of the Services.

If you are accessing the Services in your capacity as an employee, consultant or agent of a company (or other entity), you represent that you are an employee, consultant or agent of such company (or other entity) and you have the authority to agree (and be legally bound) on behalf of such company (or other entity) to all of the terms and conditions of these Terms.

From bfb0784abc70dd91da7f36aa9e4954de03e608a9 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 4 Feb 2015 15:29:24 -0500 Subject: [PATCH 23/37] Add signing to the ACI converter --- Dockerfile.web | 2 +- app.py | 3 + config.py | 2 +- data/database.py | 20 +++- data/model/legacy.py | 44 ++++++-- endpoints/verbs.py | 231 +++++++++++++++++++++++++++-------------- endpoints/web.py | 12 ++- formats/aci.py | 7 +- initdb.py | 2 + requirements-nover.txt | 1 + requirements.txt | 1 + templates/index.html | 2 + util/signing.py | 69 ++++++++++++ util/tarlayerformat.py | 5 + 14 files changed, 311 insertions(+), 90 deletions(-) create mode 100644 util/signing.py diff --git a/Dockerfile.web b/Dockerfile.web index d50256b2a..503f591cd 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -20,7 +20,7 @@ ADD requirements.txt requirements.txt RUN virtualenv --distribute venv RUN venv/bin/pip install -r requirements.txt -RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev +RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev gpgme ############################### # END COMMON SECION diff --git a/app.py b/app.py index 417722752..b73976e96 100644 --- a/app.py +++ b/app.py @@ -26,6 +26,7 @@ from data.archivedlogs import LogArchive from data.queue import WorkQueue from data.userevent import UserEventsBuilderModule from avatars.avatars import Avatar +from util.signing import Signer # pylint: disable=invalid-name,too-many-public-methods,too-few-public-methods,too-many-ancestors @@ -55,6 +56,7 @@ class Flask(BaseFlask): return Config(root_path, self.default_config) +OVERRIDE_CONFIG_DIRECTORY = 'conf/stack/' OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml' OVERRIDE_CONFIG_PY_FILENAME = 'conf/stack/config.py' @@ -135,6 +137,7 @@ build_logs = BuildLogs(app) queue_metrics = QueueMetrics(app) authentication = UserAuthentication(app) userevents = UserEventsBuilderModule(app) +signer = Signer(app, OVERRIDE_CONFIG_DIRECTORY) github_login = GithubOAuthConfig(app, 'GITHUB_LOGIN_CONFIG') github_trigger = GithubOAuthConfig(app, 'GITHUB_TRIGGER_CONFIG') diff --git a/config.py b/config.py index 39a257b15..5b24717dc 100644 --- a/config.py +++ b/config.py @@ -44,7 +44,7 @@ class DefaultConfig(object): SEND_FILE_MAX_AGE_DEFAULT = 0 POPULATE_DB_TEST_DATA = True PREFERRED_URL_SCHEME = 'http' - SERVER_HOSTNAME = 'localhost:5000' + SERVER_HOSTNAME = '10.0.2.2' AVATAR_KIND = 'local' diff --git a/data/database.py b/data/database.py index b49c8a594..0dcf9fc95 100644 --- a/data/database.py +++ b/data/database.py @@ -352,6 +352,24 @@ class ImageStorageTransformation(BaseModel): name = CharField(index=True, unique=True) +class ImageStorageSignatureKind(BaseModel): + name = CharField(index=True, unique=True) + + +class ImageStorageSignature(BaseModel): + storage = ForeignKeyField(ImageStorage, index=True) + kind = ForeignKeyField(ImageStorageSignatureKind) + signature = TextField(null=True) + uploading = BooleanField(default=True, null=True) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('kind', 'storage'), True), + ) + + class DerivedImageStorage(BaseModel): source = ForeignKeyField(ImageStorage, null=True, related_name='source') derivative = ForeignKeyField(ImageStorage, related_name='derivative') @@ -550,4 +568,4 @@ all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission, Notification, ImageStorageLocation, ImageStoragePlacement, ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification, RepositoryAuthorizedEmail, ImageStorageTransformation, DerivedImageStorage, - TeamMemberInvite] + TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind] diff --git a/data/model/legacy.py b/data/model/legacy.py index a5c779871..aee7f14b4 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -14,7 +14,8 @@ from data.database import (User, Repository, Image, AccessToken, Role, Repositor ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification, RepositoryAuthorizedEmail, TeamMemberInvite, DerivedImageStorage, ImageStorageTransformation, random_string_generator, - db, BUILD_PHASE, QuayUserField) + db, BUILD_PHASE, QuayUserField, ImageStorageSignature, + ImageStorageSignatureKind) from peewee import JOIN_LEFT_OUTER, fn from util.validation import (validate_username, validate_email, validate_password, INVALID_PASSWORD_MESSAGE) @@ -1317,7 +1318,28 @@ def find_create_or_link_image(docker_image_id, repository, username, translation ancestors='/') -def find_or_create_derived_storage(source, transformation_name, preferred_location): +def find_or_create_storage_signature(storage, signature_kind): + found = lookup_storage_signature(storage, signature_kind) + if found is None: + kind = ImageStorageSignatureKind.get(name=signature_kind) + found = ImageStorageSignature.create(storage=storage, kind=kind) + + return found + + +def lookup_storage_signature(storage, signature_kind): + kind = ImageStorageSignatureKind.get(name=signature_kind) + try: + return (ImageStorageSignature + .select() + .where(ImageStorageSignature.storage == storage, + ImageStorageSignature.kind == kind) + .get()) + except ImageStorageSignature.DoesNotExist: + return None + + +def find_derived_storage(source, transformation_name): try: found = (ImageStorage .select(ImageStorage, DerivedImageStorage) @@ -1330,11 +1352,19 @@ def find_or_create_derived_storage(source, transformation_name, preferred_locati found.locations = {placement.location.name for placement in found.imagestorageplacement_set} return found except ImageStorage.DoesNotExist: - logger.debug('Creating storage dervied from source: %s', source.uuid) - trans = ImageStorageTransformation.get(name=transformation_name) - new_storage = _create_storage(preferred_location) - DerivedImageStorage.create(source=source, derivative=new_storage, transformation=trans) - return new_storage + return None + + +def find_or_create_derived_storage(source, transformation_name, preferred_location): + existing = find_derived_storage(source, transformation_name) + if existing is not None: + return existing + + logger.debug('Creating storage dervied from source: %s', source.uuid) + trans = ImageStorageTransformation.get(name=transformation_name) + new_storage = _create_storage(preferred_location) + DerivedImageStorage.create(source=source, derivative=new_storage, transformation=trans) + return new_storage def delete_derived_storage_by_uuid(storage_uuid): diff --git a/endpoints/verbs.py b/endpoints/verbs.py index c14d8dcf0..3999d64e7 100644 --- a/endpoints/verbs.py +++ b/endpoints/verbs.py @@ -2,9 +2,9 @@ import logging import json import hashlib -from flask import redirect, Blueprint, abort, send_file +from flask import redirect, Blueprint, abort, send_file, make_response -from app import app +from app import app, signer from auth.auth import process_auth from auth.permissions import ReadRepositoryPermission from data import model @@ -53,6 +53,26 @@ def _open_stream(formatter, namespace, repository, tag, synthetic_image_id, imag return stream.read +def _sign_sythentic_image(verb, linked_storage_uuid, queue_file): + signature = None + try: + signature = signer.detached_sign(queue_file) + except: + logger.exception('Exception when signing %s image %s', verb, linked_storage_uuid) + return + + with database.UseThenDisconnect(app.config): + try: + derived = model.get_storage_by_uuid(linked_storage_uuid) + except model.InvalidImageException: + return + + signature_entry = model.find_or_create_storage_signature(derived, signer.name) + signature_entry.signature = signature + signature_entry.uploading = False + signature_entry.save() + + def _write_synthetic_image_to_storage(verb, linked_storage_uuid, linked_locations, queue_file): store = Storage(app) @@ -76,101 +96,145 @@ def _write_synthetic_image_to_storage(verb, linked_storage_uuid, linked_location # pylint: disable=too-many-locals -def _repo_verb(namespace, repository, tag, verb, formatter, checker=None, **kwargs): +def _verify_repo_verb(store, namespace, repository, tag, verb, checker=None): permission = ReadRepositoryPermission(namespace, repository) + # pylint: disable=no-member - if permission.can() or model.repository_is_public(namespace, repository): - # Lookup the requested tag. - try: - tag_image = model.get_tag_image(namespace, repository, tag) - except model.DataModelException: + if not permission.can() and not model.repository_is_public(namespace, repository): + abort(403) + + # Lookup the requested tag. + try: + tag_image = model.get_tag_image(namespace, repository, tag) + except model.DataModelException: + abort(404) + + # Lookup the tag's image and storage. + repo_image = model.get_repo_image_extended(namespace, repository, tag_image.docker_image_id) + if not repo_image: + abort(404) + + # If there is a data checker, call it first. + uuid = repo_image.storage.uuid + image_json = None + + if checker is not None: + image_json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid)) + image_json = json.loads(image_json_data) + + if not checker(image_json): + logger.debug('Check mismatch on %s/%s:%s, verb %s', namespace, repository, tag, verb) abort(404) - # Lookup the tag's image and storage. - repo_image = model.get_repo_image_extended(namespace, repository, tag_image.docker_image_id) - if not repo_image: - abort(404) + return (repo_image, tag_image, image_json) - # If there is a data checker, call it first. - store = Storage(app) - uuid = repo_image.storage.uuid - image_json = None - if checker is not None: - image_json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid)) - image_json = json.loads(image_json_data) +# pylint: disable=too-many-locals +def _repo_verb_signature(namespace, repository, tag, verb, checker=None, **kwargs): + # Verify that the image exists and that we have access to it. + store = Storage(app) + result = _verify_repo_verb(store, namespace, repository, tag, verb, checker) + (repo_image, tag_image, image_json) = result - if not checker(image_json): - logger.debug('Check mismatch on %s/%s:%s, verb %s', namespace, repository, tag, verb) - abort(404) + # Lookup the derived image storage for the verb. + derived = model.find_derived_storage(repo_image.storage, verb) + if derived is None or derived.uploading: + abort(404) - # Log the action. - track_and_log('repo_verb', repo_image.repository, tag=tag, verb=verb, **kwargs) + # Check if we have a valid signer configured. + if not signer.name: + abort(404) - derived = model.find_or_create_derived_storage(repo_image.storage, verb, - store.preferred_locations[0]) + # Lookup the signature for the verb. + signature_entry = model.lookup_storage_signature(derived, signer.name) + if signature_entry is None: + abort(404) - if not derived.uploading: - logger.debug('Derived %s image %s exists in storage', verb, derived.uuid) - derived_layer_path = store.image_layer_path(derived.uuid) - download_url = store.get_direct_download_url(derived.locations, derived_layer_path) - if download_url: - logger.debug('Redirecting to download URL for derived %s image %s', verb, derived.uuid) - return redirect(download_url) + # Return the signature. + return make_response(signature_entry.signature) - # Close the database handle here for this process before we send the long download. - database.close_db_filter(None) - logger.debug('Sending cached derived %s image %s', verb, derived.uuid) - return send_file(store.stream_read_file(derived.locations, derived_layer_path)) +# pylint: disable=too-many-locals +def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=None, **kwargs): + # Verify that the image exists and that we have access to it. + store = Storage(app) + result = _verify_repo_verb(store, namespace, repository, tag, verb, checker) + (repo_image, tag_image, image_json) = result - # Load the ancestry for the image. - logger.debug('Building and returning derived %s image %s', verb, derived.uuid) - ancestry_data = store.get_content(repo_image.storage.locations, store.image_ancestry_path(uuid)) - full_image_list = json.loads(ancestry_data) + # Log the action. + track_and_log('repo_verb', repo_image.repository, tag=tag, verb=verb, **kwargs) - # Load the image's JSON layer. - if not image_json: - image_json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid)) - image_json = json.loads(image_json_data) + # Lookup/create the derived image storage for the verb. + derived = model.find_or_create_derived_storage(repo_image.storage, verb, + store.preferred_locations[0]) - # Calculate a synthetic image ID. - synthetic_image_id = hashlib.sha256(tag_image.docker_image_id + ':' + verb).hexdigest() - - # Create a queue process to generate the data. The queue files will read from the process - # and send the results to the client and storage. - def _cleanup(): - # Close any existing DB connection once the process has exited. - database.close_db_filter(None) - - args = (formatter, namespace, repository, tag, synthetic_image_id, image_json, full_image_list) - queue_process = QueueProcess(_open_stream, - 8 * 1024, 10 * 1024 * 1024, # 8K/10M chunk/max - args, finished=_cleanup) - - client_queue_file = QueueFile(queue_process.create_queue(), 'client') - storage_queue_file = QueueFile(queue_process.create_queue(), 'storage') - - # Start building. - queue_process.run() - - # Start the storage saving. - storage_args = (verb, derived.uuid, derived.locations, storage_queue_file) - QueueProcess.run_process(_write_synthetic_image_to_storage, storage_args, finished=_cleanup) + if not derived.uploading: + logger.debug('Derived %s image %s exists in storage', verb, derived.uuid) + derived_layer_path = store.image_layer_path(derived.uuid) + download_url = store.get_direct_download_url(derived.locations, derived_layer_path) + if download_url: + logger.debug('Redirecting to download URL for derived %s image %s', verb, derived.uuid) + return redirect(download_url) # Close the database handle here for this process before we send the long download. database.close_db_filter(None) - # Return the client's data. - return send_file(client_queue_file) + logger.debug('Sending cached derived %s image %s', verb, derived.uuid) + return send_file(store.stream_read_file(derived.locations, derived_layer_path)) - abort(403) + # Load the ancestry for the image. + uuid = repo_image.storage.uuid + + logger.debug('Building and returning derived %s image %s', verb, derived.uuid) + ancestry_data = store.get_content(repo_image.storage.locations, store.image_ancestry_path(uuid)) + full_image_list = json.loads(ancestry_data) + + # Load the image's JSON layer. + if not image_json: + image_json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid)) + image_json = json.loads(image_json_data) + + # Calculate a synthetic image ID. + synthetic_image_id = hashlib.sha256(tag_image.docker_image_id + ':' + verb).hexdigest() + + def _cleanup(): + # Close any existing DB connection once the process has exited. + database.close_db_filter(None) + + # Create a queue process to generate the data. The queue files will read from the process + # and send the results to the client and storage. + args = (formatter, namespace, repository, tag, synthetic_image_id, image_json, full_image_list) + queue_process = QueueProcess(_open_stream, + 8 * 1024, 10 * 1024 * 1024, # 8K/10M chunk/max + args, finished=_cleanup) + + client_queue_file = QueueFile(queue_process.create_queue(), 'client') + storage_queue_file = QueueFile(queue_process.create_queue(), 'storage') + + # If signing is required, add a QueueFile for signing the image as we stream it out. + signing_queue_file = None + if sign and signer.name: + signing_queue_file = QueueFile(queue_process.create_queue(), 'signing') + + # Start building. + queue_process.run() + + # Start the storage saving. + storage_args = (verb, derived.uuid, derived.locations, storage_queue_file) + QueueProcess.run_process(_write_synthetic_image_to_storage, storage_args, finished=_cleanup) + + if sign and signer.name: + signing_args = (verb, derived.uuid, signing_queue_file) + QueueProcess.run_process(_sign_sythentic_image, signing_args, finished=_cleanup) + + # Close the database handle here for this process before we send the long download. + database.close_db_filter(None) + + # Return the client's data. + return send_file(client_queue_file) -@verbs.route('/aci/////aci///', methods=['GET']) -@process_auth -# pylint: disable=unused-argument -def get_aci_image(server, namespace, repository, tag, os, arch): +def os_arch_checker(os, arch): def checker(image_json): # Verify the architecture and os. operating_system = image_json.get('os', 'linux') @@ -183,8 +247,23 @@ def get_aci_image(server, namespace, repository, tag, os, arch): return True + return checker + + +@verbs.route('/aci/////sig///', methods=['GET']) +@process_auth +# pylint: disable=unused-argument +def get_aci_signature(server, namespace, repository, tag, os, arch): + return _repo_verb_signature(namespace, repository, tag, 'aci', checker=os_arch_checker(os, arch), + os=os, arch=arch) + + +@verbs.route('/aci/////aci///', methods=['GET']) +@process_auth +# pylint: disable=unused-argument +def get_aci_image(server, namespace, repository, tag, os, arch): return _repo_verb(namespace, repository, tag, 'aci', ACIImage(), - checker=checker, os=os, arch=arch) + sign=True, checker=os_arch_checker(os, arch), os=os, arch=arch) @verbs.route('/squash///', methods=['GET']) diff --git a/endpoints/web.py b/endpoints/web.py index 519fc5c5e..17454c7ba 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -1,7 +1,7 @@ import logging from flask import (abort, redirect, request, url_for, make_response, Response, - Blueprint, send_from_directory, jsonify) + Blueprint, send_from_directory, jsonify, send_file) from avatar_generator import Avatar from flask.ext.login import current_user @@ -10,7 +10,7 @@ from health.healthcheck import HealthCheck from data import model from data.model.oauth import DatabaseAuthorizationProvider -from app import app, billing as stripe, build_logs, avatar +from app import app, billing as stripe, build_logs, avatar, signer from auth.auth import require_session_login, process_oauth from auth.permissions import AdministerOrganizationPermission, ReadRepositoryPermission from util.invoice import renderInvoiceToPdf @@ -57,6 +57,14 @@ def snapshot(path = ''): abort(404) +@web.route('/aci-signing-key') +@no_cache +def aci_signing_key(): + if not signer.name: + abort(404) + + return send_file(signer.public_key_path) + @web.route('/plans/') @no_cache @route_show_if(features.BILLING) diff --git a/formats/aci.py b/formats/aci.py index ff7753652..d40c3d054 100644 --- a/formats/aci.py +++ b/formats/aci.py @@ -180,8 +180,11 @@ class ACIImage(TarImageFormatter): "group": config.get('Group', '') or 'root', "eventHandlers": [], "workingDirectory": config.get('WorkingDir', '') or '/', - "environment": [{"name": key, "value": value} - for (key, value) in [e.split('=') for e in config.get('Env')]], + # TODO(jschorr): Use the commented version once rocket has upgraded to 0.3.0. + #"environment": [{"name": key, "value": value} + # for (key, value) in [e.split('=') for e in config.get('Env')]], + "environment": {key: value + for (key, value) in [e.split('=') for e in config.get('Env')]}, "isolators": ACIImage._build_isolators(config), "mountPoints": ACIImage._build_volumes(config), "ports": ACIImage._build_ports(config), diff --git a/initdb.py b/initdb.py index 8b50431a1..15c62bb0b 100644 --- a/initdb.py +++ b/initdb.py @@ -257,6 +257,8 @@ def initialize_database(): ImageStorageTransformation.create(name='squash') ImageStorageTransformation.create(name='aci') + ImageStorageSignatureKind.create(name='gpg2') + # NOTE: These MUST be copied over to NotificationKind, since every external # notification can also generate a Quay.io notification. ExternalNotificationEvent.create(name='repo_push') diff --git a/requirements-nover.txt b/requirements-nover.txt index ca86fe72d..451363258 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -41,3 +41,4 @@ git+https://github.com/DevTable/anunidecode.git git+https://github.com/DevTable/avatar-generator.git git+https://github.com/DevTable/pygithub.git gipc +pygpgme \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 03da5b64c..758b6f136 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,6 +48,7 @@ python-dateutil==2.2 python-ldap==2.4.18 python-magic==0.4.6 pytz==2014.9 +pygpgme==0.3 raven==5.1.1 redis==2.10.3 reportlab==2.7 diff --git a/templates/index.html b/templates/index.html index 120b7865c..c37dba08c 100644 --- a/templates/index.html +++ b/templates/index.html @@ -11,6 +11,8 @@ + + {% endblock %} diff --git a/util/signing.py b/util/signing.py new file mode 100644 index 000000000..a57e4ebd7 --- /dev/null +++ b/util/signing.py @@ -0,0 +1,69 @@ +import gpgme +import os +from StringIO import StringIO + +class GPG2Signer(object): + """ Helper class for signing data using GPG2. """ + def __init__(self, app, key_directory): + if not app.config.get('GPG2_PRIVATE_KEY_NAME'): + raise Exception('Missing configuration key GPG2_PRIVATE_KEY_NAME') + + if not app.config.get('GPG2_PRIVATE_KEY_FILENAME'): + raise Exception('Missing configuration key GPG2_PRIVATE_KEY_FILENAME') + + if not app.config.get('GPG2_PUBLIC_KEY_FILENAME'): + raise Exception('Missing configuration key GPG2_PUBLIC_KEY_FILENAME') + + self._ctx = gpgme.Context() + self._ctx.armor = True + self._private_key_name = app.config['GPG2_PRIVATE_KEY_NAME'] + self._public_key_path = os.path.join(key_directory, app.config['GPG2_PUBLIC_KEY_FILENAME']) + + key_file = os.path.join(key_directory, app.config['GPG2_PRIVATE_KEY_FILENAME']) + if not os.path.exists(key_file): + raise Exception('Missing key file %s' % key_file) + + with open(key_file, 'rb') as fp: + self._ctx.import_(fp) + + @property + def name(self): + return 'gpg2' + + @property + def public_key_path(self): + return self._public_key_path + + def detached_sign(self, stream): + """ Signs the given stream, returning the signature. """ + ctx = self._ctx + ctx.signers = [ctx.get_key(self._private_key_name)] + signature = StringIO() + new_sigs = ctx.sign(stream, signature, gpgme.SIG_MODE_DETACH) + + signature.seek(0) + return signature.getvalue() + + +class Signer(object): + def __init__(self, app=None, key_directory=None): + self.app = app + if app is not None: + self.state = self.init_app(app, key_directory) + else: + self.state = None + + def init_app(self, app, key_directory): + preference = app.config.get('SIGNING_ENGINE', None) + if preference is None: + return None + + return SIGNING_ENGINES[preference](app, key_directory) + + def __getattr__(self, name): + return getattr(self.state, name, None) + + +SIGNING_ENGINES = { + 'gpg2': GPG2Signer +} \ No newline at end of file diff --git a/util/tarlayerformat.py b/util/tarlayerformat.py index 37aee16a7..3e195a616 100644 --- a/util/tarlayerformat.py +++ b/util/tarlayerformat.py @@ -44,6 +44,11 @@ class TarLayerFormat(object): # properly handle large filenames. clone = copy.deepcopy(tar_info) clone.name = os.path.join(self.path_prefix, clone.name) + + # If the entry is a link of some kind, and it is not relative, then prefix it as well. + if clone.linkname and clone.type == tarfile.LNKTYPE: + clone.linkname = os.path.join(self.path_prefix, clone.linkname) + yield clone.tobuf() else: yield tar_info.tobuf() From 925cd1f3789ec056d578b2386eeef3b2a3e36ddd Mon Sep 17 00:00:00 2001 From: Alex Malinovich Date: Wed, 4 Feb 2015 16:54:47 -0800 Subject: [PATCH 24/37] Fix date in ToS again. Again. --- templates/tos.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/tos.html b/templates/tos.html index ede476da9..b86128256 100644 --- a/templates/tos.html +++ b/templates/tos.html @@ -28,7 +28,7 @@ {% block body_content %}

CoreOS Terms of Service

-

Last Revised: February 4, 2015

+

Last Revised: February 5, 2015

These Quay.io Terms of Service (these “Terms”) apply to the features and functions provided by CoreOS, Inc. (“CoreOS,” “our,” or “we”) via quay.io (the “Site”) (collectively, the “Services”). By accessing or using the Services, you agree to be bound by these Terms. If you do not agree to these Terms, do not use any of the Services. The “Effective Date” of these Terms is the date you first access any of the Services.

If you are accessing the Services in your capacity as an employee, consultant or agent of a company (or other entity), you represent that you are an employee, consultant or agent of such company (or other entity) and you have the authority to agree (and be legally bound) on behalf of such company (or other entity) to all of the terms and conditions of these Terms.

From c7c5377285b6d3d6bc0e9161cc8df12227695467 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Thu, 5 Feb 2015 12:51:02 -0500 Subject: [PATCH 25/37] Add my key back to the ephemeral builder machines. --- buildman/templates/cloudconfig.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/buildman/templates/cloudconfig.yaml b/buildman/templates/cloudconfig.yaml index 4972e07ca..13e6894bf 100644 --- a/buildman/templates/cloudconfig.yaml +++ b/buildman/templates/cloudconfig.yaml @@ -3,6 +3,7 @@ ssh_authorized_keys: - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCC0m+hVmyR3vn/xoxJe9+atRWBxSK+YXgyufNVDMcb7H00Jfnc341QH3kDVYZamUbhVh/nyc2RP7YbnZR5zORFtgOaNSdkMYrPozzBvxjnvSUokkCCWbLqXDHvIKiR12r+UTSijPJE/Yk702Mb2ejAFuae1C3Ec+qKAoOCagDjpQ3THyb5oaKE7VPHdwCWjWIQLRhC+plu77ObhoXIFJLD13gCi01L/rp4mYVCxIc2lX5A8rkK+bZHnIZwWUQ4t8SIjWxIaUo0FE7oZ83nKuNkYj5ngmLHQLY23Nx2WhE9H6NBthUpik9SmqQPtVYbhIG+bISPoH9Xs8CLrFb0VRjz Joey's Mac - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCo6FhAP7mFFOAzM91gtaKW7saahtaN4lur42FMMztz6aqUycIltCmvxo+3FmrXgCG30maMNU36Vm1+9QRtVQEd+eRuoIWP28t+8MT01Fh4zPuE2Wca3pOHSNo3X81FfWJLzmwEHiQKs9HPQqUhezR9PcVWVkbMyAzw85c0UycGmHGFNb0UiRd9HFY6XbgbxhZv/mvKLZ99xE3xkOzS1PNsdSNvjUKwZR7pSUPqNS5S/1NXyR4GhFTU24VPH/bTATOv2ATH+PSzsZ7Qyz9UHj38tKC+ALJHEDJ4HXGzobyOUP78cHGZOfCB5FYubq0zmOudAjKIAhwI8XTFvJ2DX1P3 jimmyzelinskie +- ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNvw8qo9m8np7yQ/Smv/oklM8bo8VyNRZriGYBDuolWDL/mZpYCQnZJXphQo7RFdNABYistikjJlBuuwUohLf2uSq0iKoFa2TgwI43wViWzvuzU4nA02/ITD5BZdmWAFNyIoqeB50Ol4qUgDwLAZ+7Kv7uCi6chcgr9gTi99jY3GHyZjrMiXMHGVGi+FExFuzhVC2drKjbz5q6oRfQeLtNfG4psl5GU3MQU6FkX4fgoCx0r9R48/b7l4+TT7pWblJQiRfeldixu6308vyoTUEHasdkU3/X0OTaGz/h5XqTKnGQc6stvvoED3w+L3QFp0H5Z8sZ9stSsitmCBrmbcKZ jakemoshenko write_files: - path: /root/overrides.list From e7f7a58bcea20fb27a401e3ee2ffb0c9f89fb986 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 5 Feb 2015 13:20:40 -0500 Subject: [PATCH 26/37] Change to Rocket v0.3 env var format --- formats/aci.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/formats/aci.py b/formats/aci.py index d40c3d054..ff7753652 100644 --- a/formats/aci.py +++ b/formats/aci.py @@ -180,11 +180,8 @@ class ACIImage(TarImageFormatter): "group": config.get('Group', '') or 'root', "eventHandlers": [], "workingDirectory": config.get('WorkingDir', '') or '/', - # TODO(jschorr): Use the commented version once rocket has upgraded to 0.3.0. - #"environment": [{"name": key, "value": value} - # for (key, value) in [e.split('=') for e in config.get('Env')]], - "environment": {key: value - for (key, value) in [e.split('=') for e in config.get('Env')]}, + "environment": [{"name": key, "value": value} + for (key, value) in [e.split('=') for e in config.get('Env')]], "isolators": ACIImage._build_isolators(config), "mountPoints": ACIImage._build_volumes(config), "ports": ACIImage._build_ports(config), From 31306167ac68364c37e6fddfcb5949afeb06c48b Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 5 Feb 2015 14:38:59 -0500 Subject: [PATCH 27/37] Fix file accidentally checked in --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 5b24717dc..39a257b15 100644 --- a/config.py +++ b/config.py @@ -44,7 +44,7 @@ class DefaultConfig(object): SEND_FILE_MAX_AGE_DEFAULT = 0 POPULATE_DB_TEST_DATA = True PREFERRED_URL_SCHEME = 'http' - SERVER_HOSTNAME = '10.0.2.2' + SERVER_HOSTNAME = 'localhost:5000' AVATAR_KIND = 'local' From 555bd293eac187b673379d96bb9c1cd05ed459f1 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 5 Feb 2015 14:40:02 -0500 Subject: [PATCH 28/37] Fix tar layer format comment --- util/tarlayerformat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/tarlayerformat.py b/util/tarlayerformat.py index 3e195a616..111e0f731 100644 --- a/util/tarlayerformat.py +++ b/util/tarlayerformat.py @@ -45,7 +45,7 @@ class TarLayerFormat(object): clone = copy.deepcopy(tar_info) clone.name = os.path.join(self.path_prefix, clone.name) - # If the entry is a link of some kind, and it is not relative, then prefix it as well. + # If the entry is a *hard* link, then prefix it as well. Soft links are relative. if clone.linkname and clone.type == tarfile.LNKTYPE: clone.linkname = os.path.join(self.path_prefix, clone.linkname) From bc119aed226fa22997c96c68823f98ad2eef7b08 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 5 Feb 2015 15:00:19 -0500 Subject: [PATCH 29/37] Clarify why we need database.UserThenDisconnect --- endpoints/verbs.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/endpoints/verbs.py b/endpoints/verbs.py index 3999d64e7..091af8841 100644 --- a/endpoints/verbs.py +++ b/endpoints/verbs.py @@ -61,16 +61,19 @@ def _sign_sythentic_image(verb, linked_storage_uuid, queue_file): logger.exception('Exception when signing %s image %s', verb, linked_storage_uuid) return - with database.UseThenDisconnect(app.config): - try: - derived = model.get_storage_by_uuid(linked_storage_uuid) - except model.InvalidImageException: - return + # Setup the database (since this is a new process) and then disconnect immediately + # once the operation completes. + if not queue_file.raised_exception: + with database.UseThenDisconnect(app.config): + try: + derived = model.get_storage_by_uuid(linked_storage_uuid) + except model.InvalidImageException: + return - signature_entry = model.find_or_create_storage_signature(derived, signer.name) - signature_entry.signature = signature - signature_entry.uploading = False - signature_entry.save() + signature_entry = model.find_or_create_storage_signature(derived, signer.name) + signature_entry.signature = signature + signature_entry.uploading = False + signature_entry.save() def _write_synthetic_image_to_storage(verb, linked_storage_uuid, linked_locations, queue_file): @@ -89,6 +92,8 @@ def _write_synthetic_image_to_storage(verb, linked_storage_uuid, linked_location queue_file.close() if not queue_file.raised_exception: + # Setup the database (since this is a new process) and then disconnect immediately + # once the operation completes. with database.UseThenDisconnect(app.config): done_uploading = model.get_storage_by_uuid(linked_storage_uuid) done_uploading.uploading = False From a12bfa7623b996c2eed45d239c2987b693ce1729 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 5 Feb 2015 15:30:45 -0500 Subject: [PATCH 30/37] Add migration for the new tables for signatures --- .../5ad999136045_add_signature_storage.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 data/migrations/versions/5ad999136045_add_signature_storage.py diff --git a/data/migrations/versions/5ad999136045_add_signature_storage.py b/data/migrations/versions/5ad999136045_add_signature_storage.py new file mode 100644 index 000000000..6f5c36695 --- /dev/null +++ b/data/migrations/versions/5ad999136045_add_signature_storage.py @@ -0,0 +1,50 @@ +"""Add signature storage + +Revision ID: 5ad999136045 +Revises: 228d1af6af1c +Create Date: 2015-02-05 15:01:54.989573 + +""" + +# revision identifiers, used by Alembic. +revision = '5ad999136045' +down_revision = '228d1af6af1c' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('imagestoragesignaturekind', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_imagestoragesignaturekind')) + ) + op.create_index('imagestoragesignaturekind_name', 'imagestoragesignaturekind', ['name'], unique=True) + op.create_table('imagestoragesignature', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('storage_id', sa.Integer(), nullable=False), + sa.Column('kind_id', sa.Integer(), nullable=False), + sa.Column('signature', sa.Text(), nullable=True), + sa.Column('uploading', sa.Boolean(), nullable=True), + sa.ForeignKeyConstraint(['kind_id'], ['imagestoragesignaturekind.id'], name=op.f('fk_imagestoragesignature_kind_id_imagestoragesignaturekind')), + sa.ForeignKeyConstraint(['storage_id'], ['imagestorage.id'], name=op.f('fk_imagestoragesignature_storage_id_imagestorage')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_imagestoragesignature')) + ) + op.create_index('imagestoragesignature_kind_id', 'imagestoragesignature', ['kind_id'], unique=False) + op.create_index('imagestoragesignature_kind_id_storage_id', 'imagestoragesignature', ['kind_id', 'storage_id'], unique=True) + op.create_index('imagestoragesignature_storage_id', 'imagestoragesignature', ['storage_id'], unique=False) + ### end Alembic commands ### + + op.bulk_insert(tables.imagestoragesignaturekind, + [ + {'id': 1, 'name':'gpg2'}, + ]) + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('imagestoragesignature') + op.drop_table('imagestoragesignaturekind') + ### end Alembic commands ### From 81cebf79b7b4b989ea10d026640d5716361fd511 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 5 Feb 2015 17:15:17 -0500 Subject: [PATCH 31/37] Fix pygpgme dependency --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e87858a89..d201270b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ ENV HOME /root RUN apt-get update # 29JAN2015 # New ubuntu packages should be added as their own apt-get install lines below the existing install commands -RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev libfreetype6-dev libffi-dev gpgme +RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev libfreetype6-dev libffi-dev libgpgme11 libgpgme11-dev # Build the python dependencies ADD requirements.txt requirements.txt @@ -28,7 +28,7 @@ RUN npm install -g grunt-cli ADD grunt grunt RUN cd grunt && npm install -RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev +RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev libgpgme11-dev RUN apt-get autoremove -y RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* From 83c676f3f79b577b365a9d546cd965697e92a9b5 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 5 Feb 2015 17:17:00 -0500 Subject: [PATCH 32/37] Fix blog link --- static/partials/landing-normal.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/partials/landing-normal.html b/static/partials/landing-normal.html index 274b56ac0..9cc11834a 100644 --- a/static/partials/landing-normal.html +++ b/static/partials/landing-normal.html @@ -7,7 +7,7 @@ - Quay.io is now part of CoreOS! Read the blog post. + Quay.io is now part of CoreOS! Read the blog post.
From e1c5ccb7d65b15914c48a8214ebc9cb415dd036d Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 5 Feb 2015 17:37:58 -0500 Subject: [PATCH 33/37] Fixes --- .../versions/5ad999136045_add_signature_storage.py | 5 +++++ formats/aci.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/data/migrations/versions/5ad999136045_add_signature_storage.py b/data/migrations/versions/5ad999136045_add_signature_storage.py index 6f5c36695..f306c58b8 100644 --- a/data/migrations/versions/5ad999136045_add_signature_storage.py +++ b/data/migrations/versions/5ad999136045_add_signature_storage.py @@ -37,6 +37,11 @@ def upgrade(tables): op.create_index('imagestoragesignature_storage_id', 'imagestoragesignature', ['storage_id'], unique=False) ### end Alembic commands ### + op.bulk_insert(tables.imagestoragetransformation, + [ + {'id': 2, 'name':'aci'}, + ]) + op.bulk_insert(tables.imagestoragesignaturekind, [ {'id': 1, 'name':'gpg2'}, diff --git a/formats/aci.py b/formats/aci.py index ff7753652..62a9995d2 100644 --- a/formats/aci.py +++ b/formats/aci.py @@ -146,8 +146,8 @@ class ACIImage(TarImageFormatter): # ACI requires that the execution command be absolutely referenced. Therefore, if we find # a relative command, we give it as an argument to /bin/sh to resolve and execute for us. - entrypoint = config.get('Entrypoint') or [] - exec_path = entrypoint + config.get('Cmd') or [] + entrypoint = config.get('Entrypoint', []) or [] + exec_path = entrypoint + (config.get('Cmd', []) or []) if exec_path and not exec_path[0].startswith('/'): exec_path = ['/bin/sh', '-c', '""%s""' % ' '.join(exec_path)] From bbb127166a833f43ca8bf44830468d273a3c4973 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 5 Feb 2015 17:55:54 -0500 Subject: [PATCH 34/37] Fix template --- templates/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/index.html b/templates/index.html index c37dba08c..26178c01d 100644 --- a/templates/index.html +++ b/templates/index.html @@ -11,7 +11,7 @@ - + {% endblock %} From 5f431e966ee05583651c31f3b0594d1e6e03c788 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 6 Feb 2015 12:22:27 -0500 Subject: [PATCH 35/37] Add x86_64 compatibility check --- endpoints/verbs.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/endpoints/verbs.py b/endpoints/verbs.py index 091af8841..38cea8ff2 100644 --- a/endpoints/verbs.py +++ b/endpoints/verbs.py @@ -247,6 +247,12 @@ def os_arch_checker(os, arch): return False architecture = image_json.get('architecture', 'amd64') + + # Note: Some older Docker images have 'x86_64' rather than 'amd64'. + # We allow the conversion here. + if architecture == 'x86_64' and operating_system == 'linux': + architecture = 'amd64' + if architecture != arch: return False From 09a10b61532eec348921e6575439ca500458e376 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 6 Feb 2015 17:52:09 -0500 Subject: [PATCH 36/37] Have cache busting hashes be generated as part of the build process. --- endpoints/common.py | 43 ++++++++++++++++++++++++++++++++----------- grunt/Gruntfile.js | 15 ++++++++++++++- grunt/package.json | 3 ++- templates/base.html | 8 ++++---- 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/endpoints/common.py b/endpoints/common.py index 090708e39..3534ee072 100644 --- a/endpoints/common.py +++ b/endpoints/common.py @@ -3,6 +3,7 @@ import urlparse import json import string import datetime +import os # Register the various exceptions via decorators. import endpoints.decorated @@ -32,6 +33,23 @@ profile = logging.getLogger('application.profiler') route_data = None +CACHE_BUSTERS_JSON = 'static/dist/cachebusters.json' +CACHE_BUSTERS = None + +def get_cache_busters(): + """ Retrieves the cache busters hashes. """ + global CACHE_BUSTERS + if CACHE_BUSTERS is not None: + return CACHE_BUSTERS + + if not os.path.exists(CACHE_BUSTERS_JSON): + return {} + + with open(CACHE_BUSTERS_JSON, 'r') as f: + CACHE_BUSTERS = json.loads(f.read()) + return CACHE_BUSTERS + + class RepoPathConverter(BaseConverter): regex = '[\.a-zA-Z0-9_\-]+/[\.a-zA-Z0-9_\-]+' weight = 200 @@ -113,17 +131,15 @@ def list_files(path, extension): filepath = 'static/' + path return [join_path(dp, f) for dp, dn, files in os.walk(filepath) for f in files if matches(f)] -SAVED_CACHE_STRING = random_string() - def render_page_template(name, **kwargs): - if app.config.get('DEBUGGING', False): + debugging = app.config.get('DEBUGGING', False) + if debugging: # If DEBUGGING is enabled, then we load the full set of individual JS and CSS files # from the file system. library_styles = list_files('lib', 'css') main_styles = list_files('css', 'css') library_scripts = list_files('lib', 'js') main_scripts = list_files('js', 'js') - cache_buster = 'debugging' file_lists = [library_styles, main_styles, library_scripts, main_scripts] for file_list in file_lists: @@ -133,7 +149,6 @@ def render_page_template(name, **kwargs): main_styles = ['dist/quay-frontend.css'] library_scripts = [] main_scripts = ['dist/quay-frontend.min.js'] - cache_buster = SAVED_CACHE_STRING use_cdn = app.config.get('USE_CDN', True) if request.args.get('use_cdn') is not None: @@ -142,6 +157,12 @@ def render_page_template(name, **kwargs): external_styles = get_external_css(local=not use_cdn) external_scripts = get_external_javascript(local=not use_cdn) + def add_cachebusters(filenames): + cachebusters = get_cache_busters() + for filename in filenames: + cache_buster = cachebusters.get(filename, random_string()) if not debugging else 'debugging' + yield (filename, cache_buster) + def get_oauth_config(): oauth_config = {} for oauth_app in oauth_apps: @@ -153,13 +174,14 @@ def render_page_template(name, **kwargs): if len(app.config.get('CONTACT_INFO', [])) == 1: contact_href = app.config['CONTACT_INFO'][0] - resp = make_response(render_template(name, route_data=json.dumps(get_route_data()), + resp = make_response(render_template(name, + route_data=json.dumps(get_route_data()), external_styles=external_styles, external_scripts=external_scripts, - main_styles=main_styles, - library_styles=library_styles, - main_scripts=main_scripts, - library_scripts=library_scripts, + main_styles=add_cachebusters(main_styles), + library_styles=add_cachebusters(library_styles), + main_scripts=add_cachebusters(main_scripts), + library_scripts=add_cachebusters(library_scripts), feature_set=json.dumps(features.get_features()), config_set=json.dumps(getFrontendVisibleConfig(app.config)), oauth_set=json.dumps(get_oauth_config()), @@ -169,7 +191,6 @@ def render_page_template(name, **kwargs): sentry_public_dsn=app.config.get('SENTRY_PUBLIC_DSN', ''), is_debug=str(app.config.get('DEBUGGING', False)).lower(), show_chat=features.OLARK_CHAT, - cache_buster=cache_buster, has_billing=features.BILLING, contact_href=contact_href, hostname=app.config['SERVER_HOSTNAME'], diff --git a/grunt/Gruntfile.js b/grunt/Gruntfile.js index 5dd381e8d..e9cb14818 100644 --- a/grunt/Gruntfile.js +++ b/grunt/Gruntfile.js @@ -68,6 +68,18 @@ module.exports = function(grunt) { src: ['../static/partials/*.html', '../static/directives/*.html'], dest: '../static/dist/template-cache.js' } + }, + + cachebuster: { + build: { + options: { + format: 'json', + basedir: '../static/' + }, + src: [ '../static/dist/template-cache.js', '../static/dist/<%= pkg.name %>.min.js', + '../static/dist/<%= pkg.name %>.css' ], + dest: '../static/dist/cachebusters.json' + } } }); @@ -75,7 +87,8 @@ module.exports = function(grunt) { grunt.loadNpmTasks('grunt-contrib-concat'); grunt.loadNpmTasks('grunt-contrib-cssmin'); grunt.loadNpmTasks('grunt-angular-templates'); + grunt.loadNpmTasks('grunt-cachebuster'); // Default task(s). - grunt.registerTask('default', ['ngtemplates', 'concat', 'cssmin', 'uglify']); + grunt.registerTask('default', ['ngtemplates', 'concat', 'cssmin', 'uglify', 'cachebuster']); }; diff --git a/grunt/package.json b/grunt/package.json index e4d9836a3..0ea53569b 100644 --- a/grunt/package.json +++ b/grunt/package.json @@ -6,6 +6,7 @@ "grunt-contrib-concat": "~0.4.0", "grunt-contrib-cssmin": "~0.9.0", "grunt-angular-templates": "~0.5.4", - "grunt-contrib-uglify": "~0.4.0" + "grunt-contrib-uglify": "~0.4.0", + "grunt-cachebuster": "~0.1.5" } } diff --git a/templates/base.html b/templates/base.html index 9c4b0fed0..122a69614 100644 --- a/templates/base.html +++ b/templates/base.html @@ -28,11 +28,11 @@ - {% for style_path in main_styles %} + {% for style_path, cache_buster in main_styles %} {% endfor %} - {% for style_path in library_styles %} + {% for style_path, cache_buster in library_styles %} {% endfor %} @@ -53,7 +53,7 @@ {% endfor %} - {% for script_path in library_scripts %} + {% for script_path, cache_buster in library_scripts %} {% endfor %} @@ -61,7 +61,7 @@ {% endblock %} - {% for script_path in main_scripts %} + {% for script_path, cache_buster in main_scripts %} {% endfor %} From c081b1fa86579e21b562122571035ae29fcf5702 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 9 Feb 2015 11:10:26 -0500 Subject: [PATCH 37/37] Fix DB test and upgrade to peewee 2.4.7, which has the delete instance fix --- requirements.txt | 2 +- test/fulldbtest.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7d30ac438..73ce4da45 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,7 +38,7 @@ marisa-trie==0.7 mixpanel-py==3.2.1 mock==1.0.1 paramiko==1.15.2 -peewee==2.4.5 +peewee==2.4.7 psycopg2==2.5.4 py-bcrypt==0.4 pycrypto==2.6.1 diff --git a/test/fulldbtest.sh b/test/fulldbtest.sh index 49ad1f999..d3fa0caa7 100755 --- a/test/fulldbtest.sh +++ b/test/fulldbtest.sh @@ -2,14 +2,14 @@ set -e up_mysql() { # Run a SQL database on port 3306 inside of Docker. - docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mysql + docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mysql:5.7 # Sleep for 5s to get MySQL get started. echo 'Sleeping for 10...' sleep 10 # Add the database to mysql. - docker run --rm --link mysql:mysql mysql sh -c 'echo "create database genschema" | mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -ppassword' + docker run --rm --link mysql:mysql mysql:5.7 sh -c 'echo "create database genschema" | mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -ppassword' } down_mysql() {