diff --git a/Dockerfile b/Dockerfile index 90a17c9fd..4c5ad98b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ENV DEBIAN_FRONTEND noninteractive ENV HOME /root # Install the dependencies. -RUN apt-get update # 22OCT2015 +RUN apt-get update # 23OCT2015 # New ubuntu packages should be added as their own apt-get install lines below the existing install commands RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev libfreetype6-dev libffi-dev libgpgme11 libgpgme11-dev diff --git a/buildman/manager/executor.py b/buildman/manager/executor.py index e4f9fb7bb..54b689611 100644 --- a/buildman/manager/executor.py +++ b/buildman/manager/executor.py @@ -121,12 +121,14 @@ class EC2Executor(BuilderExecutor): block_devices = boto.ec2.blockdevicemapping.BlockDeviceMapping() block_devices['/dev/xvda'] = ssd_root_ebs - interface = boto.ec2.networkinterface.NetworkInterfaceSpecification( - subnet_id=self.executor_config['EC2_VPC_SUBNET_ID'], - groups=self.executor_config['EC2_SECURITY_GROUP_IDS'], - associate_public_ip_address=True, - ) - interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(interface) + interfaces = None + if self.executor_config.get('EC2_VPC_SUBNET_ID', None) is not None: + interface = boto.ec2.networkinterface.NetworkInterfaceSpecification( + subnet_id=self.executor_config['EC2_VPC_SUBNET_ID'], + groups=self.executor_config['EC2_SECURITY_GROUP_IDS'], + associate_public_ip_address=True, + ) + interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(interface) reservation = yield From(ec2_conn.run_instances( coreos_ami, diff --git a/data/buildlogs.py b/data/buildlogs.py index e9ea4a78f..ac58bd368 100644 --- a/data/buildlogs.py +++ b/data/buildlogs.py @@ -18,7 +18,11 @@ class RedisBuildLogs(object): PHASE = 'phase' def __init__(self, redis_config): - self._redis = redis.StrictRedis(socket_connect_timeout=5, **redis_config) + args = dict(redis_config) + args.update({'socket_connect_timeout': 5}) + + self._redis_config = redis_config + self._redis = redis.StrictRedis(**args) @staticmethod def _logs_key(build_id): @@ -94,12 +98,16 @@ class RedisBuildLogs(object): def check_health(self): try: - if not self._redis.ping() == True: + args = dict(self._redis_config) + args.update({'socket_connect_timeout': 1, 'socket_timeout': 1}) + + connection = redis.StrictRedis(**args) + if not connection.ping() == True: return False # Ensure we can write and read a key. - self._redis.set(self._health_key(), time.time()) - self._redis.get(self._health_key()) + connection.set(self._health_key(), time.time()) + connection.get(self._health_key()) return True except redis.ConnectionError: diff --git a/data/migrations/versions/2e0380215d01_backfill_image_fields_from_image_.py b/data/migrations/versions/2e0380215d01_backfill_image_fields_from_image_.py index 131c7fad0..93d89ed6e 100644 --- a/data/migrations/versions/2e0380215d01_backfill_image_fields_from_image_.py +++ b/data/migrations/versions/2e0380215d01_backfill_image_fields_from_image_.py @@ -17,8 +17,8 @@ from util.migrate.backfill_v1_metadata import backfill_v1_metadata def upgrade(tables): - backfill_image_fields() - backfill_v1_metadata() + backfill_image_fields() + backfill_v1_metadata() def downgrade(tables): - pass + pass diff --git a/data/model/image.py b/data/model/image.py index c8e3fb74e..b77bdee95 100644 --- a/data/model/image.py +++ b/data/model/image.py @@ -199,6 +199,7 @@ def _find_or_link_image(existing_image, repo_obj, username, translations, prefer command=existing_image.command, created=existing_image.created, comment=existing_image.comment, + v1_json_metadata=existing_image.v1_json_metadata, aggregate_size=existing_image.aggregate_size) diff --git a/endpoints/verbs.py b/endpoints/verbs.py index 5ba858c98..98e9063fe 100644 --- a/endpoints/verbs.py +++ b/endpoints/verbs.py @@ -22,12 +22,14 @@ verbs = Blueprint('verbs', __name__) logger = logging.getLogger(__name__) -def _open_stream(formatter, namespace, repository, tag, synthetic_image_id, image_json, - image_list): +def _open_stream(formatter, namespace, repository, tag, synthetic_image_id, image_json, repo_image): store = Storage(app) - def get_image_json(image): - return json.loads(image.v1_json_metadata) + # For performance reasons, we load the full image list here, cache it, then disconnect from + # the database. + with database.UseThenDisconnect(app.config): + image_list = list(model.image.get_parent_images(namespace, repository, repo_image)) + image_list.append(repo_image) def get_next_image(): for current_image in image_list: @@ -113,7 +115,7 @@ def _verify_repo_verb(store, namespace, repository, tag, verb, checker=None): abort(404) # Lookup the tag's image and storage. - repo_image = model.image.get_repo_image(namespace, repository, tag_image.docker_image_id) + repo_image = model.image.get_repo_image_extended(namespace, repository, tag_image.docker_image_id) if not repo_image: abort(404) @@ -199,7 +201,7 @@ def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker= # Create a queue process to generate the data. The queue files will read from the process # and send the results to the client and storage. - args = (formatter, namespace, repository, tag, synthetic_image_id, image_json, full_image_list) + args = (formatter, namespace, repository, tag, synthetic_image_id, image_json, repo_image) queue_process = QueueProcess(_open_stream, 8 * 1024, 10 * 1024 * 1024, # 8K/10M chunk/max args, finished=_cleanup) diff --git a/health/healthcheck.py b/health/healthcheck.py index c212c694d..ef700e3e4 100644 --- a/health/healthcheck.py +++ b/health/healthcheck.py @@ -10,16 +10,17 @@ def get_healthchecker(app, config_provider): class HealthCheck(object): - def __init__(self, app, config_provider): + def __init__(self, app, config_provider, instance_skips=None): self.app = app self.config_provider = config_provider + self.instance_skips = instance_skips or [] def check_instance(self): """ Conducts a check on this specific instance, returning a dict representing the HealthCheck output and a number indicating the health check response code. """ - service_statuses = check_all_services(self.app) + service_statuses = check_all_services(self.app, self.instance_skips) return self.get_instance_health(service_statuses) def check_endtoend(self): @@ -27,7 +28,7 @@ class HealthCheck(object): Conducts a check on all services, returning a dict representing the HealthCheck output and a number indicating the health check response code. """ - service_statuses = check_all_services(self.app) + service_statuses = check_all_services(self.app, []) return self.calculate_overall_health(service_statuses) def get_instance_health(self, service_statuses): @@ -80,7 +81,7 @@ class LocalHealthCheck(HealthCheck): class ProductionHealthCheck(HealthCheck): def __init__(self, app, config_provider, access_key, secret_key, db_instance='quay'): - super(ProductionHealthCheck, self).__init__(app, config_provider) + super(ProductionHealthCheck, self).__init__(app, config_provider, ['redis']) self.access_key = access_key self.secret_key = secret_key self.db_instance = db_instance @@ -92,7 +93,7 @@ class ProductionHealthCheck(HealthCheck): def get_instance_health(self, service_statuses): # Note: We skip the redis check because if redis is down, we don't want ELB taking the # machines out of service. Redis is not considered a high avaliability-required service. - skip = ['redis'] + skip = [] notes = [] # If the database is marked as unhealthy, check the status of RDS directly. If RDS is diff --git a/health/services.py b/health/services.py index ce6112651..bf108595a 100644 --- a/health/services.py +++ b/health/services.py @@ -39,10 +39,13 @@ _SERVICES = { 'redis': _check_redis } -def check_all_services(app): +def check_all_services(app, skip): """ Returns a dictionary containing the status of all the services defined. """ status = {} for name in _SERVICES: + if name in skip: + continue + status[name] = _SERVICES[name](app) - return status \ No newline at end of file + return status diff --git a/requirements-nover.txt b/requirements-nover.txt index 29628f1ba..fe4336bd8 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -36,9 +36,9 @@ git+https://github.com/DevTable/aniso8601-fake.git git+https://github.com/DevTable/anunidecode.git git+https://github.com/DevTable/pygithub.git git+https://github.com/DevTable/container-cloud-config.git -git+https://github.com/coreos/mockldap.git git+https://github.com/coreos/py-bitbucket.git git+https://github.com/coreos/pyapi-gitlab.git@timeout +git+https://github.com/coreos/mockldap.git git+https://github.com/coreos/resumablehashlib.git git+https://github.com/DevTable/python-etcd.git@sslfix gipc @@ -54,5 +54,9 @@ Flask-Testing pyjwt toposort pyjwkest +<<<<<<< HEAD rfc3987 -jsonpath-rw \ No newline at end of file +jsonpath-rw +======= +jsonpath-rw +>>>>>>> upstream/master diff --git a/requirements.txt b/requirements.txt index 83025b635..cdb79dd83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -100,6 +100,6 @@ git+https://github.com/coreos/mockldap.git git+https://github.com/coreos/py-bitbucket.git git+https://github.com/coreos/pyapi-gitlab.git@timeout git+https://github.com/coreos/resumablehashlib.git -git+https://github.com/coreos/mockldap.git +git+https://github.com/coreos/resumablehashlib.git git+https://github.com/DevTable/python-etcd.git@sslfix git+https://github.com/NateFerrero/oauth2lib.git diff --git a/test/data/test.db b/test/data/test.db index c35df7438..e5deedf5d 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/test/test_buildman.py b/test/test_buildman.py index b58eddb45..ffcbc7071 100644 --- a/test/test_buildman.py +++ b/test/test_buildman.py @@ -224,7 +224,7 @@ class TestEphemeral(unittest.TestCase): @async_test def test_change_worker(self): # Send a signal to the callback that a worker key has been changed - set_result = Mock(sepc=etcd.EtcdResult) + set_result = Mock(spec=etcd.EtcdResult) set_result.action = 'set' set_result.key = self.mock_job_key diff --git a/util/config/provider/k8sprovider.py b/util/config/provider/k8sprovider.py index 4a5a6ef9d..7fac1b179 100644 --- a/util/config/provider/k8sprovider.py +++ b/util/config/provider/k8sprovider.py @@ -14,8 +14,8 @@ KUBERNETES_API_HOST = 'kubernetes.default.svc.cluster.local' SERVICE_ACCOUNT_TOKEN_PATH = '/var/run/secrets/kubernetes.io/serviceaccount/token' -ER_NAMESPACE = 'quay' -ER_CONFIG_SECRET = 'quay-config-secret' +QE_NAMESPACE = os.environ.get('QE_K8S_NAMESPACE', 'quay-enterprise') +QE_CONFIG_SECRET = os.environ.get('QE_K8S_CONFIG_SECRET', 'quay-enterprise-config-secret') class KubernetesConfigProvider(FileConfigProvider): """ Implementation of the config provider that reads and writes configuration @@ -67,12 +67,12 @@ class KubernetesConfigProvider(FileConfigProvider): "kind": "Secret", "apiVersion": "v1", "metadata": { - "name": ER_CONFIG_SECRET + "name": QE_CONFIG_SECRET }, "data": secret_data } - secret_url = 'namespaces/%s/secrets/%s' % (ER_NAMESPACE, ER_CONFIG_SECRET) + secret_url = 'namespaces/%s/secrets/%s' % (QE_NAMESPACE, QE_CONFIG_SECRET) secret = self._lookup_secret() if not secret: self._assert_success(self._execute_k8s_api('POST', secret_url, data)) @@ -86,7 +86,7 @@ class KubernetesConfigProvider(FileConfigProvider): def _lookup_secret(self): - secret_url = 'namespaces/%s/secrets/%s' % (ER_NAMESPACE, ER_CONFIG_SECRET) + secret_url = 'namespaces/%s/secrets/%s' % (QE_NAMESPACE, QE_CONFIG_SECRET) response = self._execute_k8s_api('GET', secret_url) if response.status_code != 200: return None