diff --git a/util/dockerfileparse.py b/util/dockerfileparse.py index f4362674e..608588c2e 100644 --- a/util/dockerfileparse.py +++ b/util/dockerfileparse.py @@ -17,6 +17,10 @@ class ParsedDockerfile(object): if not image_and_tag: return None + return self.base_image_from_repo_identifier(image_and_tag) + + @staticmethod + def base_image_from_repo_identifier(image_and_tag): # Note: # Dockerfile images references can be of multiple forms: # server:port/some/path @@ -27,8 +31,8 @@ class ParsedDockerfile(object): parts = image_and_tag.strip().split(':') if len(parts) == 1: - # somepath - return parts[0] + # somepath + return parts[0] # Otherwise, determine if the last part is a port # or a tag. @@ -36,33 +40,38 @@ class ParsedDockerfile(object): # Last part is part of the hostname. return image_and_tag - return '/'.join(parts[0:-1]) + # Remaining cases: + # server/some/path:tag + # server:port/some/path:tag + return ':'.join(parts[0:-1]) def get_base_image_and_tag(self): from_commands = self.get_commands_of_kind('FROM') if not from_commands: return None - return from_commands[0]['parameters'] + return from_commands[-1]['parameters'] def strip_comments(contents): lines = [line for line in contents.split('\n') if not line.startswith(COMMENT_CHARACTER)] return '\n'.join(lines) + def join_continued_lines(contents): return LINE_CONTINUATION_REGEX.sub('', contents) + def parse_dockerfile(contents): contents = join_continued_lines(strip_comments(contents)) lines = [line for line in contents.split('\n') if len(line) > 0] commands = [] for line in lines: - m = COMMAND_REGEX.match(line) - if m: - command = m.group(1) - parameters = m.group(2) + match_command = COMMAND_REGEX.match(line) + if match_command: + command = match_command.group(1) + parameters = match_command.group(2) commands.append({ 'command': command, diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 5736ee915..52da76958 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -21,6 +21,7 @@ from data import model from workers.worker import Worker from app import app from util.safetar import safe_extractall +from util.dockerfileparse import parse_dockerfile, ParsedDockerfile root_logger = logging.getLogger('') @@ -98,7 +99,7 @@ class StreamingDockerClient(Client): class DockerfileBuildContext(object): image_id_to_cache_time = {} - public_repos = set() + private_repo_tags = set() def __init__(self, build_context_dir, dockerfile_subdir, repo, tag_names, push_token, build_uuid, pull_credentials=None): @@ -110,6 +111,7 @@ class DockerfileBuildContext(object): self._status = StatusWrapper(build_uuid) self._build_logger = partial(build_logs.append_log_message, build_uuid) self._pull_credentials = pull_credentials + self._public_repos = set() # Note: We have two different clients here because we (potentially) login # with both, but with different credentials that we do not want shared between @@ -119,7 +121,11 @@ class DockerfileBuildContext(object): dockerfile_path = os.path.join(self._build_dir, dockerfile_subdir, 'Dockerfile') - self._num_steps = DockerfileBuildContext.__count_steps(dockerfile_path) + + # Compute the number of steps + with open(dockerfile_path, 'r') as dockerfileobj: + self._parsed_dockerfile = parse_dockerfile(dockerfileobj.read()) + self._num_steps = len(self._parsed_dockerfile.commands) logger.debug('Will build and push to repo %s with tags named: %s' % (self._repo, self._tag_names)) @@ -131,20 +137,11 @@ class DockerfileBuildContext(object): return self def __exit__(self, exc_type, value, traceback): + self.__cleanup_containers() self.__cleanup() shutil.rmtree(self._build_dir) - @staticmethod - def __count_steps(dockerfile_path): - with open(dockerfile_path, 'r') as dockerfileobj: - steps = 0 - for line in dockerfileobj.readlines(): - stripped = line.strip() - if stripped and stripped[0] is not '#': - steps += 1 - return steps - @staticmethod def __total_completion(statuses, total_images): percentage_with_sizes = float(len(statuses.values()))/total_images @@ -160,6 +157,11 @@ class DockerfileBuildContext(object): self._build_cl.login(self._pull_credentials['username'], self._pull_credentials['password'], registry=self._pull_credentials['registry'], reauth=True) + # Pull the image, in case it was updated since the last build + base_image = self._parsed_dockerfile.get_base_image() + self._build_logger('Pulling base image: %s' % base_image) + self._build_cl.pull(base_image) + # Start the build itself. logger.debug('Starting build.') @@ -270,13 +272,33 @@ class DockerfileBuildContext(object): raise RuntimeError(message) def __is_repo_public(self, repo_name): - if repo_name in self.public_repos: + if repo_name in self._public_repos: return True - repo_url = 'https://index.docker.io/v1/repositories/%s/images' % repo_name - repo_info = requests.get(repo_url) + repo_portions = repo_name.split('/') + registry_hostname = 'index.docker.io' + local_repo_name = repo_name + if len(repo_portions) > 2: + registry_hostname = repo_portions[0] + local_repo_name = '/'.join(repo_portions[1:]) + + repo_url_template = '%s://%s/v1/repositories/%s/images' + protocols = ['https', 'http'] + secure_repo_url, repo_url = [repo_url_template % (protocol, registry_hostname, local_repo_name) + for protocol in protocols] + + try: + + try: + repo_info = requests.get(secure_repo_url) + except requests.exceptions.SSLError: + repo_info = requests.get(repo_url) + + except requests.exceptions.ConnectionError: + return False + if repo_info.status_code / 100 == 2: - self.public_repos.add(repo_name) + self._public_repos.add(repo_name) return True else: return False @@ -307,6 +329,11 @@ class DockerfileBuildContext(object): if expiration < now: logger.debug('Removing expired image: %s' % image_id) + + for tag in image['RepoTags']: + # We can forget about this particular tag if it was indeed one of our renamed tags + self.private_repo_tags.discard(tag) + verify_removed.add(image_id) try: self._build_cl.remove_image(image_id) @@ -320,8 +347,6 @@ class DockerfileBuildContext(object): raise RuntimeError('Image was not removed: %s' % image['Id']) def __cleanup(self): - self.__cleanup_containers() - # Iterate all of the images and rename the ones that aren't public. This should preserve # base images and also allow the cache to function. now = datetime.now() @@ -333,16 +358,18 @@ class DockerfileBuildContext(object): self.image_id_to_cache_time[image_id] = now for tag in image['RepoTags']: - # TODO this is slightly wrong, replace it with util/dockerfileparse.py when merged - tag_repo = tag.split(':')[0] + tag_repo = ParsedDockerfile.base_image_from_repo_identifier(tag) if tag_repo != '': - if self.__is_repo_public(tag_repo): + if tag_repo in self.private_repo_tags: + logger.debug('Repo is private and has already been renamed: %s' % tag_repo) + elif self.__is_repo_public(tag_repo): logger.debug('Repo was deemed public: %s', tag_repo) else: new_name = str(uuid4()) logger.debug('Private repo tag being renamed %s -> %s', tag, new_name) self._build_cl.tag(image_id, new_name) self._build_cl.remove_image(tag) + self.private_repo_tags.add(new_name) class DockerfileBuildWorker(Worker): def __init__(self, *vargs, **kwargs):