Test third party repo images for public-ness in the builder. Always clean up private images that we dont know about before build. Pull the base image to refresh before every build.
This commit is contained in:
jakedt
parent
4946dca804
commit
724fec1b74
2 changed files with 65 additions and 29 deletions
|
|
@ -17,6 +17,10 @@ class ParsedDockerfile(object):
|
|||
if not image_and_tag:
|
||||
return None
|
||||
|
||||
return self.base_image_from_repo_identifier(image_and_tag)
|
||||
|
||||
@staticmethod
|
||||
def base_image_from_repo_identifier(image_and_tag):
|
||||
# Note:
|
||||
# Dockerfile images references can be of multiple forms:
|
||||
# server:port/some/path
|
||||
|
|
@ -27,8 +31,8 @@ class ParsedDockerfile(object):
|
|||
parts = image_and_tag.strip().split(':')
|
||||
|
||||
if len(parts) == 1:
|
||||
# somepath
|
||||
return parts[0]
|
||||
# somepath
|
||||
return parts[0]
|
||||
|
||||
# Otherwise, determine if the last part is a port
|
||||
# or a tag.
|
||||
|
|
@ -36,33 +40,38 @@ class ParsedDockerfile(object):
|
|||
# Last part is part of the hostname.
|
||||
return image_and_tag
|
||||
|
||||
return '/'.join(parts[0:-1])
|
||||
# Remaining cases:
|
||||
# server/some/path:tag
|
||||
# server:port/some/path:tag
|
||||
return ':'.join(parts[0:-1])
|
||||
|
||||
def get_base_image_and_tag(self):
|
||||
from_commands = self.get_commands_of_kind('FROM')
|
||||
if not from_commands:
|
||||
return None
|
||||
|
||||
return from_commands[0]['parameters']
|
||||
return from_commands[-1]['parameters']
|
||||
|
||||
|
||||
def strip_comments(contents):
|
||||
lines = [line for line in contents.split('\n') if not line.startswith(COMMENT_CHARACTER)]
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
def join_continued_lines(contents):
|
||||
return LINE_CONTINUATION_REGEX.sub('', contents)
|
||||
|
||||
|
||||
def parse_dockerfile(contents):
|
||||
contents = join_continued_lines(strip_comments(contents))
|
||||
lines = [line for line in contents.split('\n') if len(line) > 0]
|
||||
|
||||
commands = []
|
||||
for line in lines:
|
||||
m = COMMAND_REGEX.match(line)
|
||||
if m:
|
||||
command = m.group(1)
|
||||
parameters = m.group(2)
|
||||
match_command = COMMAND_REGEX.match(line)
|
||||
if match_command:
|
||||
command = match_command.group(1)
|
||||
parameters = match_command.group(2)
|
||||
|
||||
commands.append({
|
||||
'command': command,
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from data import model
|
|||
from workers.worker import Worker
|
||||
from app import app
|
||||
from util.safetar import safe_extractall
|
||||
from util.dockerfileparse import parse_dockerfile, ParsedDockerfile
|
||||
|
||||
|
||||
root_logger = logging.getLogger('')
|
||||
|
|
@ -98,7 +99,7 @@ class StreamingDockerClient(Client):
|
|||
|
||||
class DockerfileBuildContext(object):
|
||||
image_id_to_cache_time = {}
|
||||
public_repos = set()
|
||||
private_repo_tags = set()
|
||||
|
||||
def __init__(self, build_context_dir, dockerfile_subdir, repo, tag_names,
|
||||
push_token, build_uuid, pull_credentials=None):
|
||||
|
|
@ -110,6 +111,7 @@ class DockerfileBuildContext(object):
|
|||
self._status = StatusWrapper(build_uuid)
|
||||
self._build_logger = partial(build_logs.append_log_message, build_uuid)
|
||||
self._pull_credentials = pull_credentials
|
||||
self._public_repos = set()
|
||||
|
||||
# Note: We have two different clients here because we (potentially) login
|
||||
# with both, but with different credentials that we do not want shared between
|
||||
|
|
@ -119,7 +121,11 @@ class DockerfileBuildContext(object):
|
|||
|
||||
dockerfile_path = os.path.join(self._build_dir, dockerfile_subdir,
|
||||
'Dockerfile')
|
||||
self._num_steps = DockerfileBuildContext.__count_steps(dockerfile_path)
|
||||
|
||||
# Compute the number of steps
|
||||
with open(dockerfile_path, 'r') as dockerfileobj:
|
||||
self._parsed_dockerfile = parse_dockerfile(dockerfileobj.read())
|
||||
self._num_steps = len(self._parsed_dockerfile.commands)
|
||||
|
||||
logger.debug('Will build and push to repo %s with tags named: %s' %
|
||||
(self._repo, self._tag_names))
|
||||
|
|
@ -131,20 +137,11 @@ class DockerfileBuildContext(object):
|
|||
return self
|
||||
|
||||
def __exit__(self, exc_type, value, traceback):
|
||||
self.__cleanup_containers()
|
||||
self.__cleanup()
|
||||
|
||||
shutil.rmtree(self._build_dir)
|
||||
|
||||
@staticmethod
|
||||
def __count_steps(dockerfile_path):
|
||||
with open(dockerfile_path, 'r') as dockerfileobj:
|
||||
steps = 0
|
||||
for line in dockerfileobj.readlines():
|
||||
stripped = line.strip()
|
||||
if stripped and stripped[0] is not '#':
|
||||
steps += 1
|
||||
return steps
|
||||
|
||||
@staticmethod
|
||||
def __total_completion(statuses, total_images):
|
||||
percentage_with_sizes = float(len(statuses.values()))/total_images
|
||||
|
|
@ -160,6 +157,11 @@ class DockerfileBuildContext(object):
|
|||
self._build_cl.login(self._pull_credentials['username'], self._pull_credentials['password'],
|
||||
registry=self._pull_credentials['registry'], reauth=True)
|
||||
|
||||
# Pull the image, in case it was updated since the last build
|
||||
base_image = self._parsed_dockerfile.get_base_image()
|
||||
self._build_logger('Pulling base image: %s' % base_image)
|
||||
self._build_cl.pull(base_image)
|
||||
|
||||
# Start the build itself.
|
||||
logger.debug('Starting build.')
|
||||
|
||||
|
|
@ -270,13 +272,33 @@ class DockerfileBuildContext(object):
|
|||
raise RuntimeError(message)
|
||||
|
||||
def __is_repo_public(self, repo_name):
|
||||
if repo_name in self.public_repos:
|
||||
if repo_name in self._public_repos:
|
||||
return True
|
||||
|
||||
repo_url = 'https://index.docker.io/v1/repositories/%s/images' % repo_name
|
||||
repo_info = requests.get(repo_url)
|
||||
repo_portions = repo_name.split('/')
|
||||
registry_hostname = 'index.docker.io'
|
||||
local_repo_name = repo_name
|
||||
if len(repo_portions) > 2:
|
||||
registry_hostname = repo_portions[0]
|
||||
local_repo_name = '/'.join(repo_portions[1:])
|
||||
|
||||
repo_url_template = '%s://%s/v1/repositories/%s/images'
|
||||
protocols = ['https', 'http']
|
||||
secure_repo_url, repo_url = [repo_url_template % (protocol, registry_hostname, local_repo_name)
|
||||
for protocol in protocols]
|
||||
|
||||
try:
|
||||
|
||||
try:
|
||||
repo_info = requests.get(secure_repo_url)
|
||||
except requests.exceptions.SSLError:
|
||||
repo_info = requests.get(repo_url)
|
||||
|
||||
except requests.exceptions.ConnectionError:
|
||||
return False
|
||||
|
||||
if repo_info.status_code / 100 == 2:
|
||||
self.public_repos.add(repo_name)
|
||||
self._public_repos.add(repo_name)
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
|
@ -307,6 +329,11 @@ class DockerfileBuildContext(object):
|
|||
|
||||
if expiration < now:
|
||||
logger.debug('Removing expired image: %s' % image_id)
|
||||
|
||||
for tag in image['RepoTags']:
|
||||
# We can forget about this particular tag if it was indeed one of our renamed tags
|
||||
self.private_repo_tags.discard(tag)
|
||||
|
||||
verify_removed.add(image_id)
|
||||
try:
|
||||
self._build_cl.remove_image(image_id)
|
||||
|
|
@ -320,8 +347,6 @@ class DockerfileBuildContext(object):
|
|||
raise RuntimeError('Image was not removed: %s' % image['Id'])
|
||||
|
||||
def __cleanup(self):
|
||||
self.__cleanup_containers()
|
||||
|
||||
# Iterate all of the images and rename the ones that aren't public. This should preserve
|
||||
# base images and also allow the cache to function.
|
||||
now = datetime.now()
|
||||
|
|
@ -333,16 +358,18 @@ class DockerfileBuildContext(object):
|
|||
self.image_id_to_cache_time[image_id] = now
|
||||
|
||||
for tag in image['RepoTags']:
|
||||
# TODO this is slightly wrong, replace it with util/dockerfileparse.py when merged
|
||||
tag_repo = tag.split(':')[0]
|
||||
tag_repo = ParsedDockerfile.base_image_from_repo_identifier(tag)
|
||||
if tag_repo != '<none>':
|
||||
if self.__is_repo_public(tag_repo):
|
||||
if tag_repo in self.private_repo_tags:
|
||||
logger.debug('Repo is private and has already been renamed: %s' % tag_repo)
|
||||
elif self.__is_repo_public(tag_repo):
|
||||
logger.debug('Repo was deemed public: %s', tag_repo)
|
||||
else:
|
||||
new_name = str(uuid4())
|
||||
logger.debug('Private repo tag being renamed %s -> %s', tag, new_name)
|
||||
self._build_cl.tag(image_id, new_name)
|
||||
self._build_cl.remove_image(tag)
|
||||
self.private_repo_tags.add(new_name)
|
||||
|
||||
class DockerfileBuildWorker(Worker):
|
||||
def __init__(self, *vargs, **kwargs):
|
||||
|
|
|
|||
Reference in a new issue