Test third party repo images for public-ness in the builder. Always clean up private images that we dont know about before build. Pull the base image to refresh before every build.

util/dockerfileparse.py

@@ -17,6 +17,10 @@ class ParsedDockerfile(object):
     if not image_and_tag:
       return None
 
+    return self.base_image_from_repo_identifier(image_and_tag)
+
+  @staticmethod
+  def base_image_from_repo_identifier(image_and_tag):
     # Note:
     # Dockerfile images references can be of multiple forms:
     #   server:port/some/path
@@ -36,33 +40,38 @@ class ParsedDockerfile(object):
       # Last part is part of the hostname.
       return image_and_tag
 
-    return '/'.join(parts[0:-1])
+    # Remaining cases:
+    #   server/some/path:tag
+    #   server:port/some/path:tag    
+    return ':'.join(parts[0:-1])
 
   def get_base_image_and_tag(self):
     from_commands = self.get_commands_of_kind('FROM')
     if not from_commands:
       return None
 
-    return from_commands[0]['parameters']
+    return from_commands[-1]['parameters']
 
 
 def strip_comments(contents):
   lines = [line for line in contents.split('\n') if not line.startswith(COMMENT_CHARACTER)]
   return '\n'.join(lines)
 
+
 def join_continued_lines(contents):
   return LINE_CONTINUATION_REGEX.sub('', contents)
 
+
 def parse_dockerfile(contents):
   contents = join_continued_lines(strip_comments(contents))
   lines = [line for line in contents.split('\n') if len(line) > 0]
 
   commands = []
   for line in lines:
-    m = COMMAND_REGEX.match(line)
+    match_command = COMMAND_REGEX.match(line)
-    if m:
+    if match_command:
-      command = m.group(1)
+      command = match_command.group(1)
-      parameters = m.group(2)
+      parameters = match_command.group(2)
 
       commands.append({
         'command': command,

workers/dockerfilebuild.py

@@ -21,6 +21,7 @@ from data import model
 from workers.worker import Worker
 from app import app
 from util.safetar import safe_extractall
+from util.dockerfileparse import parse_dockerfile, ParsedDockerfile
 
 
 root_logger = logging.getLogger('')
@@ -98,7 +99,7 @@ class StreamingDockerClient(Client):
 
 class DockerfileBuildContext(object):
   image_id_to_cache_time = {}
-  public_repos = set()
+  private_repo_tags = set()
 
   def __init__(self, build_context_dir, dockerfile_subdir, repo, tag_names,
                push_token, build_uuid, pull_credentials=None):
@@ -110,6 +111,7 @@ class DockerfileBuildContext(object):
     self._status = StatusWrapper(build_uuid)
     self._build_logger = partial(build_logs.append_log_message, build_uuid)
     self._pull_credentials = pull_credentials
+    self._public_repos = set()
 
     # Note: We have two different clients here because we (potentially) login
     # with both, but with different credentials that we do not want shared between
@@ -119,7 +121,11 @@ class DockerfileBuildContext(object):
 
     dockerfile_path = os.path.join(self._build_dir, dockerfile_subdir,
                                    'Dockerfile')
-    self._num_steps = DockerfileBuildContext.__count_steps(dockerfile_path)
+    
+    # Compute the number of steps
+    with open(dockerfile_path, 'r') as dockerfileobj:
+      self._parsed_dockerfile = parse_dockerfile(dockerfileobj.read())
+    self._num_steps = len(self._parsed_dockerfile.commands)
 
     logger.debug('Will build and push to repo %s with tags named: %s' %
                  (self._repo, self._tag_names))
@@ -131,20 +137,11 @@ class DockerfileBuildContext(object):
     return self
 
   def __exit__(self, exc_type, value, traceback):
+    self.__cleanup_containers()
     self.__cleanup()
 
     shutil.rmtree(self._build_dir)
 
-  @staticmethod
-  def __count_steps(dockerfile_path):
-    with open(dockerfile_path, 'r') as dockerfileobj:
-      steps = 0
-      for line in dockerfileobj.readlines():
-        stripped = line.strip()
-        if stripped and stripped[0] is not '#':
-          steps += 1
-      return steps
-
   @staticmethod
   def __total_completion(statuses, total_images):
     percentage_with_sizes = float(len(statuses.values()))/total_images
@@ -160,6 +157,11 @@ class DockerfileBuildContext(object):
       self._build_cl.login(self._pull_credentials['username'], self._pull_credentials['password'],
                            registry=self._pull_credentials['registry'], reauth=True)
 
+    # Pull the image, in case it was updated since the last build
+    base_image = self._parsed_dockerfile.get_base_image()
+    self._build_logger('Pulling base image: %s' % base_image)
+    self._build_cl.pull(base_image)
+
     # Start the build itself.
     logger.debug('Starting build.')
 
@@ -270,13 +272,33 @@ class DockerfileBuildContext(object):
         raise RuntimeError(message)
 
   def __is_repo_public(self, repo_name):
-    if repo_name in self.public_repos:
+    if repo_name in self._public_repos:
       return True
 
-    repo_url = 'https://index.docker.io/v1/repositories/%s/images' % repo_name
+    repo_portions = repo_name.split('/')
+    registry_hostname = 'index.docker.io'
+    local_repo_name = repo_name
+    if len(repo_portions) > 2:
+      registry_hostname = repo_portions[0]
+      local_repo_name = '/'.join(repo_portions[1:])
+
+    repo_url_template = '%s://%s/v1/repositories/%s/images'
+    protocols = ['https', 'http']
+    secure_repo_url, repo_url = [repo_url_template % (protocol, registry_hostname, local_repo_name)
+                                 for protocol in protocols]
+
+    try:
+
+      try:
+        repo_info = requests.get(secure_repo_url)
+      except requests.exceptions.SSLError:
         repo_info = requests.get(repo_url)
+
+    except requests.exceptions.ConnectionError:
+      return False
+
     if repo_info.status_code / 100 == 2:
-      self.public_repos.add(repo_name)
+      self._public_repos.add(repo_name)
       return True
     else:
       return False
@@ -307,6 +329,11 @@ class DockerfileBuildContext(object):
 
       if expiration < now:
         logger.debug('Removing expired image: %s' % image_id)
+
+        for tag in image['RepoTags']:
+          # We can forget about this particular tag if it was indeed one of our renamed tags
+          self.private_repo_tags.discard(tag)
+
         verify_removed.add(image_id)
         try:
           self._build_cl.remove_image(image_id)
@@ -320,8 +347,6 @@ class DockerfileBuildContext(object):
         raise RuntimeError('Image was not removed: %s' % image['Id'])
 
   def __cleanup(self):
-    self.__cleanup_containers()
-
     # Iterate all of the images and rename the ones that aren't public. This should preserve
     # base images and also allow the cache to function.
     now = datetime.now()
@@ -333,16 +358,18 @@ class DockerfileBuildContext(object):
         self.image_id_to_cache_time[image_id] = now
 
       for tag in image['RepoTags']:
-        # TODO this is slightly wrong, replace it with util/dockerfileparse.py when merged
+        tag_repo = ParsedDockerfile.base_image_from_repo_identifier(tag)
-        tag_repo = tag.split(':')[0]
         if tag_repo != '<none>':
-          if self.__is_repo_public(tag_repo):
+          if tag_repo in self.private_repo_tags:
+            logger.debug('Repo is private and has already been renamed: %s' % tag_repo)
+          elif self.__is_repo_public(tag_repo):
             logger.debug('Repo was deemed public: %s', tag_repo)
           else:
             new_name = str(uuid4())
             logger.debug('Private repo tag being renamed %s -> %s', tag, new_name)
             self._build_cl.tag(image_id, new_name)
             self._build_cl.remove_image(tag)
+            self.private_repo_tags.add(new_name)
 
 class DockerfileBuildWorker(Worker):
   def __init__(self, *vargs, **kwargs):