- Merge branch 'master' into sha-lom

- Extract out the tar handling from streamlayerformat into tarlayerformat
- Add a new tarfileappender class to make it easy to append data to gzipped tars
- Fix the gzipwrap to properly close
- Have the .git injection use the new appender

endpoints/trigger.py

@@ -3,12 +3,13 @@ import io
 import os.path
 import tarfile
 import base64
-from StringIO import StringIO
+import re
 
 from github import Github, UnknownObjectException, GithubException
 from tempfile import SpooledTemporaryFile
 
 from app import app, userfiles as user_files
+from util.tarfileappender import TarfileAppender
 
 
 client = app.config['HTTPCLIENT']
@@ -230,13 +231,35 @@ class GithubBuildTrigger(BuildTrigger):
 
     return repos_by_org
 
+  def matches_branch(self, branch_name, regex):
+    if not regex:
+      return False
+
+    m = regex.match(branch_name)
+    if not m:
+      return False
+
+    return len(m.group(0)) == len(branch_name)
+
   def list_build_subdirs(self, auth_token, config):
     gh_client = self._get_client(auth_token)
     source = config['build_source']
 
-    try:
+    try:      
       repo = gh_client.get_repo(source)
-      default_commit = repo.get_branch(repo.default_branch or 'master').commit
+
+      # Find the first matching branch.
+      branches = None
+      if 'branch_regex' in config:
+        try:
+          regex = re.compile(config['branch_regex'])
+          branches = [branch.name for branch in repo.get_branches()
+                                  if self.matches_branch(branch.name, regex)]
+        except:
+          pass
+
+      branches = branches or [repo.default_branch or 'master']
+      default_commit = repo.get_branch(branches[0]).commit
       commit_tree = repo.get_git_tree(default_commit.sha, recursive=True)
 
       return [os.path.dirname(elem.path) for elem in commit_tree.tree
@@ -298,50 +321,21 @@ class GithubBuildTrigger(BuildTrigger):
       # Seek to position 0 to make tarfile happy
       tarball.seek(0)
 
-      # Pull out the name of the subdir that GitHub generated.
+      # Pull out the name of the subdir that GitHub generated
       with tarfile.open(fileobj=tarball) as archive:
-        tarball_subdir = archive.getnames()[0]        
-        tarball_subdir_info = archive.getmember(tarball_subdir)  
+        tarball_subdir = archive.getnames()[0]
 
-      # Seek to position 0 to make tarfile happy
+      # Seek to position 0 to make tarfile happy.
       tarball.seek(0)
 
-      with SpooledTemporaryFile(CHUNK_SIZE) as updated_tarball:
-        def add_entry(arch, dir_path, base_info, contents=None):
-          info = tarfile.TarInfo(dir_path)
+      entries = {
+        tarball_subdir + '/.git/HEAD': commit_sha,
+        tarball_subdir + '/.git/objects/': None,
+        tarball_subdir + '/.git/refs/': None
+      }
 
-          info.uid = base_info.uid
-          info.gid = base_info.gid
-          info.uname = base_info.uname
-          info.gname = base_info.gname
-          info.mode = base_info.mode
-          info.mtime = base_info.mtime
-
-          info.type = tarfile.REGTYPE if contents else tarfile.DIRTYPE
-          if contents:
-            info.size = len(contents)
-
-          arch.addfile(info, fileobj=StringIO(contents) if contents else None)
-
-        with tarfile.open(fileobj=updated_tarball, mode='w|gz') as updated_archive:
-          # Copy existing members of the tar to the updated archive.
-          with tarfile.open(fileobj=tarball) as archive:
-            for tar_info in archive:
-              if tar_info.isreg():
-                updated_archive.addfile(tar_info, archive.extractfile(tar_info.name))
-              else:
-                updated_archive.addfile(tar_info)
-          
-          # Add the synthetic .git directory to the tarball, containing the commit_sha.
-          add_entry(updated_archive, tarball_subdir + '/.git/HEAD', tarball_subdir_info,
-                   contents=commit_sha)
-          add_entry(updated_archive, tarball_subdir + '/.git/objects/', tarball_subdir_info)
-          add_entry(updated_archive, tarball_subdir + '/.git/refs/', tarball_subdir_info)
-
-        # Seek to position 0 to make boto multipart happy
-        updated_tarball.seek(0)
-
-        dockerfile_id = user_files.store_file(updated_tarball, TARBALL_MIME)
+      appender = TarfileAppender(tarball, entries).get_stream()
+      dockerfile_id = user_files.store_file(appender, TARBALL_MIME)
 
     logger.debug('Successfully prepared job')
 
@@ -367,7 +361,7 @@ class GithubBuildTrigger(BuildTrigger):
     payload = request.get_json()
     if not payload or payload.get('head_commit') is None:
       raise SkipRequestException()
-    
+
     if 'zen' in payload:
       raise ValidationRequestException()
 
@@ -376,6 +370,16 @@ class GithubBuildTrigger(BuildTrigger):
     commit_sha = payload['head_commit']['id']
     commit_message = payload['head_commit'].get('message', '')
 
+    if 'branch_regex' in config:
+      try:
+        regex = re.compile(config['branch_regex'])
+      except:
+        regex = re.compile('.*')
+
+      branch = ref.split('/')[-1]
+      if not self.matches_branch(branch, regex):
+        raise SkipRequestException()
+
     if should_skip_commit(commit_message):
       raise SkipRequestException()