From b3fe3a32e45351cc558780ac71baeb8bdb0c2a10 Mon Sep 17 00:00:00 2001 From: jakedt Date: Fri, 21 Feb 2014 16:07:08 -0500 Subject: [PATCH 1/4] Fix logins for access tokens. --- endpoints/index.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/endpoints/index.py b/endpoints/index.py index 33c74d972..864805b9a 100644 --- a/endpoints/index.py +++ b/endpoints/index.py @@ -120,6 +120,11 @@ def get_user(): 'username': get_authenticated_user().username, 'email': get_authenticated_user().email, }) + elif get_validated_token(): + return jsonify({ + 'username': '$token', + 'email': None, + }) abort(404) From 13dea98499d9710cdd79f59bd6b328c1c89fd828 Mon Sep 17 00:00:00 2001 From: jakedt Date: Mon, 24 Feb 2014 16:11:23 -0500 Subject: [PATCH 2/4] Prepare the build worker to support multiple tags and subdirectories. Change the build database config to accept a job config object instead of breaking out the parameters into independent blocks. --- data/database.py | 3 +- data/model.py | 4 +-- data/queue.py | 2 +- endpoints/api.py | 22 ++++++++----- endpoints/trigger.py | 26 ++++++++++++---- endpoints/webhooks.py | 36 ++++++++++++++++------ initdb.py | 13 +++++--- test/data/test.db | Bin 167936 -> 167936 bytes workers/dockerfilebuild.py | 61 ++++++++++++++++++++++++------------- 9 files changed, 114 insertions(+), 53 deletions(-) diff --git a/data/database.py b/data/database.py index beaa033dc..dd6542106 100644 --- a/data/database.py +++ b/data/database.py @@ -237,8 +237,7 @@ class RepositoryBuild(BaseModel): uuid = CharField(default=uuid_generator, index=True) repository = ForeignKeyField(Repository, index=True) access_token = ForeignKeyField(AccessToken) - resource_key = CharField() - tag = CharField() + job_config = TextField() phase = CharField(default='waiting') started = DateTimeField(default=datetime.now) display_name = CharField() diff --git a/data/model.py b/data/model.py index 767e7ca78..b8c68ce24 100644 --- a/data/model.py +++ b/data/model.py @@ -1409,10 +1409,10 @@ def list_repository_builds(namespace_name, repository_name, return query -def create_repository_build(repo, access_token, resource_key, tag, +def create_repository_build(repo, access_token, job_config_obj, display_name, trigger=None): return RepositoryBuild.create(repository=repo, access_token=access_token, - resource_key=resource_key, tag=tag, + job_config=json.dumps(job_config_obj), display_name=display_name, trigger=trigger) diff --git a/data/queue.py b/data/queue.py index 46db150bf..09e90f1a1 100644 --- a/data/queue.py +++ b/data/queue.py @@ -68,5 +68,5 @@ class WorkQueue(object): image_diff_queue = WorkQueue('imagediff') -dockerfile_build_queue = WorkQueue('dockerfilebuild2') +dockerfile_build_queue = WorkQueue('dockerfilebuild3') webhook_queue = WorkQueue('webhook') diff --git a/endpoints/api.py b/endpoints/api.py index 877a14384..169bb5c7a 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -1144,7 +1144,7 @@ def build_status_view(build_obj, can_write=False): 'started': build_obj.started, 'display_name': build_obj.display_name, 'status': status, - 'resource_key': build_obj.resource_key if can_write else None, + 'job_config': json.loads(build_obj.job_config) if can_write else None, 'is_writer': can_write, 'trigger': trigger_view(build_obj.trigger), } @@ -1234,11 +1234,13 @@ def request_repo_build(namespace, repository): logger.debug('User requested repository initialization.') dockerfile_id = request.get_json()['file_id'] - # Check if the dockerfile resource has already been used. If so, then it can only be reused if the - # user has access to the repository for which it was used. + # Check if the dockerfile resource has already been used. If so, then it + # can only be reused if the user has access to the repository for which it + # was used. associated_repository = model.get_repository_for_resource(dockerfile_id) if associated_repository: - if not ModifyRepositoryPermission(associated_repository.namespace, associated_repository.name): + if not ModifyRepositoryPermission(associated_repository.namespace, + associated_repository.name): abort(403) # Start the build. @@ -1248,9 +1250,15 @@ def request_repo_build(namespace, repository): logger.debug('**********Md5: %s' % display_name) host = urlparse.urlparse(request.url).netloc - tag = '%s/%s/%s' % (host, repo.namespace, repo.name) - build_request = model.create_repository_build(repo, token, dockerfile_id, - tag, display_name) + repo = '%s/%s/%s' % (host, repo.namespace, repo.name) + job_config = { + 'docker_tags': ['latest'], + 'build_subdir': '', + 'repository': repo, + 'resource_key': dockerfile_id, + } + build_request = model.create_repository_build(repo, token, job_config, + display_name) dockerfile_build_queue.put(json.dumps({ 'build_uuid': build_request.uuid, 'namespace': namespace, diff --git a/endpoints/trigger.py b/endpoints/trigger.py index 990fb5278..ab902fd2e 100644 --- a/endpoints/trigger.py +++ b/endpoints/trigger.py @@ -55,7 +55,8 @@ class BuildTrigger(object): def handle_trigger_request(self, request, auth_token, config): """ - Transform the incoming request data into a set of actions. + Transform the incoming request data into a set of actions. Returns a tuple + of usefiles resource id, docker tags, build name, and resource subdir. """ raise NotImplementedError @@ -163,7 +164,7 @@ class GithubBuildTrigger(BuildTrigger): try: repo = gh_client.get_repo(source) - default_commit = repo.get_branch(repo.default_branch).commit + default_commit = repo.get_branch(repo.master_branch).commit commit_tree = repo.get_git_tree(default_commit.sha, recursive=True) return [os.path.dirname(elem.path) for elem in commit_tree.tree @@ -181,7 +182,8 @@ class GithubBuildTrigger(BuildTrigger): logger.debug('Payload %s', payload) ref = payload['ref'] - commit_id = payload['head_commit']['id'][0:7] + commit_sha = payload['head_commit']['id'] + short_sha = commit_sha[0:7] gh_client = self._get_client(auth_token) @@ -192,8 +194,7 @@ class GithubBuildTrigger(BuildTrigger): logger.debug('Github repo: %s', repo) # Prepare the download and upload URLs - branch_name = ref.split('/')[-1] - archive_link = repo.get_archive_link('zipball', branch_name) + archive_link = repo.get_archive_link('zipball', short_sha) download_archive = client.get(archive_link, stream=True) with SpooledTemporaryFile(CHUNK_SIZE) as zipball: @@ -204,4 +205,17 @@ class GithubBuildTrigger(BuildTrigger): logger.debug('Successfully prepared job') - return dockerfile_id, branch_name, commit_id + # compute the tag(s) + pushed_branch = ref.split('/')[-1] + tags = {pushed_branch} + if pushed_branch == repo.master_branch: + tags.add('latest') + logger.debug('Pushing to tags: %s' % tags) + + # compute the subdir + repo_subdir = config['subdir'] + zipball_subdir = '%s-%s-%s' % (repo.owner.login, repo.name, short_sha) + joined_subdir = os.path.join(zipball_subdir, repo_subdir) + logger.debug('Final subdir: %s' % joined_subdir) + + return dockerfile_id, list(tags), short_sha, joined_subdir diff --git a/endpoints/webhooks.py b/endpoints/webhooks.py index 0e424b052..d2beccc2d 100644 --- a/endpoints/webhooks.py +++ b/endpoints/webhooks.py @@ -1,10 +1,12 @@ import logging import stripe import urlparse +import json from flask import request, make_response, Blueprint from data import model +from data.queue import dockerfile_build_queue from auth.auth import process_auth from auth.permissions import ModifyRepositoryPermission from util.invoice import renderInvoiceToHtml @@ -62,22 +64,36 @@ def build_trigger_webhook(namespace, repository, trigger_uuid): logger.debug('Passing webhook request to handler %s', handler) try: - df_id, tag, name = handler.handle_trigger_request(request, - trigger.auth_token, - trigger.config) + specs = handler.handle_trigger_request(request, trigger.auth_token, + json.loads(trigger.config)) + dockerfile_id, tags, name, subdir = specs + except ValidationRequestException: - # This was just a validation request, don't need to build anything + # This was just a validation request, we don't need to build anything return make_response('Okay') host = urlparse.urlparse(request.url).netloc - full_tag = '%s/%s/%s:%s' % (host, trigger.repository.namespace, - trigger.repository.name, tag) + repo = '%s/%s/%s' % (host, trigger.repository.namespace, + trigger.repository.name) token = model.create_access_token(trigger.repository, 'write') - logger.debug('Creating build %s with full_tag %s and dockerfile_id %s', - name, full_tag, df_id) - model.create_repository_build(trigger.repository, token, df_id, full_tag, - name) + logger.debug('Creating build %s with repo %s tags %s and dockerfile_id %s', + name, repo, tags, dockerfile_id) + + job_config = { + 'docker_tags': tags, + 'repository': repo, + 'build_subdir': subdir, + 'resource_key': dockerfile_id, + } + build_request = model.create_repository_build(trigger.repository, token, + job_config, name) + + dockerfile_build_queue.put(json.dumps({ + 'build_uuid': build_request.uuid, + 'namespace': namespace, + 'repository': repository, + }), retries_remaining=1) return make_response('Okay') diff --git a/initdb.py b/initdb.py index 30e56efc5..35c3dd475 100644 --- a/initdb.py +++ b/initdb.py @@ -309,7 +309,6 @@ def populate_database(): False, [], (0, [], None)) token = model.create_access_token(building, 'write') - tag = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name) trigger = model.create_build_trigger(building, 'github', '123authtoken', new_user_1) @@ -318,9 +317,15 @@ def populate_database(): }) trigger.save() - build = model.create_repository_build(building, token, - '701dcc3724fb4f2ea6c31400528343cd', - tag, 'build-name', trigger) + repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name) + job_config = { + 'repository': repo, + 'docker_tags': ['latest'], + 'build_subdir': '', + 'resource_key': '701dcc3724fb4f2ea6c31400528343cd', + } + build = model.create_repository_build(building, token, job_config, + 'build-name', trigger) build.uuid = 'deadbeef-dead-beef-dead-beefdeadbeef' build.save() diff --git a/test/data/test.db b/test/data/test.db index e733ca294d1d5f8f2ae18cac7e3b72521cc6085f..8caf9ba9fa9a59ae26ce104dddf70b6291bb6ecd 100644 GIT binary patch delta 3939 zcmbtWeNO?!8HF?nOls2nmD$`5-1y47vFN`3N5&1hDFD zJ?q!$dREr+woY2@PP;jswq=I8I<4!po~`TWsd%WhTf27cv1sl5sBy!E+l&sEt=ddi)zzC>t&vnU+6m)us@bmcX}55GuG7ZoRrT$Cz5a^c{$7u~ zHB4LUG*-@Q8PK@`ZS{7l+Frs+LzOD!fihMAtp+uoSEMyR`Kqk}MyF-tfYlwYjC$ z>h%w{+3Bb=q~<)P8f~9xu)<3>8hX@POGHzms1FeeXNiXEi$?2xeGRRn6IOW|j-3}i zH9E&Pj@`N8%Yr-ac%0FH1|Jq~zti0_0FeLyJ_W;ml)TBe>HBkoMyVt)VF~!=0T2KQ z#jvD@^3yV}g7M3gDkUlNc-0;mOVXr5OsP3i$*`kezxd3?jb_$itXA4tli8><>712p zo!)As?F>=ls9{Yys$Ngo>IlkKWm0I>>V`_IPOV}TT0(7MX_rN7kx&Gwlo6;IWgbbF zl0+$`kP|FHDVVXTY;Zzcey0qvJg_ghfTB5yA_z2%lJ?1z9v?09(4K%y#S#j&n)Z1} z1v&QDV!)oi7_H4$SPCBj$%00ZbazFLW+{Bv2nv@=hVr?-)^?w}w~zC+_T-o5 z<>#-?%kS~`hWdKEes`OHARdDc>p|YU1kx4iZ4HNd67fTNP*fI=`$FC}e~&xt3HG7~ z8}r*eVSjHpzv-@iPiwffGdPo=BtfcG>alBj;L1qWI!}#~$gof4V2yc-KXTWcRF{x0KbW0d=ngy{PV5(e{ z!#(g1lL$&b0~A!B0NG+yh)LnIX2FinfCAok0yGQH-W7xwPJpmjb^Vt%K3530O(N*= zKZ2k*v+SucUW&mb7g412We^Z891&%^06caP)jRt#C|Fcl_S)+y0)ap*@Ck&?*l|n` z4uNKX3w$u`bFd6Hp996v`zNqll(x6&QCA8))`^jjJ_(A1XRi67`y?=?t$j1Umd{L1 z5;O`FOL?gfh6b^f@WYd!dDgtay^+~)@+zVmIt9w{bgA%Dc>EL?!E-)#{sK0?0(Rq) zxxcy=I}N7hq|T#_MqZlnZJ}bg^b(N4HSYkQNU{I$E3Wx)AA?bF{2fq$&t0lI52xM% zMw~q*ehJo{0gs9@@G^TO4NkHsDQ6t9N$3?6bdMvpWNA`b>@1kVQBaYf23JS#e=PPl5SW7(tl4XWTRuZcM3gi}^5Nu(xSx}# z#kK2}F>q)SwOHWHISdzHM1siOcrrYC5jaGm@&}ijav`vwiE1w)`^ER^f5vA?q1l2d z;K(K5#uwaZ&4_)DCKD0&T)Aml1wAL@wXdS&ymLo5K5Hd>?qqyduA*ecsh<|Z3s=#y z>BhfR1H(rc= z38pecC0`ugLv-Ank_pHX;qWPysVWcX41WL%turhq* z#JgwVkPLN})N$1V7t68T_@cmZ2|Oyt9HM3K-E1{!6DMC=f@O=-Q!Z!oS(?PjkCb3; zd?6WmAVy$Q64A=T&kdRCAUG5EoJGk5JF%Y6GC}j1xaTZN&N+1aeRzSz!eacZq$_-` z5pL^1hCWt`wTSb6BW&lTHE>BMim+?2ps0vGz__XtS3bT5X+8hQCvCB{aW2ui4Ntfl z6E@H~1WB^4ZG&zdvZ1JT{nzlI4k?plDH@Ad5;hE0AsZHtl3{qf3JojTAqm1}JsNiX zDPo?6LmjBKwMRqS;Nng+|LV&--+)Iu zk=exj(_fkb@tY-~F@!A;D>Jxte3n1HJ`~&?!kY0_*wf>&uDAirwuA4QdLYP(*B*!` zADKAIXLUn!7P9Z&0hG)P{niIB3}9hAf8xp*^bTSpsf%yCjlKe%-yPN^unMw6$n3Nu zr>0yT@hc|b_z-e5mHV_PwiOfMY5)5g9Gw0&*n#a9&FdKNbPdF9LcQ=<;oJ<q=s(OOJhni%vH$PzAdhsUX767TiNswa;o#j!n&rI*m%`(BBWZ-`;z`(i V50Xaxfp$x5C-$FYoVon5^1p}LwJQJs delta 3783 zcmbtVeRPx69qzq(oAe9XQfO%*g(5BeN}D&yn>TML(6sqz(x&8hoHCE=(W^+c;*pIVjyfyp939BSkK;siQ_2vTdX7#9ZS^QH=d`;qW1{Wx5BV!6 z_xauX{GR9e{gR!dNjpcAo<3`?thJgNnyamBv&KPH*J*W)QnIe9x<*~YYOM8?$}BdR zbfjG?Hrg8udbQ4Bld+8rcD+trZK7F8WAkvTrGe7;tGCz(x^#7YA$ha8qp45K%DT79 z#5F+$9qf0v1?xNMs(RN}yT)X8bZ*mXb*hH4;RBXJ${{yzp-7e9!x~werN+=ts@oN1 z)eY{hrfQR>xlht*we=buK}yPOX_DBP8j=k9hOb*nsV=k1FJWw^_HvKk)JTcF9<5PQ z=J9t{dsOzY*3sA@4%ew0Yx1Eup#sY;Ev%U)W=-GaKn(m~@%z z*zMgRjmb6;=ng7WTU?55R$8yt+UqR=ms#Jmjom){d+QpiL%)S?uJLZQ81!b3-D2~H z?OpYz{&xNLD!0O=l{Pj?jWWt)^U+#GdsmOqRx9oB4KKD)l^TtM6;mooZ(^IRR+&Vt zt})s)tgS)EP%4XFDz(&FBoc9R71LB@b2J*|1`DH-N>nzAsa5Ig8iwz(EvLF`YibOE zUR9kz=5Tw1Wj*?Un>9JiDs@{&puSfc_SE*bH<_lrd`FGY$JlmvfuOFbl|pyg7j81l-&w--Ph zNboCY+ADYYL@u|?CzAO{O5}F={UVxilOBOfN{M9@J@(Dt-d5$k_28Bqx84&2hO`vd| z0(>tADK>+uc`$sr2tB(QWY3RPE+Xk0>D=5$4l=ZWqICHj?AvCJwt$j(u#a1SJ)N3! z1-G{Zxt*YV9_)}4DCZuJPdUMBxkGatO3vk}z+7W*G@kV1aN>`{2|t8~Vnci5b-~T} zT-W&DgXb6*|Kl+6Jvio}9&gMIa^5`nO)wPlREal?A?HL8*$HMV`SD*p{gIK6gjawB z(WgL`ApNlyQn}1{6u1JUXx}N&%4?qXqsym2uRs|u8sKty=#D82J@hBgCZOu__Hsgq zR!rj)iPki{XVHThh)A+sfFM!pA;_Sh<96G2EK`bAK{ zn^?)B))zsofc(Qg#KO9OzQ&;g*gD-(fqUa6!7&`e97$K+~ zN+oK285|@kzAB<4uYlQ<)bw9pT+9g|;U+Cz_xqtQ#4ebH0&6KLiVW@iF}L?{zYg>ljGzXKIfkaqXik@tWP@z+d;zcJ<_V8x2+ zW^lAr_Ol)?Q-~~9Z1Knpa1!8j)~d*-SQ)?Um_=bMM#55DOkc&t>%raca+yW3Joa71 z#Y}1Pz3B2)JO~2EhDk1W4Z1^wAA0B-9)t|7CXW*qpcQ5KM0p*{&QDsshs{SP%J75d zuVd2}D_XlF{{pj%`Ahd*+r^TxJm>{*HIZ|0I1@P+z*<6-tvQFDTmbLq-?saM4?<dNj4$6wnkm3!X`{P-7E+@97jcXeYp}8<=Ev`e6hc2il)iDi2z8X=BIPgnDIsli_+^L@ z!Sw`>JwJj5MYyB9;a~g)EiZ!y3CdRZ0h%a-P5cd4d|qQ!Y>n&6VU{56wcVp!rYg3^ zBjpgsxB27Zh#1Z$@XI^T9X8e@a4tGW3@#>@T|C8Q8jt$ z+pD`+N6db?21D5?J$sN-gY929bXkv{)L^ltrp}`wYs~%uEw(@F`a6fwNi7zevg%V4 z{vX9+7ai`o7OBB+3X;EUf0Gk#irLOsu>xoS^vl(Z+&^&mA6S`xQ zePJ^utN8h)BGwg~?EV%!zS;5TFQCzu=tMpAYCM{0!EB->g^NNhv0a^TVzvxvwj$CB z{V4*F^+^Jk*%@_Pf*#!pTlq`+ukQ(UL}#5u@&MKkr&mm%!2oWp*tKB*Ee}S+en;Df zCW1I@rZ2jy5_GT&wi3#dUk*pQqX87;wJ$REAdne- zJB*79zWx*!Z$p+$T)a1oi?iyl_o2&S*h?fn9Oy!x0XUKxw`Zb*69&Iqs2NaK`_DyoKt6%^DbR`hJK+JoJZ}FnHXIEE|ITwTpU=y_{99vx z^v8%q!yH^nlxSNS^dyI^D$4kBZzvQEhC~B*V)eyiPt>53cVhK|M!^hf-G$XBu66z{ OvK#&`0fwUh diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 6d50601da..a6c53c798 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -50,18 +50,23 @@ class StatusWrapper(object): class DockerfileBuildContext(object): - def __init__(self, build_context_dir, tag_name, push_token, build_uuid): + def __init__(self, build_context_dir, dockerfile_subdir, repo, tag_names, + push_token, build_uuid): self._build_dir = build_context_dir - self._tag_name = tag_name + self._dockerfile_subdir = dockerfile_subdir + self._repo = repo + self._tag_names = tag_names self._push_token = push_token self._cl = Client(timeout=1200, version='1.7') self._status = StatusWrapper(build_uuid) self._build_logger = partial(build_logs.append_log_message, build_uuid) - dockerfile_path = os.path.join(self._build_dir, "Dockerfile") + dockerfile_path = os.path.join(self._build_dir, dockerfile_subdir, + "Dockerfile") self._num_steps = DockerfileBuildContext.__count_steps(dockerfile_path) - logger.debug('Will build and push to tag named: %s' % self._tag_name) + logger.debug('Will build and push to repo %s with tags named: %s' % + (self._repo, self._tag_names)) def __enter__(self): return self @@ -94,9 +99,13 @@ class DockerfileBuildContext(object): with self._status as status: status['total_commands'] = self._num_steps - logger.debug('Building to tag named: %s' % self._tag_name) - build_status = self._cl.build(path=self._build_dir, tag=self._tag_name, - stream=True) + logger.debug('Building to tags named: %s' % self._tag_names) + context_path = os.path.join(self._build_dir, self._dockerfile_subdir) + + logger.debug('Final context path: %s exists: %s' % + (context_path, os.path.exists(context_path))) + + build_status = self._cl.build(path=context_path, stream=True) current_step = 0 built_image = None @@ -128,9 +137,9 @@ class DockerfileBuildContext(object): def push(self, built_image): # Login to the registry - host = re.match(r'([a-z0-9.:]+)/.+/.+$', self._tag_name) + host = re.match(r'([a-z0-9.:]+)/.+/.+$', self._repo) if not host: - raise RuntimeError('Invalid tag name: %s' % self._tag_name) + raise RuntimeError('Invalid repo name: %s' % self._repo) for protocol in ['https', 'http']: registry_endpoint = '%s://%s/v1/' % (protocol, host.group(1)) @@ -142,13 +151,18 @@ class DockerfileBuildContext(object): except APIError: pass # Probably the wrong protocol + for tag in self._tag_names: + logger.debug('Tagging image %s as %s:%s' % + (built_image, self._repo, tag)) + self._cl.tag(built_image, self._repo, tag) + history = json.loads(self._cl.history(built_image)) num_images = len(history) with self._status as status: status['total_images'] = num_images - logger.debug('Pushing to tag name: %s' % self._tag_name) - resp = self._cl.push(self._tag_name, stream=True) + logger.debug('Pushing to repo %s' % self._repo) + resp = self._cl.push(self._repo, stream=True) for status_str in resp: status = json.loads(status_str) @@ -258,8 +272,13 @@ class DockerfileBuildWorker(Worker): job_details['repository'], job_details['build_uuid']) - resource_url = user_files.get_file_url(repository_build.resource_key) - tag_name = repository_build.tag + job_config = json.loads(repository_build.job_config) + + resource_url = user_files.get_file_url(job_config['resource_key']) + tag_names = job_config['docker_tags'] + build_subdir = job_config['build_subdir'] + repo = job_config['repository'] + access_token = repository_build.access_token.code log_appender = partial(build_logs.append_log_message, @@ -267,16 +286,15 @@ class DockerfileBuildWorker(Worker): log_appender('initializing', build_logs.PHASE) - start_msg = ('Starting job with resource url: %s tag: %s' % (resource_url, - tag_name)) - logger.debug(start_msg) + start_msg = ('Starting job with resource url: %s repo: %s' % (resource_url, + repo)) log_appender(start_msg) docker_resource = requests.get(resource_url) c_type = docker_resource.headers['content-type'] - filetype_msg = ('Request to build file of type: %s with tag: %s' % - (c_type, tag_name)) + filetype_msg = ('Request to build type: %s with repo: %s and tags: %s' % + (c_type, repo, tag_names)) logger.info(filetype_msg) log_appender(filetype_msg) @@ -288,7 +306,8 @@ class DockerfileBuildWorker(Worker): repository_build.phase = 'building' repository_build.save() - with DockerfileBuildContext(build_dir, tag_name, access_token, + with DockerfileBuildContext(build_dir, build_subdir, repo, tag_names, + access_token, repository_build.uuid) as build_ctxt: try: built_image = build_ctxt.build() @@ -298,7 +317,7 @@ class DockerfileBuildWorker(Worker): repository_build.phase = 'error' repository_build.save() log_appender('Unable to build dockerfile.', build_logs.ERROR) - return False + return True log_appender('pushing', build_logs.PHASE) repository_build.phase = 'pushing' @@ -316,7 +335,7 @@ class DockerfileBuildWorker(Worker): repository_build.phase = 'error' repository_build.save() log_appender(str(exc), build_logs.ERROR) - return False + return True return True From d505a4edbb9ad27f6da2c5cd0da87fe5fced07d8 Mon Sep 17 00:00:00 2001 From: jakedt Date: Mon, 24 Feb 2014 16:18:14 -0500 Subject: [PATCH 3/4] Slight tweak to initdb to have the subdir config. --- initdb.py | 1 + test/data/test.db | Bin 167936 -> 167936 bytes 2 files changed, 1 insertion(+) diff --git a/initdb.py b/initdb.py index 35c3dd475..198bdf598 100644 --- a/initdb.py +++ b/initdb.py @@ -314,6 +314,7 @@ def populate_database(): new_user_1) trigger.config = json.dumps({ 'build_source': 'jakedt/testconnect', + 'subdir': '', }) trigger.save() diff --git a/test/data/test.db b/test/data/test.db index 8caf9ba9fa9a59ae26ce104dddf70b6291bb6ecd..ec2d396b04feac91796fec7a9ca3d88241c74f82 100644 GIT binary patch delta 3558 zcmbtVYj6|S752SaBtK&cMs{M$n1^k^maKhgcO44IvgC&zMwV*o7%T=eRbh8pbv4yaJ+I@(GwqG6oeEVk!lnk@@E~idir6-r^e&r5 ztTZ?+o7%~?^@HKA{xVOk&Dhh~P^Z&7dJJV1hVgy&6>P5KKVukl z)m2*z_95e-ug};M-fRpD?#AxM;hM^}ey%elh7IGF?WL@X?PN`yhH-XxGY#tCAtiHZ_SA?@y7~Ek^ z(CE=bLcO+*$QFmKy?fJuma2(#^oD&Sq0RxTp+?(f7|(LB>qNJSchy=NYK&GRP3vk{ zhOc#-)|=?+I?+^V&{w#19;2?o?&512olZ|xW21|8)mf=ZQ@xqw8}#FMI`Y`oDz{<8 zA8PY7=)`)5xgs=D-)++P`9V!zsJ|{$;T-CtC=YAXnmF@-rNhNnaUJbP&N)&R-nh%l zf_L07i6b8d?~#q&=)hG2c@h9p287>B)U*UuAMGB>yy1SJX&&w-<@UL$bj7e0-~qSMwah z;0ugj?c-@uO$W5B-_L0|A0t3n8Mtx*B!T2M%Fp}QcAr|{d3>Q=%c%J_l2em(tDjK> zIDbIf>O1ny6TjQw;XIVF(V?X&J6~mYvO0lftQ>E4IcbxHW!-wC-C<-%n&#|oC)X%i zOolp718=H!)>XL~t3jVkX-P({rPLHtKR(!3uypd95-^>c7H?EwSxPtgMHOhtO6AJ6q=sa$2B>8tuqD2~(F9f^ zZ4)5S`x}8KF@iI55#I!=6Ay9vD)d+rSbW{#HAz=));5E(>ylQY-!}s?@$kcDup#js z-&CSjFJKc7@Am?I!u#Y!FL*hB>?$WdEnlVw*SVDtPW|q1%6Es8|9^PwTS0lnHF52g zQAq%oCx%Jk`tZE(4inde`GniV6*rHMfkg?@ptr7le{gOL=;h?S{QXurQe6ZT;*NuC zMXqP(X-PQ`^;`rr+Ho9siL|ehkD&|4L0GY-``Db6FGJg=acu8jKv2OheIQLzLA3Z1 zJ}Ex|0`k0PdPNbSW0!Ej*G_=tM9$mO&3n>eI|`fx`{dN$4`+!P=vXf#5qAm{64Z;3 zM=hs-S&{pq{cA~?hI&dNjov>6ymH~WKh{K2(ex~?HTpEr%Z`+^qN%6B0Ybgzg?*^y z8L*#VPdDBceHP56XR14%d|y&!;YCTJmq;aO$qZ1VTVDl!c~;u!25})e$U_R9eHHgq zP7v?3TlO#Z8g!8jp+=891xk?c1_)+mpP+b2b<0)sGFXDny@3^3u=2+x;?3yb5T-nM9u&!* z*p-1M&SP*g!!eF#&SUWGWnn3zMAO4K=ZQCgo>;YQV^j1k5KNc-^UePU?Gv( z<&mMeMHnqLG$}(iB}U`iqt8W{xcs+dLp=fa9C9IVHbztZ*>y0Q1Lx8eNzeHom-5%f zr(Y<9K}F%+&!3P~YvR*?t`G*~t0l()@rUv0A1cD!xCh<~p@|}lR`#0*0%)cPquu=M zM{J}V-TgW?@@O&Uw#J!K6sDGT;(Kf(c)fwBHRjt@ul{5~EmfFo_VHj+v>Mk` zki3B1-WbSv-kWHm8KaSx2i`?9&2Z1Wv5+~^9N*TXUJO>eU2#{m1qRX;x^sqSC1qDkZ3^Aj z4O`^7vkP}cI%CsLB3%#WzohjSFQd^O+*xMhyN6L;FXo@MRaTFV_F}UE{OC?=AhxX} zGH-@Cid#OgzbGl&W7|rh{hMJ6p*{T4#%N#6fHZYguC*TlWvuovPG0g#pj1+ZkX?zB ze>#kl75%0>bYU2Vi6wv7vkJA1zyq1ex4AA!)q8cLX7Hc8FpAA)d;d``_Qdv=L}y2_ zqYL}1mC>zGP9S+Y^rFCaxKA#7&~Zu}jv0c#^AaqS=gkEdT8CnvBNB~EFo)3IoBSC% zEMZ@ACLa1Y5{dajqLJG%d#drJ`_RRij-n0{cp(?9B; z@AEwG^Lw7(?|JTf_Z&>xb1>J2?#3+^2VG$_c9d0hlt#$Lt*!n0tSi)x9yhzipfV}SlwDk;X`9xe z@Ag@^8h3~esGW|@jZ`b?E;Tl9S4F5wZEtV5ro7YMf5W;;-Bq`}%BDA1tu?L9mNqs- zchqgu>}b+-*=)u}i>cbBjCA@92Bo3BLQ$qF4Y+J!eMi63R>X97=!|A_Ya|lfz*lzI zlop4-rZhy=^l~m&OS@UCPz1JEgEV7g+pF}Zj@q)YO3^=TL&aK*&PsE4b0<%?+Z`%n zMQ@k3E2Ic+-rCu0+-@}1F|{2Ax=GQ#p(eD=M=}mebC;<(R$a9Z3Jw~lu3?hSgX z*2z^nOF6Ax#~aG3RIJ-=qAIExyMr%v8JxDdGFxd)X(g%R85d(h^t-W_n9koJIuYM0k+P#U8A#wwMvzxom9D$TK|qn#1h<5(=s$^Pn>u6 zDlcu?d)G}2ecbezc=z27oDmQr0E83_`%$DO&tmnZt&!Bbt_LemK-DDvxN1~<;x`(< zv@^19H$(G&hNMV27v%hMhM_}pKgY0g*6$~M>X5I|A5@7kB_%J^!7-NZULveH?}oAtECK-X52bZxm= zrPF9?%ItcLnp5dWjg4pBHl0mMlN2K-Q87#&#THX!F|AUNJV_6Kkp)gk*57UZXdWP> z_yLjlz)-sM39-@(5=C{1ATi%1qFFyplO$@8VS{qU7i8r=*4HRk^Q20nVS_$OMZwV` zKo8FpflJxRw@af)N~6{cUpD}6MiR0x3bjVVanQR7RL?%2uLp&YtOq#!a1&6>_MpgY z$khYmY$FO^3I9+J7JtWhN6_t?l?_1oouEQ^xB*bJjUP9FjkC|_rVRQ$Kt0xE7H^`ht4Ses;^K&!1ftJYjKm1QK0dp<^8fs30EQz|EOA%xVu=Ns9 z!3R$Q56(^eqzPU;3A!Ze8^5U)a>a1h6!Ja&XV4_cEcx}Ykb}X+myxCT6lf$C3=mmv z08d;-2`5g0WebZ-UVA-7BoawPL6O*tjba9H6nFtH3c|Ec!BXfw2Z~_eFJM2BHdz0p zI|ZI-!zjp}2KhL9_Ie09Ik#F z1PRrlW3RgB!RI-QhU0I8W%%4B>I-n@ZD7LrG07{i`Yd>o$iPb+;WRkKqoC|@upXDP zD`@B$M`+18iD}UZFoUNqU0V4^AvgOg3^{z^71SVq9yFyARc9X;au$AzuEdJqvuSZXYWpJto#k~A3(BhiNLr+ED z15IRMeOwK7(;F`5dIoLOa3S8#OJ_Z+EMsz4VcrqNh44ee9 z{*mS8TnKEapQ>r3zT`o}&jncywAwHg9GFJx&%fD{5&abPMUeMjy=7hreWzo&uc6?) zbI1LHtPs9u6Neho0ybath#L1+o<1 zKuymb>AB_3hbI&Wec}e9JhSQM%hAulOa`I+?ASgx6<0%(f-S@I2UfoiJt>$8U-c&e zz#}QxlSIOSpX7xVaW(qpV(al`oq;#u=v-_7U+dZQ2=u06`|&I)v@e>5AzdWm8JCd5 z&1N5IpNR#D1?kQE+?qK1XEU*UeAUh~H{nbsLR;BfR0OMK2o2vUem=~^>0g+IX>n!p zy1meoh0sb&t~Js5*vuTsiWd)&LhicwhVc1VlceOt-#!p>O5z(bnU6IR>4l&9+-u_- z5-C8vtxEl%0Uj(sXv=Tm&%o&dgjQgFZD+U`K6(z3J+eI3+e6z1qbsmRT#{fL3kWhp zoc4pOFb^^B%n%Hh$98~$yc{dR3n$;1fPHdQT4L)pA6%qBL@aC^mBJAP<|LM$ztv*a z#dp3+iDgOBQ?6tQGHrb42b7oxUqFQ)j*{4nlqfv*VxPGhg0nHzc@#|Ylj{VT8CuWA zROeA}_R-OI;YA+nlHgw?UKMgpa91mm^r>R3S(5iV@m3+H5-x5-7JfC>L=>=xId?^T z%g0wEvgaND*Ous-7#6X1!vS|)TnAQcioq?u~K&$E}F} z)i=Q#`tdbl=&>V|`41fKh}On(uUJ3yA0amo zA9B7Pshzp<{l|nHe|*R%>k-;o`%72dzW9(w8W7szzRyp=gAE8Rujua{INgB#BH@j< zgm6QAYezf?tmw*#il`TBoFiFD=W7MomKfVK{MA;>OK|)DZ4YmbO+ETbwxa$if_nz` zwW7+_jI{59i`r2CtFG*Q1CF#IvB`O7J~ubUww8jXc5J?c$?&fgWTDvj&~Se{=EYZH z&x}X6#RTAX9eKyR9RgV_cTX(%_~e8j>ws1nlJ9{Y6wLMgJ_s-NU|smVlUIjfpcflR zU3Bv;^a`|nyIIrdM}Y4`Vy7J+n{l_s_Lzd>eMr$%|0e~}ofv_qee)U|`TlFL2is5F z(>mVf?up5SY7wyfxf$j`c~wMu`w0+

+)XoIEaI^Tivx{{fE(NHjX>(Bg17CK?5M n??coq?>w>uj^2l;5vNP0p!a@6jplvb=ICDRzsWdv Date: Mon, 24 Feb 2014 16:36:49 -0500 Subject: [PATCH 4/4] Add code that will allow build triggers to be deactivated or deleted from their services. --- data/model.py | 10 ---------- endpoints/api.py | 26 +++++++++++++++++++++----- endpoints/trigger.py | 35 +++++++++++++++++++++++++++++++++-- 3 files changed, 54 insertions(+), 17 deletions(-) diff --git a/data/model.py b/data/model.py index b8c68ce24..938efb640 100644 --- a/data/model.py +++ b/data/model.py @@ -1508,16 +1508,6 @@ def list_build_triggers(namespace_name, repository_name): Repository.name == repository_name)) -def delete_build_trigger(namespace_name, repository_name, trigger_uuid): - trigger = get_build_trigger(namespace_name, repository_name, trigger_uuid) - - # Delete the access token created for this trigger, and the trigger itself - if trigger.write_token and trigger.write_token.code: - trigger.write_token.delete_instance() - - trigger.delete_instance() - - def list_trigger_builds(namespace_name, repository_name, trigger_uuid, limit=None): query = (list_repository_builds(namespace_name, repository_name) diff --git a/endpoints/api.py b/endpoints/api.py index 169bb5c7a..ebe147840 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -29,7 +29,8 @@ from auth.permissions import (ReadRepositoryPermission, ViewTeamPermission, UserPermission) from endpoints.common import common_login, get_route_data, truthy_param -from endpoints.trigger import BuildTrigger, TriggerActivationException +from endpoints.trigger import (BuildTrigger, TriggerActivationException, + TriggerDeactivationException) from util.cache import cache_control from datetime import datetime, timedelta @@ -1413,15 +1414,15 @@ def activate_build_trigger(namespace, repository, trigger_uuid): token.code, app.config['URL_HOST'], path) - handler.activate(trigger.uuid, authed_url, trigger.auth_token, - new_config_dict) + final_config = handler.activate(trigger.uuid, authed_url, + trigger.auth_token, new_config_dict) except TriggerActivationException as e: token.delete_instance() abort(400, message = e.msg) return # Save the updated config. - trigger.config = json.dumps(new_config_dict) + trigger.config = json.dumps(final_config) trigger.write_token = token trigger.save() @@ -1492,7 +1493,22 @@ def list_build_triggers(namespace, repository): def delete_build_trigger(namespace, repository, trigger_uuid): permission = AdministerRepositoryPermission(namespace, repository) if permission.can(): - model.delete_build_trigger(namespace, repository, trigger_uuid) + try: + trigger = model.get_build_trigger(namespace, repository, trigger_uuid) + except model.InvalidBuildTriggerException: + abort(404) + return + + handler = BuildTrigger.get_trigger_for_service(trigger.service.name) + config_dict = json.loads(trigger.config) + if handler.is_active(config_dict): + try: + handler.deactivate(trigger.auth_token, config_dict) + except TriggerDeactivationException as ex: + # We are just going to eat this error + logger.warning('Trigger deactivation problem.', ex) + + trigger.delete_instance() log_action('delete_repo_trigger', namespace, {'repo': repository, 'trigger_id': trigger_uuid}, repo=model.get_repository(namespace, repository)) diff --git a/endpoints/trigger.py b/endpoints/trigger.py index ab902fd2e..acd1ab127 100644 --- a/endpoints/trigger.py +++ b/endpoints/trigger.py @@ -28,6 +28,9 @@ class InvalidServiceException(Exception): class TriggerActivationException(Exception): pass +class TriggerDeactivationException(Exception): + pass + class ValidationRequestException(Exception): pass @@ -70,6 +73,15 @@ class BuildTrigger(object): def activate(self, trigger_uuid, standard_webhook_url, auth_token, config): """ Activates the trigger for the service, with the given new configuration. + Returns new configuration that should be stored if successful. + """ + raise NotImplementedError + + def deactivate(self, auth_token, config): + """ + Deactivates the trigger for the service, removing any hooks installed in + the remote service. Returns the new config that should be stored if this + trigger is going to be re-activated. """ raise NotImplementedError @@ -104,7 +116,7 @@ class GithubBuildTrigger(BuildTrigger): return 'github' def is_active(self, config): - return 'build_source' in config and len(config['build_source']) > 0 + return 'hook_id' in config def activate(self, trigger_uuid, standard_webhook_url, auth_token, config): new_build_source = config['build_source'] @@ -122,11 +134,30 @@ class GithubBuildTrigger(BuildTrigger): } try: - to_add_webhook.create_hook('web', webhook_config) + hook = to_add_webhook.create_hook('web', webhook_config) + config['hook_id'] = hook.id except GithubException: msg = 'Unable to create webhook on repository: %s' raise TriggerActivationException(msg % new_build_source) + return config + + def deactivate(self, auth_token, config): + gh_client = self._get_client(auth_token) + + try: + repo = gh_client.get_repo(config['build_source']) + to_delete = repo.get_hook(config['hook_id']) + to_delete.delete() + except GithubException: + msg = 'Unable to remove hook: %s' % config['hook_id'] + raise TriggerDeactivationException(msg) + + config.pop('hook_id', None) + + return config + + def list_build_sources(self, auth_token): gh_client = self._get_client(auth_token) usr = gh_client.get_user()