From f07690956dc7c562ffaa851168d623378f274f93 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Fri, 15 Nov 2013 15:49:26 -0500 Subject: [PATCH] Add a fixed number of retries to each item that gets put in the work queue.: --- data/database.py | 6 ++++++ data/queue.py | 8 ++++++-- test/data/test.db | Bin 96256 -> 97280 bytes 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/data/database.py b/data/database.py index fc5a68454..e6bda976a 100644 --- a/data/database.py +++ b/data/database.py @@ -104,6 +104,11 @@ class Repository(BaseModel): ) +class Webhook(BaseModel): + repository = ForeignKeyField(Repository) + parameters = TextField() + + class Role(BaseModel): name = CharField(index=True) @@ -198,6 +203,7 @@ class QueueItem(BaseModel): available_after = DateTimeField(default=datetime.now, index=True) available = BooleanField(default=True, index=True) processing_expires = DateTimeField(null=True, index=True) + retries_remaining = IntegerField(default=5) all_models = [User, Repository, Image, AccessToken, Role, diff --git a/data/queue.py b/data/queue.py index 8e63c4d17..5f292a1d7 100644 --- a/data/queue.py +++ b/data/queue.py @@ -7,7 +7,7 @@ class WorkQueue(object): def __init__(self, queue_name): self.queue_name = queue_name - def put(self, message, available_after=0): + def put(self, message, available_after=0, retries_remaining=5): """ Put an item, if it shouldn't be processed for some number of seconds, specify that amount as available_after. @@ -16,6 +16,7 @@ class WorkQueue(object): params = { 'queue_name': self.queue_name, 'body': message, + 'retries_remaining': retries_remaining, } if available_after: @@ -37,7 +38,8 @@ class WorkQueue(object): # have one worker. avail = QueueItem.select().where(QueueItem.queue_name == self.queue_name, QueueItem.available_after <= now, - available_or_expired) + available_or_expired, + QueueItem.retries_remaining > 0) found = list(avail.limit(1).order_by(QueueItem.available_after)) @@ -45,6 +47,7 @@ class WorkQueue(object): item = found[0] item.available = False item.processing_expires = now + timedelta(seconds=processing_time) + item.retries_remaining -= 1 item.save() return item @@ -61,3 +64,4 @@ class WorkQueue(object): image_diff_queue = WorkQueue('imagediff') dockerfile_build_queue = WorkQueue('dockerfilebuild') +webhook_queue = WorkQueue('webhook') diff --git a/test/data/test.db b/test/data/test.db index 20ed3819d36d71d54cdf20cd477623d14002ea5f..fc3f5c088710af863c3692ba8f133ab1364f8a62 100644 GIT binary patch delta 1567 zcmbu9|4&m_6u{qk_t6%p2uOt~lTuXVTY3FPDd@Dt(n3qWfA>pQXa^5kT1rvCY!T-V z+kTkJHL1VMWcX!E++yU-U>2GDw4XLL8&k7ni8EQu5@0MQ&NyOr3vv4cc5~n7oOjO6 zxi{yWdrz+BoqUmZwhF@Mq~^*3beAH-jQ&Q?(Vu7uJw;E@FX$2a5#8lWm>$%u*Af3- z4S`lcz%~)o*Ampo2)0!cY$+osC?d!g%rZ=6v4A>5QD>kGPSKapCNzmlm=jnheO7R4 z2NIT7Rl(^(+)}-|yHP4j$1-UyHX2UH5)p2g8}64$yKEk#$>@^W>>jDjYqj#bWk;c6 z6A3S(Iw&&lKm&}x@AQ6TLZ6b@4f&HoVfp?ln5x07tO8%DIaEF<5|&FOFs02zdgHNl za^oxHr}osNnVrmRbD zmrWZPll$N4J$P^kkF?cld;{a!pvB!kY3@mKDu=yCukZ84deu68EMu`N2a+A`(82CR zN6eWB462St`kE)S5&V5yohF&o_VwG+iIzlvpDjK-kZ_uYyfN*lBJDHl62?~65gi*1 zxZII3jiIY`e8@L$nuy?r_FbBhj&ZKfsTuWq{fb1?t@XEdN3C|nxT|-l|5()S>@jBg zJsxLAy3b&7_D^=HGnpud?^!DFC+*ubk#vGn`Q38eupyM`>Krj@Y)Q2}nHu7BR&U(V z!Wy{_!TcPEXLU&gmuiT7$RJRxBv5 zmcSS{9F2{SjmFa9TovOvtqhl}NU+J_0Li5b4m4vH&7K#Cc){V9!h(%bhA1k8MR5nd z;2zk~vsF;OO9ILDNSRn95zdt?%tdqhcKkSQfl9tH@iWZ}qA!KBDS=w-rzk&l6TYNt z>FcBkjJ)S^kfv?5czIF1N#5Zy>ODrOM{lqi@tNgq*!|A z&=BA8=>>*o9Ll&FY6jB|KX|}=z(TDeY+M~*Qb#|!g2XKyW_1ONBLGYjJ4^^;L7d4yn|w$i_ns!P%P1xoysY}d}0eiH$a zN?2}cfGHIYFG}#)JC(3+BkLdC*@7J_#rXbWerY@U_zzchz|40daURJ<5&AmU7e{&ViyO#!A7A3+39K}9R3H9aL11T delta 1504 zcmbu9e@s(X6vy9r_d#nx23md$m=wh!2<_{S*B^9>ZDEvJ`k?gDA1K(;+FGH2bkgc# zB>IPcm`%Bh-9H-BEo_;~T<8{-1xnri*gq~yrZH1ovMtM&I3zlg&1}Z)mi;mJ_g+pu z=iYmA&Ux>9Zr;hq1t%XAoU4PO^D=Ao26TfWj~V@menSt@D*74yfWAfFpc|Po#)X<+ z(vqlFNx-%cG&d4#+e%PdOHf`;Q1l$Z2GKmj)NT?{28vh~TIfdlJpB}%#%0Vl{JQLs zWF~<`m34J+svV!Le~?LU9Rh4=cs0F)soF>ih^Z(=>=x9*8*q>Q71_}hlIz6F4HY2C z+-hh>xXCKax9lp>ygZ_F27@DA+@#r~2`2^Hq)Q0!W>chVTzz!4pfyJLKJ?k+?q;J{C>dVgq5;;|O(mUEz+&fj-Y< zWJ;R|*$#T;_{+8??iGDh?-gP~xa;WDwBD;q2BRJyADH4t^j58)9q5gjxK5pN?1td~r4r7(1++jvS1{)#FhQA4$Yb9GhS}gHD6JKcduWOgK&QV0^@`Kcvw*;=G%STEo-F zlFlCexWi)5h9iDeX4NPyp1saw82b~^tgNuKf}$#@gYX(%Mqfa0QWQR6t%1#WPklMQ zYV8GSx*K6%Fp_qarXyl;ejFm|GYU(By;#OXxAasLR&Fnc@k6m-C=n0&$A*uLkA{X1 z1@r4_JQLiE9|iSLlV4@pLtoHu6;5c1T@>Y_=HWfMfqtJ%(wuR>?WO6yMm%_XE66h^ zZp&f*U8Xq0+`NqT89c6jm$}pG6YQ2gtG;i*ZgZR59ZHkW=4Cr=8pnXkYj*Lx-K%B$ zm0laKC9?B6pOf!0xXi3jsA84sR;99))yY)qHkF}`)ykEePODea^LKN|`F|VaMJb@p zQN;d$1pJ65$v2&#BRlX1cN#&luH`RxTA&G6t+1fP*H&t9a778t_`*svsK~w@os#Ut z>eXgYXNFe&04+FoUk@7maIp&Cy1zNqPD5d8PaR0GX02*AN5=QvlcMwk@uq)z3lt?# z_h=)19qlK*Eof#_E9TeQfLpH+S}Vs3YdcFEa#1D6!Rhw6KQJ1?^-EIx^_&!RcxbgM z^Vb}JWTr$Rs_f)oMuAUet8vF-4d_XUZTQq8D`uY!4KIuE-9;%FNVs3D6jk<-tgiK} zD@z;k)vOfS)-$gxwWU82r*p-#R;HwY8l#9kh0p0T=pFK^gQTVE-@5g|DWrY5LgRnLO?k}NTt;*}}2l zsAV`n2k6`A08ykf^*g|eU`pixq^S>S*tkHZe+?D_Ti_BD=gSw+r}O297K+7ioCaU& ij0E<=KM+#Mx7$dAE1rlOnK}E-*59GWa)G^1s6X