From ef9fe871fcf3c888682965996c7d6747d071c8a5 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Fri, 2 May 2014 16:59:46 -0400 Subject: [PATCH] Move the upload flag to the database, and use the database stored image size rather than going to s3. --- data/database.py | 3 ++- data/model/legacy.py | 2 +- endpoints/registry.py | 50 ++++++++++++++++++++++++++--------------- storage/basestorage.py | 3 --- storage/local.py | 4 ---- storage/s3.py | 9 -------- test/data/test.db | Bin 200704 -> 200704 bytes 7 files changed, 35 insertions(+), 36 deletions(-) diff --git a/data/database.py b/data/database.py index c4a96c0d0..eaf2f0ff0 100644 --- a/data/database.py +++ b/data/database.py @@ -220,7 +220,8 @@ class ImageStorage(BaseModel): created = DateTimeField(null=True) comment = TextField(null=True) command = TextField(null=True) - image_size = BigIntegerField(null=True) + image_size = BigIntegerField(null=True) + uploading = BooleanField(default=True, null=True) class Image(BaseModel): diff --git a/data/model/legacy.py b/data/model/legacy.py index 296b6b1f5..1c207cb79 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -817,7 +817,7 @@ def get_repository(namespace_name, repository_name): def get_repo_image(namespace_name, repository_name, image_id): query = (Image - .select() + .select(Image, ImageStorage) .join(Repository) .switch(Image) .join(ImageStorage, JOIN_LEFT_OUTER) diff --git a/endpoints/registry.py b/endpoints/registry.py index 602c45036..d701fd140 100644 --- a/endpoints/registry.py +++ b/endpoints/registry.py @@ -40,16 +40,35 @@ class SocketReader(object): return buf +def image_is_uploading(namespace, repository, image_id, repo_image): + if repo_image and repo_image.storage and repo_image.storage.uploading is not None: + return repo_image.storage.uploading + + logger.warning('Setting legacy upload flag') + uuid = repo_image and repo_image.storage and repo_image.storage.uuid + mark_path = store.image_mark_path(namespace, repository, image_id, uuid) + return store.exists(mark_path) + + +def mark_upload_complete(namespace, repository, image_id, repo_image): + if repo_image and repo_image.storage and repo_image.storage.uploading is not None: + repo_image.storage.uploading = False + repo_image.storage.save() + else: + logger.warning('Removing legacy upload flag') + uuid = repo_image and repo_image.storage and repo_image.storage.uuid + mark_path = store.image_mark_path(namespace, repository, image_id, uuid) + if store.exists(mark_path): + store.remove(mark_path) + + def require_completion(f): """This make sure that the image push correctly finished.""" @wraps(f) def wrapper(namespace, repository, *args, **kwargs): image_id = kwargs['image_id'] repo_image = model.get_repo_image(namespace, repository, image_id) - uuid = repo_image and repo_image.storage and repo_image.storage.uuid - - if store.exists(store.image_mark_path(namespace, repository, image_id, - uuid)): + if image_is_uploading(namespace, repository, image_id, repo_image): abort(400, 'Image %(image_id)s is being uploaded, retry later', issue='upload-in-progress', image_id=kwargs['image_id']) @@ -139,9 +158,9 @@ def put_image_layer(namespace, repository, image_id): profile.debug('Retrieving image path info') layer_path = store.image_layer_path(namespace, repository, image_id, uuid) - mark_path = store.image_mark_path(namespace, repository, image_id, uuid) - if store.exists(layer_path) and not store.exists(mark_path): + if (store.exists(layer_path) and not + image_is_uploading(namespace, repository, image_id, repo_image)): abort(409, 'Image already exists', issue='image-exists', image_id=image_id) profile.debug('Storing layer data') @@ -192,7 +211,7 @@ def put_image_layer(namespace, repository, image_id): issue='checksum-mismatch', image_id=image_id) # Checksum is ok, we remove the marker - store.remove(mark_path) + mark_upload_complete(namespace, repository, image_id, repo_image) # The layer is ready for download, send a job to the work queue to # process it. @@ -234,8 +253,7 @@ def put_image_checksum(namespace, repository, image_id): abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id) profile.debug('Marking image path') - mark_path = store.image_mark_path(namespace, repository, image_id, uuid) - if not store.exists(mark_path): + if not image_is_uploading(namespace, repository, image_id, repo_image): abort(409, 'Cannot set checksum for image %(image_id)s', issue='image-write-error', image_id=image_id) @@ -252,7 +270,7 @@ def put_image_checksum(namespace, repository, image_id): issue='checksum-mismatch', image_id=image_id) # Checksum is ok, we remove the marker - store.remove(mark_path) + mark_upload_complete(namespace, repository, image_id, repo_image) # The layer is ready for download, send a job to the work queue to # process it. @@ -291,8 +309,7 @@ def get_image_json(namespace, repository, image_id, headers): profile.debug('Looking up repo layer size') try: - size = store.get_size(store.image_layer_path(namespace, repository, - image_id, uuid)) + size = repo_image.image_size or repo_image.storage.image_size headers['X-Docker-Size'] = str(size) except OSError: pass @@ -429,13 +446,13 @@ def put_image_json(namespace, repository, image_id): parent_uuid))): abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s', issue='invalid-request', image_id=image_id, parent_id=parent_id) - + profile.debug('Looking up image storage paths') json_path = store.image_json_path(namespace, repository, image_id, uuid) - mark_path = store.image_mark_path(namespace, repository, image_id, uuid) profile.debug('Checking if image already exists') - if store.exists(json_path) and not store.exists(mark_path): + if (store.exists(json_path) and not + image_is_uploading(namespace, repository, image_id, repo_image)): abort(409, 'Image already exists', issue='image-exists', image_id=image_id) # If we reach that point, it means that this is a new image or a retry @@ -449,9 +466,6 @@ def put_image_json(namespace, repository, image_id): data.get('created'), data.get('comment'), command, parent_image) - profile.debug('Putting mark path') - store.put_content(mark_path, 'true') - profile.debug('Putting json path') store.put_content(json_path, request.data) diff --git a/storage/basestorage.py b/storage/basestorage.py index c8cec5931..1e924ed1a 100644 --- a/storage/basestorage.py +++ b/storage/basestorage.py @@ -92,6 +92,3 @@ class BaseStorage(object): def remove(self, path): raise NotImplementedError - - def get_size(self, path): - raise NotImplementedError diff --git a/storage/local.py b/storage/local.py index c1df70297..361d76403 100644 --- a/storage/local.py +++ b/storage/local.py @@ -80,7 +80,3 @@ class LocalStorage(BaseStorage): os.remove(path) except OSError: pass - - def get_size(self, path): - path = self._init_path(path) - return os.path.getsize(path) diff --git a/storage/s3.py b/storage/s3.py index ddb7b7f57..1747d34dc 100644 --- a/storage/s3.py +++ b/storage/s3.py @@ -171,12 +171,3 @@ class S3Storage(BaseStorage): path += '/' for key in self._s3_bucket.list(prefix=path): key.delete() - - def get_size(self, path): - self._initialize_s3() - path = self._init_path(path) - # Lookup does a HEAD HTTP Request on the object - key = self._s3_bucket.lookup(path) - if not key: - raise OSError('No such key: \'{0}\''.format(path)) - return key.size diff --git a/test/data/test.db b/test/data/test.db index 664547addc99b805de62c2b01bcdebdfadd9199b..3ea2e8dfa3f00e22b1b077f0e2a6d5fc9d862abe 100644 GIT binary patch delta 7734 zcmbtYd2|$Iw(qKHLN;i^Vzb#DLPxe#Qu`7dNOyWmy3^@SC!M8bsqX4dASOUU(h15y z%^-|Aj$)9)Wds@DjOz>#PczOzn75pF6yI^2d5Fv5>G3G)c}GVC-P>`~B*7@AuWWbzkw;eZ>zY&`;2V^8MJ9S#wM#Q^_=V6c)oHfAl--4@Go0 z{4*=VV{`>P4#eQ`d?P%b@xkNQHSpLk!ebu=kH?n4W6%82&n?r7Cr^TZ=b1K|OdFBO z$ipL(OVgwGmD-VlP1rL;OPBj1IVGd zdFGM4g;}Iv9uF?@7N7lUqLTU<>Ze_J5Z5vvWxS~^Ig z*%c%FQ9tJ;VnmE&+5@o=SKrLlcZj0LYc5v#U8UV389+9wRM4L zi!Qj6T?JUXbCos z9ADeDJXRO+#~SK5DjX9CQDoU@D^pL02wx}=iw2yG-|KO4WNR~D8)pv?_iq&x9b#q$2bODYHeBW?UIRI+khzdXw_B z`;oCsD0(B2N^hjnp`n8OiHpz=v6rz8qh#@)QFUJpnzseLSvQ$|6F@cOhp9$x$*G2o zC%Re9P_G84;D(Cng3Y;(uMTayaLYm`=Wg@2bI!VksHe702-h}+qh2OJIr*kgi`Q4% z;)d4nd7LB19xos9Ki;^IXlUlSpqq;YYTYg-6pn@>gtLvWZ)}2rH8!$8i!)C3q2MWG;=11k2+rCGZR-F;p@|^`+LYtCr-RHN6{C{Tnx~>rVAj z1j*n8ixX5SDONLVHN{jC1kLcmp%+d~bbOzzhh93+7-kc!z$Q5kXXuoKGXz8762lTW zpJLMjN3s;d(b!#O{p)&CW!0r+gVLP?sj|(+>RdG|R5N5HFR%n9X3fawMbpP{T@jgr zXu!Tlz3fb67~2A}P%_EmOgf#w87@iVB9lnqG($6_MDi?=6fm?3%{L(}1<<#J>7*#J z96{oABALROR4Rohl4%f9;3PTCm5E4Neo(m0z0MI@31j&dp z0qzha3aTdwJR!&w=p@U$M92h_64V#H$XrK4AO%WHYM}(JnF$zC2?-ZC*cmuh=2o!0PW=jk0(e8ey8}P$fVVFAF|jXunC4qvn0-fSnyf_M}ih5 z@Bu^eL?S861gZYU2YRNHY(f%~5aKvZC5FVM6sW`#bV3p+=p{zV`jMxr;-O$P%7-I@ z*IO5fQL$QQLns)GH8eG|G0Nv|aJ7c(!gQM}S|171vAQ zR4-3BGUl;Ws~}SHJgl-)XiZ7w=RX-%?AZ7!Tw5hW>`@BsEFljZeOsFuKEp?{v;Ys({#% zNs4Wycj<$~HT@PAjAu4tp4ox-C*3i&ijdp7|tyoLO^XV^~fY>%Ky2`?={O7{V zlpJla$=Z@-$=@Q1-C_WzJ7kv+?J~1l_Z-c_nbkaXn+e;2eu4fHtwHtzwQZ&rcwseT z+=?=D4LLO=$AVy@Mbs0oD)u?X#2ON@0Zhcm1OK7eW*b-omJ0P$1F&WuJA}k%q%>oWabtFLd_ya8qpdv+WE}% zY#22cPoHQ?ny@r>3;HrT6Bz`8NfQGE?U!Djb-ppR)D6&m@+ zycZSwJR^Ax(7_czo=Ts4N3oR}$!n0BR)W*bZ~goiaeI!u?%6L_0(s=YbSz}ck=Jb! zT7}LixowH?o?^Y#fKWHe-c{%d^U^;J?ai)6&E|RermzV+fJMVqIh`tGRkd3WS;P$_LM67LKjb zMhk$2(UJyXmVNpGs@NA8gaNQSOY+K|s#9#Y88y+2RNV=hOnB?P2jb-hT$+vE>4ff| z*Zk(UA%_8+W}>rCqBG282bG>|7n(PH>hk$&V!ttqcRZ$x8e4a%NvV7?izCXXsM!H_xf zV{P8t1D$7|zppdS=V=ax2h1C!eb375&$Ll`{drT9%c&zdwhNraN=yDh zu`f3y1_J}LM__*E4^23&*p?Ys>pnj92(Yfs|LKwVk8-SaIGNqRI=|olbm;aRYuz9x zc573%lij3Pml{~>7AaTZWedCaIy2Qdg!)c#M1^@Xd3*o2**(yVH=Z{>V`{Cf_C5h^ zHSNtWS7yAq)H)y@hH}ZA4`2AJV)q!x>wpga5?*=XH{0G&Y;GfY&9zNWg43A;+b_gx zbL4f;o_P|SzHGyhO(B*1S#8p{{Cs65IJQb_GyND;`W(pP*m-udVhJg_3`|0T_a5=Sy^;etsmGRyFx>qQfz3gr;h7ra5b|-0jf!+e7^3k(7*=u(|rX$Z+%8fr0K# z!|Y&Pw8ve!v5l|utRJH5`|3NU^$qgvv2{JI1hJ8i48}x1=~I7y2wlK7kxYi|az|aQ za({2MGSc7PCsn1^M0u*o*A`k6X!S9j-Hmc5eA`7kdi$d3bjNV3I(-ynZ~ER*y>}Fy z%hFv#Qg3rtuqT*G`J{S(_pm?@4K%Om=oi;E`y`^DrR4TRhb!FKQYQ@r2fQMc96tQs zD7qc1x!Jonb>CsMwB`rhR~oJ>v%lB2nd|SMF+=0%4fp5~Z3n$5{3H(dFaQ6UqMezz zS74}G+Rr~2Y98+%Xi{jl3iXj!ptUQM4}yv{I-YNQD}f#K6=>}_rz^k8G>xs&chIA- zl`MI2_PvTdGQL3jOwnie*GHkNXEv|#Dz>lzoIWgfAA=oq+L@9>_K#>@{tb4}6^6`u z`|o)b#;JL5@*%}`hcUA@0G7T6dr9%E{Zc%hqpjmP`5Mr6Tx^~a>d4X7(a^_XFPZ=A zB37}skLS~9>vM4Xao9nN&b_fY)0U$Ra}YjMusPL=z10X#8-b6!j&_=>*C=PQ|E5v9 z{thbVa_UHqoq#E6IoJa`Xwr}vknAj3TKb<)DYk@xus)7Yy#<7qtvE3&F69X8`^V5p zn34-3joU-3bA)wooHz-Dg-YQi#k$HsSRclYw_%4XoL^dyS(yW;yW{BF=rZ%PZ{N3O z-}zpL^=@NoLfbq)hSZifgTKnG&7}rFV9C>{vxJ+vbH8F=W8e*dD%8=_aB|>W&)O7Q zmw`M$TBJrkK|3wC|NHiD;++QaGDN=o1m5=STJPUNX#@G~5D9#W&MaAc>Wy;6nlc~+ zk5s68KZV13K@Ytw`x$JW-}kWIZ>W6+4(r@+b_|C4a%FX>+*eT6QErFN!VQM9U}!k3 ze+9zaa#!Jfne}6<^leiEYdZ_Vn0JNXu)fY941fhgTsY-XY`q3ez(b2v@f>J^PMz~W zyeEeXjv;Uba1JzK8n>E4cjmxFz5(8mk73)ipMCD|cjwTFHySmY3>D%0wn*LcKTvUz z>vKzd*icc^Q#=o+h`D!NJR2G^RMb#Dd>-UrL!U$yYsMglj`O4QAjgD)`aouIY?VH0 zzJZ?0KmW6TRqO)>CBW&6)UUsRQDfg$_D_m!qY<1JVvk&aeq^40eM0sk^rLwtTn>zq z#m(3x#eSDDkM>7q-O@-g?dBkvbr3ss^3GpncbvGQyG z4XmE63I*8mYlk7lBK4&L?3TY9u9em0!bi{qzNEm zu_7wF>KLFrf`|-`j*nR(ZX+Xt&a87Bc65)dinDrLbwQjRnPq&%J<7#v4Sa6Ef!1R1bF1;!6SR*9QKP`x(7Zc zWq6D%fyc99csvz=#}i(7{InV#dqsHcq2Te*Ja{~4ANh1#QQm>+8P=?OES7r^JMzSE zesO%bI?pyTP&^IEma)tm%F2*`%pKlU^4`e3rK^#>bF2>#)?ustc_eH2!=)34kK1oX zaz-l4-az-xFFLT`6Du-)_~`6OBS_6Tbgy^PUfKGHr!4) zS{xp4&>5<2jEJsge`brv=M6*}hmS4qT0}OGO%BEt@%U?r`sO;JroGMWsI8;g0s()0 zV=zqBI(?+m9jvF@B3`;a>}2RBPGnl^DZ0I(wtaY2H&^Wov#v-a)LiHEwt4EB8>vRG zgQyGgB-<2n5L9z09ON90zDBV%)YRG_a7|*J$0@Y?I9I5qxwh3Ya=QE5oa-%p{vc&G>e8H7#*2=o8iQV^Jndd&G!u$mOQh-7Qt7}@c2?#p>=61PRz5=Jy@6_b zs?m&%kofxCWc7>z)sP>inz+$a4LhIcdNo7cI-u-pDrN{a^(wwPaL2c{IqhK==??HA zo+3kphp2Z^?&hX2&9^nQw+L)|n6B~G2ijX(hmYUK%mEFn;iXa)BU~rP1P6|~FQ$-OK9L;c?c;M;xGVR|d>w)K< z4Kxz89G9iI0`;gk&M15g7bG4&2~nCSV=0QPUCaW?7I>HtgG+I2DsQl2LFO!Z^dmSzL+%6FRQ2f+PuCOr%h(5-}Da79>X? zl{iTVG|n>;3kz5o7b!7@$D)E1Wi!|~EFcg5Z7LF&W9OnGBQh}t7iCeGDNf)LE7G_^ zNKu)O$6_K$YO9=x)6PjU!*Wptmw2AS8Is^}AqJ>OhKUn#lA&al(0=DcSi8)VJRyrR z&I+^wo#*1X#LyHTl@*$hVL4Aw8s$Q6vI|sHQpji&=UCk%ybReHp2I;1P9X$^;zdOp zb|DTsNyTNEmLyzZSq5h$E{Y2>9mk__PNpeYBI5$7eeFW#*-17oDXhXkT*pZRH$ed3 z(J>z8E*X!3#+uiS)YvIOiAu5r!U$3Lrbx%|s2I~N!b?n4AO(SqYR|ZlV!O!jWRzn> zT$U&jXBbApqdcf2#$;9!Vo{2f6zz-~@!Azp;F!3;;&hbMn?*CYq$mQ;Mp;f&L?RlG zky^V4nQP~HQH(MiiPJI#j5vYBqk<^l3PqA}iGieqtUd10^<;TUrh$>dMLEF9D!3>} z9L`b_#R#GlV|gO!MfO$xs3z3X;_|p#Il@&RuA}OGTzxp)PC0|F8pcO8G%@Y14l3dm zg5ij_JroSIIl>Mf%aHCc-R`Vm#H`Ew&kz;RMId$X+#2L%tonK@ByU9?8fO;-Im*#7 z7LSX1H)#%fBys}I#}rnSN%$xV2ajw;noHAn9AM!HnrB7L{v>i@LfT%`QHlb^X!%be zZ(g#56-k~U4}SF&@~C5D`WmuApa@=|^GdNAa=iK1k+G??1{8`# z?fK-1O`ZNcld=xvUKX8HK$LgBq}p=RDCp65`LDs_C zwyCArCgoc`hjgEFXjj49XO0!CC0Qo6v&SIa4jzqL**ouBlN8X5yuzpj>p)MV4ahm< zC%`OfVSv-L3(QIVl`X+F6n!X*k!(z-XMc%aXDp}gs8>9{m( zH-jFU2UV1*Rtt6)`hVyi)QRi^W?L;wfSGrcSx%xjmDYfAs2snUY3X2lKWc`aJu@Up$e?Dav5e`P!NWbY}k4UiWwYNh!*P zGmbX^<*7loPPN@&QZ~F{4?v$MywbTTVM}2%O!`6qt+7rhFRM;AqSm|_S(d35tP`7r z9!86i^}uX7{KI1L+f$W{O?0XgDvd3DXt!#+(X0gM?#`d}<^FpUGgCzjVAK*2VEpXa zxLP{H6uCL+TkyE@RN){Fx8q2yB3YAwpNTrxl&x2%Qc$>*CFnU^YN0NgA? zvC2jJpZCu-i|Oz_lwns*Mm{*A+RDv}20%suwK47&1&KMMYhaLojpu5=RX_+dvB|EM z&NhWGP`cybgsQu0&#EP}Od$;a{5}p&7%ME{B4sIThLN{)qGi@;+E?!;yU>iH!fOl? z%1mp38NL7<}62VLi&YiDun_-Z!&%LlA!ly_ZVE%J6^w|#gDnFMZ zZ9pOJ1kzcOSf$$7^T}Y47~a@)C-j-8pDju-CTZOpled5y3%+>Rqn6TX*mU!5--32p z`B{IiPu``E$g2;M>Qq_-%AsvwALP9Qlv=vTOlv@4wu6?7-o1UNTCy;WvJU1g2<1=s zyk~RdCX=!bWpD>j&fF9X_!p!o8_qbs11Qg&nCw$+|6o!!ym6xjeJ=Xryq$@v6gI;f zhc&d?N=8FN7v|s zL_Zu}`xyu!zVXpXwY1I@!Z5P;NpQk~PkwzuE%BK`fPe1Kx4!z|q4wJIurS@j4cMZW zZ%RImK5AtLBqAg&3n`v~A*a+D5o((xVwEx|23OWA{;rint&YmZia}oJ>yI@z^otFH zQn**r-hLLH!w&UTwi1r$a-TEN&3S@kh^h##4k*iOR(fd4G2l=52ZSiu5|Y=+D_a{D zSM>UVHN6_WADz#}+&!y_?il005G*Hu;yu3AT|YxXvGvyo7=DZ+L)^$vwA zIft{or@P)SFVnW|M{y?FAUYJTH9X|=cQ>wH<|DkF>jJ(&ux-fY47CrfZ6$mA{q+*z zXcXcRx69+?8#wLD{V2KddS99>vF2K)sbgS-XJuu@S1bKpgZ;{CJ0-zyO1rlW-VV5H zht|}(=t0Lod|gjXr?aAWsG(=256#m;3}m%(c8N$=IUGB%4yFZKxebd z$kt^w(NIfYsI#k$iZ=xMv3=g|7!%j69WX-_ijwva}b?g{e#{oroB!~KCd4b*EkBp=Eg9#)9_)K5Lvq? z&yR%8?~e0738)ugL~{MF{J|d_4ZzO`*h=%YAHN8jcEOR!AFH+&bJN0(ny>xOi|CU4 zxmzQzB$~}ls}H-OBQRQ~?Y6>E*p#+ucfp#kj-X0G=GNd;wWKi(n|^25aTJb1K7Ze# ztNpc4O}p>Cts}c zM^dDX@k<6KyHdFx{3Hl=c4{uG6h1EaT>~ z?cdQZD@!|1C4Z}vy80<7r!pES4!sGpFXO>U_o$^YGp&JwehUtPoL66igFkArHKy!4 zZ^6MovE|m>h-9+WXY9b+Fqg%m>i_mHHQDMj_W0Ytc0$>GGgMoL$<~;$_7iaM+m8PS zp1382&9KFh6X-%KLJU5ceCKM{>7G<2z*erE{2WffdHbrsOxBUARhyEAXcv5a*J-#;k4-JdFE z0KDTY6tgAE9`mnG6*J&{cosBRaB|UM)was4XaJ=D1m2qUrPr6}8(pKHf~Uc#um1_| zmVEBVe^pCYrU{{=bbkgykp2Jmh+4A36axHHu6^+fVZ&6$Gsm}o5~1x80gg3!1ksc+j{*&DYgcNd%p&@*_)@otlHL? zYyky~kFQ~TOsahShD0L8)^N$tH_(&Z!@mrurGsf~Pr{n7zJXnitow4IS~8G^O*hN- zZ{faK)%(k~T@Sv%k-`3RX8!t&J8^=1}B=WUr-m({v+!5urYF#kobye_<2(xfcR zdF9Ca+1MP7$i}SNsVuDQ%HM(2Q#2tPTXf|xq$t;(%f@VfH(XsvUiQLD$23LrX$GvPbFjHr3VxD|dnn}+%QH$nc6;}%G&Beqk{QuyYT+D-A8M${H_Q0&~ K^MZ2bhW`hjJm_Ko