From 5c18ffe67d664dcb8b7397d2759b9fc6ac4f6111 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Thu, 2 Oct 2014 10:46:20 -0400 Subject: [PATCH 01/10] Allow the namespace column to be null, and also non-unique. Fix the uncompressed size clobbering the size on the wire field. Add metadata constraints so that foreign key constraints get predictable names. Fix all downgrade migrations. --- ...4671_backfill_the_namespace_user_fields.py | 1 + ...0e7e6f_email_invites_for_joining_a_team.py | 3 -- ...b007d_allow_the_namespace_column_to_be_.py | 28 ++++++++++++++++++ data/model/legacy.py | 14 ++++----- data/model/sqlalchemybridge.py | 7 ++++- endpoints/registry.py | 8 +---- initdb.py | 5 ++-- test/data/test.db | Bin 630784 -> 626688 bytes util/gzipstream.py | 15 ++++++---- 9 files changed, 53 insertions(+), 28 deletions(-) create mode 100644 data/migrations/versions/9a1087b007d_allow_the_namespace_column_to_be_.py diff --git a/data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py b/data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py index cf0b90199..6f40f4fc0 100644 --- a/data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py +++ b/data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py @@ -22,4 +22,5 @@ def upgrade(tables): def downgrade(tables): + op.drop_constraint('fk_repository_namespace_user_id_user', table_name='repository', type_='foreignkey') op.drop_index('repository_namespace_user_id_name', table_name='repository') diff --git a/data/migrations/versions/51d04d0e7e6f_email_invites_for_joining_a_team.py b/data/migrations/versions/51d04d0e7e6f_email_invites_for_joining_a_team.py index 2e4242d8a..c18335adb 100644 --- a/data/migrations/versions/51d04d0e7e6f_email_invites_for_joining_a_team.py +++ b/data/migrations/versions/51d04d0e7e6f_email_invites_for_joining_a_team.py @@ -74,8 +74,5 @@ def downgrade(tables): .where(tables.notificationkind.c.name == op.inline_literal('org_team_invite'))) ) - op.drop_index('teammemberinvite_user_id', table_name='teammemberinvite') - op.drop_index('teammemberinvite_team_id', table_name='teammemberinvite') - op.drop_index('teammemberinvite_inviter_id', table_name='teammemberinvite') op.drop_table('teammemberinvite') ### end Alembic commands ### diff --git a/data/migrations/versions/9a1087b007d_allow_the_namespace_column_to_be_.py b/data/migrations/versions/9a1087b007d_allow_the_namespace_column_to_be_.py new file mode 100644 index 000000000..9b63ae190 --- /dev/null +++ b/data/migrations/versions/9a1087b007d_allow_the_namespace_column_to_be_.py @@ -0,0 +1,28 @@ +"""Allow the namespace column to be nullable. + +Revision ID: 9a1087b007d +Revises: 3f4fe1194671 +Create Date: 2014-10-01 16:11:21.277226 + +""" + +# revision identifiers, used by Alembic. +revision = '9a1087b007d' +down_revision = '3f4fe1194671' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + op.drop_index('repository_namespace_name', table_name='repository') + op.alter_column('repository', 'namespace', nullable=True, existing_type=sa.String(length=255), + server_default=sa.text('NULL')) + + +def downgrade(tables): + conn = op.get_bind() + conn.execute('update repository set namespace = (select username from user where user.id = repository.namespace_user_id) where namespace is NULL') + + op.create_index('repository_namespace_name', 'repository', ['namespace', 'name'], unique=True) + op.alter_column('repository', 'namespace', nullable=False, existing_type=sa.String(length=255)) diff --git a/data/model/legacy.py b/data/model/legacy.py index ba1fdd81f..87b4349fb 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -1223,8 +1223,7 @@ def get_storage_by_uuid(storage_uuid): return found -def set_image_size(docker_image_id, namespace_name, repository_name, - image_size): +def set_image_size(docker_image_id, namespace_name, repository_name, image_size, uncompressed_size): try: image = (Image .select(Image, ImageStorage) @@ -1233,18 +1232,15 @@ def set_image_size(docker_image_id, namespace_name, repository_name, .switch(Image) .join(ImageStorage, JOIN_LEFT_OUTER) .where(Repository.name == repository_name, Namespace.username == namespace_name, - Image.docker_image_id == docker_image_id) + Image.docker_image_id == docker_image_id) .get()) except Image.DoesNotExist: raise DataModelException('No image with specified id and repository') - if image.storage and image.storage.id: - image.storage.image_size = image_size - image.storage.save() - else: - image.image_size = image_size - image.save() + image.storage.image_size = image_size + image.storage.uncompressed_size = uncompressed_size + image.storage.save() return image diff --git a/data/model/sqlalchemybridge.py b/data/model/sqlalchemybridge.py index 46809fb21..8b7d8b664 100644 --- a/data/model/sqlalchemybridge.py +++ b/data/model/sqlalchemybridge.py @@ -17,7 +17,12 @@ OPTION_TRANSLATIONS = { def gen_sqlalchemy_metadata(peewee_model_list): - metadata = MetaData() + metadata = MetaData(naming_convention={ + "ix": 'ix_%(column_0_label)s', + "uq": "uq_%(table_name)s_%(column_0_name)s", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s" + }) for model in peewee_model_list: meta = model._meta diff --git a/endpoints/registry.py b/endpoints/registry.py index 14bd88ce0..5699f0db2 100644 --- a/endpoints/registry.py +++ b/endpoints/registry.py @@ -220,7 +220,7 @@ def put_image_layer(namespace, repository, image_id): image_size = tmp.tell() # Save the size of the image. - model.set_image_size(image_id, namespace, repository, image_size) + model.set_image_size(image_id, namespace, repository, image_size, uncompressed_size_info.size) tmp.seek(0) csums.append(checksums.compute_tarsum(tmp, json_data)) @@ -229,12 +229,6 @@ def put_image_layer(namespace, repository, image_id): logger.debug('put_image_layer: Error when computing tarsum ' '{0}'.format(e)) - # Write the uncompressed image size, if any. - if uncompressed_size_info['size'] > 0: - profile.debug('Storing uncompressed layer size: %s' % uncompressed_size_info['size']) - repo_image.storage.uncompressed_size = uncompressed_size_info['size'] - repo_image.storage.save() - if repo_image.storage.checksum is None: # We don't have a checksum stored yet, that's fine skipping the check. # Not removing the mark though, image is not downloadable yet. diff --git a/initdb.py b/initdb.py index 6be72f6ff..6fa8efe98 100644 --- a/initdb.py +++ b/initdb.py @@ -82,8 +82,9 @@ def __create_subtree(repo, structure, creator_username, parent): new_image = model.set_image_metadata(docker_image_id, repo.namespace_user.username, repo.name, str(creation_time), 'no comment', command, parent) - model.set_image_size(docker_image_id, repo.namespace_user.username, repo.name, - random.randrange(1, 1024 * 1024 * 1024)) + compressed_size = random.randrange(1, 1024 * 1024 * 1024) + model.set_image_size(docker_image_id, repo.namespace_user.username, repo.name, compressed_size, + int(compressed_size * 1.4)) # Populate the diff file diff_path = store.image_file_diffs_path(new_image.storage.uuid) diff --git a/test/data/test.db b/test/data/test.db index af85d23e2016a9254ab3ad3b01876942f8d50c4f..4be5d0b90e533ee3120dc04b47e6b0836a596007 100644 GIT binary patch delta 11514 zcmeI2d3+OP{>L*(CX=*HQbaMQ3>bl6T2nqta;sJ$4QP>56-!myRQPg#R{Qlg&G$ixP z^E~hG=XpNQ^UOTY#^0xGyf5V^i|5%zBGK*e|4V<@ZONqIMf`&$D7++7e;_1W{bb*e z8td(YUst=Hno%dTaks0_s!ypuQ_n=K58o3`g#9v1qj*knm3&IB#TmMK6%=otBnD@-X+W0 zSMmDA7jxlKDYzd~WK8<|+qLY6i>*};KEIRK#av9zfXQu21wOf8z^%zGx9o2)-A=bH znk{)LEcu3kD$VklS24SF;%(c)+3O;B{rLKf$@zoP+T75BO;l(ORs=2!uVLYnh~E45_ib&HDj9RCNe*9KuHH}>I^<3k>;x^z>o*8Axj zye=w)wOR%<#|?WmY0qqQEn5?#Z_@->Nk2aM{BW|ygspkGBFt&A%@6XP7%tF;KXiH` zvU;hNkDV7}omHoIq*X(0)t$B&E~Jb{24Mr*C`NO&OB&xFp-Xcf;`Qnf*c%4$$!kYF z+W5G-r_<9Svw58=2piBSWvy2uedP&>?8#J9n=;6H=h`aGyyT~U8hy@HZi|@XoK;6Z z7Oh?USWQyO^Zr}-*zh21K%KQ?EgJkT&Cm1R<&TYr1z7{yM6O$@RX@D&O#3wDb z$%3#@os3C*eYAG@ug+q1=U;l8k3~aZZy3m!+_=6~6Z?5;bn10%;<$7U7HZR_<^H~P zdri3a170r);k0 zu=j#|VH9*17=a(RHo=dd}`IQ9)hJxlgtFg{h-%w^S+c}H1*g%jpP7*j_NF?cHH2jaq z;uzA%P~c~0kmP2zi6PhmLoG)XHX18vL%q>jYpAzWH`F&Z+w090#g@`iYf&R*vM?pp z1$Np}TV7DnTutUT+SvL!Gx4qEQ?!X@fc`9!r~PKtBuh~o7(I(5PLyW(>$^(boUf59 z?I`dx(Z1sBGEbq`Q&?mrZCwRf1~!Xw8C~Y0VHeqGtu_>oby<6ThLwXIj^3_;ESIn6 zvf3#wi!hi7)-)nliWc!`nnEOpfBF3$huhPyH#qd9LEky-DOf`{R5UhsnC({1&vd%7 z2Rf?S3kw}H9P@9g zpbfNv1xLO@lCH81vIX7M!+xf*yR*+w*gxVoHPvMqyNmpdU2E#<`bP=cUO<<#nk1k!)B2rN16=a<8w$NLvsUN4utORL}WFO*!)aMeRD@UQ&QPjJ7)3Mk{-(0>-3ga_`&k?Nc`8j6=lj| zNroUv;5(1RG1-o;I!m25f5hUeDe)Aznv6_&;gF@Otjg>i>n$!d5Pnx>t$nO=w0jL> zZ1Oa6S$&=04>0RQXlI3`LYuG#W<;zM;UZos7s(Z{C%HYtgAQl6C&NA1?{oSFO$A)F zxzY(K+Pm)BtiEDB!TgJvY3)l!je_gSld-yR+n9t?H$dkcQ^Mm4YKWpgRY^< zjtNa9#bT{n zZ0BGE0qowhFmO6Ecp)I(tdEyo<83y4o}3MQKzZi zSNorum$*wcQG&v_5y_QyIO)Z5{!{rTzBzqKE*5m9%8AWFV2bY&?UH6m|8_W-`ib~# zWol7tHnMafQ=M<5X?wN3u%eN!DXOPUCDSm=iR>Dh_Rt1{%Yhphj{~QT zloMx-ZVs;INy^}04NiyG*zf7{EpxbgI(r8_1A~LUE>CaBXhoWqQRFfMTu#8;q;V=z zvR|$LA;oS>ln^qtL_H+B20mdVtGoqRG?|3(SZA->GYXzdmgInYk|gJ@hac1TBuV32 z^aMqES+~oDJ7|*!r&*%|cX&w>CuyUXbGTrdhXaQ;NQsHMj-kv%xY((34JHb(7gs$L zmxvSaFF7+YdC=GDb_{xwmn9}&em5hmk>>7R9~v()o0LUS1jDkt6j6xmh=ND+ynr_~idq&37#C3Zznm+zKk>P_jWdctTh(wvcV;FQZj!3JhbIO}5FIBagl;9@;4 zsINqvEtZQAy8;fLVFtI?ZJ@n0?s7W}I88d;IBTL^xX0rr2$P#2DU(wy$wmSh5D^mQ zg5@(Fio~5RcrWcTdTcB`ZO^}3R`ustueB;6!MCu*tHENmaFRETus`53Z zUzr&3`-mIh7}gcBk_pE|iUdSt9~kQM^pAA9Jz}rXX{4QQH%>Tc2B&GxggaagLg)mv z!(%eAPS&(K3=XYnh|La$x=|RT%R_Ut3);?Qgpbn>2ZM7ixK`&l$Y(H7oQrZ*DPWex zNTD4Z>_+vt!A+80&S`+P2+}y=W4OcU^5PWdAXvgd7~L+WK@Q>Kgj%>BG!yrFX_BIZ zSP2KTIpKyC&>YOdx}CJiZDLFg&s7-AvIue5!JR!QIbq}of+ILswucd7aZ)(zb#gdk zaMNCfbvca;S0saAaY%+;;qv*qg>gb`^YwTVg_#lD;8lcc)9pSPfcw(WM^3#K0Z_9d6=CBUFd+;w%eCE0dF? zOCC zu`TdVAku-(L0|vcEW+i0ce^OuWT0878p*-PAv)o8cu234@ESa1r7*cBAvtzgzoXaf z>j~%vUh<&}bxtqsGEr{WUP8F=cHBv_1nx0WCXz4`UY4e&`cPTowCq0gO!h;E&`3fF z%|B1<7m5Aiqmq8>AHO#)%N(J1j%h0_<_M8$|W&JJ$+D{2CU zzoHr7*k92V$PX(4?06HEgI^z&%>%7(qv7DOJtziBccW?G{kPC$aLaCVF^J!d>cQW3 zqlCN(i>`qLB?3VVSo1cj1ipP}7+A3vemt=oT>`q^MswuNA)qlJelIL?=i6u$Fux58 z3j9>EJz@|MgK=@4ydcMRW zLHr(c0eEaL8l`N(qB#z^&KFqr1IK=N{lmTJ0>mXqxDa&hMxasqoWiP1nuCRjX3-`+I%+_;o za)HANd`sbBNt|Zj=+7}Uoa3ENrp7%A)Z&1azFn;z9IjY$3S!G?pykQ`5{%{Z)!F>=V5f9jQ5KV`xIoRa;52AX>wV7UU-}^Ea>7MOr|M*yj3WZ{T zI08R@b4d30>5o5~k91MrPW_7O~k-{ygLc0t2T z%!I@u1=~KBXzOlF9@H3G`UA%0MPSlu>FWELHKEs`=Fa(89B3M>=Y8=9a#W=%R%~td@R^;6yCjb zW;XiNsS`3q<)z2xz*i`~|kP>W6qBd3{}Y7+n2`5K+FUT@iNqN{yZ?)bS`*_#O2fcnnJg#ABEg9DWqj zf5%NQe+*zB!*ahP9Tt)Xo_-9|es_BQJT{S>RWB4P29!OHrOr2)E=vr6J^DD7{%wh$ zJdWAs8;qA_f}mR9d;-h*w#4mEVBEJ{pFZ{krdj#4d|UjbcRpXojEI-yBbSeTm!b&& z{^YRlPgaB`PY5q&zh1dKH2v~qY%ZDLaKmK^%f6O;na1chl41OTx5Ae~lYTAzGPNb& zpB#LA5ZEruE$mZ?)u)7~mTkhOlxn?dT9YXMNG;pi2(N9KzK|_PB57e|vq1E7k-SHK zQZW(!c0^vp`hQ5K;Nne+yiQf~m$b{(UaI7B==WBQw7TumF z-h_eAzQ7Vd&KDSt+T|hCFPTNDQLZb6(Rp~>Ao27w+k39 z-DKe<;F%Lx9&*#{_`Ci~OanHY#|}v1c`@*x!~f;v9JW9*%!@$wNjTU1Xcl1fB$ka#fJaVZOOflru9MhGKx{|iz%ysD zUrP9d4PeU|j6tpg{)dqmu>B02hc2AKNU>)%X88 z#4tg^OZ?!D85si#XW*!Id`3nh{UGs}%mj7{7nUEBMSxAmWNC1yee{?t78#p<6()+s z?StU{FCoi1@amV?N@NIxoq{I+)0c4Wvz~%V2By+jA-$?!0Uj(6y56Rb-_0fzJ9p;hSRoa6PWxLW|~?l z4^KpHn~rFcx2lm_K(|kBMJA_z?URRBFKXn<8yc-8l@@cMrL4S!%0OfyguxZ!z?;0- z!aBlSYHzH_Hy2bC5+zI+>~doD9+7%Ry$3$!y{=A+lHkZU3lVE1xZ;~xr{0uH6LaI> zhN=1<)jw4=%D*Y;h=c#xUMuXqy~_V?p8j|9G-N+g*joPY=IOuLJk^&giyYzew;nGL zCXIN}Dv|mZ>QdDM%CD3gpb2k`NDbcH`sD`;K71^gszm{E?o={ zJS-KzBu+p^MQ22-w&X`c7v_XC0_o2Hzd;nO8np98HFnFqsPSdNU17Kr4Qyh?O0E9M z=N_h7bK`hD7u*^aHluN{ORVr{nR7R=)@>;~uVX_thud0U`k+MN)@Y>W%$R#`?Q1gy zcZcD2G?*_{=ryS|8Mm|k2Yfc;Y~sc944_Y{h}I@Vo`}uZ5c>rm%Y;zFO=<9=6jHBf z{PiyT2X{4$({t3qo^+R1_w(BK$m4@zUQY#yXLqDgMY}eyXTcMVug~1f>kJ{laLay+ zCL#0lnv4f+xou>SRZwX$23cj9hg0MauKbIQ2(k()t;Ha#I&B2eE>j|WY<7@UQ0Yd^ z;@21aHs#ZVx5l%AoB?^-<%(P_UG#UN6wi(1^>_%ZAkh1A2uu6S#ibp$PV&0UAgpkl z0bHX{cr{T!?z<-TWy#UDj3BTekUk6o8y`7oEx-QyowoEjU_l@pTob0yY9nU0spl_5 zEqrWRkT{^y?l7pX*!f(aq#^9C@zfwS)CdQC;gFj6S(m9O{;Q3=erX6bJoN!)!eO6C zxOQ)OX40S5@Vb-`YIrmTY>t3kBKs9-0S5Ia7Sj;vPk z`lJwEcs2%nse-&qpLuJZ>FyUc@VdkxZ{UCf*rtXuC$^qGjN6vJ+mZ zte2#UUlu1KW1_D_EnD&zLGw0+#2qki4Q#0FjC+#`6bD`zZw$sAFmEl4JL$nK4`#<) zh~ae&m*SSdQ%DVhd21D(sLt2-if)N>@VfetxZBnX-nGGoN;&WT&7|EaBRF8F}peRa?e^cywjN@wzJ-WmcM@NOf7U5H(un)FlMJ-n_g2s`WDCfKPo`xow9{KjwX zt)&-%CDO@&cbgUZ`RgCWMNfPdZnMns3i?3nHbd*`HfV>)9oO{nu_Zy^S?e}Kaie~> zB_~edo*6d>p=YhT3PLMizB^e{THxgM#UapwQb$%n=-3N)`%`aML+GLqXhEwVUj?B} z-Y4I$*!F{ztZgAT}K83badJJQPPClx^l?^FxRQ zr4F<};)TQ?_a?n}$T7Zhj(FC)t6?9et4z|;oTs{ZeO?H#pw!9L5IDvDyH@5G%WHUD zZV-5OoV%?R_F?0yuI%zf+f;2U=7Tm8w?2_j+g1cKcHyW7k+$=rf};FAL|dHzH@ClcwrDyr^`|+mhJAk zXS{EY8jwTK!o*9r-c?kxYmnFbg2Yhrbg*Yg(V@*+Yj_|(Qu8FQ>kR>}9#(9fFGV(I z-^)1KdV;8eNHHT2HDljrQ!CmNNLzOhRS;=(M6oDp{Ucoo$94rt1&yGAr5){M z-0D-~Yl4&kaonR&Tn)avJLa7~x_NzP2&y2@p;3ssAoJDngz$}HysjgN8W8B}wNTn6 z;Dm|3|LTox?Ll5a9^Dw^U3g|+OY%Se{41Mxj#rQeI(Q5^SQaFBzO?>gwVSjSK|`c@|9L-mYI6y% zbB0g{96SykoPA*MXrXjPX`5q?dUis{UJo6dd~#|las1C-TU!tt4tNDx)ic9qqwZdr z$;Y;a5Cnz_Y{MZ4=PJxmmR<8k7&&+g8mH&pNmUIvDFXlo8f?-*Q`X5I7)JXAN}l zvS&r_)p7_kwNW9BjF2k7nwy2;gYb+}$?MmBjVK*_eh+MTop#Vi#w$8_-B>7*(0sSu z3yHWt{VJmR@M}eFYeR|Bfx-`4KBe%bt<2V_+ijyEM8h!a8b2h8{pnrrr!M+g@Bad! C7}|RP delta 11555 zcmeI2d3YP;oyRrO%xL6mEF8zqVM})G#Mjt!A0d}~AF}03@e$|HF>)N=w`@q1fo@^b zmLwSR!nPqmfx<3yDW!H>*g}t|TXuUSB#?w$9BnDxmKIt-*@!ByYF4O``MNImRUC%42FM#f0tjk>@VWrMe5aD3|_*?S9Qbg zZw-x;=D{)e@0P%~|3KQdx&Mj0K%OVxBL9$Zcl^8Yt#L2Kt+%{s*+;xd)R~W%Mf^AT zE!c0cQuJpiWjbxzWjt+EkY|t_!%@Sy?(vFO$gSmvFU;7dQ{%gAaR#@6%UXpJ@H+*x zOGGxAvGk{}x-#bt29y8|6_!Hq$|WMrn(xituBB(hhs$Ug(Dv3lMB&WuhPIN+Pc{BN zxO~3QR%qCJfuLq?8D3|NANbyTntjze!b9@XrrF+8y0GkN2x)m-vsNV6@9@`PJH^NaBSDg5ou_qtLdQ1i_m zp2#6nzT3LZbzPJy-0T-7Qmy7EUVBbUx5iLint)VGAKDO1j!?k{Q!~#^uCq#q$7(e@ z83Uy^yDHdijo&){s%A@yfeJVL&0v6B`>f-wuEYowY&JD>$5b|%d)Ifix)P#H;ijLT zO0^d5di$uB9v{PGpWZ_jzuJ`=jEgYMH~M$e8?5WVsZE;Q5`~&?_VydPtuR)gEC$N4 za}Swt-<{tGqEV=Dqb<8OSmSQVtCO&kZ9|o#qM}kMcQll;?bU6P zgBMuFS+}v(Q6_aXadLY{O>0NBv%Z>R>t*;ZaZO?=SIsw;x?3Arn&By0p;+F|GOGn~ zHOm$Yf=mkxc=&xJyHj*GvW<r+2kH7oEO2Z9$6d#I(SymwF%6}4UV z?PzQ6aZP#K$BRoGy%o;-DVi-&dKqbKV8FTECzrV;U#HYv+gmj-$ZitJL9_yDLR^g|ol7Q66p{9~urfz##J}HM6oJFdP{80I_$K(w?T# zvH&|i*;832m$#HPb5kYi#Nb4*vvhjIJuo#g-XYfc%Vl4Ehddgr9cUR2z$%1Ks)A99 z6`5ms@cR#ttS-Oe^~rr3D<|dNx^XApyS=5ee{6hQuB`S40-mbr_T~yFJ2m1iQ36fw z?xyzop6#u}1B3jh)F8?Nqk!%Yk(HfvYgvU-&Q?`9Y1&!mW}2Gal29tR8(bS}>+0%S z>lzzqcXMZHb%)g2R#{b6*U?zr-dXpovedx zW&QFLCrVNYGv2AXI-82U6K-c~X)hP>PB;0R`Rz@kjTOa}BbA*~lWkzlMI`fzu!3GW znNbu43|>Ssgo%<)Un?JI92HCJipMrKR8RLZ!zx?VOZPRk?`UtLTV$`lajdOk%u}M2 z);T9P)|R({AHt%c*kFZTX-VJ|Wzv{vpbT0fVIV9plKkq#n8(|v7W&6V2EBu0bshDt z+Ln4ncJQ=V*4F6WNY~Z6nwT=Nu3l+!ch)yL+Fa$eLY<3qmvT}?XG^(D6x(ZEOl32? z)e$2SIiI0=&7#cl0_eMh*o0C?S?PH5WP3@Kz?F-G1I4xes?y?~?#Y&t@ur?;|A4Qi zyL_a+y0x)m+&Sg0aW?SNDtPu1Qh23Gazio!ik&OlmVhsm7*|w$-VQRRnFNcj=b~BKc)a*? zi*>`!I+NLeAX=g|-t_lt7P@l$h2D${-Z0bWl@2;X7MvMPAER6TXO8T-i(8HHcJnZq zPkt}ym2X?^N0LHS7;HCU`n&;$F8KL-}X&9xmoTQ6{KuC#-7OVwOUv*6y?;i#E|MgCmUBncia;AbR?_DKrG zdj*M-8C9SHqQEO~R;ML4Frp3(uJ-u*dj`hT(Xp|?UUeWcS_yWVUd?l>X?;1tG9o>j zY&uQmd=|F{>?Ych^i}dM!x!LE2i#{jeF%o~O(-~&ZSqII-U7eQ9m+;Cx^j4r8emkJ zqh!D6rFhxTQC<)2r5MKNR|A5=(QvSQc^68D+Ejm$9jv(>GIt>BhBUQj5Kd5Rv3{{1Pw(j1~+2SFb5%swCdr#9HlH3 zd>q9qUWxLGT!4}wG|Tg%-=})PBO$bO(fkY(8dV7o+CWqzP%?%?ygFZ>t zf0lkQt}?kTQAIvpklDj%kPI<`-|yuD0wwzWD#fcFg;E5MkMenC7OtlxmSwyqV~LS4 zAZ`mBHRDvC5d$;>!@$q_C|+bm%FFsxiW4NS&o98WgwJC%l_22)hyfN*Jt`vz3>63n zJjHup4KfQC#XdFQ^O_ae?^OcQ+{JS!u{%B6Kt4!rOnM{f;Uq`mpA-KvaaF?46TAu8 zaPUGCTGquAh75}hF}N=;0!GJ&)RD;^ziNaj6JQkRJ3zyZhat*PiX<|WUr-pDVFj<( z!)}iw3^pVmX>ohUrv~~wBi$-^t`Fm+0Pj~6FU?Ub=MiE2NGeQ5kDrqH07FZ>pm=y+ zhea>E3@M8#oZpY-h)_OR@bP}iBluuppCrOG@h}u82B4<^&GY`iHbN)QfHhwx2h;sn z5e!U8_NuU~#4-W}7cf4`>!WpakJqQjj3A4=YSzmxK|Im2NBc2GW)&|l$q-fbGalFx zUWM`sF!FsIT;nmc>=!($3nvU&NCr~q9vmMVg*pw6bOUApTfz$r&--YVreOBM@dvKi zDaFUru=5o~daX&J=G1>_6&@Tdj^cGBFjEj^-w+-UKG!Iy_AO+1j@tsS)TE*0iTEO zHR%qPA-$1C!>5g*J~e0rX@givh!Opa0u!4Gz(*;H=UJXo6h1)FEXODkBl@7S^)P64 z;h2bz(H`B0u_gRr*N4~9p0UA^sbXH>y|5o(0QfoRj@5;rWjHv)z!6l*$HV^ZfFYfQ zWFgnPM?A!Uf3SaiR2>1|8^m%}h+;rwc#+})dR|D1K*^%4P=ZJDi*R(7MTMCi!f<;! zY9N=9GTA`>fP9GblSfIE43hiE*I|7BHYv0i&9EVt10)jK@yk(LJ`#yo&%of=c69`H zq|P6girSV%jSE*Tsc1$<#FKZtr_bLr&`qSE8A~EpN4I-MRKGb5&B{RLgCnFzhi`DO zPj`%`qD!)n`F@{&*RNt}Xhv4VMYK=RMQFyd$n}M;GF@5Uy!Pp6R+eSNb6LOn|H3UX z%o+D#7faYgII1o(TrwCgAyYM6jp?6 z1J;xHVzB-+tY+~Uc-49O3bf?YSTd-{CCE5!xyPJVuOQlbaPKKdw&o0GIdBGB44%}B z!EYkrHK*0V8?K+f0U9%r?I7zYmJa@O3VMC%4D1^pAkSjii9Kb_6^_;ld#j_=Spl9c zBo=|GpCR_Sk+WDX0-5hRhpj?(%)DYs1uvY#3XM0aAoCbbgP$J5mxCS0a0Us0uN}h+ zkUro#hA#uJ9>a4UdKODQ@T?h~Ltn=3J=ED)+fY%lk#kmYE>0G!7@?Fatra?}BsW95 z%UOpYRf%#7B~5NgH{ap&MqyD$_p{bpy)_xC|~H#|iM=!)A z8ws8}i>D$(;M`e!HF#9FUpR}Cpz0i6gbeGYmV^N3ZD_k129A=43T4AkWlV&<4R4Gy)p-VpvxqHnNU9T*## zy8L;=+@X9zGS0q?TN)OA3|Lrt!c{w*@nr>M7MqV>Wr9&5F(9twS|*WJ_KxLpbwf+ z@SCrhb3W!KSbiyx51Q9~j2$}32TvX}r+&QMVWlrW1jJ%31-18>^R8o*tBhf=NAEEg zT+8^od(17@G4fS~AZRM^9x@kS%Xt4GvvQ5=bALQ!PT3&p6ITjqHDHHu0-wa+Fni2D zByuf7mj8%b6?a=aozR=`>%{&flH7#;7PXp+j9)h*hVQ_14WTzxYbbann^+HaXA{I+ z-U#Zqq!Le*5wlE2m;!22^yfe zLumnbx925>kRhVrf9v;xU1i9A$m(#+tScA^E@t5;J3j@gNng~=%!7VOR+ZzZ&f6_jIdU zN8$Poeu95rMP!2NJR;Yq?gIa^ipWL22#%~G)|)h22*j;|dfoztzkw_TovR2JdXsdu zSnrx9w2x?-k7}6qnYv>!s%2`|?$8a_=zSw%%F-~fCYvY+hqDL*+?q|eXJ0lS$Xxi1 zbYU2|oCpb-CI@o$jCI)@|HfawrnC^j4TwNljNuCJOne6Jv^rvV64Y?6U-FFSWsaUh;m@ zkhD4RwZ!U#cmMuvVmKa^{QqXKzB&E>o5BCbHiN;jKM^_p_g!&Gv|KM^m-9gK6r6Bkb65u(I=hq(-%xG2N!#a5zULRZ5R7pslHn1S4cFtBhd zbaLe@a^eEOJ?Y$zG0({8cz+>$5@+GYu$Sfo@W%#5@p<9mU*g#S!!WYom;JKpV*>t9 zvv;k%VrLtKT$UVi#n`6vt~HUZ_Jso+E=!tKnRz-{OTRw4*{*NtQ{c`&6Pro?&3Z#{ zb#%*pZ=Hn#>|4Y-YteG<4$ZE_?772Deen2OM7P!U7W=4XlcO8&`o2DBdz%Q5#lxEh zx}+!`JkbCyyiMegC1LT3(ZdPxkLuE7iEAsX@aeW}6X2wGTyk#~s=)`XcYbDEus zA=5Q@`&~$;?92OsW-E#z3#)MV-ym7ebwo;6VT3HK!P@traJECQx?BZOy08ZKy$9(G zPnZ8fOV5v?(=|B%9;90-e*Z!+FG4r3z|8v)Z|R!2A8YniQM~Z^4v=yI@-1&CvNYSu z7(RHwV?U|fz4FGc+$b76$^*820M87qw%>cel@mdO8c^Uz9}tU^zj6W5(pSWQ=@O)U z2*Jz`=HvylqhMhHp8ODkT`x_+Gagw{F#Tu?5HAv{HT&cZH)=L}3>QqQ{b-(e#{*AV z@C}{+yV)yAD%>;<2myE~z{Ak+1VliP6qcn0zt77_@b$^|u1!%X^$rUS!t6yJdwZR$ zGa{wlVH9|6(2{Ch_Rz!$ExjWu=6r|UxxuoPG@N+qrC@th&Aq*_6z{OCv#x*Vm`$^{ z#i*%A?h_76x7GY#=_#6RV^qy>?6#L$0;HU9=GR^B2p%|6YH^W^geJw+8byO$2K&DZ zqOG0Qs<2`ORtI{3ukQ&BwPLTtt*0+5weA>t%YcK#yd}H z_KFy^@DQF_2hnWb|N3K^tvm(|4q*GqB9+G7|58e%Kh?32xZ9ODQZ^(XFv#*bmg*D(CV0Vg7DG8cwT?`l; zyY{1L-&j5}+i1a)Hy!(*&bl`vtl=0uk9gJV6Q^B~p?xa+jUVpPcop8y_-V-x_q=2e zuZjUqWk1c{H5FCUK3xEI?X~2R@jtw!#ubdHsrOj{czLfS-O3%VSf-^-UY64Y&rPZ@ zobKFa*_<@6wk9|cl@kuL1)#)ZaaffXMoww=@fbNTO%#BC^;ouB%^NnosoBP&a>AY~ z02{rQZc^GMe6edZO6S;F0N(dPI{db-CtM>@I@oUopvwp8)_zcUKuaIKOa~XiXGaUb zw|$mv=r-F-K=}?PR%|TL#OL-*bnLQs!Ap_+d!0VUWrN7vOQ_b z*&lcHN9c4JT;z2VtA6C_i;{&U*cq@aNml3Jpwb%yrYmqP0Ku+%`iUx}G+Yftt2b7K8`@}CbyDy5jFz5EdBq?bO z!O_?o!v_bk{iLN}mAlImMGI$dpFUCi`vJRaTLf)Bd;4JarUf6vwe+n~viZ?#Nxx+? zneyD7{}S91MS~JhV76bMCmO}aH2dZ#+I;#BK(e*>f8`F%))hkr2eJL=>Vw~%m>sg9 t$%HZKsc-_r$Jc*NO4RH(T;ZhvaJNDIx%@xDxqCX+%Wcrhy1!a7{~I@QlfVD~ diff --git a/util/gzipstream.py b/util/gzipstream.py index 7dbb2fc3e..8f5aa9f6d 100644 --- a/util/gzipstream.py +++ b/util/gzipstream.py @@ -9,17 +9,20 @@ import zlib # http://stackoverflow.com/questions/3122145/zlib-error-error-3-while-decompressing-incorrect-header-check/22310760#22310760 ZLIB_GZIP_WINDOW = zlib.MAX_WBITS | 32 +class SizeInfo(object): + def __init__(self): + self.size = 0 + def calculate_size_handler(): """ Returns an object and a SocketReader handler. The handler will gunzip the data it receives, adding the size found to the object. """ - uncompressed_size_info = { - 'size': 0 - } - + + size_info = SizeInfo() + decompressor = zlib.decompressobj(ZLIB_GZIP_WINDOW) def fn(buf): - uncompressed_size_info['size'] += len(decompressor.decompress(buf)) + size_info.size += len(decompressor.decompress(buf)) - return uncompressed_size_info, fn + return size_info, fn From c6828998616894d49591650820bd205ad04291ba Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 2 Oct 2014 14:49:18 -0400 Subject: [PATCH 02/10] Add a feature flag to disable user creation --- config.py | 3 +++ endpoints/api/user.py | 1 + endpoints/callbacks.py | 12 +++++++++--- endpoints/index.py | 4 ++++ static/directives/signup-form.html | 2 +- static/directives/user-setup.html | 2 +- templates/ologinerror.html | 4 +++- 7 files changed, 22 insertions(+), 6 deletions(-) diff --git a/config.py b/config.py index 4fe5b2cd5..6742d1a43 100644 --- a/config.py +++ b/config.py @@ -165,6 +165,9 @@ class DefaultConfig(object): # Feature Flag: Whether emails are enabled. FEATURE_MAILING = True + # Feature Flag: Whether users can be created (by non-super users). + FEATURE_USER_CREATION = True + DISTRIBUTED_STORAGE_CONFIG = { 'local_eu': ['LocalStorage', {'storage_path': 'test/data/registry/eu'}], 'local_us': ['LocalStorage', {'storage_path': 'test/data/registry/us'}], diff --git a/endpoints/api/user.py b/endpoints/api/user.py index 7747addcc..130bb9124 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -195,6 +195,7 @@ class User(ApiResource): return user_view(user) + @show_if(features.USER_CREATION) @nickname('createNewUser') @parse_args @query_param('inviteCode', 'Invitation code given for creating the user.', type=str, diff --git a/endpoints/callbacks.py b/endpoints/callbacks.py index 1cbd46192..637033ab6 100644 --- a/endpoints/callbacks.py +++ b/endpoints/callbacks.py @@ -26,7 +26,8 @@ def render_ologin_error(service_name, error_message='Could not load user data. The token may have expired.'): return render_page_template('ologinerror.html', service_name=service_name, error_message=error_message, - service_url=get_app_url()) + service_url=get_app_url(), + user_creation=features.USER_CREATION) def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_encode=False, redirect_suffix=''): @@ -85,7 +86,12 @@ def get_google_user(token): def conduct_oauth_login(service_name, user_id, username, email, metadata={}): to_login = model.verify_federated_login(service_name.lower(), user_id) if not to_login: - # try to create the user + # See if we can create a new user. + if not features.USER_CREATION: + error_message = 'User creation is disabled. Please contact your administrator' + return render_ologin_error(service_name, error_message) + + # Try to create the user try: valid = next(generate_valid_usernames(username)) to_login = model.create_federated_user(valid, email, service_name.lower(), @@ -147,7 +153,7 @@ def github_oauth_callback(): token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB') user_data = get_github_user(token) - if not user_data: + if not user_data or not 'login' in user_data: return render_ologin_error('GitHub') username = user_data['login'] diff --git a/endpoints/index.py b/endpoints/index.py index 46c5b9771..5f0a88695 100644 --- a/endpoints/index.py +++ b/endpoints/index.py @@ -19,6 +19,7 @@ from auth.permissions import (ModifyRepositoryPermission, UserAdminPermission, from util.http import abort from endpoints.notificationhelper import spawn_notification +import features logger = logging.getLogger(__name__) profile = logging.getLogger('application.profiler') @@ -65,6 +66,9 @@ def generate_headers(role='read'): @index.route('/users', methods=['POST']) @index.route('/users/', methods=['POST']) def create_user(): + if not features.USER_CREATION: + abort(400, 'User creation is disabled. Please speak to your administrator.') + user_data = request.get_json() if not 'username' in user_data: abort(400, 'Missing username') diff --git a/static/directives/signup-form.html b/static/directives/signup-form.html index ba4efe287..9117e880a 100644 --- a/static/directives/signup-form.html +++ b/static/directives/signup-form.html @@ -1,4 +1,4 @@ - -
+
diff --git a/templates/ologinerror.html b/templates/ologinerror.html index 304b7f554..74f51987e 100644 --- a/templates/ologinerror.html +++ b/templates/ologinerror.html @@ -8,17 +8,19 @@
-

There was an error logging in with {{ service_name }}.

+

There was an error logging in with {{ service_name }}.

{% if error_message %}
{{ error_message }}
{% endif %} + {% if user_creation %}
Please register using the registration form to continue. You will be able to connect your account to your Quay.io account in the user settings.
+ {% endif %}
From a1470460a76e724fce901d14d197a98fb133fe6f Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 2 Oct 2014 16:04:23 -0400 Subject: [PATCH 03/10] Move the /static handler into the base and have nginx serve the Docker ping endpoint --- conf/nginx-nossl.conf | 5 ----- conf/nginx.conf | 5 ----- conf/server-base.conf | 10 ++++++++++ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/conf/nginx-nossl.conf b/conf/nginx-nossl.conf index 73a9c7605..fbcce63c0 100644 --- a/conf/nginx-nossl.conf +++ b/conf/nginx-nossl.conf @@ -13,10 +13,5 @@ http { include server-base.conf; listen 80 default; - - location /static/ { - # checks for static file, if not found proxy to app - alias /static/; - } } } diff --git a/conf/nginx.conf b/conf/nginx.conf index 43c21b6ca..e208d30e0 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -23,10 +23,5 @@ http { ssl_protocols SSLv3 TLSv1; ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; ssl_prefer_server_ciphers on; - - location /static/ { - # checks for static file, if not found proxy to app - alias /static/; - } } } diff --git a/conf/server-base.conf b/conf/server-base.conf index a13cf1424..dc38992bc 100644 --- a/conf/server-base.conf +++ b/conf/server-base.conf @@ -24,4 +24,14 @@ location / { proxy_pass http://app_server; proxy_read_timeout 2000; proxy_temp_path /var/log/nginx/proxy_temp 1 2; +} + +location /static/ { + # checks for static file, if not found proxy to app + alias /static/; +} + +location /v1/_ping { + add_header Content-Type text/plain; + return 200 'okay'; } \ No newline at end of file From e654b2b6080b6d946f97c0937983800524f8b5b8 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 3 Oct 2014 12:35:58 -0400 Subject: [PATCH 04/10] Branch.name has to use a dot accessor, not a dict lookup --- endpoints/trigger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/endpoints/trigger.py b/endpoints/trigger.py index 4a10485ae..c7c47db79 100644 --- a/endpoints/trigger.py +++ b/endpoints/trigger.py @@ -377,7 +377,7 @@ class GithubBuildTrigger(BuildTrigger): gh_client = self._get_client(auth_token) source = config['build_source'] repo = gh_client.get_repo(source) - branches = [branch['name'] for branch in repo.get_branches()] + branches = [branch.name for branch in repo.get_branches()] if not repo.default_branch in branches: branches.insert(0, repo.default_branch) From ec42303750c5edfa2cef1631443566f2a99e0804 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 3 Oct 2014 13:00:41 -0400 Subject: [PATCH 05/10] image_and_tag must be before we use it --- workers/dockerfilebuild.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index a45d82b67..60b29a109 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -223,6 +223,13 @@ class DockerfileBuildContext(object): raise RuntimeError(message) def pull(self): + image_and_tag_tuple = self._parsed_dockerfile.get_image_and_tag() + if image_and_tag_tuple is None or image_and_tag_tuple[0] is None: + self._build_logger('Missing FROM command in Dockerfile', build_logs.ERROR) + raise JobException('Missing FROM command in Dockerfile') + + image_and_tag = ':'.join(image_and_tag_tuple) + # Login with the specified credentials (if any). if self._pull_credentials: logger.debug('Logging in with pull credentials: %s@%s', @@ -238,13 +245,6 @@ class DockerfileBuildContext(object): registry=self._pull_credentials['registry'], reauth=True) # Pull the image, in case it was updated since the last build - image_and_tag_tuple = self._parsed_dockerfile.get_image_and_tag() - if image_and_tag_tuple is None or image_and_tag_tuple[0] is None: - self._build_logger('Missing FROM command in Dockerfile', build_logs.ERROR) - raise JobException('Missing FROM command in Dockerfile') - - image_and_tag = ':'.join(image_and_tag_tuple) - self._build_logger('Pulling base image: %s' % image_and_tag, log_data = { 'phasestep': 'pull', 'repo_url': image_and_tag From e0993b26af4f59ef7e7442f5cb801116132be1a8 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 3 Oct 2014 15:05:34 -0400 Subject: [PATCH 06/10] Make query params only read from query params, not JSON as well --- endpoints/api/__init__.py | 6 ++++-- endpoints/api/user.py | 11 ++++++----- static/js/app.js | 2 +- test/test_api_usage.py | 4 ++-- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/endpoints/api/__init__.py b/endpoints/api/__init__.py index 2f5e2045e..1943051e0 100644 --- a/endpoints/api/__init__.py +++ b/endpoints/api/__init__.py @@ -27,8 +27,8 @@ api_bp = Blueprint('api', __name__) api = Api() api.init_app(api_bp) api.decorators = [csrf_protect, - process_oauth, - crossdomain(origin='*', headers=['Authorization', 'Content-Type'])] + crossdomain(origin='*', headers=['Authorization', 'Content-Type']), + process_oauth] class ApiException(Exception): @@ -90,6 +90,7 @@ def handle_api_error(error): if error.error_type is not None: response.headers['WWW-Authenticate'] = ('Bearer error="%s" error_description="%s"' % (error.error_type, error.error_description)) + return response @@ -191,6 +192,7 @@ def query_param(name, help_str, type=reqparse.text_type, default=None, 'default': default, 'choices': choices, 'required': required, + 'location': ('args') }) return func return add_param diff --git a/endpoints/api/user.py b/endpoints/api/user.py index 130bb9124..3fd59d86e 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -120,6 +120,10 @@ class User(ApiResource): 'type': 'string', 'description': 'The user\'s email address', }, + 'invite_code': { + 'type': 'string', + 'description': 'The optional invite code' + } } }, 'UpdateUser': { @@ -197,15 +201,12 @@ class User(ApiResource): @show_if(features.USER_CREATION) @nickname('createNewUser') - @parse_args - @query_param('inviteCode', 'Invitation code given for creating the user.', type=str, - default='') @internal_only @validate_json_request('NewUser') - def post(self, args): + def post(self): """ Create a new user. """ user_data = request.get_json() - invite_code = args['inviteCode'] + invite_code = user_data.get('invite_code', '') existing_user = model.get_user(user_data['username']) if existing_user: diff --git a/static/js/app.js b/static/js/app.js index 27b086bab..823de0ad0 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -2813,7 +2813,7 @@ quayApp.directive('signupForm', function () { $scope.registering = true; if ($scope.inviteCode) { - $scope.newUser['inviteCode'] = $scope.inviteCode; + $scope.newUser['invite_code'] = $scope.inviteCode; } ApiService.createNewUser($scope.newUser).then(function(resp) { diff --git a/test/test_api_usage.py b/test/test_api_usage.py index 2d73008f0..05fe525be 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -173,7 +173,7 @@ class ApiTestCase(unittest.TestCase): if memberData['name'] == membername: return - self.fail(membername + ' not found in team: ' + json.dumps(data)) + self.fail(membername + ' not found in team: ' + py_json.dumps(data)) def login(self, username, password='password'): return self.postJsonResponse(Signin, data=dict(username=username, password=password)) @@ -405,7 +405,7 @@ class TestCreateNewUser(ApiTestCase): invite = model.add_or_invite_to_team(inviter, team, None, 'foo@example.com') details = { - 'inviteCode': invite.invite_token + 'invite_code': invite.invite_token } details.update(NEW_USER_DETAILS); From 45208983bf3d3e62be33e7f35a90f95405790d18 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 3 Oct 2014 15:07:50 -0400 Subject: [PATCH 07/10] Update the backfill script to always read the size from the layer data --- tools/uncompressedsize.py | 106 +++++++++++--------------------------- 1 file changed, 30 insertions(+), 76 deletions(-) diff --git a/tools/uncompressedsize.py b/tools/uncompressedsize.py index 56a22756b..2e8955761 100644 --- a/tools/uncompressedsize.py +++ b/tools/uncompressedsize.py @@ -1,92 +1,47 @@ import json import logging +import zlib from data import model from data.database import ImageStorage from app import app, storage as store from data.database import db -from gzip import GzipFile -from tempfile import SpooledTemporaryFile +from util.gzipstream import ZLIB_GZIP_WINDOW logger = logging.getLogger(__name__) - -def backfill_sizes_from_json(): - query = (ImageStorage - .select() - .where(ImageStorage.uncompressed_size == None, ImageStorage.uploading == False) - .limit(100)) - - total = 0 - missing = 0 - batch_processed = 1 - - while batch_processed > 0: - batch_processed = 0 - with app.config['DB_TRANSACTION_FACTORY'](db): - for image_storage in query.clone(): - total += 1 - batch_processed += 1 - - if (total - 1) % 100 == 0: - logger.debug('Storing entry: %s', total) - - # Lookup the JSON for the image. - uuid = image_storage.uuid - with_locations = model.get_storage_by_uuid(uuid) - - try: - json_string = store.get_content(with_locations.locations, store.image_json_path(uuid)) - json_data = json.loads(json_string) - size = json_data.get('Size', json_data.get('size', -1)) - except IOError: - logger.debug('Image storage with no json %s', uuid) - size = -1 - - if size == -1: - missing += 1 - - logger.debug('Missing entry %s (%s/%s)', uuid, missing, total) - - image_storage.uncompressed_size = size - image_storage.save() - - def backfill_sizes_from_data(): - storage_ids = list(ImageStorage - .select(ImageStorage.uuid) - .where(ImageStorage.uncompressed_size == -1, ImageStorage.uploading == False)) - - counter = 0 - for uuid in [s.uuid for s in storage_ids]: - counter += 1 - - # Load the storage with locations. - logger.debug('Loading entry: %s (%s/%s)', uuid, counter, len(storage_ids)) - with_locations = model.get_storage_by_uuid(uuid) - layer_size = -2 - - # Read the layer from backing storage and calculate the uncompressed size. + while True: + # Load the record from the DB. try: - logger.debug('Loading data: %s (%s bytes)', uuid, with_locations.image_size) - CHUNK_SIZE = 512 * 1024 - with SpooledTemporaryFile(CHUNK_SIZE) as tarball: - layer_data = store.get_content(with_locations.locations, store.image_layer_path(uuid)) - tarball.write(layer_data) - tarball.seek(0) + record = (ImageStorage + .select(ImageStorage.uuid) + .where(ImageStorage.uncompressed_size == None, ImageStorage.uploading == False) + .get()) + except ImageStorage.DoesNotExist: + # We're done! + return - with GzipFile(fileobj=tarball, mode='rb') as gzip_file: - gzip_file.read() - layer_size = gzip_file.size + uuid = record.uuid + + # Read the layer from backing storage and calculate the uncompressed size. + logger.debug('Loading data: %s (%s bytes)', uuid, with_locations.image_size) + decompressor = zlib.decompressobj(ZLIB_GZIP_WINDOW) + stream = store.read_stream(with_locations.locations, store.image_layer_path(uuid)) + + uncompressed_size = 0 + CHUNK_SIZE = 512 * 1024 * 1024 + while True: + current_data = stream.read(CHUNK_SIZE) + if len(current_data) == 0: + break + + uncompressed_size += len(decompressor.decompress(current_data)) - except Exception as ex: - logger.debug('Could not gunzip entry: %s. Reason: %s', uuid, ex) - continue - # Write the size to the image storage. We do so under a transaction AFTER checking to # make sure the image storage still exists and has not changed. - logger.debug('Writing entry: %s. Size: %s', uuid, layer_size) + logger.debug('Writing entry: %s. Size: %s', uuid, uncompressed_size) with app.config['DB_TRANSACTION_FACTORY'](db): try: current_record = model.get_storage_by_uuid(uuid) @@ -94,14 +49,13 @@ def backfill_sizes_from_data(): # Record no longer exists. continue - if not current_record.uploading and current_record.uncompressed_size == -1: - current_record.uncompressed_size = layer_size - current_record.save() + if not current_record.uploading and current_record.uncompressed_size == None: + current_record.uncompressed_size = uncompressed_size + #current_record.save() if __name__ == "__main__": logging.basicConfig(level=logging.DEBUG) logging.getLogger('boto').setLevel(logging.CRITICAL) - backfill_sizes_from_json() backfill_sizes_from_data() From d0b93146dee1cbf85f3e63885333a9e0a638876d Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Fri, 3 Oct 2014 16:14:24 -0400 Subject: [PATCH 08/10] Fixes for the uncompressedsize backfill script. --- tools/uncompressedsize.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/tools/uncompressedsize.py b/tools/uncompressedsize.py index 2e8955761..1b6c0e468 100644 --- a/tools/uncompressedsize.py +++ b/tools/uncompressedsize.py @@ -1,4 +1,3 @@ -import json import logging import zlib @@ -11,47 +10,52 @@ from util.gzipstream import ZLIB_GZIP_WINDOW logger = logging.getLogger(__name__) + +CHUNK_SIZE = 512 * 1024 * 1024 + + def backfill_sizes_from_data(): while True: # Load the record from the DB. try: record = (ImageStorage - .select(ImageStorage.uuid) - .where(ImageStorage.uncompressed_size == None, ImageStorage.uploading == False) - .get()) + .select(ImageStorage.uuid) + .where(ImageStorage.uncompressed_size >> None, ImageStorage.uploading == False) + .get()) except ImageStorage.DoesNotExist: # We're done! return uuid = record.uuid + with_locations = model.get_storage_by_uuid(uuid) + # Read the layer from backing storage and calculate the uncompressed size. logger.debug('Loading data: %s (%s bytes)', uuid, with_locations.image_size) decompressor = zlib.decompressobj(ZLIB_GZIP_WINDOW) - stream = store.read_stream(with_locations.locations, store.image_layer_path(uuid)) uncompressed_size = 0 - CHUNK_SIZE = 512 * 1024 * 1024 - while True: - current_data = stream.read(CHUNK_SIZE) - if len(current_data) == 0: - break + with store.stream_read_file(with_locations.locations, store.image_layer_path(uuid)) as stream: + while True: + current_data = stream.read(CHUNK_SIZE) + if len(current_data) == 0: + break - uncompressed_size += len(decompressor.decompress(current_data)) + uncompressed_size += len(decompressor.decompress(current_data)) # Write the size to the image storage. We do so under a transaction AFTER checking to # make sure the image storage still exists and has not changed. logger.debug('Writing entry: %s. Size: %s', uuid, uncompressed_size) with app.config['DB_TRANSACTION_FACTORY'](db): - try: + try: current_record = model.get_storage_by_uuid(uuid) - except: - # Record no longer exists. + except model.InvalidImageException: + logger.warning('Storage with uuid no longer exists: %s', uuid) continue if not current_record.uploading and current_record.uncompressed_size == None: current_record.uncompressed_size = uncompressed_size - #current_record.save() + current_record.save() if __name__ == "__main__": From dd6f31cba4330f468e5db40d1abbc191a02b8088 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Fri, 3 Oct 2014 16:41:16 -0400 Subject: [PATCH 09/10] Fix the docker registry headers for _ping. --- conf/server-base.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/server-base.conf b/conf/server-base.conf index dc38992bc..4636afdde 100644 --- a/conf/server-base.conf +++ b/conf/server-base.conf @@ -33,5 +33,7 @@ location /static/ { location /v1/_ping { add_header Content-Type text/plain; + add_header X-Docker-Registry-Version 0.6.0; + add_header X-Docker-Registry-Standalone 0; return 200 'okay'; } \ No newline at end of file From 6bc982b77b0eb44817d2909a2e97d251a69a0f64 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Mon, 6 Oct 2014 10:24:40 -0400 Subject: [PATCH 10/10] Add a note about updating nginx config when updating the _ping endpoint. --- endpoints/index.py | 1 + 1 file changed, 1 insertion(+) diff --git a/endpoints/index.py b/endpoints/index.py index 5f0a88695..eb52971cf 100644 --- a/endpoints/index.py +++ b/endpoints/index.py @@ -458,6 +458,7 @@ def get_search(): @index.route('/_ping') @index.route('/_ping') def ping(): + # NOTE: any changes made here must also be reflected in the nginx config response = make_response('true', 200) response.headers['X-Docker-Registry-Version'] = '0.6.0' response.headers['X-Docker-Registry-Standalone'] = '0'