From 847e91b696956be2a593d5ef9ec9810672a4505f Mon Sep 17 00:00:00 2001 From: yackob03 Date: Thu, 24 Oct 2013 16:37:03 -0400 Subject: [PATCH] Add the first part of the build worker that runs on quay. --- buildserver/buildserver.py | 2 +- certs/digital_ocean | 27 ++++++++ certs/digital_ocean.pub | 1 + config.py | 13 +++- data/database.py | 9 ++- data/model.py | 12 ++++ data/queue.py | 1 + test/data/test.db | Bin 75776 -> 77824 bytes workers/diffsworker.py | 6 -- workers/dockerfilebuild.py | 136 +++++++++++++++++++++++++++++++++++++ 10 files changed, 196 insertions(+), 11 deletions(-) create mode 100644 certs/digital_ocean create mode 100644 certs/digital_ocean.pub create mode 100644 workers/dockerfilebuild.py diff --git a/buildserver/buildserver.py b/buildserver/buildserver.py index 94bf2dbe2..b6de17da5 100644 --- a/buildserver/buildserver.py +++ b/buildserver/buildserver.py @@ -195,4 +195,4 @@ def get_all_status(): if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT) - app.run(host='0.0.0.0', port=5002, debug=True) \ No newline at end of file + app.run(host='0.0.0.0', port=5002, debug=True) diff --git a/certs/digital_ocean b/certs/digital_ocean new file mode 100644 index 000000000..700f2b586 --- /dev/null +++ b/certs/digital_ocean @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAwjlIK0HodmDNrZAmaALtr9RLriRSeeLh76gV8KHmjRweeT7v +dmhKeGP1nOAs17caZkcwsW0tiDbCeIv2MisV405sScjPOxFivWpY8tL72sgVuOAl +ReZauOGZ4M1ZcSa/YbT7tnFCIayYE9pde4ih5LmYZqKsBsaNq3ErcMnAzqG77D95 +8swuVwhz/INioBWwe4FjO76/0DqS357hT5yHDWthJD6UUH12VajPKBtXEvGNUtNL +vdq+drm9omt2y0seMn47fZXiNIulLv7ojsWKwtRMTsGcjnv6VMZAVAuX11u4cJd+ +oPTbDl0D+02B7XYcxABqdMZcOc1/7VTUlFFd4wIDAQABAoIBAAs4V+z3z8AW84rV +SwKzOJvxvbV/r6wO6VJ4+Vt/XtxEBZanhhnnCHZP//5iDPUhRMsnza5SSlEWKMHi +BAT97DPHcgYJLb+Rz4x1ulG80oPfDzIw8LZLCm6nycXs1v/sZx3z4J63iER9vgNX +mBLs371g42b6esmhasm+re3EGflV0LeY1IX0MY40pqGndmW8Fly1QH179TrMzVUJ +btu3i2JrwWmKk5zO5YGm0SYY5QQGCdjPj6SL+idDniAefEvbjJYz2qOaPOF3wj/7 +r8dAnmyaP10Q3JojT01Et5ltMfr0oF2/pic9tWYGrgn/aIuoXUXj0SF3Pfgrb/4L +Et1kzFECgYEA8Tb/9bYzQgtaQTQfzFU/KnsIKKnrxh73rZwnIxG59WvN0Ws41Byf +rv8fEbXWU8Yj0drxRSud9fADr99lZGWFxle8rSW5+qqoUxG8n/fkktzHxyPE/9Mh +pZW7un7a5/glKgUpHpjaOCZj9rhdF1AwdUXLSo1sFc7VBsKvKiKJAT0CgYEAziDt +A9h5lOgiLGf1xdBq3qmLIlARz7fivAcZ5acSGN5k6MFFxjHNqhcXRusqs7g+hvCN +eRupdwfgSdLwrTfvxuY4pCcddfYIZO3uUZYs/glvYRtIxaP2kMBkZTs9KzI02Bjv +zT3NPReR/46SqW0zvYTlRFSY7VZ0eRED/5xnjZ8CgYAZdlrSjyceA6DFXUE2CpGe +ZFpaIIW45i/y7ZbcBtUAaR7SymS3T0Yz7M5UykMTmMjTMC9jw9Tqzyk0eXp0fJsA +cuaByIe3RCh8jFTC9iH0tsWH6eizsI/OsN2eNCHbdsBFjUHn7u6qGrNWqeN5wIc8 ++d8ZwY/1RV4LVqWy5u5baQKBgHLFvJMWluQFuPl2zU9etBLU3ma1pKU/I11EqvPH +afk044UCEKLBml1pzAkt6jH1lcM2798OOvbPCOCyNlaMvdLG36TvLqU+3/+qx7bf +4p90i3LLaWK64BBLP9tp9640n13vzJ5AGiY5GI7uSNVTu6p789hvLlOAfwvmII7T +/IjLAoGBAO6iU8i6pAOaKa7+/uExXx6xwk3vqQtovxByo1/m7NpyUtT+ElDSq+t9 +7f+3TzzPB6ggdMl8d+PSyHR3o7KjVPgOSe7zld7eePhUrLjwZ4lh5ohcvhvYfaRL +0EgRTaTb+zLtCAvJS/ilNnJoIcxUmD8u5uSXpY7vAleSOiQTJRTh +-----END RSA PRIVATE KEY----- diff --git a/certs/digital_ocean.pub b/certs/digital_ocean.pub new file mode 100644 index 000000000..95db83601 --- /dev/null +++ b/certs/digital_ocean.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDCOUgrQeh2YM2tkCZoAu2v1EuuJFJ54uHvqBXwoeaNHB55Pu92aEp4Y/Wc4CzXtxpmRzCxbS2INsJ4i/YyKxXjTmxJyM87EWK9aljy0vvayBW44CVF5lq44ZngzVlxJr9htPu2cUIhrJgT2l17iKHkuZhmoqwGxo2rcStwycDOobvsP3nyzC5XCHP8g2KgFbB7gWM7vr/QOpLfnuFPnIcNa2EkPpRQfXZVqM8oG1cS8Y1S00u92r52ub2ia3bLSx4yfjt9leI0i6Uu/uiOxYrC1ExOwZyOe/pUxkBUC5fXW7hwl36g9NsOXQP7TYHtdhzEAGp0xlw5zX/tVNSUUV3j jake@coreserver diff --git a/config.py b/config.py index 3ae57769a..71297a00b 100644 --- a/config.py +++ b/config.py @@ -85,8 +85,15 @@ class GitHubProdConfig(GitHubTestConfig): GITHUB_CLIENT_SECRET = 'f89d8bb28ea3bd4e1c68808500d185a816be53b1' +class DigitalOceanConfig(): + DO_CLIENT_ID = 'LJ44y2wwYj1MD0BRxS6qHA' + DO_CLIENT_SECRET = 'b9357a6f6ff45a33bb03f6dbbad135f9' + DO_SSH_KEY_ID = '46986' + + class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB, - StripeTestConfig, MixpanelTestConfig, GitHubTestConfig): + StripeTestConfig, MixpanelTestConfig, GitHubTestConfig, + DigitalOceanConfig): REGISTRY_SERVER = 'localhost:5000' LOGGING_CONFIG = { 'level': logging.DEBUG, @@ -98,7 +105,7 @@ class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB, class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelTestConfig, - GitHubProdConfig): + GitHubProdConfig, DigitalOceanConfig): REGISTRY_SERVER = 'localhost:5000' LOGGING_CONFIG = { 'level': logging.DEBUG, @@ -109,7 +116,7 @@ class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, class ProductionConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelProdConfig, - GitHubProdConfig): + GitHubProdConfig, DigitalOceanConfig): REGISTRY_SERVER = 'quay.io' LOGGING_CONFIG = { 'stream': sys.stderr, diff --git a/data/database.py b/data/database.py index 0f3aef888..38bc2d2b7 100644 --- a/data/database.py +++ b/data/database.py @@ -150,6 +150,13 @@ class RepositoryTag(BaseModel): ) +class RepositoryBuild(BaseModel): + digitalocean_build_node_id = IntegerField(null=True) + phase = CharField(default='waiting') + status_url = CharField(null=True) + repository = ForeignKeyField(Repository) + + class QueueItem(BaseModel): queue_name = CharField(index=True) body = TextField() @@ -162,7 +169,7 @@ def initialize_db(): create_model_tables([User, Repository, Image, AccessToken, Role, RepositoryPermission, Visibility, RepositoryTag, EmailConfirmation, FederatedLogin, LoginService, - QueueItem]) + QueueItem, RepositoryBuild]) Role.create(name='admin') Role.create(name='write') Role.create(name='read') diff --git a/data/model.py b/data/model.py index 15827cca4..182ee2a21 100644 --- a/data/model.py +++ b/data/model.py @@ -30,6 +30,10 @@ class InvalidTokenException(DataModelException): pass +class InvalidRepositoryBuildException(DataModelException): + pass + + def create_user(username, password, email): if not validate_email(email): raise InvalidEmailAddressException('Invalid email address: %s' % email) @@ -548,3 +552,11 @@ def load_token_data(code): return fetched[0] else: raise InvalidTokenException('Invalid delegate token code: %s' % code) + + +def get_repository_build(request_dbid): + try: + return RepositoryBuild.get(RepositoryBuild == request_dbid) + except RepositoryBuild.DoesNotExist: + msg = 'Unable to locate a build by id: %s' % request_dbid + raise InvalidRepositoryBuildException(msg) diff --git a/data/queue.py b/data/queue.py index 0e1383aa2..3038b8389 100644 --- a/data/queue.py +++ b/data/queue.py @@ -56,3 +56,4 @@ class WorkQueue(object): image_diff_queue = WorkQueue('imagediff') +dockerfile_build_queue = WorkQueue('dockerfilebuild') diff --git a/test/data/test.db b/test/data/test.db index 96e878b95e9124c1618e41e662406d431a5af964..83688fab4d7c9acbd3b2a6402aa7b0b0cc274e8b 100644 GIT binary patch delta 925 zcmZ9KT})D87{||Z&Ou!w&1QqL@<@JwfXG*B^dS&Jf>|dJy>NzdG!n%SN~|rhF1oBe z7l~PGdC|v}aB6c|&CRxMnsm{+s;#=MRX5dIt@XmHHQ(+1?fKZA_y4@lGyR;Me#2g> zEZ?Gq3XrQ)c`m1NGuNCaSkg#t$EF42~F`y#sc8;AqR~ zn9(;F+}ZQ-@CZ^m>0ZfH|uIubZ}5Jl#E0!s!9`Xs6+X zNobw)@Vu`#;TuX?Tus5Qv6eyq$O(Qb86OHXb+`IM{-YzEpGVJhwj)|RV>$N;JiN`9Eb=pVLUn!jz?q3fkZeGTIbd}c&E+h=lCnow6 zv51oEw($NE)CcaJYs4Q4YPW7MkyfpPpLp8pfF@p+A&wpW3G9|vBt;RF6 zmH0Sai_5cm*hwOLpdnum+tOC3!Lw;QX#V)6cr9IqpVGRUmr8EVOJ!wb=nSk_6o1S~fDY8- zjZ78jaV66%%`cUVd5~W!uPD8pVUBp(h}~f&a5odUj0EjE0=t^PR!h)UO|UP&se88& z?5QAV-AK@~fuLDJ69SVXqz*7SqV@?RacG%7K*Lw%mCq6=0%i7IDGLWh>C#ggocQ8_ z2lgYk>;WrA2zQH1%Wk$J-*yqzOTjm~kNySI)OV_v%*)B7w(D>(*9yD-_5Lt-fJw1h ze6?l<3;D$^I-$YPnjXx=*@la2YC5FC@0l8CB8^wzVzvzTXE{*gv8)*kg*tb$-DDyl T)`1aK;;RLAMJlq-e+GU7y+<8n delta 697 zcmXxgOH5Ni6adh7XI^cBrKSXJYe7EpD}~a}E2V-;3*}Q9D}vMzQCcVuY^8k4r);Fb zwUOL~u^1La7r3L(g)uIO5TIbJi7s?OqO3G=;m(*4;t+K*b24-9By;D^f8pof^AB|@ z9mjDVw*C*pVwIHzs}IU$EF`D<#|dF=Y@9-cbU>puwgLBqIBW6Dgx!&_f7&eiY+lox z&p6s8v<_VNS_5I%q)V8J3_FJ=n}c?vzs}cV@?)l6gUeQ}%@`W&w%-xMq1h|KOqZ>x zW5!Zj6S7UmBGzj`qu?DDqr<(f_K01)YKk~}$2)skhvL!?>shbDz;On+2j7WEJ}f%5 zQp@}*A(aKFdMu!BL5sH@@27Hkg~UHOl;>o#vSpl~WAhutNgnx%D&9>K%0kQ}i|}?* z2W4nZYtWQB$xd>7mefNL%QZm7zjzMTF6p5d`HQX?xB*tI_{xIQoLBBo$ZZm-&W6Pu`}EjMsqcz zQO~$s%D7a-Xvk;OYZ(`E8TM0*+AM}mMwmb|OwL6!l8WD?GG`U-12i8MKl+&sCsf79 z-{)bOgJt+d`p6Nx^#F*!6j!AJsKT^lCxd19J6!@sJh$tFa_rv~poabGG(O+$VLgDo zGB9D@f(9G+HYNVGoQGCPvA#{92|vE+fQwkT(E$#OFzck38*O~*Ew9L;B*>8<`AqxS zlifr)rI)K6<&aR1j1GooaXo!TIT0F*PL9N)6Y*F;G_z*s;k&e+CiqG`+!4USB0ieP XH{i`32UuB21=yZe;nt3I<