From c1ea6263e17d5a7f480b4ac890eaca1747ab01e3 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Fri, 15 Nov 2013 16:45:02 -0500 Subject: [PATCH] Flesh out the webworkers a bit. --- data/database.py | 12 ++++---- data/model.py | 27 ++++++++++++++++++ endpoints/api.py | 52 +++++++++++++++++++++++++++++++++- test/data/test.db | Bin 97280 -> 97280 bytes workers/webhookworker.py | 59 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 144 insertions(+), 6 deletions(-) create mode 100644 workers/webhookworker.py diff --git a/data/database.py b/data/database.py index e6bda976a..18b6b8bad 100644 --- a/data/database.py +++ b/data/database.py @@ -104,11 +104,6 @@ class Repository(BaseModel): ) -class Webhook(BaseModel): - repository = ForeignKeyField(Repository) - parameters = TextField() - - class Role(BaseModel): name = CharField(index=True) @@ -135,6 +130,13 @@ def random_string_generator(length=16): return random_string +class Webhook(BaseModel): + public_id = CharField(default=random_string_generator(length=64), + unique=True, index=True) + repository = ForeignKeyField(Repository) + parameters = TextField() + + class AccessToken(BaseModel): friendly_name = CharField(null=True) code = CharField(default=random_string_generator(length=64), unique=True, diff --git a/data/model.py b/data/model.py index cfbb9340c..47af6c181 100644 --- a/data/model.py +++ b/data/model.py @@ -2,6 +2,7 @@ import bcrypt import logging import dateutil.parser import operator +import json from database import * from util.validation import * @@ -42,6 +43,10 @@ class InvalidRepositoryBuildException(DataModelException): pass +class InvalidWebhookException(DataModelException): + pass + + def create_user(username, password, email): if not validate_email(email): raise InvalidEmailAddressException('Invalid email address: %s' % email) @@ -935,3 +940,25 @@ def list_repository_builds(namespace_name, repository_name, def create_repository_build(repo, access_token, resource_key, tag): return RepositoryBuild.create(repository=repo, access_token=access_token, resource_key=resource_key, tag=tag) + + +def create_webhook(repo, params_obj): + return Webhook.create(repository=repo, parameters=json.dumps(params_obj)) + + +def get_webhook(namespace_name, repository_name, public_id): + joined = Webhook.select().join(Repository) + found = list(joined.where(Repository.namespace == namespace_name, + Repository.name == repository_name, + Webhook.public_id == public_id)) + + if not found: + raise InvalidWebhookException('No webhook found with id: %s' % public_id) + + return found[0] + + +def list_webhooks(namespace_name, repository_name): + joined = Webhook.select().join(Repository) + return joined.where(Repository.namespace == namespace_name, + Repository.name == repository_name) diff --git a/endpoints/api.py b/endpoints/api.py index e3f2dab82..d9a06e768 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -4,7 +4,7 @@ import requests import urlparse import json -from flask import request, make_response, jsonify, abort +from flask import request, make_response, jsonify, abort, url_for from flask.ext.login import current_user, logout_user from flask.ext.principal import identity_changed, AnonymousIdentity from functools import wraps @@ -799,6 +799,56 @@ def request_repo_build(namespace, repository): abort(403) # Permissions denied +def webhook_view(webhook): + return { + 'public_id': webhook.public_id, + 'parameters': json.loads(webhook.parameters), + } + + +@app.route('/api/repository//webhook/', methods=['POST']) +@api_login_required +@parse_repository_name +def create_webhook(namespace, repository): + permission = ModifyRepositoryPermission(namespace, repository) + if permission.can(): + repo = model.get_repository(namespace, repository) + webhook = model.create_webhook(repo, request.get_json()) + resp = jsonify(webhook_view(webhook)) + repo_string = '%s/%s' % (namespace, repository) + resp.headers['Location'] = url_for('get_webhook', repository=repo_string, + public_id=webhook.public_id) + + abort(403) # Permissions denied + + +@app.route('/api/repository//webhook/', + methods=['GET']) +@api_login_required +@parse_repository_name +def get_webhook(namespace, repository, public_id): + permission = ModifyRepositoryPermission(namespace, repository) + if permission.can(): + webhook = model.get_webhook(namespace, repository, public_id) + return jsonify(webhook_view(webhook)) + + abort(403) # Permission denied + + +@app.route('/api/repository//webhook/', methods=['GET']) +@api_login_required +@parse_repository_name +def list_webhooks(namespace, repository): + permission = ModifyRepositoryPermission(namespace, repository) + if permission.can(): + webhooks = model.list_webhooks(namespace, repository) + return jsonify({ + 'webhooks': [webhook_view(webhook) for webhook in webhooks] + }) + + abort(403) # Permission denied + + @app.route('/api/filedrop/', methods=['POST']) @api_login_required def get_filedrop_url(): diff --git a/test/data/test.db b/test/data/test.db index fc3f5c088710af863c3692ba8f133ab1364f8a62..c300b0c1c1a6981ac4f12223a93f5df684883c52 100644 GIT binary patch delta 469 zcmbu&OOlg7007X{=oO}1xzVaAYY5%VKV_9Lgd|OprU}plR|Ls0Oh{)iAINI%z*60K z0;^`@)&p2B-Fg5IFgx`QUf{j^$Fuv#voEg=`5+ct$|8S?(Z=PPcl4Jv7A7HRTtU=} zl}Fds@@U@{>1vqji=CUAS=-gdOSur|ciO@CaG2njmHK`i4@h3LDobP5;s6IVI-|y8{^5Wu-tY2&D`w1~deUKumLQa>wEIEM zyHEXRGC#l)P$%8+YC)HCgQ;z2xSPV@wd09^ub e3FPNMF(hQjk^nUwiRj_a&u>rP{BQ92*T=sPfR?EM delta 469 zcmbu&J(819007`>^$L!ajm|jEvOM7B0U2imLI@$@@lU{t0FoENLI~`Vt@aL-YU2r< z*~ZobIF_~^zyoZjhw!_=cmH^C|9J8Btq7w%F__$^nl#?f#9o|BRcd?cDr$~5N!@G& zwi#q?k8~3yJr#b~&SoVUTk#bPb_0qGx42uZYrF4MK+F&76Do1(nDG2hcXzn zb27e}zJy0*AgK*cp&x=Ob5Y1IvlRw|vn`984BKn9D?HCu+Eg~|;!fwfE+gYl{A~O@ zeGZehB5deV!nGWC`uwOuv*vK!ltfth2WvvBM3H;kO@c#ZO@iBaZzJ;XXZr2qdlh3C z^N|@Z<&|b}$W#5mSp(1HJ%1j|Bh3qS({mjZFtK6!TFj`vZ*X|Nv?a%2S5Fuku$KT_ e0{8+zk>f=gP!P~Gq#yqN`u_CY{{~-vfBFaMH=6AL diff --git a/workers/webhookworker.py b/workers/webhookworker.py new file mode 100644 index 000000000..73e549138 --- /dev/null +++ b/workers/webhookworker.py @@ -0,0 +1,59 @@ +import logging +import daemon +import argparse +import requests + +from data.queue import webhook_queue +from workers.worker import Worker + + +root_logger = logging.getLogger('') +root_logger.setLevel(logging.DEBUG) + +FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - %(funcName)s - %(message)s' +formatter = logging.Formatter(FORMAT) + +logger = logging.getLogger(__name__) + + +class WebhookWorker(Worker): + def process_queue_item(self, job_details): + url = job_details['url'] + payload = job_details['payload'] + + try: + resp = requests.post(url, data=payload) + if resp.status_code/100 != 2: + logger.error('%s response for webhook to url: %s' % (resp.status_code, + url)) + return False + except requests.exceptions.RequestException as ex: + logger.exception('Webhook was unable to be sent: %s' % ex.message) + return False + + return True + + +parser = argparse.ArgumentParser(description='Worker daemon to send webhooks') +parser.add_argument('-D', action='store_true', default=False, + help='Run the worker in daemon mode.') +parser.add_argument('--log', default='webhooks.log', + help='Specify the log file for the worker as a daemon.') +args = parser.parse_args() + + +worker = WebhookWorker(webhook_queue, poll_period_seconds=15, + reservation_seconds=3600) + +if args.D: + handler = logging.FileHandler(args.log) + handler.setFormatter(formatter) + root_logger.addHandler(handler) + with daemon.DaemonContext(files_preserve=[handler.stream]): + worker.start() + +else: + handler = logging.StreamHandler() + handler.setFormatter(formatter) + root_logger.addHandler(handler) + worker.start() \ No newline at end of file