From 4c6012f756b6ebeb440419a16eb5a46ba57c161c Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 18 Nov 2013 14:49:54 -0500 Subject: [PATCH 01/21] Add ability to download receipts in PDF form --- endpoints/web.py | 24 +++++++++++++++++++++--- requirements.txt | 1 + static/partials/org-admin.html | 12 +++++++++--- util/invoice.py | 16 ++++++++++++++++ 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/endpoints/web.py b/endpoints/web.py index 9f8712629..ff6f1790a 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -1,15 +1,16 @@ import logging import requests +import stripe from flask import (abort, redirect, request, url_for, render_template, - make_response) + make_response, Response) from flask.ext.login import login_user, UserMixin, login_required from flask.ext.principal import identity_changed, Identity, AnonymousIdentity from data import model from app import app, login_manager, mixpanel -from auth.permissions import QuayDeferredPermissionUser - +from auth.permissions import QuayDeferredPermissionUser, AdministerOrganizationPermission +from util.invoice import renderInvoiceToPdf logger = logging.getLogger(__name__) @@ -102,6 +103,23 @@ def privacy(): return render_template('privacy.html') +@app.route('/receipt', methods=['GET']) +def receipt(): + id = request.args.get('id') + if id: + invoice = stripe.Invoice.retrieve(id) + if invoice: + org = model.get_user_or_org_by_customer_id(invoice.customer) + if org and org.organization: + admin_org = AdministerOrganizationPermission(org.username) + if admin_org.can(): + file_data = renderInvoiceToPdf(invoice, org) + return Response(file_data, + mimetype="application/pdf", + headers={"Content-Disposition": + "attachment;filename=receipt.pdf"}) + abort(404) + def common_login(db_user): if login_user(_LoginWrappedDBUser(db_user.username, db_user)): logger.debug('Successfully signed in as: %s' % db_user.username) diff --git a/requirements.txt b/requirements.txt index 494ed9198..dc2c2a6ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,3 +31,4 @@ requests==2.0.0 six==1.4.1 stripe==1.9.8 wsgiref==0.1.2 +xhtml2pdf \ No newline at end of file diff --git a/static/partials/org-admin.html b/static/partials/org-admin.html index 46e3af4ea..d31d10279 100644 --- a/static/partials/org-admin.html +++ b/static/partials/org-admin.html @@ -49,12 +49,13 @@ Billing Date/Time Amount Due Status + - - {{ invoice.date * 1000 | date:'medium' }} - {{ invoice.amount_due / 100 }} + + {{ invoice.date * 1000 | date:'medium' }} + {{ invoice.amount_due / 100 }} Paid - Thank you! @@ -63,6 +64,11 @@ Payment pending + + + + + diff --git a/util/invoice.py b/util/invoice.py index 773c428f1..86dd2eea4 100644 --- a/util/invoice.py +++ b/util/invoice.py @@ -1,5 +1,7 @@ from datetime import datetime from jinja2 import Environment, FileSystemLoader +from xhtml2pdf import pisa +import StringIO jinja_options = { "loader": FileSystemLoader('util'), @@ -7,6 +9,20 @@ jinja_options = { env = Environment(**jinja_options) + +def renderInvoiceToPdf(invoice, user): + """ Renders a nice PDF display for the given invoice. """ + sourceHtml = renderInvoiceToHtml(invoice, user) + output = StringIO.StringIO() + pisaStatus = pisa.CreatePDF(sourceHtml, dest=output) + if pisaStatus.err: + return None + + value = output.getvalue() + output.close() + return value + + def renderInvoiceToHtml(invoice, user): """ Renders a nice HTML display for the given invoice. """ def get_price(price): From bde0a29296262a49409e30383e451d012d1bb7da Mon Sep 17 00:00:00 2001 From: yackob03 Date: Mon, 18 Nov 2013 15:00:08 -0500 Subject: [PATCH 02/21] Switcht the requirements and -nover files to the standard formatting. --- requirements-nover.txt | 3 ++- requirements.txt | 16 ++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/requirements-nover.txt b/requirements-nover.txt index 8d95e3e48..dc0d069a0 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -16,4 +16,5 @@ marisa-trie apscheduler python-daemon paramiko -python-digitalocean \ No newline at end of file +python-digitalocean +xhtml2pdf \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index dc2c2a6ec..48a6fffa3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,30 +5,34 @@ Flask-Mail==0.9.0 Flask-Principal==0.4.0 Jinja2==2.7.1 MarkupSafe==0.18 +Pillow==2.2.1 PyMySQL==0.6.1 Werkzeug==0.9.4 argparse==1.2.1 beautifulsoup4==4.3.2 blinker==1.3 -boto==2.15.0 +boto==2.17.0 distribute==0.6.34 ecdsa==0.10 eventlet==0.14.0 greenlet==0.4.1 gunicorn==18.0 +html5lib==1.0b3 itsdangerous==0.23 lockfile==0.9.1 marisa-trie==0.5.1 mixpanel-py==3.0.0 paramiko==1.12.0 -peewee==2.1.4 +peewee==2.1.5 py-bcrypt==0.4 +pyPdf==1.13 pycrypto==2.6.1 python-daemon==1.6 -python-dateutil==2.1 -python-digitalocean==0.5 -requests==2.0.0 +python-dateutil==2.2 +python-digitalocean==0.5.1 +reportlab==2.7 +requests==2.0.1 six==1.4.1 stripe==1.9.8 wsgiref==0.1.2 -xhtml2pdf \ No newline at end of file +xhtml2pdf==0.0.5 From a079919a771df88b08909e6f372b6f68601e1459 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Mon, 18 Nov 2013 15:58:42 -0500 Subject: [PATCH 03/21] Add the required apt packages for proper PIL usage. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 35ea35a87..6b394eecf 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ to prepare a new host: sudo apt-get install software-properties-common sudo apt-add-repository -y ppa:nginx/stable sudo apt-get update -sudo apt-get install -y git python-virtualenv python-dev phantomjs +sudo apt-get install -y git python-virtualenv python-dev phantomjs libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev sudo apt-get install -y nginx-full ``` From 949a2de01c7cbc02ccbfd26264363e86fca42440 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Mon, 18 Nov 2013 15:59:07 -0500 Subject: [PATCH 04/21] Add a script to detect freeloaders for our own entertainment. --- tools/freeloaders.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tools/freeloaders.py diff --git a/tools/freeloaders.py b/tools/freeloaders.py new file mode 100644 index 000000000..ad6b37722 --- /dev/null +++ b/tools/freeloaders.py @@ -0,0 +1,25 @@ +from data import model +from data.database import User +from app import stripe +from data.plans import get_plan + +def get_private_allowed(customer): + if not customer.stripe_id: + return 0 + + subscription = stripe.Customer.retrieve(customer.stripe_id).subscription + if subscription is None: + return 0 + + plan = get_plan(subscription.plan.id) + return plan['privateRepos'] + +# Find customers who have more private repositories than their plans allow +users = User.select() + +usage = [(user.username, model.get_private_repo_count(user.username), + get_private_allowed(user)) for user in users] + +for username, used, allowed in usage: + if used > allowed: + print('Violation: %s %s > %s' % (username, used, allowed)) From 738973cf3957c30285aca4b6e42d2b45673ff27e Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 18 Nov 2013 17:11:06 -0500 Subject: [PATCH 05/21] Add the snapshot endpoint to web.py and have the phantomjs running only load the page's HTML once there are no further pending XHR requests --- endpoints/web.py | 12 ++++++++++++ static/js/app.js | 12 +++++++++++- util/phantomjs-runner.js | 37 +++++++++++++++++++++++++++++++++++++ util/seo.py | 27 +++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 util/phantomjs-runner.js create mode 100644 util/seo.py diff --git a/endpoints/web.py b/endpoints/web.py index ff6f1790a..c76d2f60b 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -11,6 +11,7 @@ from data import model from app import app, login_manager, mixpanel from auth.permissions import QuayDeferredPermissionUser, AdministerOrganizationPermission from util.invoice import renderInvoiceToPdf +from util.seo import renderSnapshot logger = logging.getLogger(__name__) @@ -49,6 +50,17 @@ def index(path): return render_template('index.html') +@app.route('/snapshot', methods=['GET']) +@app.route('/snapshot/', methods=['GET']) +@app.route('/snapshot/', methods=['GET']) +def snapshot(path = ''): + result = renderSnapshot(path) + if result: + return result + + abort(404) + + @app.route('/plans/') def plans(): return index('') diff --git a/static/js/app.js b/static/js/app.js index 26d7aa5da..13b63a5b6 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -1053,7 +1053,8 @@ quayApp.directive('ngBlur', function() { }; }); -quayApp.run(['$location', '$rootScope', 'Restangular', 'UserService', function($location, $rootScope, Restangular, UserService) { +quayApp.run(['$location', '$rootScope', 'Restangular', 'UserService', '$http', + function($location, $rootScope, Restangular, UserService, $http) { Restangular.setErrorInterceptor(function(response) { if (response.status == 401) { $('#sessionexpiredModal').modal({}); @@ -1068,4 +1069,13 @@ quayApp.run(['$location', '$rootScope', 'Restangular', 'UserService', function($ $rootScope.title = current.$$route.title; } }); + + var initallyChecked = false; + window.__isLoading = function() { + if (!initallyChecked) { + initallyChecked = true; + return true; + } + return $http.pendingRequests.length > 0; + }; }]); diff --git a/util/phantomjs-runner.js b/util/phantomjs-runner.js new file mode 100644 index 000000000..30b0439fa --- /dev/null +++ b/util/phantomjs-runner.js @@ -0,0 +1,37 @@ +var system = require('system'); +var url = system.args[1] || ''; +if(url.length > 0) { + var page = require('webpage').create(); + page.open(url, function (status) { + if (status == 'success') { + var delay, checker = (function() { + var html = page.evaluate(function () { + var found = document.getElementsByTagName('html')[0].outerHTML || ''; + if (window.__isLoading && !window.__isLoading()) { + return found; + } + if (found.indexOf('404 Not Found') > 0) { + return found; + } + return null; + }); + + if (html) { + if (html.indexOf('404 Not Found') > 0) { + console.log('Not Found'); + phantom.exit(); + return; + } + + clearTimeout(delay); + console.log(html); + phantom.exit(); + } + }); + delay = setInterval(checker, 100); + } else { + console.log('Not Found'); + phantom.exit(); + } + }); +} \ No newline at end of file diff --git a/util/seo.py b/util/seo.py new file mode 100644 index 000000000..6958e2c67 --- /dev/null +++ b/util/seo.py @@ -0,0 +1,27 @@ +import subprocess +import urllib +import os +import logging +import codecs + +from bs4 import BeautifulSoup + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.DEBUG) + +def renderSnapshot(path): + final_url = 'http://localhost:5000/' + path + logger.info('Snapshotting url: %s -> %s' % (path, final_url)) + out_html = subprocess.check_output(['phantomjs', '--ignore-ssl-errors=yes', + 'util/phantomjs-runner.js', final_url]) + + if not out_html or out_html.strip() == 'Not Found': + return None + + # Remove script tags + soup = BeautifulSoup(out_html) + to_extract = soup.findAll('script') + for item in to_extract: + item.extract() + + return soup.prettify() From af4c67d7cb45a546dda8c31a0436f7eaf6b4d7f2 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Mon, 18 Nov 2013 18:42:02 -0500 Subject: [PATCH 06/21] Switch from eventlet to gevent, it seems to work better with flask static files. --- README.md | 2 +- requirements-nover.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6b394eecf..a37e0d31d 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ running: ``` sudo nginx -c `pwd`/nginx.conf -STACK=prod gunicorn -D --workers 4 -b unix:/tmp/gunicorn.sock --worker-class eventlet -t 2000 application:application +STACK=prod gunicorn -D --workers 4 -b unix:/tmp/gunicorn.sock --worker-class gevent -t 2000 application:application ``` set up the snapshot script: diff --git a/requirements-nover.txt b/requirements-nover.txt index dc0d069a0..f42a8a90d 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -9,7 +9,7 @@ boto pymysql stripe gunicorn -eventlet +gevent mixpanel-py beautifulsoup4 marisa-trie diff --git a/requirements.txt b/requirements.txt index 48a6fffa3..712c818c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ blinker==1.3 boto==2.17.0 distribute==0.6.34 ecdsa==0.10 -eventlet==0.14.0 +gevent==0.13.8 greenlet==0.4.1 gunicorn==18.0 html5lib==1.0b3 From 6355b4a217cfc2b8f0150e013ee29d46ce41d42a Mon Sep 17 00:00:00 2001 From: yackob03 Date: Mon, 18 Nov 2013 18:42:27 -0500 Subject: [PATCH 07/21] Fix some things with the seo snapshots and use the pep8 style guite. --- endpoints/web.py | 4 ++-- static/partials/header.html | 20 +++++++++++--------- util/seo.py | 29 +++++++++++++---------------- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/endpoints/web.py b/endpoints/web.py index c76d2f60b..188533721 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -11,7 +11,7 @@ from data import model from app import app, login_manager, mixpanel from auth.permissions import QuayDeferredPermissionUser, AdministerOrganizationPermission from util.invoice import renderInvoiceToPdf -from util.seo import renderSnapshot +from util.seo import render_snapshot logger = logging.getLogger(__name__) @@ -54,7 +54,7 @@ def index(path): @app.route('/snapshot/', methods=['GET']) @app.route('/snapshot/', methods=['GET']) def snapshot(path = ''): - result = renderSnapshot(path) + result = render_snapshot(path) if result: return result diff --git a/static/partials/header.html b/static/partials/header.html index 34e2d2231..1bb22afbf 100644 --- a/static/partials/header.html +++ b/static/partials/header.html @@ -22,15 +22,17 @@ From 5f9da10a36709f00c8b0f92b6af33174289adecf Mon Sep 17 00:00:00 2001 From: yackob03 Date: Fri, 22 Nov 2013 17:45:22 -0500 Subject: [PATCH 21/21] Make the asterisk smaller. --- static/css/quay.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/static/css/quay.css b/static/css/quay.css index 15ae81f66..593c17020 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -523,8 +523,8 @@ html, body { } .jumbotron .disclaimer-link { - font-size: .5em; - vertical-align: top; + font-size: .3em; + vertical-align: 23px; } .jumbotron .disclaimer-link:hover {