From 5168f473212b447d8a8dc008b881084b04d5a0e0 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 18 Dec 2014 18:08:50 +0200 Subject: [PATCH 01/39] Only lookup if the repo is public if necessary --- endpoints/index.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/endpoints/index.py b/endpoints/index.py index 85afc6710..00221b8cc 100644 --- a/endpoints/index.py +++ b/endpoints/index.py @@ -305,9 +305,7 @@ def get_repository_images(namespace, repository): permission = ReadRepositoryPermission(namespace, repository) # TODO invalidate token? - profile.debug('Looking up public status of repository') - is_public = model.repository_is_public(namespace, repository) - if permission.can() or is_public: + if permission.can() or model.repository_is_public(namespace, repository): # We can't rely on permissions to tell us if a repo exists anymore profile.debug('Looking up repository') repo = model.get_repository(namespace, repository) From 84f6cc7385b461ca64afc88da6c025967bcc327f Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 23 Dec 2014 18:34:02 -0500 Subject: [PATCH 02/39] add percona to migration tests --- data/migrations/migration.sh | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/data/migrations/migration.sh b/data/migrations/migration.sh index 183955207..17901e130 100755 --- a/data/migrations/migration.sh +++ b/data/migrations/migration.sh @@ -2,13 +2,14 @@ set -e DOCKER_IP=`echo $DOCKER_HOST | sed 's/tcp:\/\///' | sed 's/:.*//'` MYSQL_CONFIG_OVERRIDE="{\"DB_URI\":\"mysql+pymysql://root:password@$DOCKER_IP/genschema\"}" +PERCONA_CONFIG_OVERRIDE="{\"DB_URI\":\"mysql+pymysql://root@$DOCKER_IP/genschema\"}" PGSQL_CONFIG_OVERRIDE="{\"DB_URI\":\"postgresql://postgres@$DOCKER_IP/genschema\"}" up_mysql() { # Run a SQL database on port 3306 inside of Docker. docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mysql - # Sleep for 5s to get MySQL get started. + # Sleep for 10s to get MySQL get started. echo 'Sleeping for 10...' sleep 10 @@ -25,12 +26,12 @@ up_mariadb() { # Run a SQL database on port 3306 inside of Docker. docker run --name mariadb -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mariadb - # Sleep for 5s to get MySQL get started. + # Sleep for 10s to get MySQL get started. echo 'Sleeping for 10...' sleep 10 # Add the database to mysql. - docker run --rm --link mariadb:mysql mariadb sh -c 'echo "create database genschema" | mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -ppassword' + docker run --rm --link mariadb:mariadb mariadb sh -c 'echo "create database genschema" | mysql -h"$MARIADB_PORT_3306_TCP_ADDR" -P"$MARIADB_PORT_3306_TCP_PORT" -uroot -ppassword' } down_mariadb() { @@ -38,6 +39,23 @@ down_mariadb() { docker rm mariadb } +up_percona() { + # Run a SQL database on port 3306 inside of Docker. + docker run --name percona -p 3306:3306 -d dockerfile/percona + + # Sleep for 10s + echo 'Sleeping for 10...' + sleep 10 + + # Add the daabase to mysql. + docker run --rm --link percona:percona dockerfile/percona sh -c 'echo "create database genschema" | mysql -h $PERCONA_PORT_3306_TCP_ADDR' +} + +down_percona() { + docker kill percona + docker rm percona +} + up_postgres() { # Run a SQL database on port 5432 inside of Docker. docker run --name postgres -p 5432:5432 -d postgres @@ -93,12 +111,23 @@ down_mysql # Test via MariaDB. echo '> Starting MariaDB' up_mariadb + echo '> Testing Migration (mariadb)' set +e test_migrate $MYSQL_CONFIG_OVERRIDE set -e down_mariadb +# Test via Percona. +echo '> Starting Percona' +up_percona + +echo '> Testing Migration (percona)' +set +e +test_migrate $PERCONA_CONFIG_OVERRIDE +set -e +down_percona + # Test via Postgres. echo '> Starting Postgres' up_postgres From 8085ff81a81315a8cba5ace318a64771d2e7aa13 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Sun, 4 Jan 2015 14:50:08 -0500 Subject: [PATCH 03/39] Only call get_authenticated_* once in the track_and_log, and add better logging to it --- endpoints/trackhelper.py | 41 ++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/endpoints/trackhelper.py b/endpoints/trackhelper.py index d26db7e7e..83b9d4270 100644 --- a/endpoints/trackhelper.py +++ b/endpoints/trackhelper.py @@ -19,20 +19,23 @@ def track_and_log(event_name, repo, **kwargs): analytics_id = 'anonymous' + authenticated_oauth_token = get_validated_oauth_token() + authenticated_user = get_authenticated_user() + authenticated_token = get_validated_token() if not authenticated_user else None + profile.debug('Logging the %s to Mixpanel and the log system', event_name) - if get_validated_oauth_token(): - oauth_token = get_validated_oauth_token() - metadata['oauth_token_id'] = oauth_token.id - metadata['oauth_token_application_id'] = oauth_token.application.client_id - metadata['oauth_token_application'] = oauth_token.application.name - analytics_id = 'oauth:' + oauth_token.id - elif get_authenticated_user(): - metadata['username'] = get_authenticated_user().username - analytics_id = get_authenticated_user().username - elif get_validated_token(): - metadata['token'] = get_validated_token().friendly_name - metadata['token_code'] = get_validated_token().code - analytics_id = 'token:' + get_validated_token().code + if authenticated_oauth_token: + metadata['oauth_token_id'] = authenticated_oauth_token.id + metadata['oauth_token_application_id'] = authenticated_oauth_token.application.client_id + metadata['oauth_token_application'] = authenticated_oauth_token.application.name + analytics_id = 'oauth:' + authenticated_oauth_token.id + elif authenticated_user: + metadata['username'] = authenticated_user.username + analytics_id = authenticated_user.username + elif authenticated_token: + metadata['token'] = authenticated_token.friendly_name + metadata['token_code'] = authenticated_token.code + analytics_id = 'token:' + authenticated_token.code else: metadata['public'] = True analytics_id = 'anonymous' @@ -42,21 +45,27 @@ def track_and_log(event_name, repo, **kwargs): } # Publish the user event (if applicable) - if get_authenticated_user(): + profile.debug('Checking publishing %s to the user events system', event_name) + if authenticated_user: + profile.debug('Publishing %s to the user events system', event_name) user_event_data = { 'action': event_name, 'repository': repository, 'namespace': namespace } - event = userevents.get_event(get_authenticated_user().username) + event = userevents.get_event(authenticated_user.username) event.publish_event_data('docker-cli', user_event_data) # Save the action to mixpanel. + profile.debug('Logging the %s to Mixpanel', event_name) analytics.track(analytics_id, event_name, extra_params) # Log the action to the database. + profile.debug('Logging the %s to logs system', event_name) model.log_action(event_name, namespace, - performer=get_authenticated_user(), + performer=authenticated_user, ip=request.remote_addr, metadata=metadata, repository=repo) + + profile.debug('Track and log of %s complete', event_name) From 27e74475692a048c4a902ff345f81a6ec3c3f3f6 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 5 Jan 2015 12:12:00 -0500 Subject: [PATCH 04/39] Fix spacing --- endpoints/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/endpoints/web.py b/endpoints/web.py index 913a6905a..519fc5c5e 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -162,7 +162,7 @@ def health(): check = HealthCheck.get_check(app.config['HEALTH_CHECKER'][0], app.config['HEALTH_CHECKER'][1]) (data, is_healthy) = check.conduct_healthcheck(db_healthy, buildlogs_healthy) - response = jsonify(dict(data = data, is_healthy = is_healthy)) + response = jsonify(dict(data=data, is_healthy=is_healthy)) response.status_code = 200 if is_healthy else 503 return response From 88a60d05b6e0fba46a178d0b16465aed62a74002 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Mon, 5 Jan 2015 14:51:00 -0500 Subject: [PATCH 05/39] fix shadowed variable 'tag' was being used for both a string value and a boolean value --- util/names.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/util/names.py b/util/names.py index 67ceed982..f37f0135d 100644 --- a/util/names.py +++ b/util/names.py @@ -4,7 +4,7 @@ from functools import wraps from uuid import uuid4 -def parse_namespace_repository(repository, tag=False): +def parse_namespace_repository(repository, include_tag=False): parts = repository.rstrip('/').split('/', 1) if len(parts) < 2: namespace = 'library' @@ -12,15 +12,15 @@ def parse_namespace_repository(repository, tag=False): else: (namespace, repository) = parts - if tag: + if include_tag: parts = repository.split(':', 1) if len(parts) < 2: - tag = None + tag = 'latest' else: (repository, tag) = parts repository = urllib.quote_plus(repository) - if tag: + if include_tag: return (namespace, repository, tag) return (namespace, repository) @@ -34,7 +34,7 @@ def parse_repository_name(f): def parse_repository_name_and_tag(f): @wraps(f) def wrapper(repository, *args, **kwargs): - (namespace, repository, tag) = parse_namespace_repository(repository, tag=True) + namespace, repository, tag = parse_namespace_repository(repository, include_tag=True) return f(namespace, repository, tag, *args, **kwargs) return wrapper From baa4918d89af14a3646a65332fa40ca2e68c0555 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 6 Jan 2015 14:55:31 -0500 Subject: [PATCH 06/39] Explicitly declare length of large MySQL indices. Most MySQL installs silently truncate indices at a specific number of bytes. This value is 767 bytes unless either the innodb_large_prefix option is turned on or the innodb_page_size has been changed. This change explicitly limits the size of large indices to 767 characters due to the latin1 charset being 1 byte per char. --- .../228d1af6af1c_mysql_max_index_lengths.py | 22 +++++++++++++++++++ .../5a07499ce53f_set_up_initial_database.py | 4 ++-- 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 data/migrations/versions/228d1af6af1c_mysql_max_index_lengths.py diff --git a/data/migrations/versions/228d1af6af1c_mysql_max_index_lengths.py b/data/migrations/versions/228d1af6af1c_mysql_max_index_lengths.py new file mode 100644 index 000000000..0b70acbb3 --- /dev/null +++ b/data/migrations/versions/228d1af6af1c_mysql_max_index_lengths.py @@ -0,0 +1,22 @@ +"""mysql max index lengths + +Revision ID: 228d1af6af1c +Revises: 5b84373e5db +Create Date: 2015-01-06 14:35:24.651424 + +""" + +# revision identifiers, used by Alembic. +revision = '228d1af6af1c' +down_revision = '5b84373e5db' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(tables): + op.drop_index('queueitem_queue_name', table_name='queueitem') + op.create_index('queueitem_queue_name', 'queueitem', ['queue_name'], unique=False, mysql_length=767) + +def downgrade(tables): + pass diff --git a/data/migrations/versions/5a07499ce53f_set_up_initial_database.py b/data/migrations/versions/5a07499ce53f_set_up_initial_database.py index 0e786de14..c1d082066 100644 --- a/data/migrations/versions/5a07499ce53f_set_up_initial_database.py +++ b/data/migrations/versions/5a07499ce53f_set_up_initial_database.py @@ -53,7 +53,7 @@ def upgrade(tables): op.create_index('queueitem_available', 'queueitem', ['available'], unique=False) op.create_index('queueitem_available_after', 'queueitem', ['available_after'], unique=False) op.create_index('queueitem_processing_expires', 'queueitem', ['processing_expires'], unique=False) - op.create_index('queueitem_queue_name', 'queueitem', ['queue_name'], unique=False) + op.create_index('queueitem_queue_name', 'queueitem', ['queue_name'], unique=False, mysql_length=767) op.create_table('role', sa.Column('id', sa.Integer(), nullable=False), sa.Column('name', sa.String(length=255), nullable=False), @@ -376,7 +376,7 @@ def upgrade(tables): sa.ForeignKeyConstraint(['storage_id'], ['imagestorage.id'], ), sa.PrimaryKeyConstraint('id') ) - op.create_index('image_ancestors', 'image', ['ancestors'], unique=False) + op.create_index('image_ancestors', 'image', ['ancestors'], unique=False, mysql_length=767) op.create_index('image_repository_id', 'image', ['repository_id'], unique=False) op.create_index('image_repository_id_docker_image_id', 'image', ['repository_id', 'docker_image_id'], unique=True) op.create_index('image_storage_id', 'image', ['storage_id'], unique=False) From b18986c72bcec60f8e032222ee6c32ae9326510a Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 6 Jan 2015 15:11:37 -0500 Subject: [PATCH 07/39] Continuation of baa4918d89. --- .../versions/228d1af6af1c_mysql_max_index_lengths.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data/migrations/versions/228d1af6af1c_mysql_max_index_lengths.py b/data/migrations/versions/228d1af6af1c_mysql_max_index_lengths.py index 0b70acbb3..2f6ff722b 100644 --- a/data/migrations/versions/228d1af6af1c_mysql_max_index_lengths.py +++ b/data/migrations/versions/228d1af6af1c_mysql_max_index_lengths.py @@ -18,5 +18,8 @@ def upgrade(tables): op.drop_index('queueitem_queue_name', table_name='queueitem') op.create_index('queueitem_queue_name', 'queueitem', ['queue_name'], unique=False, mysql_length=767) + op.drop_index('image_ancestors', table_name='image') + op.create_index('image_ancestors', 'image', ['ancestors'], unique=False, mysql_length=767) + def downgrade(tables): pass From f4d078e87a9e9eb6f6e4e058d6560e054e8d4362 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 6 Jan 2015 15:41:29 -0500 Subject: [PATCH 08/39] vendor pygithub It's going to be a long time before the developer decides to merge our changes upstream. --- requirements-nover.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-nover.txt b/requirements-nover.txt index c1bf6c19f..ca86fe72d 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -22,7 +22,6 @@ xhtml2pdf redis hiredis docker-py -pygithub flask-restful==0.2.12 jsonschema git+https://github.com/NateFerrero/oauth2lib.git @@ -40,4 +39,5 @@ pyyaml git+https://github.com/DevTable/aniso8601-fake.git git+https://github.com/DevTable/anunidecode.git git+https://github.com/DevTable/avatar-generator.git +git+https://github.com/DevTable/pygithub.git gipc diff --git a/requirements.txt b/requirements.txt index 86e69157c..03da5b64c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,6 @@ LogentriesLogger==0.2.1 Mako==1.0.0 MarkupSafe==0.23 Pillow==2.6.1 -PyGithub==1.25.2 PyMySQL==0.6.2 PyPDF2==1.23 PyYAML==3.11 @@ -19,6 +18,7 @@ alembic==0.7.0 git+https://github.com/DevTable/aniso8601-fake.git git+https://github.com/DevTable/anunidecode.git git+https://github.com/DevTable/avatar-generator.git +git+https://github.com/DevTable/pygithub.git aiowsgi==0.3 autobahn==0.9.3-3 backports.ssl-match-hostname==3.4.0.2 From 5a05c6560b0a1bf8df3bf138fbf82c9cdeaf19c5 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Thu, 8 Jan 2015 16:23:43 -0500 Subject: [PATCH 09/39] Always use default twitter image for twitter-view. It was generating a very low resolution (16x16) avatar for the null user and blowing it up to fit. It's much better to just use what would've been the fallback. --- static/directives/twitter-view.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/directives/twitter-view.html b/static/directives/twitter-view.html index 748a68ab5..e78776ea1 100644 --- a/static/directives/twitter-view.html +++ b/static/directives/twitter-view.html @@ -4,7 +4,7 @@

- + {{ authorName }} (@{{authorUser}}) {{ messageDate }} From 42ea3b835ca9a5883dd44cfd8d326cb525e04575 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 12 Jan 2015 11:42:09 -0500 Subject: [PATCH 10/39] Fix NPE --- auth/permissions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/auth/permissions.py b/auth/permissions.py index ae398092d..f970450af 100644 --- a/auth/permissions.py +++ b/auth/permissions.py @@ -89,6 +89,8 @@ class QuayDeferredPermissionUser(Identity): if not self._permissions_loaded: logger.debug('Loading user permissions after deferring.') user_object = model.get_user_by_uuid(self.id) + if user_object is None: + return super(QuayDeferredPermissionUser, self).can(permission) # Add the superuser need, if applicable. if (user_object.username is not None and From 5a0277b4654b64f178e76b11aa01ee1cfd7028de Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 12 Jan 2015 12:17:00 -0500 Subject: [PATCH 11/39] Fix plans table on mobile --- static/css/quay.css | 16 ++++++++++ static/directives/plans-table.html | 50 ++++++++++++++++++++---------- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/static/css/quay.css b/static/css/quay.css index 4df625503..bc23177df 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -3547,6 +3547,22 @@ p.editable:hover i { font-size: 16px; } +.plans-table ul { + margin-top: 10px; + padding: 0px; +} + + +.plans-table ul li { + padding: 4px; + margin: 0px; +} + +.plans-table ul li .plan-info { + padding: 4px; +} + + .repo-breadcrumb-element .crumb { cursor: pointer; } diff --git a/static/directives/plans-table.html b/static/directives/plans-table.html index 2870a06de..d4badb3c6 100644 --- a/static/directives/plans-table.html +++ b/static/directives/plans-table.html @@ -1,23 +1,41 @@
- - - - - - - +
    +
  • -
- - - - - -
PlanPrivate RepositoriesPrice
{{ plan.title }}{{ plan.privateRepos }}
${{ plan.price / 100 }}
 - {{ currentPlan == plan ? 'Selected' : 'Choose' }} + {{ plan.title }} -
+ +
+ ${{ plan.price / 100 }} / month - + {{ plan.privateRepos }} repositories +
+ + + +
From 2878fd28c69bd58936aa3205617849981b1a8de4 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 12 Jan 2015 12:26:57 -0500 Subject: [PATCH 12/39] Fix header and some other minor fixes on mobile --- static/css/quay.css | 1 - static/directives/header-bar.html | 15 ++++++++++++++- static/partials/new-organization.html | 4 ++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/static/css/quay.css b/static/css/quay.css index bc23177df..e0aafa8e8 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -1528,7 +1528,6 @@ i.toggle-icon:hover { } .landing .call-to-action { - height: 40px; font-size: 18px; padding-left: 14px; padding-right: 14px; diff --git a/static/directives/header-bar.html b/static/directives/header-bar.html index 903de59bd..dc82a425a 100644 --- a/static/directives/header-bar.html +++ b/static/directives/header-bar.html @@ -19,8 +19,21 @@
  • Organizations
    • + + -
      + +
    +
    + - - + + From df9a417207df17ad3fe4dcf17d79eae84ac67d1c Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 12 Jan 2015 12:37:01 -0500 Subject: [PATCH 14/39] Fix NPE --- static/js/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/js/app.js b/static/js/app.js index 43fa2f5d4..69f135538 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -2645,7 +2645,7 @@ quayApp.directive('focusablePopoverContent', ['$timeout', '$popover', function ( if (!scope) { return; } scope.$apply(function() { - if (!scope) { return; } + if (!scope || !$scope.$hide) { return; } scope.$hide(); }); }; From f268a5d66114724c21bbe0046fb86233431f0373 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 13 Jan 2015 11:02:08 -0500 Subject: [PATCH 15/39] Fix twitter-view once and for all! One image URL was broken and it was accidentally using the avatar directive, so the class has been changed to 'twitter-avatar' and made explicit. --- static/css/quay.css | 2 +- static/directives/twitter-view.html | 2 +- static/partials/landing-normal.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/static/css/quay.css b/static/css/quay.css index 4df625503..08438df08 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -1667,7 +1667,7 @@ i.toggle-icon:hover { padding-left: 70px; } -.landing-page .twitter-tweet .avatar img { +.landing-page .twitter-tweet .twitter-avatar img { border-radius: 4px; border: 2px solid rgb(70, 70, 70); width: 50px; diff --git a/static/directives/twitter-view.html b/static/directives/twitter-view.html index e78776ea1..68e30f366 100644 --- a/static/directives/twitter-view.html +++ b/static/directives/twitter-view.html @@ -4,7 +4,7 @@

    - + {{ authorName }} (@{{authorUser}}) {{ messageDate }} diff --git a/static/partials/landing-normal.html b/static/partials/landing-normal.html index 0a0dedc3a..274b56ac0 100644 --- a/static/partials/landing-normal.html +++ b/static/partials/landing-normal.html @@ -207,7 +207,7 @@
  • -
    • PlanPrivate RepositoriesPrice + + + $/mo