From d71fbcab1f3d9e6323a972551fda76ee82fffbb3 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 25 May 2016 16:17:44 -0400 Subject: [PATCH 1/6] dockerfile: add check for GPL pip packages --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index cb20b2746..18850f0cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,6 +79,10 @@ ADD .git/HEAD GIT_HEAD # Add all of the files! ADD . . +# If a pip package is licensed as GPL, fail the build. +# If this fails the build, change the last grep to be "grep -B 10 GPL" to see what package is GPLed. +RUN test -z "$(cat requirements.txt | grep -v "^-e" | awk -F'==' '{print $1}' | xargs pip show --disable-pip-version-check | grep -v LGPL | grep GPL)" + # Run the tests ARG RUN_TESTS=true ENV RUN_TESTS ${RUN_TESTS} From 70f794b0af2e400228495061a24991c56bda23e8 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Thu, 26 May 2016 13:29:48 -0400 Subject: [PATCH 2/6] replace rfc3987 library with urlparse The former is GPL licensed. --- endpoints/api/build.py | 9 +++++---- requirements-nover.txt | 1 - requirements.txt | 1 - 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/endpoints/api/build.py b/endpoints/api/build.py index 972255955..4f3307ab5 100644 --- a/endpoints/api/build.py +++ b/endpoints/api/build.py @@ -1,18 +1,19 @@ """ Create, list, cancel and get status/logs of repository builds. """ +from urlparse import urlparse + import logging import json import datetime import hashlib from flask import request -from rfc3987 import parse as uri_parse from app import app, userfiles as user_files, build_logs, log_archive, dockerfile_build_queue from buildtrigger.basehandler import BuildTriggerHandler from endpoints.api import (RepositoryParamResource, parse_args, query_param, nickname, resource, require_repo_read, require_repo_write, validate_json_request, - ApiResource, internal_only, format_date, api, path_param, + ApiResource, internal_only, format_date, api, path_param, require_repo_admin) from endpoints.exception import Unauthorized, NotFound, InvalidRequest from endpoints.building import start_build, PreparedBuild @@ -220,14 +221,14 @@ class RepositoryBuildList(RepositoryParamResource): if archive_url: archive_match = None try: - archive_match = uri_parse(archive_url, 'URI') + archive_match = urlparse(archive_url) except ValueError: pass if not archive_match: raise InvalidRequest('Invalid Archive URL: Must be a valid URI') - scheme = archive_match.get('scheme', None) + scheme = archive_match.scheme if scheme != 'http' and scheme != 'https': raise InvalidRequest('Invalid Archive URL: Must be http or https') diff --git a/requirements-nover.txt b/requirements-nover.txt index d19715a0a..3898df463 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -54,7 +54,6 @@ Flask-Testing pyjwt toposort pyjwkest -rfc3987 jsonpath-rw bintrees redlock diff --git a/requirements.txt b/requirements.txt index ce982e43e..1e2557ac8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -101,7 +101,6 @@ reportlab==2.7 requests==2.9.1 requests-oauthlib==0.6.1 -e git+https://github.com/coreos/resumablehashlib.git@b1b631249589b07adf40e0ee545b323a501340b4#egg=resumablehashlib -rfc3987==1.3.5 semantic-version==2.5.0 six==1.10.0 SQLAlchemy==1.0.12 From 56b2655326ec9f09a1f9052fb34d1001098073ec Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Thu, 26 May 2016 17:28:57 -0400 Subject: [PATCH 3/6] dockerfile: 16.04 baseimage, add GPL check --- Dockerfile | 57 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 18850f0cc..566c9bb72 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,51 @@ # vim:ft=dockerfile -FROM phusion/baseimage:0.9.18 +FROM quay.io/jzelinskie/phusion-baseimage-xenial:e436d1c ENV DEBIAN_FRONTEND noninteractive ENV HOME /root -# Install the dependencies. -RUN apt-get update # 07APR2016 +# Install system packages +RUN apt-get update # 02JUN2016 +RUN apt-get install -y \ + g++ \ + gdebi-core \ + git \ + libevent-2.0.5 \ + libevent-dev \ + libffi-dev \ + libfreetype6-dev \ + libgpgme11 \ + libgpgme11-dev \ + libjpeg62 \ + libjpeg62-dev \ + libjpeg8 \ + libldap-2.4-2 \ + libldap2-dev \ + libmagic1 \ + libpq-dev \ + libpq5 \ + libsasl2-dev \ + libsasl2-modules \ + nodejs \ + npm \ + phantomjs \ + python-dev \ + python-pip \ + python-virtualenv -# New ubuntu packages should be added as their own apt-get install lines below the existing install commands -RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev libfreetype6-dev libffi-dev libgpgme11 libgpgme11-dev - -# Build the python dependencies +# Install python dependencies ADD requirements.txt requirements.txt RUN virtualenv --distribute venv -RUN venv/bin/pip install -r requirements.txt # 01MAR2016 +RUN venv/bin/pip install -r requirements.txt # 02JUN2016 RUN venv/bin/pip freeze +# Check python dependencies for the GPL +# Due to the following bug, pip results must be piped to a file before grepping: +# https://github.com/pypa/pip/pull/3304 +RUN cat requirements.txt | grep -v "^-e" | awk -F'==' '{print $1}' | xargs venv/bin/pip --disable-pip-version-check show > pipinfo.txt +RUN test -z $(cat pipinfo.txt | grep GPL | grep -v LGPL) + # Install the binary dependencies ADD binary_dependencies binary_dependencies RUN gdebi --n binary_dependencies/*.deb @@ -26,13 +55,13 @@ RUN mkdir /gocode ENV GOPATH /gocode RUN curl -O https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz && \ tar -xvf go1.6.linux-amd64.tar.gz && \ - sudo mv go /usr/local && \ + mv go /usr/local && \ rm -rf go1.6.linux-amd64.tar.gz && \ /usr/local/go/bin/go get -u github.com/cloudflare/cfssl/cmd/cfssl && \ /usr/local/go/bin/go get -u github.com/cloudflare/cfssl/cmd/cfssljson && \ - sudo cp /gocode/bin/cfssljson /bin/cfssljson && \ - sudo cp /gocode/bin/cfssl /bin/cfssl && \ - sudo rm -rf /gocode && sudo rm -rf /usr/local/go + cp /gocode/bin/cfssljson /bin/cfssljson && \ + cp /gocode/bin/cfssl /bin/cfssl && \ + rm -rf /gocode && rm -rf /usr/local/go # Install jwtproxy RUN curl -L -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.1/jwtproxy-linux-x64 @@ -79,10 +108,6 @@ ADD .git/HEAD GIT_HEAD # Add all of the files! ADD . . -# If a pip package is licensed as GPL, fail the build. -# If this fails the build, change the last grep to be "grep -B 10 GPL" to see what package is GPLed. -RUN test -z "$(cat requirements.txt | grep -v "^-e" | awk -F'==' '{print $1}' | xargs pip show --disable-pip-version-check | grep -v LGPL | grep GPL)" - # Run the tests ARG RUN_TESTS=true ENV RUN_TESTS ${RUN_TESTS} From 8810157586b9b016aef44a27bea2cd757fcdddf6 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 27 May 2016 12:08:40 -0400 Subject: [PATCH 4/6] remove GPL'd timeparse library --- requirements-nover.txt | 1 - requirements.txt | 1 - util/generatepresharedkey.py | 18 ++++++++++++++---- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/requirements-nover.txt b/requirements-nover.txt index 3898df463..786792fb5 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -62,4 +62,3 @@ bencode cryptography httmock moto -timeparse diff --git a/requirements.txt b/requirements.txt index 1e2557ac8..2d81b6b60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -107,7 +107,6 @@ SQLAlchemy==1.0.12 stevedore==1.12.0 stringscore==0.1.0 stripe==1.32.0 -timeparse==0.5.5 toposort==1.4 trollius==2.1 tzlocal==1.2.2 diff --git a/util/generatepresharedkey.py b/util/generatepresharedkey.py index 4343deb51..99b5eff22 100644 --- a/util/generatepresharedkey.py +++ b/util/generatepresharedkey.py @@ -1,10 +1,12 @@ +import argparse + +from dateutil.parser import parse as parse_date + from app import app from data import model from data.database import ServiceKeyApprovalType from data.model.log import log_action -from timeparse import ParseDatetime -import argparse def generate_key(service, name, expiration_date=None, notes=None): metadata = { @@ -34,12 +36,20 @@ def generate_key(service, name, expiration_date=None, notes=None): return private_key, key.kid +def valid_date(s): + try: + return parse_date(s) + except ValueError: + msg = "Not a valid date: '{0}'.".format(s) + raise argparse.ArgumentTypeError(msg) + + if __name__ == '__main__': parser = argparse.ArgumentParser(description='Generates a preshared key') parser.add_argument('service', help='The service name for which the key is being generated') parser.add_argument('name', help='The friendly name for the key') - parser.add_argument('--expiration', help='The optional expiration date/time for the key', - default=None, action=ParseDatetime) + parser.add_argument('--expiration', default=None, type=valid_date, + help='The optional expiration date for the key') parser.add_argument('--notes', help='Optional notes about the key', default=None) args = parser.parse_args() From 0637091a8b0e2caef27429e12667d50b821aac23 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 27 May 2016 13:33:31 -0400 Subject: [PATCH 5/6] dockerfile: combine GPL check into one line --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 566c9bb72..dc46019c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,8 +43,9 @@ RUN venv/bin/pip freeze # Check python dependencies for the GPL # Due to the following bug, pip results must be piped to a file before grepping: # https://github.com/pypa/pip/pull/3304 -RUN cat requirements.txt | grep -v "^-e" | awk -F'==' '{print $1}' | xargs venv/bin/pip --disable-pip-version-check show > pipinfo.txt -RUN test -z $(cat pipinfo.txt | grep GPL | grep -v LGPL) +RUN cat requirements.txt | grep -v "^-e" | awk -F'==' '{print $1}' | xargs venv/bin/pip --disable-pip-version-check show > pipinfo.txt && + test -z $(cat pipinfo.txt | grep GPL | grep -v LGPL) && + rm pipinfo.txt # Install the binary dependencies ADD binary_dependencies binary_dependencies From e5241c6d8803dd29aa229cf0c5d9c5665f6f5416 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Thu, 2 Jun 2016 11:59:16 -0400 Subject: [PATCH 6/6] tests: simple test for BuildRequest w/ archive URL --- test/test_api_usage.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/test/test_api_usage.py b/test/test_api_usage.py index b3c5f3a87..c7eaf9d6d 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -247,11 +247,11 @@ class TestDiscovery(ApiTestCase): class TestErrorDescription(ApiTestCase): - def test_get_error(self): - json = self.getJsonResponse(Error, params=dict(error_type='not_found')) - assert json['title'] == 'not_found' - assert 'type' in json - assert 'description' in json + def test_get_error(self): + json = self.getJsonResponse(Error, params=dict(error_type='not_found')) + assert json['title'] == 'not_found' + assert 'type' in json + assert 'description' in json class TestPlans(ApiTestCase): @@ -455,9 +455,11 @@ class TestConvertToOrganization(ApiTestCase): self.assertEquals(True, json['is_admin']) # Verify the now-org has no permissions. - count = (database.RepositoryPermission.select() - .where(database.RepositoryPermission.user == organization) - .count()) + count = (database + .RepositoryPermission + .select() + .where(database.RepositoryPermission.user == organization) + .count()) self.assertEquals(0, count) def test_convert_via_email(self): @@ -1814,6 +1816,15 @@ class TestGetRepository(ApiTestCase): class TestRepositoryBuildResource(ApiTestCase): + + def test_repo_build_invalid_url(self): + self.login(ADMIN_ACCESS_USER) + + self.postJsonResponse(RepositoryBuildList, + params=dict(repository=ADMIN_ACCESS_USER + '/simple'), + data=dict(archive_url='hppt://quay.io'), + expected_code=400) + def test_cancel_invalidbuild(self): self.login(ADMIN_ACCESS_USER)