diff --git a/data/model/legacy.py b/data/model/legacy.py index 98583907d..296b6b1f5 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -61,8 +61,10 @@ class InvalidBuildTriggerException(DataModelException): def create_user(username, password, email, is_organization=False): if not validate_email(email): raise InvalidEmailAddressException('Invalid email address: %s' % email) - if not validate_username(username): - raise InvalidUsernameException('Invalid username: %s' % username) + + (username_valid, username_issue) = validate_username(username) + if not username_valid: + raise InvalidUsernameException('Invalid username %s: %s' % (username, username_issue)) # We allow password none for the federated login case. if password is not None and not validate_password(password): @@ -125,9 +127,10 @@ def create_organization(name, email, creating_user): def create_robot(robot_shortname, parent): - if not validate_username(robot_shortname): - raise InvalidRobotException('The name for the robot \'%s\' is invalid.' % - robot_shortname) + (username_valid, username_issue) = validate_username(robot_shortname) + if not username_valid: + raise InvalidRobotException('The name for the robot \'%s\' is invalid: %s' % + (robot_shortname, username_issue)) username = format_robot_username(parent.username, robot_shortname) @@ -214,8 +217,9 @@ def convert_user_to_organization(user, admin_user): def create_team(name, org, team_role_name, description=''): - if not validate_username(name): - raise InvalidTeamException('Invalid team name: %s' % name) + (username_valid, username_issue) = validate_username(name) + if not username_valid: + raise InvalidTeamException('Invalid team name %s: %s' % (name, username_issue)) if not org.organization: raise InvalidOrganizationException('User with name %s is not an org.' % diff --git a/endpoints/api/trigger.py b/endpoints/api/trigger.py index 13e2658b0..20ab04330 100644 --- a/endpoints/api/trigger.py +++ b/endpoints/api/trigger.py @@ -16,8 +16,9 @@ from endpoints.trigger import (BuildTrigger as BuildTriggerBase, TriggerDeactiva TriggerActivationException, EmptyRepositoryException, RepositoryReadException) from data import model -from auth.permissions import UserAdminPermission, AdministerOrganizationPermission +from auth.permissions import UserAdminPermission, AdministerOrganizationPermission, ReadRepositoryPermission from util.names import parse_robot_username +from util.dockerfileparse import parse_dockerfile logger = logging.getLogger(__name__) @@ -231,6 +232,141 @@ class BuildTriggerActivate(RepositoryParamResource): raise Unauthorized() +@resource('/v1/repository//trigger//analyze') +@internal_only +class BuildTriggerAnalyze(RepositoryParamResource): + """ Custom verb for analyzing the config for a build trigger and suggesting various changes + (such as a robot account to use for pulling) + """ + schemas = { + 'BuildTriggerAnalyzeRequest': { + 'id': 'BuildTriggerAnalyzeRequest', + 'type': 'object', + 'required': [ + 'config' + ], + 'properties': { + 'config': { + 'type': 'object', + 'description': 'Arbitrary json.', + } + } + }, + } + + @require_repo_admin + @nickname('analyzeBuildTrigger') + @validate_json_request('BuildTriggerAnalyzeRequest') + def post(self, namespace, repository, trigger_uuid): + """ Analyze the specified build trigger configuration. """ + try: + trigger = model.get_build_trigger(namespace, repository, trigger_uuid) + except model.InvalidBuildTriggerException: + raise NotFound() + + handler = BuildTriggerBase.get_trigger_for_service(trigger.service.name) + new_config_dict = request.get_json()['config'] + + try: + # Load the contents of the Dockerfile. + contents = handler.load_dockerfile_contents(trigger.auth_token, new_config_dict) + if not contents: + return { + 'status': 'error', + 'message': 'Could not read the Dockerfile for the trigger' + } + + # Parse the contents of the Dockerfile. + parsed = parse_dockerfile(contents) + if not parsed: + return { + 'status': 'error', + 'message': 'Could not parse the Dockerfile specified' + } + + # Determine the base image (i.e. the FROM) for the Dockerfile. + base_image = parsed.get_base_image() + if not base_image: + return { + 'status': 'warning', + 'message': 'No FROM line found in the Dockerfile' + } + + # Check to see if the base image lives in Quay. + quay_registry_prefix = '%s/' % (app.config['SERVER_HOSTNAME']) + + if not base_image.startswith(quay_registry_prefix): + return { + 'status': 'publicbase' + } + + # Lookup the repository in Quay. + result = base_image[len(quay_registry_prefix):].split('/', 2) + if len(result) != 2: + return { + 'status': 'warning', + 'message': '"%s" is not a valid Quay repository path' % (base_image) + } + + (base_namespace, base_repository) = result + found_repository = model.get_repository(base_namespace, base_repository) + if not found_repository: + return { + 'status': 'error', + 'message': 'Repository "%s" was not found' % (base_image) + } + + # If the repository is private and the user cannot see that repo, then + # mark it as not found. + can_read = ReadRepositoryPermission(base_namespace, base_repository) + if found_repository.visibility.name != 'public' and not can_read: + return { + 'status': 'error', + 'message': 'Repository "%s" was not found' % (base_image) + } + + # Check to see if the repository is public. If not, we suggest the + # usage of a robot account to conduct the pull. + read_robots = [] + + if AdministerOrganizationPermission(base_namespace).can(): + def robot_view(robot): + return { + 'name': robot.username, + 'kind': 'user', + 'is_robot': True + } + + def is_valid_robot(user): + # Make sure the user is a robot. + if not user.robot: + return False + + # Make sure the current user can see/administer the robot. + (robot_namespace, shortname) = parse_robot_username(user.username) + return AdministerOrganizationPermission(robot_namespace).can() + + repo_perms = model.get_all_repo_users(base_namespace, base_repository) + read_robots = [robot_view(perm.user) for perm in repo_perms if is_valid_robot(perm.user)] + + return { + 'namespace': base_namespace, + 'name': base_repository, + 'is_public': found_repository.visibility.name == 'public', + 'robots': read_robots, + 'status': 'analyzed', + 'dockerfile_url': handler.dockerfile_url(trigger.auth_token, new_config_dict) + } + + except RepositoryReadException as rre: + return { + 'status': 'error', + 'message': rre.message + } + + raise NotFound() + + @resource('/v1/repository//trigger//start') class ActivateBuildTrigger(RepositoryParamResource): """ Custom verb to manually activate a build trigger. """ diff --git a/endpoints/trigger.py b/endpoints/trigger.py index ddd68deac..77a818206 100644 --- a/endpoints/trigger.py +++ b/endpoints/trigger.py @@ -2,6 +2,7 @@ import logging import io import os.path import tarfile +import base64 from github import Github, UnknownObjectException, GithubException from tempfile import SpooledTemporaryFile @@ -45,6 +46,19 @@ class BuildTrigger(object): def __init__(self): pass + def dockerfile_url(self, auth_token, config): + """ + Returns the URL at which the Dockerfile for the trigger can be found or None if none/not applicable. + """ + return None + + def load_dockerfile_contents(self, auth_token, config): + """ + Loads the Dockerfile found for the trigger's config and returns them or None if none could + be found/loaded. + """ + return None + def list_build_sources(self, auth_token): """ Take the auth information for the specific trigger type and load the @@ -167,7 +181,6 @@ class GithubBuildTrigger(BuildTrigger): return config - def list_build_sources(self, auth_token): gh_client = self._get_client(auth_token) usr = gh_client.get_user() @@ -218,6 +231,41 @@ class GithubBuildTrigger(BuildTrigger): raise RepositoryReadException(message) + def dockerfile_url(self, auth_token, config): + source = config['build_source'] + subdirectory = config.get('subdir', '') + path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile' + + gh_client = self._get_client(auth_token) + try: + repo = gh_client.get_repo(source) + master_branch = repo.master_branch or 'master' + return 'https://github.com/%s/blob/%s/%s' % (source, master_branch, path) + except GithubException as ge: + return None + + def load_dockerfile_contents(self, auth_token, config): + gh_client = self._get_client(auth_token) + + source = config['build_source'] + subdirectory = config.get('subdir', '') + path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile' + + try: + repo = gh_client.get_repo(source) + file_info = repo.get_file_contents(path) + if file_info is None: + return None + + content = file_info.content + if file_info.encoding == 'base64': + content = base64.b64decode(content) + return content + + except GithubException as ge: + message = ge.data.get('message', 'Unable to read Dockerfile: %s' % source) + raise RepositoryReadException(message) + @staticmethod def _prepare_build(config, repo, commit_sha, build_name, ref): # Prepare the download and upload URLs diff --git a/endpoints/web.py b/endpoints/web.py index 04b0326ef..a9cace61d 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -9,6 +9,7 @@ from urlparse import urlparse from data import model from data.model.oauth import DatabaseAuthorizationProvider from app import app, billing as stripe +from auth.auth import require_session_login from auth.permissions import AdministerOrganizationPermission from util.invoice import renderInvoiceToPdf from util.seo import render_snapshot @@ -161,6 +162,7 @@ def privacy(): @web.route('/receipt', methods=['GET']) @route_show_if(features.BILLING) +@require_session_login def receipt(): if not current_user.is_authenticated(): abort(401) diff --git a/initdb.py b/initdb.py index 9381ea282..2f8a77429 100644 --- a/initdb.py +++ b/initdb.py @@ -292,7 +292,7 @@ def populate_database(): __generate_repository(new_user_1, 'complex', 'Complex repository with many branches and tags.', - False, [(new_user_2, 'read')], + False, [(new_user_2, 'read'), (dtrobot[0], 'read')], (2, [(3, [], 'v2.0'), (1, [(1, [(1, [], ['prod'])], 'staging'), diff --git a/static/css/quay.css b/static/css/quay.css index b00e09492..8eb25f740 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -2406,12 +2406,19 @@ p.editable:hover i { text-align: center; } -#image-history-container .tags .tag, #confirmdeleteTagModal .tag { +.tags .tag, #confirmdeleteTagModal .tag { border-radius: 10px; margin-right: 4px; cursor: pointer; } +.tooltip-tags { + display: block; + margin-top: 10px; + border-top: 1px dotted #aaa; + padding-top: 10px; +} + #changes-tree-container { overflow: hidden; } @@ -3451,7 +3458,7 @@ pre.command:before { position: relative; - height: 100px; + height: 75px; opacity: 1; } @@ -3602,10 +3609,10 @@ pre.command:before { margin-right: 34px; } -.trigger-option-section:not(:last-child) { - border-bottom: 1px solid #eee; - padding-bottom: 16px; - margin-bottom: 16px; +.trigger-option-section:not(:first-child) { + border-top: 1px solid #eee; + padding-top: 16px; + margin-top: 10px; } .trigger-option-section .entity-search-element .twitter-typeahead { diff --git a/static/directives/billing-invoices.html b/static/directives/billing-invoices.html index fc64b1abf..2e2bfd469 100644 --- a/static/directives/billing-invoices.html +++ b/static/directives/billing-invoices.html @@ -30,7 +30,7 @@ - + diff --git a/static/directives/copy-box.html b/static/directives/copy-box.html index 204bd3a57..1d996cc31 100644 --- a/static/directives/copy-box.html +++ b/static/directives/copy-box.html @@ -2,7 +2,7 @@
- +
diff --git a/static/directives/delete-ui.html b/static/directives/delete-ui.html index d04e840f0..275c25b50 100644 --- a/static/directives/delete-ui.html +++ b/static/directives/delete-ui.html @@ -1,4 +1,4 @@ - + diff --git a/static/directives/entity-reference.html b/static/directives/entity-reference.html index 12ee81f86..6aecc1ebe 100644 --- a/static/directives/entity-reference.html +++ b/static/directives/entity-reference.html @@ -1,14 +1,14 @@ - + {{entity.name}} {{entity.name}} - - + + {{ getPrefix(entity.name) }}+{{ getShortenedName(entity.name) }} @@ -22,6 +22,6 @@ + data-title="This user is not a member of the organization" bs-tooltip="tooltip.title" data-container="body"> diff --git a/static/directives/entity-search.html b/static/directives/entity-search.html index 04dac6c0f..bc1c0a94e 100644 --- a/static/directives/entity-search.html +++ b/static/directives/entity-search.html @@ -1,5 +1,5 @@ - +
@@ -55,7 +55,7 @@
-
on behalf of
diff --git a/static/directives/namespace-selector.html b/static/directives/namespace-selector.html index 98d91394d..98f6a353a 100644 --- a/static/directives/namespace-selector.html +++ b/static/directives/namespace-selector.html @@ -18,7 +18,7 @@ diff --git a/static/directives/plan-manager.html b/static/directives/plan-manager.html index f2ad26df4..5978043a7 100644 --- a/static/directives/plan-manager.html +++ b/static/directives/plan-manager.html @@ -38,7 +38,7 @@ {{ plan.title }}
- Discontinued Plan + Discontinued Plan
{{ plan.privateRepos }} diff --git a/static/directives/prototype-manager.html b/static/directives/prototype-manager.html index 26cec48c8..f5d5b0f9c 100644 --- a/static/directives/prototype-manager.html +++ b/static/directives/prototype-manager.html @@ -3,7 +3,7 @@
- Default permissions provide a means of specifying additional permissions that should be granted automatically to a repository. + Default permissions provide a means of specifying additional permissions that should be granted automatically to a repository.
@@ -17,13 +17,13 @@ Repository Creator - Applies To User/Robot/Team diff --git a/static/directives/repo-circle.html b/static/directives/repo-circle.html index ca5cbddf4..82648863e 100644 --- a/static/directives/repo-circle.html +++ b/static/directives/repo-circle.html @@ -1,2 +1,2 @@ - + diff --git a/static/directives/setup-trigger-dialog.html b/static/directives/setup-trigger-dialog.html new file mode 100644 index 000000000..1e84a782a --- /dev/null +++ b/static/directives/setup-trigger-dialog.html @@ -0,0 +1,100 @@ +
+ + + +
diff --git a/static/directives/signup-form.html b/static/directives/signup-form.html index 0e87f17e5..fb0ccc6fa 100644 --- a/static/directives/signup-form.html +++ b/static/directives/signup-form.html @@ -1,13 +1,15 @@ -
{{ shownToken.friendlyName }}
- - - + +
diff --git a/static/partials/repo-list.html b/static/partials/repo-list.html index 2c7da9c9d..5c4e5d5ae 100644 --- a/static/partials/repo-list.html +++ b/static/partials/repo-list.html @@ -4,7 +4,7 @@
@@ -60,11 +60,11 @@
- - diff --git a/static/partials/user-admin.html b/static/partials/user-admin.html index 0ad550232..25ef0ec32 100644 --- a/static/partials/user-admin.html +++ b/static/partials/user-admin.html @@ -69,10 +69,10 @@ + data-title="{{ authInfo.application.description || authInfo.application.name }}" bs-tooltip> {{ authInfo.application.name }} - + {{ authInfo.application.name }} {{ authInfo.application.organization.name }} @@ -80,7 +80,7 @@ + ng-repeat="scopeInfo in authInfo.scopes" data-title="{{ scopeInfo.description }}" bs-tooltip> {{ scopeInfo.scope }} @@ -124,8 +124,8 @@
Change e-mail address
- + @@ -146,11 +146,12 @@ Password changed successfully
-
- + + + match="cuser.password" required ng-pattern="/^.{8,}$/">
@@ -169,7 +170,7 @@
GitHub Login:
- + {{githubLogin}}
diff --git a/static/partials/view-repo.html b/static/partials/view-repo.html index b5db777de..51f34204c 100644 --- a/static/partials/view-repo.html +++ b/static/partials/view-repo.html @@ -17,7 +17,7 @@
-
+
- +
@@ -145,10 +145,10 @@
- + {{getTagCount(repo)}} - + {{imageHistory.value.length}} @@ -162,7 +162,7 @@
Total Compressed Size
{{ getTotalSize(currentTag) | bytes }}
@@ -171,7 +171,7 @@
+ bs-tooltip="" data-title="{{ image.size | bytes }}"> {{ image.id.substr(0, 12) }}
@@ -199,14 +199,14 @@
{{ currentImage.id }}
Compressed Image Size
{{ currentImage.size | bytes }}
Command
{{ getFormattedCommand(currentImage) }}
@@ -216,17 +216,17 @@
- {{currentImageChanges.added.length}} - {{currentImageChanges.removed.length}} - {{currentImageChanges.changed.length}} @@ -237,15 +237,15 @@
- {{file}} + {{file}}
- {{file}} + {{file}}
- {{file}} + {{file}}
@@ -296,7 +296,7 @@ - {{ image.id.substr(0, 12) }} diff --git a/templates/base.html b/templates/base.html index de435f460..ee5629dc8 100644 --- a/templates/base.html +++ b/templates/base.html @@ -57,7 +57,7 @@ - + {% if mixpanel_key %} diff --git a/templates/index.html b/templates/index.html index 83b87457f..45f44fbd6 100644 --- a/templates/index.html +++ b/templates/index.html @@ -7,7 +7,7 @@ {% block added_meta %} - + {% endblock %} diff --git a/test/data/test.db b/test/data/test.db index fc0078c61..75c2fe8aa 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/test/test_api_security.py b/test/test_api_security.py index 8c46e9b11..4edd351ee 100644 --- a/test/test_api_security.py +++ b/test/test_api_security.py @@ -17,7 +17,7 @@ from endpoints.api.build import (FileDropResource, RepositoryBuildStatus, Reposi from endpoints.api.robot import UserRobotList, OrgRobot, OrgRobotList, UserRobot from endpoints.api.trigger import (BuildTriggerActivate, BuildTriggerSources, BuildTriggerSubdirs, TriggerBuildList, ActivateBuildTrigger, BuildTrigger, - BuildTriggerList) + BuildTriggerList, BuildTriggerAnalyze) from endpoints.api.webhook import Webhook, WebhookList from endpoints.api.user import (PrivateRepositories, ConvertToOrganization, Recovery, Signout, Signin, User, UserAuthorizationList, UserAuthorization) @@ -87,6 +87,9 @@ class ApiTestCase(unittest.TestCase): rv = client.open(final_url, **open_kwargs) msg = '%s %s: %s expected: %s' % (method, final_url, rv.status_code, expected_status) + if rv.status_code != expected_status: + print rv.data + self.assertEqual(rv.status_code, expected_status, msg) def setUp(self): @@ -1198,6 +1201,130 @@ class TestActivateBuildTrigger0byeBuynlargeOrgrepo(ApiTestCase): def test_post_devtable(self): self._run_test('POST', 404, 'devtable', None) +class TestActivateBuildTrigger0byeDevtableShared(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(ActivateBuildTrigger, trigger_uuid="0BYE", repository="devtable/shared") + + def test_post_anonymous(self): + self._run_test('POST', 401, None, None) + + def test_post_freshuser(self): + self._run_test('POST', 403, 'freshuser', None) + + def test_post_reader(self): + self._run_test('POST', 403, 'reader', None) + + def test_post_devtable(self): + self._run_test('POST', 404, 'devtable', None) + + +class TestActivateBuildTrigger0byeBuynlargeOrgrepo(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(ActivateBuildTrigger, trigger_uuid="0BYE", repository="buynlarge/orgrepo") + + def test_post_anonymous(self): + self._run_test('POST', 401, None, None) + + def test_post_freshuser(self): + self._run_test('POST', 403, 'freshuser', None) + + def test_post_reader(self): + self._run_test('POST', 403, 'reader', None) + + def test_post_devtable(self): + self._run_test('POST', 404, 'devtable', None) + + +class TestBuildTriggerAnalyze0byePublicPublicrepo(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(BuildTriggerAnalyze, trigger_uuid="0BYE", repository="public/publicrepo") + + def test_post_anonymous(self): + self._run_test('POST', 401, None, None) + + def test_post_freshuser(self): + self._run_test('POST', 403, 'freshuser', None) + + def test_post_reader(self): + self._run_test('POST', 403, 'reader', None) + + def test_post_devtable(self): + self._run_test('POST', 403, 'devtable', {'config': {}}) + + +class TestBuildTriggerAnalyze0byeDevtableShared(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(BuildTriggerAnalyze, trigger_uuid="0BYE", repository="devtable/shared") + + def test_post_anonymous(self): + self._run_test('POST', 401, None, None) + + def test_post_freshuser(self): + self._run_test('POST', 403, 'freshuser', None) + + def test_post_reader(self): + self._run_test('POST', 403, 'reader', None) + + def test_post_devtable(self): + self._run_test('POST', 404, 'devtable', {'config': {}}) + + +class TestBuildTriggerAnalyze0byeBuynlargeOrgrepo(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(BuildTriggerAnalyze, trigger_uuid="0BYE", repository="buynlarge/orgrepo") + + def test_post_anonymous(self): + self._run_test('POST', 401, None, None) + + def test_post_freshuser(self): + self._run_test('POST', 403, 'freshuser', None) + + def test_post_reader(self): + self._run_test('POST', 403, 'reader', None) + + def test_post_devtable(self): + self._run_test('POST', 404, 'devtable', {'config': {}}) + +class TestBuildTriggerAnalyze0byeDevtableShared(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(BuildTriggerAnalyze, trigger_uuid="0BYE", repository="devtable/shared") + + def test_post_anonymous(self): + self._run_test('POST', 401, None, None) + + def test_post_freshuser(self): + self._run_test('POST', 403, 'freshuser', None) + + def test_post_reader(self): + self._run_test('POST', 403, 'reader', None) + + def test_post_devtable(self): + self._run_test('POST', 404, 'devtable', {'config': {}}) + + +class TestBuildTriggerAnalyze0byeBuynlargeOrgrepo(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(BuildTriggerAnalyze, trigger_uuid="0BYE", repository="buynlarge/orgrepo") + + def test_post_anonymous(self): + self._run_test('POST', 401, None, None) + + def test_post_freshuser(self): + self._run_test('POST', 403, 'freshuser', None) + + def test_post_reader(self): + self._run_test('POST', 403, 'reader', None) + + def test_post_devtable(self): + self._run_test('POST', 404, 'devtable', {'config': {}}) + class TestRepositoryImageChangesPtsgPublicPublicrepo(ApiTestCase): def setUp(self): diff --git a/test/test_api_usage.py b/test/test_api_usage.py index f03a48d87..08eb6f1c1 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -19,7 +19,7 @@ from endpoints.api.build import RepositoryBuildStatus, RepositoryBuildLogs, Repo from endpoints.api.robot import UserRobotList, OrgRobot, OrgRobotList, UserRobot from endpoints.api.trigger import (BuildTriggerActivate, BuildTriggerSources, BuildTriggerSubdirs, TriggerBuildList, ActivateBuildTrigger, BuildTrigger, - BuildTriggerList) + BuildTriggerList, BuildTriggerAnalyze) from endpoints.api.webhook import Webhook, WebhookList from endpoints.api.user import (PrivateRepositories, ConvertToOrganization, Signout, Signin, User, UserAuthorizationList, UserAuthorization) @@ -292,6 +292,24 @@ class TestCreateNewUser(ApiTestCase): expected_code=400) self.assertEquals('The username already exists', json['message']) + + def test_trycreatetooshort(self): + json = self.postJsonResponse(User, + data=dict(username='a', + password='password', + email='test@example.com'), + expected_code=400) + + self.assertEquals('Invalid username a: Username must be between 4 and 30 characters in length', json['error_description']) + + def test_trycreateregexmismatch(self): + json = self.postJsonResponse(User, + data=dict(username='auserName', + password='password', + email='test@example.com'), + expected_code=400) + + self.assertEquals('Invalid username auserName: Username must match expression [a-z0-9_]+', json['error_description']) def test_createuser(self): data = self.postResponse(User, @@ -1605,6 +1623,15 @@ class FakeBuildTrigger(BuildTriggerBase): def manual_start(self, auth_token, config): return ('foo', ['bar'], 'build-name', 'subdir') + def dockerfile_url(self, auth_token, config): + return 'http://some/url' + + def load_dockerfile_contents(self, auth_token, config): + if not 'dockerfile' in config: + return None + + return config['dockerfile'] + class TestBuildTriggers(ApiTestCase): def test_list_build_triggers(self): @@ -1653,6 +1680,82 @@ class TestBuildTriggers(ApiTestCase): self.assertEquals(0, len(json['triggers'])) + def test_analyze_fake_trigger(self): + self.login(ADMIN_ACCESS_USER) + + database.BuildTriggerService.create(name='fakeservice') + + # Add a new fake trigger. + repo = model.get_repository(ADMIN_ACCESS_USER, 'simple') + user = model.get_user(ADMIN_ACCESS_USER) + trigger = model.create_build_trigger(repo, 'fakeservice', 'sometoken', user) + + # Analyze the trigger's dockerfile: First, no dockerfile. + trigger_config = {} + analyze_json = self.postJsonResponse(BuildTriggerAnalyze, + params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid), + data={'config': trigger_config}) + + self.assertEquals('error', analyze_json['status']) + self.assertEquals('Could not read the Dockerfile for the trigger', analyze_json['message']) + + # Analyze the trigger's dockerfile: Second, missing FROM in dockerfile. + trigger_config = {'dockerfile': 'MAINTAINER me'} + analyze_json = self.postJsonResponse(BuildTriggerAnalyze, + params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid), + data={'config': trigger_config}) + + self.assertEquals('warning', analyze_json['status']) + self.assertEquals('No FROM line found in the Dockerfile', analyze_json['message']) + + # Analyze the trigger's dockerfile: Third, dockerfile with public repo. + trigger_config = {'dockerfile': 'FROM somerepo'} + analyze_json = self.postJsonResponse(BuildTriggerAnalyze, + params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid), + data={'config': trigger_config}) + + self.assertEquals('publicbase', analyze_json['status']) + + # Analyze the trigger's dockerfile: Fourth, dockerfile with private repo with an invalid path. + trigger_config = {'dockerfile': 'FROM localhost:5000/somepath'} + analyze_json = self.postJsonResponse(BuildTriggerAnalyze, + params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid), + data={'config': trigger_config}) + + self.assertEquals('warning', analyze_json['status']) + self.assertEquals('"localhost:5000/somepath" is not a valid Quay repository path', analyze_json['message']) + + # Analyze the trigger's dockerfile: Fifth, dockerfile with private repo that does not exist. + trigger_config = {'dockerfile': 'FROM localhost:5000/nothere/randomrepo'} + analyze_json = self.postJsonResponse(BuildTriggerAnalyze, + params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid), + data={'config': trigger_config}) + + self.assertEquals('error', analyze_json['status']) + self.assertEquals('Repository "localhost:5000/nothere/randomrepo" was not found', analyze_json['message']) + + # Analyze the trigger's dockerfile: Sixth, dockerfile with private repo that the user cannot see. + trigger_config = {'dockerfile': 'FROM localhost:5000/randomuser/randomrepo'} + analyze_json = self.postJsonResponse(BuildTriggerAnalyze, + params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid), + data={'config': trigger_config}) + + self.assertEquals('error', analyze_json['status']) + self.assertEquals('Repository "localhost:5000/randomuser/randomrepo" was not found', analyze_json['message']) + + # Analyze the trigger's dockerfile: Seventh, dockerfile with private repo that the user see. + trigger_config = {'dockerfile': 'FROM localhost:5000/devtable/complex'} + analyze_json = self.postJsonResponse(BuildTriggerAnalyze, + params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid), + data={'config': trigger_config}) + + self.assertEquals('analyzed', analyze_json['status']) + self.assertEquals('devtable', analyze_json['namespace']) + self.assertEquals('complex', analyze_json['name']) + self.assertEquals(False, analyze_json['is_public']) + self.assertEquals(ADMIN_ACCESS_USER + '+dtrobot', analyze_json['robots'][0]['name']) + + def test_fake_trigger(self): self.login(ADMIN_ACCESS_USER) diff --git a/tools/renderinvoice.py b/tools/renderinvoice.py new file mode 100644 index 000000000..a7fd1382c --- /dev/null +++ b/tools/renderinvoice.py @@ -0,0 +1,36 @@ +from app import stripe +from app import app + +from util.invoice import renderInvoiceToPdf + +from data import model + +import argparse + +from flask import Flask, current_app +from flask_mail import Mail + +def sendInvoice(invoice_id): + invoice = stripe.Invoice.retrieve(invoice_id) + if not invoice['customer']: + print 'No customer found' + return + + customer_id = invoice['customer'] + user = model.get_user_or_org_by_customer_id(customer_id) + if not user: + print 'No user found for customer %s' % (customer_id) + return + + + with app.app_context(): + file_data = renderInvoiceToPdf(invoice, user) + with open('invoice.pdf', 'wb') as f: + f.write(file_data) + + print 'Invoice output as invoice.pdf' + +parser = argparse.ArgumentParser(description='Generate an invoice') +parser.add_argument('invoice_id', help='The invoice ID') +args = parser.parse_args() +sendInvoice(args.invoice_id) diff --git a/util/dockerfileparse.py b/util/dockerfileparse.py new file mode 100644 index 000000000..608588c2e --- /dev/null +++ b/util/dockerfileparse.py @@ -0,0 +1,81 @@ +import re + +LINE_CONTINUATION_REGEX = re.compile('\s*\\\s*\n') +COMMAND_REGEX = re.compile('([A-Z]+)\s(.*)') + +COMMENT_CHARACTER = '#' + +class ParsedDockerfile(object): + def __init__(self, commands): + self.commands = commands + + def get_commands_of_kind(self, kind): + return [command for command in self.commands if command['command'] == kind] + + def get_base_image(self): + image_and_tag = self.get_base_image_and_tag() + if not image_and_tag: + return None + + return self.base_image_from_repo_identifier(image_and_tag) + + @staticmethod + def base_image_from_repo_identifier(image_and_tag): + # Note: + # Dockerfile images references can be of multiple forms: + # server:port/some/path + # somepath + # server/some/path + # server/some/path:tag + # server:port/some/path:tag + parts = image_and_tag.strip().split(':') + + if len(parts) == 1: + # somepath + return parts[0] + + # Otherwise, determine if the last part is a port + # or a tag. + if parts[-1].find('/') >= 0: + # Last part is part of the hostname. + return image_and_tag + + # Remaining cases: + # server/some/path:tag + # server:port/some/path:tag + return ':'.join(parts[0:-1]) + + def get_base_image_and_tag(self): + from_commands = self.get_commands_of_kind('FROM') + if not from_commands: + return None + + return from_commands[-1]['parameters'] + + +def strip_comments(contents): + lines = [line for line in contents.split('\n') if not line.startswith(COMMENT_CHARACTER)] + return '\n'.join(lines) + + +def join_continued_lines(contents): + return LINE_CONTINUATION_REGEX.sub('', contents) + + +def parse_dockerfile(contents): + contents = join_continued_lines(strip_comments(contents)) + lines = [line for line in contents.split('\n') if len(line) > 0] + + commands = [] + for line in lines: + match_command = COMMAND_REGEX.match(line) + if match_command: + command = match_command.group(1) + parameters = match_command.group(2) + + commands.append({ + 'command': command, + 'parameters': parameters + }) + + return ParsedDockerfile(commands) diff --git a/util/validation.py b/util/validation.py index 895767d98..8b5b8400d 100644 --- a/util/validation.py +++ b/util/validation.py @@ -10,10 +10,16 @@ def validate_email(email_address): def validate_username(username): - # Minimum length of 2, maximum length of 255, no url unsafe characters - return (re.search(r'[^a-z0-9_]', username) is None and - len(username) >= 4 and - len(username) <= 30) + # Based off the restrictions defined in the Docker Registry API spec + regex_match = (re.search(r'[^a-z0-9_]', username) is None) + if not regex_match: + return (False, 'Username must match expression [a-z0-9_]+') + + length_match = (len(username) >= 4 and len(username) <= 30) + if not length_match: + return (False, 'Username must be between 4 and 30 characters in length') + + return (True, '') def validate_password(password): diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index b16008ba1..ac77c412e 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -14,12 +14,14 @@ from zipfile import ZipFile from functools import partial from datetime import datetime, timedelta from threading import Event +from uuid import uuid4 from data.queue import dockerfile_build_queue from data import model from workers.worker import Worker from app import app, userfiles as user_files from util.safetar import safe_extractall +from util.dockerfileparse import parse_dockerfile, ParsedDockerfile root_logger = logging.getLogger('') @@ -33,6 +35,7 @@ logger = logging.getLogger(__name__) build_logs = app.config['BUILDLOGS'] TIMEOUT_PERIOD_MINUTES = 20 +CACHE_EXPIRATION_PERIOD_HOURS = 24 class StatusWrapper(object): @@ -94,6 +97,9 @@ class StreamingDockerClient(Client): class DockerfileBuildContext(object): + image_id_to_cache_time = {} + private_repo_tags = set() + def __init__(self, build_context_dir, dockerfile_subdir, repo, tag_names, push_token, build_uuid, pull_credentials=None): self._build_dir = build_context_dir @@ -104,6 +110,7 @@ class DockerfileBuildContext(object): self._status = StatusWrapper(build_uuid) self._build_logger = partial(build_logs.append_log_message, build_uuid) self._pull_credentials = pull_credentials + self._public_repos = set() # Note: We have two different clients here because we (potentially) login # with both, but with different credentials that we do not want shared between @@ -113,29 +120,27 @@ class DockerfileBuildContext(object): dockerfile_path = os.path.join(self._build_dir, dockerfile_subdir, 'Dockerfile') - self._num_steps = DockerfileBuildContext.__count_steps(dockerfile_path) + + # Compute the number of steps + with open(dockerfile_path, 'r') as dockerfileobj: + self._parsed_dockerfile = parse_dockerfile(dockerfileobj.read()) + self._num_steps = len(self._parsed_dockerfile.commands) logger.debug('Will build and push to repo %s with tags named: %s' % (self._repo, self._tag_names)) def __enter__(self): + self.__cleanup_containers() + self.__evict_expired_images() + self.__cleanup() return self def __exit__(self, exc_type, value, traceback): + self.__cleanup_containers() self.__cleanup() shutil.rmtree(self._build_dir) - @staticmethod - def __count_steps(dockerfile_path): - with open(dockerfile_path, 'r') as dockerfileobj: - steps = 0 - for line in dockerfileobj.readlines(): - stripped = line.strip() - if stripped and stripped[0] is not '#': - steps += 1 - return steps - @staticmethod def __total_completion(statuses, total_images): percentage_with_sizes = float(len(statuses.values()))/total_images @@ -151,6 +156,11 @@ class DockerfileBuildContext(object): self._build_cl.login(self._pull_credentials['username'], self._pull_credentials['password'], registry=self._pull_credentials['registry'], reauth=True) + # Pull the image, in case it was updated since the last build + base_image = self._parsed_dockerfile.get_base_image() + self._build_logger('Pulling base image: %s' % base_image) + self._build_cl.pull(base_image) + # Start the build itself. logger.debug('Starting build.') @@ -260,7 +270,39 @@ class DockerfileBuildContext(object): raise RuntimeError(message) - def __cleanup(self): + def __is_repo_public(self, repo_name): + if repo_name in self._public_repos: + return True + + repo_portions = repo_name.split('/') + registry_hostname = 'index.docker.io' + local_repo_name = repo_name + if len(repo_portions) > 2: + registry_hostname = repo_portions[0] + local_repo_name = '/'.join(repo_portions[1:]) + + repo_url_template = '%s://%s/v1/repositories/%s/images' + protocols = ['https', 'http'] + secure_repo_url, repo_url = [repo_url_template % (protocol, registry_hostname, local_repo_name) + for protocol in protocols] + + try: + + try: + repo_info = requests.get(secure_repo_url) + except requests.exceptions.SSLError: + repo_info = requests.get(repo_url) + + except requests.exceptions.ConnectionError: + return False + + if repo_info.status_code / 100 == 2: + self._public_repos.add(repo_name) + return True + else: + return False + + def __cleanup_containers(self): # First clean up any containers that might be holding the images for running in self._build_cl.containers(quiet=True): logger.debug('Killing container: %s' % running['Id']) @@ -271,40 +313,62 @@ class DockerfileBuildContext(object): logger.debug('Removing container: %s' % container['Id']) self._build_cl.remove_container(container['Id']) - # Iterate all of the images and remove the ones that the public registry - # doesn't know about, this should preserve base images. - images_to_remove = set() - repos = set() + def __evict_expired_images(self): + logger.debug('Cleaning images older than %s hours.', CACHE_EXPIRATION_PERIOD_HOURS) + now = datetime.now() + verify_removed = set() + for image in self._build_cl.images(): - images_to_remove.add(image['Id']) + image_id = image[u'Id'] + created = datetime.fromtimestamp(image[u'Created']) - for tag in image['RepoTags']: - tag_repo = tag.split(':')[0] - if tag_repo != '': - repos.add(tag_repo) + # If we don't have a cache time, use the created time (e.g. worker reboot) + cache_time = self.image_id_to_cache_time.get(image_id, created) + expiration = cache_time + timedelta(hours=CACHE_EXPIRATION_PERIOD_HOURS) - for repo in repos: - repo_url = 'https://index.docker.io/v1/repositories/%s/images' % repo - repo_info = requests.get(repo_url) - if repo_info.status_code / 100 == 2: - for repo_image in repo_info.json(): - if repo_image['id'] in images_to_remove: - logger.debug('Image was deemed public: %s' % repo_image['id']) - images_to_remove.remove(repo_image['id']) + if expiration < now: + logger.debug('Removing expired image: %s' % image_id) - for to_remove in images_to_remove: - logger.debug('Removing private image: %s' % to_remove) - try: - self._build_cl.remove_image(to_remove) - except APIError: - # Sometimes an upstream image removed this one - pass + for tag in image['RepoTags']: + # We can forget about this particular tag if it was indeed one of our renamed tags + self.private_repo_tags.discard(tag) + + verify_removed.add(image_id) + try: + self._build_cl.remove_image(image_id) + except APIError: + # Sometimes an upstream image removed this one + pass # Verify that our images were actually removed for image in self._build_cl.images(): - if image['Id'] in images_to_remove: + if image['Id'] in verify_removed: raise RuntimeError('Image was not removed: %s' % image['Id']) + def __cleanup(self): + # Iterate all of the images and rename the ones that aren't public. This should preserve + # base images and also allow the cache to function. + now = datetime.now() + for image in self._build_cl.images(): + image_id = image[u'Id'] + + if image_id not in self.image_id_to_cache_time: + logger.debug('Setting image %s cache time to %s', image_id, now) + self.image_id_to_cache_time[image_id] = now + + for tag in image['RepoTags']: + tag_repo = ParsedDockerfile.base_image_from_repo_identifier(tag) + if tag_repo != '': + if tag_repo in self.private_repo_tags: + logger.debug('Repo is private and has already been renamed: %s' % tag_repo) + elif self.__is_repo_public(tag_repo): + logger.debug('Repo was deemed public: %s', tag_repo) + else: + new_name = str(uuid4()) + logger.debug('Private repo tag being renamed %s -> %s', tag, new_name) + self._build_cl.tag(image_id, new_name) + self._build_cl.remove_image(tag) + self.private_repo_tags.add(new_name) class DockerfileBuildWorker(Worker): def __init__(self, *vargs, **kwargs): @@ -317,6 +381,7 @@ class DockerfileBuildWorker(Worker): 'application/octet-stream': DockerfileBuildWorker.__prepare_dockerfile, 'application/x-tar': DockerfileBuildWorker.__prepare_tarball, 'application/gzip': DockerfileBuildWorker.__prepare_tarball, + 'application/x-gzip': DockerfileBuildWorker.__prepare_tarball, } self._timeout = Event() @@ -327,7 +392,7 @@ class DockerfileBuildWorker(Worker): # Save the zip file to temp somewhere with TemporaryFile() as zip_file: - zip_file.write(request_file.raw) + zip_file.write(request_file.content) to_extract = ZipFile(zip_file) to_extract.extractall(build_dir)