diff --git a/.gitignore b/.gitignore index 9c8de6681..d5a2ef909 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ htmlcov .tox .cache .npm-debug.log +Dockerfile-e diff --git a/buildman/component/buildcomponent.py b/buildman/component/buildcomponent.py index e7730cd81..075285334 100644 --- a/buildman/component/buildcomponent.py +++ b/buildman/component/buildcomponent.py @@ -1,4 +1,5 @@ import datetime +import os import time import logging import json @@ -122,9 +123,21 @@ class BuildComponent(BaseComponent): # base_image: The image name and credentials to use to conduct the base image pull. # username: The username for pulling the base image (if any). # password: The password for pulling the base image (if any). + + subdir, dockerfile_name = os.path.split(build_config.get('build_subdir', '/Dockerfile')) + + # HACK HACK HACK HACK HACK HACK HACK + # TODO: FIX THIS in the database and then turn the split back on. + if dockerfile_name.find('Dockerfile') < 0: + # This is a *HACK* for the broken path handling. To be fixed ASAP. + subdir = build_config.get('build_subdir') or '/' + dockerfile_name = 'Dockerfile' + # /HACK HACK HACK HACK HACK HACK HACK + build_arguments = { 'build_package': build_job.get_build_package_url(self.user_files), - 'sub_directory': build_config.get('build_subdir', ''), + 'sub_directory': subdir, + 'dockerfile_name': dockerfile_name, 'repository': repository_name, 'registry': self.registry_hostname, 'pull_token': build_job.repo_build.access_token.code, diff --git a/buildman/manager/executor.py b/buildman/manager/executor.py index 202cf0921..585267c05 100644 --- a/buildman/manager/executor.py +++ b/buildman/manager/executor.py @@ -276,6 +276,7 @@ class PopenExecutor(BuilderExecutor): 'DOCKER_TLS_VERIFY': os.environ.get('DOCKER_TLS_VERIFY', ''), 'DOCKER_CERT_PATH': os.environ.get('DOCKER_CERT_PATH', ''), 'DOCKER_HOST': os.environ.get('DOCKER_HOST', ''), + 'PATH': "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" } logpipe = LogPipe(logging.INFO) diff --git a/buildtrigger/basehandler.py b/buildtrigger/basehandler.py index f8ed97563..64647e06c 100644 --- a/buildtrigger/basehandler.py +++ b/buildtrigger/basehandler.py @@ -258,6 +258,11 @@ class BuildTriggerHandler(object): can be called in a loop, so it should be as fast as possible. """ pass + @classmethod + def path_is_dockerfile(cls, file_name): + """ Returns whether the file is named Dockerfile or follows the convention .Dockerfile""" + return file_name.endswith(".Dockerfile") or u"Dockerfile" == file_name + @classmethod def service_name(cls): """ @@ -285,14 +290,10 @@ class BuildTriggerHandler(object): def get_dockerfile_path(self): """ Returns the normalized path to the Dockerfile found in the subdirectory in the config. """ - subdirectory = self.config.get('subdir', '') - if subdirectory == '/': - subdirectory = '' - else: - if not subdirectory.endswith('/'): - subdirectory = subdirectory + '/' - - return subdirectory + 'Dockerfile' + dockerfile_path = self.config.get('subdir') or 'Dockerfile' + if dockerfile_path[0] == '/': + dockerfile_path = dockerfile_path[1:] + return dockerfile_path def prepare_build(self, metadata, is_manual=False): # Ensure that the metadata meets the scheme. diff --git a/buildtrigger/bitbuckethandler.py b/buildtrigger/bitbuckethandler.py index 02e6b228f..68290cde4 100644 --- a/buildtrigger/bitbuckethandler.py +++ b/buildtrigger/bitbuckethandler.py @@ -1,23 +1,21 @@ import logging +import os import re - from calendar import timegm import dateutil.parser - +from bitbucket import BitBucket from jsonschema import validate + +from app import app, get_app_url +from buildtrigger.basehandler import BuildTriggerHandler from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException, TriggerDeactivationException, TriggerStartException, InvalidPayloadException, TriggerProviderException, determine_build_ref, raise_if_skipped_build, find_matching_branches) - -from buildtrigger.basehandler import BuildTriggerHandler - -from app import app, get_app_url -from bitbucket import BitBucket -from util.security.ssh import generate_ssh_keypair from util.dict_wrappers import JSONPathDict, SafeDictSetter +from util.security.ssh import generate_ssh_keypair logger = logging.getLogger(__name__) @@ -455,10 +453,7 @@ class BitbucketBuildTrigger(BuildTriggerHandler): raise RepositoryReadException(err_msg) files = set([f['path'] for f in data['files']]) - if 'Dockerfile' in files: - return [''] - - return [] + return ["/" + file_path for file_path in files if self.path_is_dockerfile(os.path.basename(file_path))] def load_dockerfile_contents(self): repository = self._get_repository_client() diff --git a/buildtrigger/githubhandler.py b/buildtrigger/githubhandler.py index a6df8a979..f6687798e 100644 --- a/buildtrigger/githubhandler.py +++ b/buildtrigger/githubhandler.py @@ -1,6 +1,7 @@ import logging import os.path import base64 +import re from calendar import timegm from functools import wraps @@ -348,9 +349,8 @@ class GithubBuildTrigger(BuildTriggerHandler): default_commit = repo.get_branch(branches[0]).commit commit_tree = repo.get_git_tree(default_commit.sha, recursive=True) - return [os.path.dirname(elem.path) for elem in commit_tree.tree - if (elem.type == u'blob' and - os.path.basename(elem.path) == u'Dockerfile')] + return [elem.path for elem in commit_tree.tree + if (elem.type == u'blob' and self.path_is_dockerfile(os.path.basename(elem.path)))] except GithubException as ghe: message = ghe.data.get('message', 'Unable to list contents of repository: %s' % source) if message == 'Branch not found': @@ -371,6 +371,9 @@ class GithubBuildTrigger(BuildTriggerHandler): raise RepositoryReadException(message) path = self.get_dockerfile_path() + if not path: + return None + try: file_info = repo.get_file_contents(path) except GithubException as ghe: diff --git a/buildtrigger/gitlabhandler.py b/buildtrigger/gitlabhandler.py index 5d221874b..0bd47c910 100644 --- a/buildtrigger/gitlabhandler.py +++ b/buildtrigger/gitlabhandler.py @@ -1,4 +1,5 @@ import logging +import os from calendar import timegm from functools import wraps @@ -341,11 +342,7 @@ class GitLabBuildTrigger(BuildTriggerHandler): msg = 'Unable to find GitLab repository tree for source: %s' % new_build_source raise RepositoryReadException(msg) - for node in repo_tree: - if node['name'] == 'Dockerfile': - return [''] - - return [] + return ["/"+node['name'] for node in repo_tree if self.path_is_dockerfile(node['name'])] @_catch_timeouts def load_dockerfile_contents(self): diff --git a/buildtrigger/test/bitbucketmock.py b/buildtrigger/test/bitbucketmock.py index 93d66d143..c53f4a57f 100644 --- a/buildtrigger/test/bitbucketmock.py +++ b/buildtrigger/test/bitbucketmock.py @@ -23,7 +23,7 @@ def get_repo_path_contents(path, revision): return (True, data, None) def get_raw_path_contents(path, revision): - if path == '/Dockerfile': + if path == 'Dockerfile': return (True, 'hello world', None) if path == 'somesubdir/Dockerfile': diff --git a/buildtrigger/test/githubmock.py b/buildtrigger/test/githubmock.py index 77c8a7a1f..f42ec57ca 100644 --- a/buildtrigger/test/githubmock.py +++ b/buildtrigger/test/githubmock.py @@ -119,7 +119,7 @@ def get_mock_github(): return [master, otherbranch] def get_file_contents_mock(filepath): - if filepath == '/Dockerfile': + if filepath == 'Dockerfile': m = Mock() m.content = 'hello world' return m diff --git a/buildtrigger/test/gitlabmock.py b/buildtrigger/test/gitlabmock.py index 8a2212be9..c7d4a5865 100644 --- a/buildtrigger/test/gitlabmock.py +++ b/buildtrigger/test/gitlabmock.py @@ -151,7 +151,7 @@ def gettag_mock(repo_id, tag): } def getrawfile_mock(repo_id, branch_name, path): - if path == '/Dockerfile': + if path == 'Dockerfile': return 'hello world' if path == 'somesubdir/Dockerfile': diff --git a/buildtrigger/test/test_basehandler.py b/buildtrigger/test/test_basehandler.py new file mode 100644 index 000000000..d8955740a --- /dev/null +++ b/buildtrigger/test/test_basehandler.py @@ -0,0 +1,15 @@ +import pytest + +from buildtrigger.basehandler import BuildTriggerHandler + + +@pytest.mark.parametrize('input,output', [ + ("Dockerfile", True), + ("server.Dockerfile", True), + (u"Dockerfile", True), + (u"server.Dockerfile", True), + ("bad file name", False), + (u"bad file name", False), +]) +def test_path_is_dockerfile(input, output): + assert BuildTriggerHandler.path_is_dockerfile(input) == output diff --git a/buildtrigger/test/test_bitbuckethandler.py b/buildtrigger/test/test_bitbuckethandler.py index 12c653db5..320a18906 100644 --- a/buildtrigger/test/test_bitbuckethandler.py +++ b/buildtrigger/test/test_bitbuckethandler.py @@ -13,12 +13,12 @@ def bitbucket_trigger(): def test_list_build_subdirs(bitbucket_trigger): - assert bitbucket_trigger.list_build_subdirs() == [''] + assert bitbucket_trigger.list_build_subdirs() == ["/Dockerfile"] @pytest.mark.parametrize('subdir, contents', [ - ('', 'hello world'), - ('somesubdir', 'hi universe'), + ('/Dockerfile', 'hello world'), + ('somesubdir/Dockerfile', 'hi universe'), ('unknownpath', None), ]) def test_load_dockerfile_contents(subdir, contents): diff --git a/buildtrigger/test/test_githubhandler.py b/buildtrigger/test/test_githubhandler.py index 5f9a1d786..4ca91ece4 100644 --- a/buildtrigger/test/test_githubhandler.py +++ b/buildtrigger/test/test_githubhandler.py @@ -68,8 +68,8 @@ def test_handle_trigger_request(github_trigger, payload, expected_error, expecte @pytest.mark.parametrize('subdir, contents', [ - ('', 'hello world'), - ('somesubdir', 'hi universe'), + ('/Dockerfile', 'hello world'), + ('somesubdir/Dockerfile', 'hi universe'), ('unknownpath', None), ]) def test_load_dockerfile_contents(subdir, contents): @@ -86,4 +86,4 @@ def test_lookup_user(username, expected_response, github_trigger): def test_list_build_subdirs(github_trigger): - assert github_trigger.list_build_subdirs() == ['', 'somesubdir'] + assert github_trigger.list_build_subdirs() == ['Dockerfile', 'somesubdir/Dockerfile'] diff --git a/buildtrigger/test/test_gitlabhandler.py b/buildtrigger/test/test_gitlabhandler.py index c88c591d6..d20a690be 100644 --- a/buildtrigger/test/test_gitlabhandler.py +++ b/buildtrigger/test/test_gitlabhandler.py @@ -13,12 +13,12 @@ def gitlab_trigger(): def test_list_build_subdirs(gitlab_trigger): - assert gitlab_trigger.list_build_subdirs() == [''] + assert gitlab_trigger.list_build_subdirs() == ['/Dockerfile'] @pytest.mark.parametrize('subdir, contents', [ - ('', 'hello world'), - ('somesubdir', 'hi universe'), + ('/Dockerfile', 'hello world'), + ('somesubdir/Dockerfile', 'hi universe'), ('unknownpath', None), ]) def test_load_dockerfile_contents(subdir, contents): diff --git a/ci/tasks/karma.yaml b/ci/tasks/karma.yaml index b7f336028..ce8f4c28a 100644 --- a/ci/tasks/karma.yaml +++ b/ci/tasks/karma.yaml @@ -9,6 +9,5 @@ run: - | set -eux cd quay-pull-request - npm install - npm link typescript - npm test + yarn install --ignore-engines + yarn test diff --git a/data/model/tag.py b/data/model/tag.py index 239bc3468..98d0579fa 100644 --- a/data/model/tag.py +++ b/data/model/tag.py @@ -2,7 +2,7 @@ import logging from uuid import uuid4 -from peewee import IntegrityError +from peewee import IntegrityError, JOIN_LEFT_OUTER, fn from data.model import (image, db_transaction, DataModelException, _basequery, InvalidManifestException, TagAlreadyCreatedException, StaleTagException) from data.database import (RepositoryTag, Repository, Image, ImageStorage, Namespace, TagManifest, @@ -13,6 +13,40 @@ from data.database import (RepositoryTag, Repository, Image, ImageStorage, Names logger = logging.getLogger(__name__) +def get_max_id_for_sec_scan(): + """ Gets the maximum id for security scanning """ + return RepositoryTag.select(fn.Max(RepositoryTag.id)).scalar() + + +def get_min_id_for_sec_scan(version): + """ Gets the minimum id for a security scanning """ + return (RepositoryTag + .select(fn.Min(RepositoryTag.id)) + .join(Image) + .where(Image.security_indexed_engine < version) + .scalar()) + + +def get_tag_pk_field(): + """ Returns the primary key for Image DB model """ + return RepositoryTag.id + + +def get_tags_images_eligible_for_scan(clair_version): + Parent = Image.alias() + ParentImageStorage = ImageStorage.alias() + + return _tag_alive(RepositoryTag + .select(Image, ImageStorage, Parent, ParentImageStorage, RepositoryTag) + .join(Image, on=(RepositoryTag.image == Image.id)) + .join(ImageStorage, on=(Image.storage == ImageStorage.id)) + .switch(Image) + .join(Parent, JOIN_LEFT_OUTER, on=(Image.parent == Parent.id)) + .join(ParentImageStorage, JOIN_LEFT_OUTER, on=(ParentImageStorage.id == Parent.storage)) + .where(RepositoryTag.hidden == False) + .where(Image.security_indexed_engine < clair_version)) + + def _tag_alive(query, now_ts=None): if now_ts is None: now_ts = get_epoch_timestamp() diff --git a/endpoints/api/trigger.py b/endpoints/api/trigger.py index 3dec076b6..95328818d 100644 --- a/endpoints/api/trigger.py +++ b/endpoints/api/trigger.py @@ -404,7 +404,7 @@ class ActivateBuildTrigger(RepositoryParamResource): 'description': '(Custom Only) If specified, the ref/SHA1 used to checkout a git repository.' }, 'refs': { - 'type': 'object', + 'type': ['object', 'null'], 'description': '(SCM Only) If specified, the ref to build.' } }, diff --git a/package.json b/package.json index a18699fd4..4fb9f445c 100644 --- a/package.json +++ b/package.json @@ -15,11 +15,11 @@ }, "homepage": "https://github.com/coreos-inc/quay#readme", "dependencies": { - "angular": "1.5.3", - "angular-animate": "^1.5.3", - "angular-cookies": "^1.5.3", - "angular-route": "^1.5.3", - "angular-sanitize": "^1.5.3", + "angular": "1.6.2", + "angular-animate": "1.6.2", + "angular-cookies": "1.6.2", + "angular-route": "1.6.2", + "angular-sanitize": "1.6.2", "bootbox": "^4.1.0", "bootstrap": "^3.3.2", "bootstrap-datepicker": "^1.6.4", @@ -35,15 +35,16 @@ "underscore": "^1.5.2" }, "devDependencies": { - "@types/angular": "1.5.16", + "@types/angular": "1.6.2", "@types/angular-mocks": "^1.5.8", "@types/angular-route": "^1.3.3", "@types/angular-sanitize": "^1.3.4", "@types/es6-shim": "^0.31.32", "@types/jasmine": "^2.5.41", + "@types/jquery": "^2.0.40", "@types/react": "0.14.39", "@types/react-dom": "0.14.17", - "angular-mocks": "^1.5.3", + "angular-mocks": "1.6.2", "angular-ts-decorators": "0.0.19", "css-loader": "0.25.0", "jasmine-core": "^2.5.2", diff --git a/static/css/directives/ui/manage-trigger-control.css b/static/css/directives/ui/manage-trigger-control.css index 708852881..5f4a399ac 100644 --- a/static/css/directives/ui/manage-trigger-control.css +++ b/static/css/directives/ui/manage-trigger-control.css @@ -30,6 +30,12 @@ height: 28px; } +@media (max-width: 768px) { + .manage-trigger-control .co-top-bar { + margin-bottom: 80px; + } +} + .manage-trigger-control .namespace-avatar { margin-left: 2px; margin-right: 2px; diff --git a/static/directives/manual-trigger-build-dialog.html b/static/directives/manual-trigger-build-dialog.html index 56f59ea14..8a8b8ee9e 100644 --- a/static/directives/manual-trigger-build-dialog.html +++ b/static/directives/manual-trigger-build-dialog.html @@ -26,7 +26,8 @@ icon-key="kind" icon-map="field.iconMap" items="fieldOptions[field.name]" - ng-show="fieldOptions[field.name]"> + ng-show="fieldOptions[field.name]" + clear-value="counter"> diff --git a/static/js/directives/ui/dockerfile-build-form.js b/static/js/directives/ui/dockerfile-build-form.js index 8fae11bb7..136f29118 100644 --- a/static/js/directives/ui/dockerfile-build-form.js +++ b/static/js/directives/ui/dockerfile-build-form.js @@ -39,28 +39,7 @@ angular.module('quay').directive('dockerfileBuildForm', function () { $scope.state = 'checking'; $scope.selectedFiles = files; - // FIXME: Remove this - // DockerfileService.getDockerfile(files[0], function(df) { - // var baseImage = df.getRegistryBaseImage(); - // if (baseImage) { - // checkPrivateImage(baseImage); - // } else { - // $scope.state = 'ready'; - // } - // - // $scope.$apply(function() { - // opt_callback && opt_callback(true, 'Dockerfile found and valid') - // }); - // }, function(msg) { - // $scope.state = 'empty'; - // $scope.privateBaseRepository = null; - // - // $scope.$apply(function() { - // opt_callback && opt_callback(false, msg || 'Could not find valid Dockerfile'); - // }); - // }); - - DockerfileService.extractDockerfile(files[0]) + DockerfileService.getDockerfile(files[0]) .then(function(dockerfileInfo) { var baseImage = dockerfileInfo.getRegistryBaseImage(); if (baseImage) { diff --git a/static/js/directives/ui/manage-trigger-githost/manage-trigger-githost.component.html b/static/js/directives/ui/manage-trigger-githost/manage-trigger-githost.component.html index b3fd10bd7..4fcdf417f 100644 --- a/static/js/directives/ui/manage-trigger-githost/manage-trigger-githost.component.html +++ b/static/js/directives/ui/manage-trigger-githost/manage-trigger-githost.component.html @@ -25,7 +25,7 @@ - +
@@ -39,7 +39,7 @@
-
+

Verification Warning

{{ $ctrl.local.triggerAnalysis.message }}
-
+

Ready to go!

Click "Create Trigger" to complete setup of this build trigger
-
+

Robot Account Required

The selected Dockerfile in the selected repository depends upon a private base image

A robot account with access to the base image is required to setup this trigger, but you are not the administrator of this namespace.

@@ -310,7 +313,8 @@
-
+

Select Robot Account

The selected Dockerfile in the selected repository depends upon a private base image. Select a robot account with access: @@ -338,7 +342,7 @@
@@ -355,8 +359,8 @@
-
+
No matching robot accounts found.
Try expanding your filtering terms.
diff --git a/static/js/directives/ui/manage-trigger-githost/manage-trigger-githost.component.ts b/static/js/directives/ui/manage-trigger-githost/manage-trigger-githost.component.ts index a28f1e2f4..f83e58d96 100644 --- a/static/js/directives/ui/manage-trigger-githost/manage-trigger-githost.component.ts +++ b/static/js/directives/ui/manage-trigger-githost/manage-trigger-githost.component.ts @@ -62,14 +62,8 @@ export class ManageTriggerGithostComponent implements ng.IComponentController { public $onInit(): void { // TODO: Replace $scope.$watch with @Output methods for child component mutations or $onChanges for parent mutations - this.$scope.$watch(() => this.trigger, (trigger) => { - if (trigger && this.repository) { - this.config = trigger['config'] || {}; - this.namespaceTitle = 'organization'; - this.local.selectedNamespace = null; - this.loadNamespaces(); - } - }); + this.$scope.$watch(() => this.trigger, this.initialSetup.bind(this)); + this.$scope.$watch(() => this.repository, this.initialSetup.bind(this)); this.$scope.$watch(() => this.local.selectedNamespace, (namespace) => { if (namespace) { @@ -102,6 +96,20 @@ export class ManageTriggerGithostComponent implements ng.IComponentController { this.$scope.$watch(() => this.local.robotOptions.filter, this.buildOrderedRobotAccounts); } + private initialSetup(): void { + if (!this.repository || !this.trigger) { return; } + + if (this.namespaceTitle) { + // Already setup. + return; + } + + this.config = this.trigger['config'] || {}; + this.namespaceTitle = 'organization'; + this.local.selectedNamespace = null; + this.loadNamespaces(); + } + public getTriggerIcon(): any { return this.TriggerService.getIcon(this.trigger.service); } diff --git a/static/js/directives/ui/manual-trigger-build-dialog.js b/static/js/directives/ui/manual-trigger-build-dialog.js index a0f09cae2..80e1c631a 100644 --- a/static/js/directives/ui/manual-trigger-build-dialog.js +++ b/static/js/directives/ui/manual-trigger-build-dialog.js @@ -57,6 +57,8 @@ angular.module('quay').directive('manualTriggerBuildDialog', function () { $scope.fieldOptions[parameter['name']] = resp['values']; }); } + + delete $scope.parameters[parameter['name']]; } $scope.runParameters = parameters; diff --git a/static/js/quay.module.ts b/static/js/quay.module.ts index 3b558e5d9..47bb4954f 100644 --- a/static/js/quay.module.ts +++ b/static/js/quay.module.ts @@ -49,7 +49,7 @@ export class quay { // TODO: Make injected values into services and move to NgModule.providers, as constants are not supported in Angular 2 angular .module(quay.name) - .factory("FileReaderFactory", () => () => new FileReader()) + .factory("fileReaderFactory", () => () => new FileReader()) .constant('NAME_PATTERNS', NAME_PATTERNS) .constant('INJECTED_CONFIG', INJECTED_CONFIG) .constant('INJECTED_FEATURES', INJECTED_FEATURES) diff --git a/static/js/services/datafile-service.js b/static/js/services/datafile-service.js index 6461a8e99..3b8368076 100644 --- a/static/js/services/datafile-service.js +++ b/static/js/services/datafile-service.js @@ -76,14 +76,15 @@ angular.module('quay').factory('DataFileService', [function() { return parts.join('/'); }; - var handler = new Untar(new Uint8Array(buf)); - handler.process(function(status, read, files, err) { - switch (status) { - case 'error': - failure(err); - break; + try { + var handler = new Untar(new Uint8Array(buf)); + handler.process(function(status, read, files, err) { + switch (status) { + case 'error': + failure(err); + break; - case 'done': + case 'done': var processed = []; for (var i = 0; i < files.length; ++i) { var currentFile = files[i]; @@ -104,8 +105,12 @@ angular.module('quay').factory('DataFileService', [function() { } success(processed); break; - } - }); + } + }); + } catch (e) { + failure(); + } + }; dataFileService.blobToString = function(blob, callback) { diff --git a/static/js/services/dockerfile-service.js b/static/js/services/dockerfile-service.js deleted file mode 100644 index 4495d6917..000000000 --- a/static/js/services/dockerfile-service.js +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Service which provides helper methods for extracting information out from a Dockerfile - * or an archive containing a Dockerfile. - */ -angular.module('quay').factory('DockerfileServiceOld', ['DataFileService', 'Config', function(DataFileService, Config) { - var dockerfileService = {}; - - function DockerfileInfo(contents) { - this.contents = contents; - } - - DockerfileInfo.prototype.getRegistryBaseImage = function() { - var baseImage = this.getBaseImage(); - if (!baseImage) { - return null; - } - - if (baseImage.indexOf(Config.getDomain() + '/') != 0) { - return null; - } - - return baseImage.substring(Config.getDomain().length + 1); - }; - - DockerfileInfo.prototype.getBaseImage = function() { - var imageAndTag = this.getBaseImageAndTag(); - if (!imageAndTag) { - return null; - } - - // Note, we have to handle a few different cases here: - // 1) someimage - // 2) someimage:tag - // 3) host:port/someimage - // 4) host:port/someimage:tag - var lastIndex = imageAndTag.lastIndexOf(':'); - if (lastIndex < 0) { - return imageAndTag; - } - - // Otherwise, check if there is a / in the portion after the split point. If so, - // then the latter is part of the path (and not a tag). - var afterColon = imageAndTag.substring(lastIndex + 1); - if (afterColon.indexOf('/') >= 0) { - return imageAndTag; - } - - return imageAndTag.substring(0, lastIndex); - }; - - DockerfileInfo.prototype.getBaseImageAndTag = function() { - var fromIndex = this.contents.indexOf('FROM '); - if (fromIndex < 0) { - return null; - } - - var newline = this.contents.indexOf('\n', fromIndex); - if (newline < 0) { - newline = this.contents.length; - } - - return $.trim(this.contents.substring(fromIndex + 'FROM '.length, newline)); - }; - - DockerfileInfo.forData = function(contents) { - if (contents.indexOf('FROM ') < 0) { - return; - } - - return new DockerfileInfo(contents); - }; - - var processFiles = function(files, dataArray, success, failure) { - // The files array will be empty if the submitted file was not an archive. We therefore - // treat it as a single Dockerfile. - if (files.length == 0) { - DataFileService.arrayToString(dataArray, function(c) { - var result = DockerfileInfo.forData(c); - if (!result) { - failure('File chosen is not a valid Dockerfile'); - return; - } - - success(result); - }); - return; - } - - var found = false; - files.forEach(function(file) { - if (file['name'] == 'Dockerfile') { - DataFileService.blobToString(file.toBlob(), function(c) { - var result = DockerfileInfo.forData(c); - if (!result) { - failure('Dockerfile inside archive is not a valid Dockerfile'); - return; - } - - success(result); - }); - found = true; - } - }); - - if (!found) { - failure('No Dockerfile found in root of archive'); - } - }; - - dockerfileService.getDockerfile = function(file, success, failure) { - var reader = new FileReader(); - reader.onload = function(e) { - var dataArray = reader.result; - DataFileService.readDataArrayAsPossibleArchive(dataArray, function(files) { - processFiles(files, dataArray, success, failure); - }, function() { - // Not an archive. Read directly as a single file. - processFiles([], dataArray, success, failure); - }); - }; - - reader.onerror = failure; - reader.readAsArrayBuffer(file); - }; - - return dockerfileService; -}]); \ No newline at end of file diff --git a/static/js/services/dockerfile/dockerfile.service.impl.spec.ts b/static/js/services/dockerfile/dockerfile.service.impl.spec.ts index 20ad4f6f1..edbb8175a 100644 --- a/static/js/services/dockerfile/dockerfile.service.impl.spec.ts +++ b/static/js/services/dockerfile/dockerfile.service.impl.spec.ts @@ -52,157 +52,7 @@ describe("DockerfileServiceImpl", () => { }); it("calls datafile service to read given file as possible archive file", (done) => { - dockerfileServiceImpl.getDockerfile(file, - (dockerfile: DockerfileInfoImpl) => { - expect(readAsFileBufferSpy.calls.argsFor(0)[0]).toEqual(file); - expect(dataFileServiceMock.readDataArrayAsPossibleArchive).toHaveBeenCalled(); - done(); - }, - (error: Event | string) => { - fail("Should not invoke failure callback"); - done(); - }); - }); - - it("calls datafile service to convert file to string if given file is not an archive", (done) => { - dockerfileServiceImpl.getDockerfile(file, - (dockerfile: DockerfileInfoImpl) => { - expect(dataFileServiceMock.arrayToString.calls.argsFor(0)[0]).toEqual(file); - done(); - }, - (error: Event | string) => { - fail("Should not invoke success callback"); - done(); - }); - }); - - it("calls failure callback if given non-archive file that is not a valid Dockerfile", (done) => { - forDataSpy.and.returnValue(null); - dockerfileServiceImpl.getDockerfile(file, - (dockerfile: DockerfileInfoImpl) => { - fail("Should not invoke success callback"); - done(); - }, - (error: Event | string) => { - expect(error).toEqual('File chosen is not a valid Dockerfile'); - done(); - }); - }); - - it("calls success callback with new DockerfileInfoImpl instance if given valid Dockerfile", (done) => { - dockerfileServiceImpl.getDockerfile(file, - (dockerfile: DockerfileInfoImpl) => { - expect(dockerfile).toBeDefined(); - done(); - }, - (error: Event | string) => { - fail('Should not invoke failure callback'); - done(); - }); - }); - - it("calls failure callback if given archive file with no Dockerfile present in root directory", (done) => { - dataFileServiceMock.readDataArrayAsPossibleArchive.and.callFake((buf, success, failure) => { - success(invalidArchiveFile); - }); - - dockerfileServiceImpl.getDockerfile(file, - (dockerfile: DockerfileInfoImpl) => { - fail("Should not invoke success callback"); - done(); - }, - (error: Event | string) => { - expect(error).toEqual('No Dockerfile found in root of archive'); - done(); - }); - }); - - it("calls datafile service to convert blob to string if given file is an archive", (done) => { - dataFileServiceMock.readDataArrayAsPossibleArchive.and.callFake((buf, success, failure) => { - success(validArchiveFile); - }); - - dockerfileServiceImpl.getDockerfile(file, - (dockerfile: DockerfileInfoImpl) => { - expect(validArchiveFile[0].toBlob).toHaveBeenCalled(); - expect(dataFileServiceMock.blobToString.calls.argsFor(0)[0]).toEqual(validArchiveFile[0].toBlob()); - done(); - }, - (error: Event | string) => { - fail("Should not invoke success callback"); - done(); - }); - }); - - it("calls failure callback if given archive file with invalid Dockerfile", (done) => { - forDataSpy.and.returnValue(null); - invalidArchiveFile[0].name = 'Dockerfile'; - dataFileServiceMock.readDataArrayAsPossibleArchive.and.callFake((buf, success, failure) => { - success(invalidArchiveFile); - }); - - dockerfileServiceImpl.getDockerfile(file, - (dockerfile: DockerfileInfoImpl) => { - fail("Should not invoke success callback"); - done(); - }, - (error: Event | string) => { - expect(error).toEqual('Dockerfile inside archive is not a valid Dockerfile'); - done(); - }); - }); - - it("calls success callback with new DockerfileInfoImpl instance if given archive with valid Dockerfile", (done) => { - dataFileServiceMock.readDataArrayAsPossibleArchive.and.callFake((buf, success, failure) => { - success(validArchiveFile); - }); - - dockerfileServiceImpl.getDockerfile(file, - (dockerfile: DockerfileInfoImpl) => { - expect(dockerfile).toBeDefined(); - done(); - }, - (error: Event | string) => { - fail('Should not invoke failure callback'); - done(); - }); - }); - }); - - describe("extractDockerfile", () => { - var file: any; - var invalidArchiveFile: any[]; - var validArchiveFile: any[]; - var readAsFileBufferSpy: Spy; - var forDataSpy: Spy; - - beforeEach(() => { - dataFileServiceMock.readDataArrayAsPossibleArchive.and.callFake((buf, success, failure) => { - failure([]); - }); - - dataFileServiceMock.arrayToString.and.callFake((buf, callback) => { - var contents: string = ""; - callback(contents); - }); - - dataFileServiceMock.blobToString.and.callFake((blob, callback) => { - callback(blob.toString()); - }); - - forDataSpy = spyOn(DockerfileInfoImpl, "forData").and.returnValue(new DockerfileInfoImpl(file, configMock)); - readAsFileBufferSpy = spyOn(fileReaderMock, "readAsArrayBuffer").and.callFake(() => { - var event: any = {target: {result: file}}; - fileReaderMock.onload(event); - }); - - file = "FROM quay.io/coreos/nginx:latest"; - validArchiveFile = [{name: 'Dockerfile', toBlob: jasmine.createSpy('toBlobSpy').and.returnValue(file)}]; - invalidArchiveFile = [{name: 'main.exe', toBlob: jasmine.createSpy('toBlobSpy').and.returnValue("")}]; - }); - - it("calls datafile service to read given file as possible archive file", (done) => { - dockerfileServiceImpl.extractDockerfile(file) + dockerfileServiceImpl.getDockerfile(file) .then((dockerfile: DockerfileInfoImpl) => { expect(readAsFileBufferSpy.calls.argsFor(0)[0]).toEqual(file); expect(dataFileServiceMock.readDataArrayAsPossibleArchive).toHaveBeenCalled(); @@ -215,31 +65,107 @@ describe("DockerfileServiceImpl", () => { }); it("calls datafile service to convert file to string if given file is not an archive", (done) => { - done(); + dockerfileServiceImpl.getDockerfile(file) + .then((dockerfile: DockerfileInfoImpl) => { + expect(dataFileServiceMock.arrayToString.calls.argsFor(0)[0]).toEqual(file); + done(); + }) + .catch((error: string) => { + fail('Promise should be resolved'); + done(); + }); }); it("returns rejected promise if given non-archive file that is not a valid Dockerfile", (done) => { - done(); + forDataSpy.and.returnValue(null); + dockerfileServiceImpl.getDockerfile(file) + .then((dockerfile: DockerfileInfoImpl) => { + fail("Promise should be rejected"); + done(); + }) + .catch((error: string) => { + expect(error).toEqual('File chosen is not a valid Dockerfile'); + done(); + }); }); it("returns resolved promise with new DockerfileInfoImpl instance if given valid Dockerfile", (done) => { - done(); + dockerfileServiceImpl.getDockerfile(file) + .then((dockerfile: DockerfileInfoImpl) => { + expect(dockerfile).toBeDefined(); + done(); + }) + .catch((error: string) => { + fail('Promise should be resolved'); + done(); + }); }); it("returns rejected promise if given archive file with no Dockerfile present in root directory", (done) => { - done(); + dataFileServiceMock.readDataArrayAsPossibleArchive.and.callFake((buf, success, failure) => { + success(invalidArchiveFile); + }); + + dockerfileServiceImpl.getDockerfile(file) + .then((dockerfile: DockerfileInfoImpl) => { + fail('Promise should be rejected'); + done(); + }) + .catch((error: string) => { + expect(error).toEqual('No Dockerfile found in root of archive'); + done(); + }); }); it("calls datafile service to convert blob to string if given file is an archive", (done) => { - done(); + dataFileServiceMock.readDataArrayAsPossibleArchive.and.callFake((buf, success, failure) => { + success(validArchiveFile); + }); + + dockerfileServiceImpl.getDockerfile(file) + .then((dockerfile: DockerfileInfoImpl) => { + expect(validArchiveFile[0].toBlob).toHaveBeenCalled(); + expect(dataFileServiceMock.blobToString.calls.argsFor(0)[0]).toEqual(validArchiveFile[0].toBlob()); + done(); + }) + .catch((error: string) => { + fail('Promise should be resolved'); + done(); + }); }); it("returns rejected promise if given archive file with invalid Dockerfile", (done) => { - done(); + forDataSpy.and.returnValue(null); + invalidArchiveFile[0].name = 'Dockerfile'; + dataFileServiceMock.readDataArrayAsPossibleArchive.and.callFake((buf, success, failure) => { + success(invalidArchiveFile); + }); + + dockerfileServiceImpl.getDockerfile(file) + .then((dockerfile: DockerfileInfoImpl) => { + fail('Promise should be rejected'); + done(); + }) + .catch((error: string) => { + expect(error).toEqual('Dockerfile inside archive is not a valid Dockerfile'); + done(); + }); }); it("returns resolved promise of new DockerfileInfoImpl instance if given archive with valid Dockerfile", (done) => { - done(); + dataFileServiceMock.readDataArrayAsPossibleArchive.and.callFake((buf, success, failure) => { + success(validArchiveFile); + }); + + dockerfileServiceImpl.getDockerfile(file) + .then((dockerfile: DockerfileInfoImpl) => { + expect(dockerfile).toBeDefined(); + done(); + }) + .catch((error: string) => { + fail('Promise should be resolved'); + done(); + }); }); }); }); diff --git a/static/js/services/dockerfile/dockerfile.service.impl.ts b/static/js/services/dockerfile/dockerfile.service.impl.ts index f865ae70f..5cb861749 100644 --- a/static/js/services/dockerfile/dockerfile.service.impl.ts +++ b/static/js/services/dockerfile/dockerfile.service.impl.ts @@ -5,21 +5,30 @@ import { Injectable } from 'angular-ts-decorators'; @Injectable(DockerfileService.name) export class DockerfileServiceImpl implements DockerfileService { - constructor(private DataFileService: any, private Config: any, private FileReaderFactory: () => FileReader) { + constructor(private DataFileService: any, + private Config: any, + private fileReaderFactory: () => FileReader) { } - public extractDockerfile(file: any): Promise { + public getDockerfile(file: any): Promise { return new Promise((resolve, reject) => { - var reader: FileReader = this.FileReaderFactory(); + var reader: FileReader = this.fileReaderFactory(); reader.onload = (event: any) => { + // FIXME: Debugging + console.log(event.target.result); + this.DataFileService.readDataArrayAsPossibleArchive(event.target.result, (files: any[]) => { - this.processFiles1(files); + this.processFiles(files) + .then((dockerfileInfo: DockerfileInfoImpl) => resolve(dockerfileInfo)) + .catch((error: string) => reject(error)); }, () => { // Not an archive. Read directly as a single file. - this.processFile1(event.target.result); + this.processFile(event.target.result) + .then((dockerfileInfo: DockerfileInfoImpl) => resolve(dockerfileInfo)) + .catch((error: string) => reject(error)); }); }; @@ -28,40 +37,7 @@ export class DockerfileServiceImpl implements DockerfileService { }); } - public getDockerfile(file: any, - success: (dockerfile: DockerfileInfoImpl) => void, - failure: (error: Event | string) => void): void { - var reader: FileReader = this.FileReaderFactory(); - reader.onload = (event: any) => { - this.DataFileService.readDataArrayAsPossibleArchive(event.target.result, - (files: any[]) => { - this.processFiles(files, success, failure); - }, - () => { - // Not an archive. Read directly as a single file. - this.processFile(event.target.result, success, failure); - }); - }; - - reader.onerror = failure; - reader.readAsArrayBuffer(file); - } - - private processFile(dataArray: any, - success: (dockerfile: DockerfileInfoImpl) => void, - failure: (error: ErrorEvent | string) => void): void { - this.DataFileService.arrayToString(dataArray, (contents: string) => { - var result: DockerfileInfoImpl | null = DockerfileInfoImpl.forData(contents, this.Config); - if (result == null) { - failure('File chosen is not a valid Dockerfile'); - } - else { - success(result); - } - }); - } - - private processFile1(dataArray: any): Promise { + private processFile(dataArray: any): Promise { return new Promise((resolve, reject) => { this.DataFileService.arrayToString(dataArray, (contents: string) => { var result: DockerfileInfoImpl | null = DockerfileInfoImpl.forData(contents, this.Config); @@ -75,31 +51,7 @@ export class DockerfileServiceImpl implements DockerfileService { }); } - private processFiles(files: any[], - success: (dockerfile: DockerfileInfoImpl) => void, - failure: (error: ErrorEvent | string) => void): void { - var found: boolean = false; - files.forEach((file) => { - if (file['name'] == 'Dockerfile') { - this.DataFileService.blobToString(file.toBlob(), (contents: string) => { - var result: DockerfileInfoImpl | null = DockerfileInfoImpl.forData(contents, this.Config); - if (result == null) { - failure('Dockerfile inside archive is not a valid Dockerfile'); - } - else { - success(result); - } - }); - found = true; - } - }); - - if (!found) { - failure('No Dockerfile found in root of archive'); - } - } - - private processFiles1(files: any[]): Promise { + private processFiles(files: any[]): Promise { return new Promise((resolve, reject) => { var found: boolean = false; files.forEach((file) => { diff --git a/static/js/services/dockerfile/dockerfile.service.ts b/static/js/services/dockerfile/dockerfile.service.ts index 3af34b0ee..3c5186b6b 100644 --- a/static/js/services/dockerfile/dockerfile.service.ts +++ b/static/js/services/dockerfile/dockerfile.service.ts @@ -7,19 +7,9 @@ export abstract class DockerfileService { /** * Retrieve Dockerfile from given file. * @param file Dockerfile or archive file containing Dockerfile. - * @param success Success callback with retrieved Dockerfile as parameter. - * @param failure Failure callback with failure message as parameter. + * @return promise Promise which resolves to new DockerfileInfo instance or rejects with error message. */ - public abstract getDockerfile(file: any, - success: (dockerfile: DockerfileInfo) => void, - failure: (error: ErrorEvent | string) => void): void; - - /** - * Retrieve Dockerfile from given file. - * @param file Dockerfile or archive file containing Dockerfile. - * @return promise Promise resolving to new DockerfileInfo instance or rejecting to error message. - */ - public abstract extractDockerfile(file: any): Promise; + public abstract getDockerfile(file: any): Promise; } diff --git a/static/js/services/trigger-service.js b/static/js/services/trigger-service.js index 6406fec10..3d8c3d143 100644 --- a/static/js/services/trigger-service.js +++ b/static/js/services/trigger-service.js @@ -208,10 +208,6 @@ angular.module('quay').factory('TriggerService', ['UtilService', '$sanitize', 'K return '//Dockerfile'; } - if (subdirectory[subdirectory.length - 1] != '/') { - subdirectory = subdirectory + '/'; - } - return '//' + subdirectory.replace(new RegExp('(^\/+|\/+$)'), '') + 'Dockerfile'; }; diff --git a/test/test_api_usage.py b/test/test_api_usage.py index eead6487f..0291fba4c 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -4059,11 +4059,18 @@ class TestBuildTriggers(ApiTestCase): self.assertEquals('bar', py_json.loads(build_obj.job_config)['trigger_metadata']['foo']) # Start another manual build, with a ref. - start_json = self.postJsonResponse(ActivateBuildTrigger, - params=dict(repository=ADMIN_ACCESS_USER + '/simple', - trigger_uuid=trigger.uuid), - data=dict(refs={'kind': 'branch', 'name': 'foobar'}), - expected_code=201) + self.postJsonResponse(ActivateBuildTrigger, + params=dict(repository=ADMIN_ACCESS_USER + '/simple', + trigger_uuid=trigger.uuid), + data=dict(refs={'kind': 'branch', 'name': 'foobar'}), + expected_code=201) + + # Start another manual build with a null ref. + self.postJsonResponse(ActivateBuildTrigger, + params=dict(repository=ADMIN_ACCESS_USER + '/simple', + trigger_uuid=trigger.uuid), + data=dict(refs=None), + expected_code=201) def test_invalid_robot_account(self): self.login(ADMIN_ACCESS_USER) diff --git a/workers/securityworker.py b/workers/securityworker.py index 9c92c1f19..964d3c5b1 100644 --- a/workers/securityworker.py +++ b/workers/securityworker.py @@ -6,8 +6,8 @@ import features from app import app, secscan_api, prometheus from workers.worker import Worker from data.database import UseThenDisconnect -from data.model.image import (get_images_eligible_for_scan, get_max_id_for_sec_scan, - get_min_id_for_sec_scan, get_image_id) +from data.model.tag import (get_tags_images_eligible_for_scan, get_tag_pk_field, + get_max_id_for_sec_scan, get_min_id_for_sec_scan) from util.secscan.api import SecurityConfigValidator from util.secscan.analyzer import LayerAnalyzer, PreemptedException from util.migrate.allocator import yield_random_entries @@ -43,7 +43,7 @@ class SecurityWorker(Worker): def _index_images(self): def batch_query(): - return get_images_eligible_for_scan(self._target_version) + return get_tags_images_eligible_for_scan(self._target_version) # Get the ID of the last image we can analyze. Will be None if there are no images in the # database. @@ -56,14 +56,14 @@ class SecurityWorker(Worker): with UseThenDisconnect(app.config): to_scan_generator = yield_random_entries( batch_query, - get_image_id(), + get_tag_pk_field(), BATCH_SIZE, max_id, self._min_id, ) for candidate, abt, num_remaining in to_scan_generator: try: - self._analyzer.analyze_recursively(candidate) + self._analyzer.analyze_recursively(candidate.image) except PreemptedException: logger.info('Another worker pre-empted us for layer: %s', candidate.id) abt.set() diff --git a/yarn.lock b/yarn.lock index 08b9adf09..2d7325650 100644 --- a/yarn.lock +++ b/yarn.lock @@ -20,9 +20,9 @@ dependencies: "@types/angular" "*" -"@types/angular@*", "@types/angular@1.5.16": - version "1.5.16" - resolved "https://registry.yarnpkg.com/@types/angular/-/angular-1.5.16.tgz#02a56754b50dbf9209266b4339031a54317702d9" +"@types/angular@*", "@types/angular@1.6.2": + version "1.6.2" + resolved "https://registry.yarnpkg.com/@types/angular/-/angular-1.6.2.tgz#a5c323ea5d4426ad18984cc8167fa091f7c8201b" dependencies: "@types/jquery" "*" @@ -36,7 +36,7 @@ dependencies: typescript ">=2.1.4" -"@types/jquery@*": +"@types/jquery@*", "@types/jquery@^2.0.40": version "2.0.40" resolved "https://registry.yarnpkg.com/@types/jquery/-/jquery-2.0.40.tgz#acdd69e29b74cdec15dc3c074bcb064bc1b87213" @@ -109,23 +109,23 @@ amdefine@>=0.0.4: version "1.0.1" resolved "https://registry.yarnpkg.com/amdefine/-/amdefine-1.0.1.tgz#4a5282ac164729e93619bcfd3ad151f817ce91f5" -angular-animate@^1.5.3: +angular-animate@1.6.2: version "1.6.2" resolved "https://registry.yarnpkg.com/angular-animate/-/angular-animate-1.6.2.tgz#def2a8b9ede53b4b6e234c25f5c64e4b4385df15" -angular-cookies@^1.5.3: +angular-cookies@1.6.2: version "1.6.2" resolved "https://registry.yarnpkg.com/angular-cookies/-/angular-cookies-1.6.2.tgz#ffb69f35f84d1efe71addac20a2905476b658884" -angular-mocks@^1.5.3: +angular-mocks@1.6.2: version "1.6.2" resolved "https://registry.yarnpkg.com/angular-mocks/-/angular-mocks-1.6.2.tgz#fbb28208e74d3512769afdb8771f5cc5a99f9128" -angular-route@^1.5.3: +angular-route@1.6.2: version "1.6.2" resolved "https://registry.yarnpkg.com/angular-route/-/angular-route-1.6.2.tgz#95a349de2e73674f3dd783bb21e8d7b3fc526312" -angular-sanitize@^1.5.3: +angular-sanitize@1.6.2: version "1.6.2" resolved "https://registry.yarnpkg.com/angular-sanitize/-/angular-sanitize-1.6.2.tgz#8a327c1acb2c14f50da5b5cad5ea452750a1a375" @@ -135,9 +135,9 @@ angular-ts-decorators@0.0.19: dependencies: reflect-metadata "^0.1.8" -angular@1.5.3: - version "1.5.3" - resolved "https://registry.yarnpkg.com/angular/-/angular-1.5.3.tgz#37c2f198ae76c2d6f3717a4ecef1cddcb048af79" +angular@1.6.2: + version "1.6.2" + resolved "https://registry.yarnpkg.com/angular/-/angular-1.6.2.tgz#d0b677242ac4bf9ae81424297c6320973af4bb5a" ansi-align@^1.1.0: version "1.1.0"