refactored DockerfileServiceImpl to return promise instead of callbacks

This commit is contained in:
alecmerdler 2017-03-09 01:26:19 -08:00
commit 4e913f106d
34 changed files with 299 additions and 490 deletions

1
.gitignore vendored
View file

@ -22,3 +22,4 @@ htmlcov
.tox
.cache
.npm-debug.log
Dockerfile-e

View file

@ -1,4 +1,5 @@
import datetime
import os
import time
import logging
import json
@ -122,9 +123,21 @@ class BuildComponent(BaseComponent):
# base_image: The image name and credentials to use to conduct the base image pull.
# username: The username for pulling the base image (if any).
# password: The password for pulling the base image (if any).
subdir, dockerfile_name = os.path.split(build_config.get('build_subdir', '/Dockerfile'))
# HACK HACK HACK HACK HACK HACK HACK
# TODO: FIX THIS in the database and then turn the split back on.
if dockerfile_name.find('Dockerfile') < 0:
# This is a *HACK* for the broken path handling. To be fixed ASAP.
subdir = build_config.get('build_subdir') or '/'
dockerfile_name = 'Dockerfile'
# /HACK HACK HACK HACK HACK HACK HACK
build_arguments = {
'build_package': build_job.get_build_package_url(self.user_files),
'sub_directory': build_config.get('build_subdir', ''),
'sub_directory': subdir,
'dockerfile_name': dockerfile_name,
'repository': repository_name,
'registry': self.registry_hostname,
'pull_token': build_job.repo_build.access_token.code,

View file

@ -276,6 +276,7 @@ class PopenExecutor(BuilderExecutor):
'DOCKER_TLS_VERIFY': os.environ.get('DOCKER_TLS_VERIFY', ''),
'DOCKER_CERT_PATH': os.environ.get('DOCKER_CERT_PATH', ''),
'DOCKER_HOST': os.environ.get('DOCKER_HOST', ''),
'PATH': "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
}
logpipe = LogPipe(logging.INFO)

View file

@ -258,6 +258,11 @@ class BuildTriggerHandler(object):
can be called in a loop, so it should be as fast as possible. """
pass
@classmethod
def path_is_dockerfile(cls, file_name):
""" Returns whether the file is named Dockerfile or follows the convention <name>.Dockerfile"""
return file_name.endswith(".Dockerfile") or u"Dockerfile" == file_name
@classmethod
def service_name(cls):
"""
@ -285,14 +290,10 @@ class BuildTriggerHandler(object):
def get_dockerfile_path(self):
""" Returns the normalized path to the Dockerfile found in the subdirectory
in the config. """
subdirectory = self.config.get('subdir', '')
if subdirectory == '/':
subdirectory = ''
else:
if not subdirectory.endswith('/'):
subdirectory = subdirectory + '/'
return subdirectory + 'Dockerfile'
dockerfile_path = self.config.get('subdir') or 'Dockerfile'
if dockerfile_path[0] == '/':
dockerfile_path = dockerfile_path[1:]
return dockerfile_path
def prepare_build(self, metadata, is_manual=False):
# Ensure that the metadata meets the scheme.

View file

@ -1,23 +1,21 @@
import logging
import os
import re
from calendar import timegm
import dateutil.parser
from bitbucket import BitBucket
from jsonschema import validate
from app import app, get_app_url
from buildtrigger.basehandler import BuildTriggerHandler
from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException,
TriggerDeactivationException, TriggerStartException,
InvalidPayloadException, TriggerProviderException,
determine_build_ref, raise_if_skipped_build,
find_matching_branches)
from buildtrigger.basehandler import BuildTriggerHandler
from app import app, get_app_url
from bitbucket import BitBucket
from util.security.ssh import generate_ssh_keypair
from util.dict_wrappers import JSONPathDict, SafeDictSetter
from util.security.ssh import generate_ssh_keypair
logger = logging.getLogger(__name__)
@ -455,10 +453,7 @@ class BitbucketBuildTrigger(BuildTriggerHandler):
raise RepositoryReadException(err_msg)
files = set([f['path'] for f in data['files']])
if 'Dockerfile' in files:
return ['']
return []
return ["/" + file_path for file_path in files if self.path_is_dockerfile(os.path.basename(file_path))]
def load_dockerfile_contents(self):
repository = self._get_repository_client()

View file

@ -1,6 +1,7 @@
import logging
import os.path
import base64
import re
from calendar import timegm
from functools import wraps
@ -348,9 +349,8 @@ class GithubBuildTrigger(BuildTriggerHandler):
default_commit = repo.get_branch(branches[0]).commit
commit_tree = repo.get_git_tree(default_commit.sha, recursive=True)
return [os.path.dirname(elem.path) for elem in commit_tree.tree
if (elem.type == u'blob' and
os.path.basename(elem.path) == u'Dockerfile')]
return [elem.path for elem in commit_tree.tree
if (elem.type == u'blob' and self.path_is_dockerfile(os.path.basename(elem.path)))]
except GithubException as ghe:
message = ghe.data.get('message', 'Unable to list contents of repository: %s' % source)
if message == 'Branch not found':
@ -371,6 +371,9 @@ class GithubBuildTrigger(BuildTriggerHandler):
raise RepositoryReadException(message)
path = self.get_dockerfile_path()
if not path:
return None
try:
file_info = repo.get_file_contents(path)
except GithubException as ghe:

View file

@ -1,4 +1,5 @@
import logging
import os
from calendar import timegm
from functools import wraps
@ -341,11 +342,7 @@ class GitLabBuildTrigger(BuildTriggerHandler):
msg = 'Unable to find GitLab repository tree for source: %s' % new_build_source
raise RepositoryReadException(msg)
for node in repo_tree:
if node['name'] == 'Dockerfile':
return ['']
return []
return ["/"+node['name'] for node in repo_tree if self.path_is_dockerfile(node['name'])]
@_catch_timeouts
def load_dockerfile_contents(self):

View file

@ -23,7 +23,7 @@ def get_repo_path_contents(path, revision):
return (True, data, None)
def get_raw_path_contents(path, revision):
if path == '/Dockerfile':
if path == 'Dockerfile':
return (True, 'hello world', None)
if path == 'somesubdir/Dockerfile':

View file

@ -119,7 +119,7 @@ def get_mock_github():
return [master, otherbranch]
def get_file_contents_mock(filepath):
if filepath == '/Dockerfile':
if filepath == 'Dockerfile':
m = Mock()
m.content = 'hello world'
return m

View file

@ -151,7 +151,7 @@ def gettag_mock(repo_id, tag):
}
def getrawfile_mock(repo_id, branch_name, path):
if path == '/Dockerfile':
if path == 'Dockerfile':
return 'hello world'
if path == 'somesubdir/Dockerfile':

View file

@ -0,0 +1,15 @@
import pytest
from buildtrigger.basehandler import BuildTriggerHandler
@pytest.mark.parametrize('input,output', [
("Dockerfile", True),
("server.Dockerfile", True),
(u"Dockerfile", True),
(u"server.Dockerfile", True),
("bad file name", False),
(u"bad file name", False),
])
def test_path_is_dockerfile(input, output):
assert BuildTriggerHandler.path_is_dockerfile(input) == output

View file

@ -13,12 +13,12 @@ def bitbucket_trigger():
def test_list_build_subdirs(bitbucket_trigger):
assert bitbucket_trigger.list_build_subdirs() == ['']
assert bitbucket_trigger.list_build_subdirs() == ["/Dockerfile"]
@pytest.mark.parametrize('subdir, contents', [
('', 'hello world'),
('somesubdir', 'hi universe'),
('/Dockerfile', 'hello world'),
('somesubdir/Dockerfile', 'hi universe'),
('unknownpath', None),
])
def test_load_dockerfile_contents(subdir, contents):

View file

@ -68,8 +68,8 @@ def test_handle_trigger_request(github_trigger, payload, expected_error, expecte
@pytest.mark.parametrize('subdir, contents', [
('', 'hello world'),
('somesubdir', 'hi universe'),
('/Dockerfile', 'hello world'),
('somesubdir/Dockerfile', 'hi universe'),
('unknownpath', None),
])
def test_load_dockerfile_contents(subdir, contents):
@ -86,4 +86,4 @@ def test_lookup_user(username, expected_response, github_trigger):
def test_list_build_subdirs(github_trigger):
assert github_trigger.list_build_subdirs() == ['', 'somesubdir']
assert github_trigger.list_build_subdirs() == ['Dockerfile', 'somesubdir/Dockerfile']

View file

@ -13,12 +13,12 @@ def gitlab_trigger():
def test_list_build_subdirs(gitlab_trigger):
assert gitlab_trigger.list_build_subdirs() == ['']
assert gitlab_trigger.list_build_subdirs() == ['/Dockerfile']
@pytest.mark.parametrize('subdir, contents', [
('', 'hello world'),
('somesubdir', 'hi universe'),
('/Dockerfile', 'hello world'),
('somesubdir/Dockerfile', 'hi universe'),
('unknownpath', None),
])
def test_load_dockerfile_contents(subdir, contents):

View file

@ -9,6 +9,5 @@ run:
- |
set -eux
cd quay-pull-request
npm install
npm link typescript
npm test
yarn install --ignore-engines
yarn test

View file

@ -2,7 +2,7 @@ import logging
from uuid import uuid4
from peewee import IntegrityError
from peewee import IntegrityError, JOIN_LEFT_OUTER, fn
from data.model import (image, db_transaction, DataModelException, _basequery,
InvalidManifestException, TagAlreadyCreatedException, StaleTagException)
from data.database import (RepositoryTag, Repository, Image, ImageStorage, Namespace, TagManifest,
@ -13,6 +13,40 @@ from data.database import (RepositoryTag, Repository, Image, ImageStorage, Names
logger = logging.getLogger(__name__)
def get_max_id_for_sec_scan():
""" Gets the maximum id for security scanning """
return RepositoryTag.select(fn.Max(RepositoryTag.id)).scalar()
def get_min_id_for_sec_scan(version):
""" Gets the minimum id for a security scanning """
return (RepositoryTag
.select(fn.Min(RepositoryTag.id))
.join(Image)
.where(Image.security_indexed_engine < version)
.scalar())
def get_tag_pk_field():
""" Returns the primary key for Image DB model """
return RepositoryTag.id
def get_tags_images_eligible_for_scan(clair_version):
Parent = Image.alias()
ParentImageStorage = ImageStorage.alias()
return _tag_alive(RepositoryTag
.select(Image, ImageStorage, Parent, ParentImageStorage, RepositoryTag)
.join(Image, on=(RepositoryTag.image == Image.id))
.join(ImageStorage, on=(Image.storage == ImageStorage.id))
.switch(Image)
.join(Parent, JOIN_LEFT_OUTER, on=(Image.parent == Parent.id))
.join(ParentImageStorage, JOIN_LEFT_OUTER, on=(ParentImageStorage.id == Parent.storage))
.where(RepositoryTag.hidden == False)
.where(Image.security_indexed_engine < clair_version))
def _tag_alive(query, now_ts=None):
if now_ts is None:
now_ts = get_epoch_timestamp()

View file

@ -404,7 +404,7 @@ class ActivateBuildTrigger(RepositoryParamResource):
'description': '(Custom Only) If specified, the ref/SHA1 used to checkout a git repository.'
},
'refs': {
'type': 'object',
'type': ['object', 'null'],
'description': '(SCM Only) If specified, the ref to build.'
}
},

View file

@ -15,11 +15,11 @@
},
"homepage": "https://github.com/coreos-inc/quay#readme",
"dependencies": {
"angular": "1.5.3",
"angular-animate": "^1.5.3",
"angular-cookies": "^1.5.3",
"angular-route": "^1.5.3",
"angular-sanitize": "^1.5.3",
"angular": "1.6.2",
"angular-animate": "1.6.2",
"angular-cookies": "1.6.2",
"angular-route": "1.6.2",
"angular-sanitize": "1.6.2",
"bootbox": "^4.1.0",
"bootstrap": "^3.3.2",
"bootstrap-datepicker": "^1.6.4",
@ -35,15 +35,16 @@
"underscore": "^1.5.2"
},
"devDependencies": {
"@types/angular": "1.5.16",
"@types/angular": "1.6.2",
"@types/angular-mocks": "^1.5.8",
"@types/angular-route": "^1.3.3",
"@types/angular-sanitize": "^1.3.4",
"@types/es6-shim": "^0.31.32",
"@types/jasmine": "^2.5.41",
"@types/jquery": "^2.0.40",
"@types/react": "0.14.39",
"@types/react-dom": "0.14.17",
"angular-mocks": "^1.5.3",
"angular-mocks": "1.6.2",
"angular-ts-decorators": "0.0.19",
"css-loader": "0.25.0",
"jasmine-core": "^2.5.2",

View file

@ -30,6 +30,12 @@
height: 28px;
}
@media (max-width: 768px) {
.manage-trigger-control .co-top-bar {
margin-bottom: 80px;
}
}
.manage-trigger-control .namespace-avatar {
margin-left: 2px;
margin-right: 2px;

View file

@ -26,7 +26,8 @@
icon-key="kind"
icon-map="field.iconMap"
items="fieldOptions[field.name]"
ng-show="fieldOptions[field.name]"></div>
ng-show="fieldOptions[field.name]"
clear-value="counter"></div>
</div>
<!-- Option -->

View file

@ -39,28 +39,7 @@ angular.module('quay').directive('dockerfileBuildForm', function () {
$scope.state = 'checking';
$scope.selectedFiles = files;
// FIXME: Remove this
// DockerfileService.getDockerfile(files[0], function(df) {
// var baseImage = df.getRegistryBaseImage();
// if (baseImage) {
// checkPrivateImage(baseImage);
// } else {
// $scope.state = 'ready';
// }
//
// $scope.$apply(function() {
// opt_callback && opt_callback(true, 'Dockerfile found and valid')
// });
// }, function(msg) {
// $scope.state = 'empty';
// $scope.privateBaseRepository = null;
//
// $scope.$apply(function() {
// opt_callback && opt_callback(false, msg || 'Could not find valid Dockerfile');
// });
// });
DockerfileService.extractDockerfile(files[0])
DockerfileService.getDockerfile(files[0])
.then(function(dockerfileInfo) {
var baseImage = dockerfileInfo.getRegistryBaseImage();
if (baseImage) {

View file

@ -25,7 +25,7 @@
</div>
</div>
<table class="co-table" style="margin-top: 20px;">
<table class="co-table">
<thead>
<td class="checkbox-col"></td>
<td ng-class="$ctrl.TableService.tablePredicateClass('id', $ctrl.local.namespaceOptions.predicate, $ctrl.local.namespaceOptions.reverse)">
@ -39,7 +39,7 @@
<tr class="co-checkable-row"
ng-repeat="namespace in $ctrl.local.orderedNamespaces.visibleEntries | slice:($ctrl.namespacesPerPage * $ctrl.local.namespaceOptions.page):($ctrl.namespacesPerPage * ($ctrl.local.namespaceOptions.page + 1))"
ng-class="$ctrl.local.selectedNamespace == $ctrl.namespace ? 'checked' : ''"
ng-class="$ctrl.local.selectedNamespace == namespace ? 'checked' : ''"
bindonce>
<td>
<input type="radio"
@ -290,19 +290,22 @@
</div>
<!-- Warning -->
<div class="col-lg-7 col-md-7 col-sm-12 main-col" ng-if="$ctrl.local.triggerAnalysis.status == 'warning'">
<div class="col-lg-7 col-md-7 col-sm-12 main-col"
ng-if="$ctrl.local.triggerAnalysis.status == 'warning'">
<h3 class="warning"><i class="fa fa-exclamation-triangle"></i> Verification Warning</h3>
{{ $ctrl.local.triggerAnalysis.message }}
</div>
<!-- Public base -->
<div class="col-lg-7 col-md-7 col-sm-12 main-col" ng-if="$ctrl.local.triggerAnalysis.status == 'publicbase'">
<div class="col-lg-7 col-md-7 col-sm-12 main-col"
ng-if="$ctrl.local.triggerAnalysis.status == 'publicbase'">
<h3 class="success"><i class="fa fa-check-circle"></i> Ready to go!</h3>
<strong>Click "Create Trigger" to complete setup of this build trigger</strong>
</div>
<!-- Requires robot and is not admin -->
<div class="col-lg-7 col-md-7 col-sm-12 main-col" ng-if="$ctrl.local.triggerAnalysis.status == 'requiresrobot' && !$ctrl.local.triggerAnalysis.is_admin">
<div class="col-lg-7 col-md-7 col-sm-12 main-col"
ng-if="$ctrl.local.triggerAnalysis.status == 'requiresrobot' && !$ctrl.local.triggerAnalysis.is_admin">
<h3>Robot Account Required</h3>
<p>The selected Dockerfile in the selected repository depends upon a private base image</p>
<p>A robot account with access to the base image is required to setup this trigger, but you are not the administrator of this namespace.</p>
@ -310,7 +313,8 @@
</div>
<!-- Requires robot and is admin -->
<div class="col-lg-7 col-md-7 col-sm-12 main-col" ng-if="$ctrl.local.triggerAnalysis.status == 'requiresrobot' && $ctrl.local.triggerAnalysis.is_admin">
<div class="col-lg-7 col-md-7 col-sm-12 main-col"
ng-if="$ctrl.local.triggerAnalysis.status == 'requiresrobot' && $ctrl.local.triggerAnalysis.is_admin">
<h3>Select Robot Account</h3>
<strong>
The selected Dockerfile in the selected repository depends upon a private base image. Select a robot account with access:
@ -338,7 +342,7 @@
</thead>
<tr class="co-checkable-row"
ng-repeat="robot in $ctrl.local.orderedRobotAccounts.visibleEntries | slice:($ctrl.robotsPerPage * $ctrl.local.namespaceOptions.page):(robotsPerPage * ($ctrl.local.robotOptions.page + 1))"
ng-repeat="robot in $ctrl.local.orderedRobotAccounts.visibleEntries | slice:($ctrl.robotsPerPage * $ctrl.local.namespaceOptions.page):($ctrl.robotsPerPage * ($ctrl.local.robotOptions.page + 1))"
ng-class="$ctrl.local.robotAccount == robot ? 'checked' : ''"
bindonce>
<td>
@ -355,8 +359,8 @@
</td>
</tr>
</table>
<div class="empty" ng-if="$ctrl.local.triggerAnalysis.robots.length && !$ctrl.local.orderedRobotAccounts.entries.length"
style="margin-top: 20px;">
<div class="empty" style="margin-top: 20px;"
ng-if="$ctrl.local.triggerAnalysis.robots.length && !$ctrl.local.orderedRobotAccounts.entries.length">
<div class="empty-primary-msg">No matching robot accounts found.</div>
<div class="empty-secondary-msg">Try expanding your filtering terms.</div>
</div>

View file

@ -62,14 +62,8 @@ export class ManageTriggerGithostComponent implements ng.IComponentController {
public $onInit(): void {
// TODO: Replace $scope.$watch with @Output methods for child component mutations or $onChanges for parent mutations
this.$scope.$watch(() => this.trigger, (trigger) => {
if (trigger && this.repository) {
this.config = trigger['config'] || {};
this.namespaceTitle = 'organization';
this.local.selectedNamespace = null;
this.loadNamespaces();
}
});
this.$scope.$watch(() => this.trigger, this.initialSetup.bind(this));
this.$scope.$watch(() => this.repository, this.initialSetup.bind(this));
this.$scope.$watch(() => this.local.selectedNamespace, (namespace) => {
if (namespace) {
@ -102,6 +96,20 @@ export class ManageTriggerGithostComponent implements ng.IComponentController {
this.$scope.$watch(() => this.local.robotOptions.filter, this.buildOrderedRobotAccounts);
}
private initialSetup(): void {
if (!this.repository || !this.trigger) { return; }
if (this.namespaceTitle) {
// Already setup.
return;
}
this.config = this.trigger['config'] || {};
this.namespaceTitle = 'organization';
this.local.selectedNamespace = null;
this.loadNamespaces();
}
public getTriggerIcon(): any {
return this.TriggerService.getIcon(this.trigger.service);
}

View file

@ -57,6 +57,8 @@ angular.module('quay').directive('manualTriggerBuildDialog', function () {
$scope.fieldOptions[parameter['name']] = resp['values'];
});
}
delete $scope.parameters[parameter['name']];
}
$scope.runParameters = parameters;

View file

@ -49,7 +49,7 @@ export class quay {
// TODO: Make injected values into services and move to NgModule.providers, as constants are not supported in Angular 2
angular
.module(quay.name)
.factory("FileReaderFactory", () => () => new FileReader())
.factory("fileReaderFactory", () => () => new FileReader())
.constant('NAME_PATTERNS', NAME_PATTERNS)
.constant('INJECTED_CONFIG', INJECTED_CONFIG)
.constant('INJECTED_FEATURES', INJECTED_FEATURES)

View file

@ -76,6 +76,7 @@ angular.module('quay').factory('DataFileService', [function() {
return parts.join('/');
};
try {
var handler = new Untar(new Uint8Array(buf));
handler.process(function(status, read, files, err) {
switch (status) {
@ -106,6 +107,10 @@ angular.module('quay').factory('DataFileService', [function() {
break;
}
});
} catch (e) {
failure();
}
};
dataFileService.blobToString = function(blob, callback) {

View file

@ -1,127 +0,0 @@
/**
* Service which provides helper methods for extracting information out from a Dockerfile
* or an archive containing a Dockerfile.
*/
angular.module('quay').factory('DockerfileServiceOld', ['DataFileService', 'Config', function(DataFileService, Config) {
var dockerfileService = {};
function DockerfileInfo(contents) {
this.contents = contents;
}
DockerfileInfo.prototype.getRegistryBaseImage = function() {
var baseImage = this.getBaseImage();
if (!baseImage) {
return null;
}
if (baseImage.indexOf(Config.getDomain() + '/') != 0) {
return null;
}
return baseImage.substring(Config.getDomain().length + 1);
};
DockerfileInfo.prototype.getBaseImage = function() {
var imageAndTag = this.getBaseImageAndTag();
if (!imageAndTag) {
return null;
}
// Note, we have to handle a few different cases here:
// 1) someimage
// 2) someimage:tag
// 3) host:port/someimage
// 4) host:port/someimage:tag
var lastIndex = imageAndTag.lastIndexOf(':');
if (lastIndex < 0) {
return imageAndTag;
}
// Otherwise, check if there is a / in the portion after the split point. If so,
// then the latter is part of the path (and not a tag).
var afterColon = imageAndTag.substring(lastIndex + 1);
if (afterColon.indexOf('/') >= 0) {
return imageAndTag;
}
return imageAndTag.substring(0, lastIndex);
};
DockerfileInfo.prototype.getBaseImageAndTag = function() {
var fromIndex = this.contents.indexOf('FROM ');
if (fromIndex < 0) {
return null;
}
var newline = this.contents.indexOf('\n', fromIndex);
if (newline < 0) {
newline = this.contents.length;
}
return $.trim(this.contents.substring(fromIndex + 'FROM '.length, newline));
};
DockerfileInfo.forData = function(contents) {
if (contents.indexOf('FROM ') < 0) {
return;
}
return new DockerfileInfo(contents);
};
var processFiles = function(files, dataArray, success, failure) {
// The files array will be empty if the submitted file was not an archive. We therefore
// treat it as a single Dockerfile.
if (files.length == 0) {
DataFileService.arrayToString(dataArray, function(c) {
var result = DockerfileInfo.forData(c);
if (!result) {
failure('File chosen is not a valid Dockerfile');
return;
}
success(result);
});
return;
}
var found = false;
files.forEach(function(file) {
if (file['name'] == 'Dockerfile') {
DataFileService.blobToString(file.toBlob(), function(c) {
var result = DockerfileInfo.forData(c);
if (!result) {
failure('Dockerfile inside archive is not a valid Dockerfile');
return;
}
success(result);
});
found = true;
}
});
if (!found) {
failure('No Dockerfile found in root of archive');
}
};
dockerfileService.getDockerfile = function(file, success, failure) {
var reader = new FileReader();
reader.onload = function(e) {
var dataArray = reader.result;
DataFileService.readDataArrayAsPossibleArchive(dataArray, function(files) {
processFiles(files, dataArray, success, failure);
}, function() {
// Not an archive. Read directly as a single file.
processFiles([], dataArray, success, failure);
});
};
reader.onerror = failure;
reader.readAsArrayBuffer(file);
};
return dockerfileService;
}]);

View file

@ -52,157 +52,7 @@ describe("DockerfileServiceImpl", () => {
});
it("calls datafile service to read given file as possible archive file", (done) => {
dockerfileServiceImpl.getDockerfile(file,
(dockerfile: DockerfileInfoImpl) => {
expect(readAsFileBufferSpy.calls.argsFor(0)[0]).toEqual(file);
expect(dataFileServiceMock.readDataArrayAsPossibleArchive).toHaveBeenCalled();
done();
},
(error: Event | string) => {
fail("Should not invoke failure callback");
done();
});
});
it("calls datafile service to convert file to string if given file is not an archive", (done) => {
dockerfileServiceImpl.getDockerfile(file,
(dockerfile: DockerfileInfoImpl) => {
expect(dataFileServiceMock.arrayToString.calls.argsFor(0)[0]).toEqual(file);
done();
},
(error: Event | string) => {
fail("Should not invoke success callback");
done();
});
});
it("calls failure callback if given non-archive file that is not a valid Dockerfile", (done) => {
forDataSpy.and.returnValue(null);
dockerfileServiceImpl.getDockerfile(file,
(dockerfile: DockerfileInfoImpl) => {
fail("Should not invoke success callback");
done();
},
(error: Event | string) => {
expect(error).toEqual('File chosen is not a valid Dockerfile');
done();
});
});
it("calls success callback with new DockerfileInfoImpl instance if given valid Dockerfile", (done) => {
dockerfileServiceImpl.getDockerfile(file,
(dockerfile: DockerfileInfoImpl) => {
expect(dockerfile).toBeDefined();
done();
},
(error: Event | string) => {
fail('Should not invoke failure callback');
done();
});
});
it("calls failure callback if given archive file with no Dockerfile present in root directory", (done) => {
dataFileServiceMock.readDataArrayAsPossibleArchive.and.callFake((buf, success, failure) => {
success(invalidArchiveFile);
});
dockerfileServiceImpl.getDockerfile(file,
(dockerfile: DockerfileInfoImpl) => {
fail("Should not invoke success callback");
done();
},
(error: Event | string) => {
expect(error).toEqual('No Dockerfile found in root of archive');
done();
});
});
it("calls datafile service to convert blob to string if given file is an archive", (done) => {
dataFileServiceMock.readDataArrayAsPossibleArchive.and.callFake((buf, success, failure) => {
success(validArchiveFile);
});
dockerfileServiceImpl.getDockerfile(file,
(dockerfile: DockerfileInfoImpl) => {
expect(validArchiveFile[0].toBlob).toHaveBeenCalled();
expect(dataFileServiceMock.blobToString.calls.argsFor(0)[0]).toEqual(validArchiveFile[0].toBlob());
done();
},
(error: Event | string) => {
fail("Should not invoke success callback");
done();
});
});
it("calls failure callback if given archive file with invalid Dockerfile", (done) => {
forDataSpy.and.returnValue(null);
invalidArchiveFile[0].name = 'Dockerfile';
dataFileServiceMock.readDataArrayAsPossibleArchive.and.callFake((buf, success, failure) => {
success(invalidArchiveFile);
});
dockerfileServiceImpl.getDockerfile(file,
(dockerfile: DockerfileInfoImpl) => {
fail("Should not invoke success callback");
done();
},
(error: Event | string) => {
expect(error).toEqual('Dockerfile inside archive is not a valid Dockerfile');
done();
});
});
it("calls success callback with new DockerfileInfoImpl instance if given archive with valid Dockerfile", (done) => {
dataFileServiceMock.readDataArrayAsPossibleArchive.and.callFake((buf, success, failure) => {
success(validArchiveFile);
});
dockerfileServiceImpl.getDockerfile(file,
(dockerfile: DockerfileInfoImpl) => {
expect(dockerfile).toBeDefined();
done();
},
(error: Event | string) => {
fail('Should not invoke failure callback');
done();
});
});
});
describe("extractDockerfile", () => {
var file: any;
var invalidArchiveFile: any[];
var validArchiveFile: any[];
var readAsFileBufferSpy: Spy;
var forDataSpy: Spy;
beforeEach(() => {
dataFileServiceMock.readDataArrayAsPossibleArchive.and.callFake((buf, success, failure) => {
failure([]);
});
dataFileServiceMock.arrayToString.and.callFake((buf, callback) => {
var contents: string = "";
callback(contents);
});
dataFileServiceMock.blobToString.and.callFake((blob, callback) => {
callback(blob.toString());
});
forDataSpy = spyOn(DockerfileInfoImpl, "forData").and.returnValue(new DockerfileInfoImpl(file, configMock));
readAsFileBufferSpy = spyOn(fileReaderMock, "readAsArrayBuffer").and.callFake(() => {
var event: any = {target: {result: file}};
fileReaderMock.onload(event);
});
file = "FROM quay.io/coreos/nginx:latest";
validArchiveFile = [{name: 'Dockerfile', toBlob: jasmine.createSpy('toBlobSpy').and.returnValue(file)}];
invalidArchiveFile = [{name: 'main.exe', toBlob: jasmine.createSpy('toBlobSpy').and.returnValue("")}];
});
it("calls datafile service to read given file as possible archive file", (done) => {
dockerfileServiceImpl.extractDockerfile(file)
dockerfileServiceImpl.getDockerfile(file)
.then((dockerfile: DockerfileInfoImpl) => {
expect(readAsFileBufferSpy.calls.argsFor(0)[0]).toEqual(file);
expect(dataFileServiceMock.readDataArrayAsPossibleArchive).toHaveBeenCalled();
@ -215,31 +65,107 @@ describe("DockerfileServiceImpl", () => {
});
it("calls datafile service to convert file to string if given file is not an archive", (done) => {
dockerfileServiceImpl.getDockerfile(file)
.then((dockerfile: DockerfileInfoImpl) => {
expect(dataFileServiceMock.arrayToString.calls.argsFor(0)[0]).toEqual(file);
done();
})
.catch((error: string) => {
fail('Promise should be resolved');
done();
});
});
it("returns rejected promise if given non-archive file that is not a valid Dockerfile", (done) => {
forDataSpy.and.returnValue(null);
dockerfileServiceImpl.getDockerfile(file)
.then((dockerfile: DockerfileInfoImpl) => {
fail("Promise should be rejected");
done();
})
.catch((error: string) => {
expect(error).toEqual('File chosen is not a valid Dockerfile');
done();
});
});
it("returns resolved promise with new DockerfileInfoImpl instance if given valid Dockerfile", (done) => {
dockerfileServiceImpl.getDockerfile(file)
.then((dockerfile: DockerfileInfoImpl) => {
expect(dockerfile).toBeDefined();
done();
})
.catch((error: string) => {
fail('Promise should be resolved');
done();
});
});
it("returns rejected promise if given archive file with no Dockerfile present in root directory", (done) => {
dataFileServiceMock.readDataArrayAsPossibleArchive.and.callFake((buf, success, failure) => {
success(invalidArchiveFile);
});
dockerfileServiceImpl.getDockerfile(file)
.then((dockerfile: DockerfileInfoImpl) => {
fail('Promise should be rejected');
done();
})
.catch((error: string) => {
expect(error).toEqual('No Dockerfile found in root of archive');
done();
});
});
it("calls datafile service to convert blob to string if given file is an archive", (done) => {
dataFileServiceMock.readDataArrayAsPossibleArchive.and.callFake((buf, success, failure) => {
success(validArchiveFile);
});
dockerfileServiceImpl.getDockerfile(file)
.then((dockerfile: DockerfileInfoImpl) => {
expect(validArchiveFile[0].toBlob).toHaveBeenCalled();
expect(dataFileServiceMock.blobToString.calls.argsFor(0)[0]).toEqual(validArchiveFile[0].toBlob());
done();
})
.catch((error: string) => {
fail('Promise should be resolved');
done();
});
});
it("returns rejected promise if given archive file with invalid Dockerfile", (done) => {
forDataSpy.and.returnValue(null);
invalidArchiveFile[0].name = 'Dockerfile';
dataFileServiceMock.readDataArrayAsPossibleArchive.and.callFake((buf, success, failure) => {
success(invalidArchiveFile);
});
dockerfileServiceImpl.getDockerfile(file)
.then((dockerfile: DockerfileInfoImpl) => {
fail('Promise should be rejected');
done();
})
.catch((error: string) => {
expect(error).toEqual('Dockerfile inside archive is not a valid Dockerfile');
done();
});
});
it("returns resolved promise of new DockerfileInfoImpl instance if given archive with valid Dockerfile", (done) => {
dataFileServiceMock.readDataArrayAsPossibleArchive.and.callFake((buf, success, failure) => {
success(validArchiveFile);
});
dockerfileServiceImpl.getDockerfile(file)
.then((dockerfile: DockerfileInfoImpl) => {
expect(dockerfile).toBeDefined();
done();
})
.catch((error: string) => {
fail('Promise should be resolved');
done();
});
});
});
});

View file

@ -5,21 +5,30 @@ import { Injectable } from 'angular-ts-decorators';
@Injectable(DockerfileService.name)
export class DockerfileServiceImpl implements DockerfileService {
constructor(private DataFileService: any, private Config: any, private FileReaderFactory: () => FileReader) {
constructor(private DataFileService: any,
private Config: any,
private fileReaderFactory: () => FileReader) {
}
public extractDockerfile(file: any): Promise<DockerfileInfoImpl | string> {
public getDockerfile(file: any): Promise<DockerfileInfoImpl | string> {
return new Promise((resolve, reject) => {
var reader: FileReader = this.FileReaderFactory();
var reader: FileReader = this.fileReaderFactory();
reader.onload = (event: any) => {
// FIXME: Debugging
console.log(event.target.result);
this.DataFileService.readDataArrayAsPossibleArchive(event.target.result,
(files: any[]) => {
this.processFiles1(files);
this.processFiles(files)
.then((dockerfileInfo: DockerfileInfoImpl) => resolve(dockerfileInfo))
.catch((error: string) => reject(error));
},
() => {
// Not an archive. Read directly as a single file.
this.processFile1(event.target.result);
this.processFile(event.target.result)
.then((dockerfileInfo: DockerfileInfoImpl) => resolve(dockerfileInfo))
.catch((error: string) => reject(error));
});
};
@ -28,40 +37,7 @@ export class DockerfileServiceImpl implements DockerfileService {
});
}
public getDockerfile(file: any,
success: (dockerfile: DockerfileInfoImpl) => void,
failure: (error: Event | string) => void): void {
var reader: FileReader = this.FileReaderFactory();
reader.onload = (event: any) => {
this.DataFileService.readDataArrayAsPossibleArchive(event.target.result,
(files: any[]) => {
this.processFiles(files, success, failure);
},
() => {
// Not an archive. Read directly as a single file.
this.processFile(event.target.result, success, failure);
});
};
reader.onerror = failure;
reader.readAsArrayBuffer(file);
}
private processFile(dataArray: any,
success: (dockerfile: DockerfileInfoImpl) => void,
failure: (error: ErrorEvent | string) => void): void {
this.DataFileService.arrayToString(dataArray, (contents: string) => {
var result: DockerfileInfoImpl | null = DockerfileInfoImpl.forData(contents, this.Config);
if (result == null) {
failure('File chosen is not a valid Dockerfile');
}
else {
success(result);
}
});
}
private processFile1(dataArray: any): Promise<DockerfileInfoImpl | string> {
private processFile(dataArray: any): Promise<DockerfileInfoImpl | string> {
return new Promise((resolve, reject) => {
this.DataFileService.arrayToString(dataArray, (contents: string) => {
var result: DockerfileInfoImpl | null = DockerfileInfoImpl.forData(contents, this.Config);
@ -75,31 +51,7 @@ export class DockerfileServiceImpl implements DockerfileService {
});
}
private processFiles(files: any[],
success: (dockerfile: DockerfileInfoImpl) => void,
failure: (error: ErrorEvent | string) => void): void {
var found: boolean = false;
files.forEach((file) => {
if (file['name'] == 'Dockerfile') {
this.DataFileService.blobToString(file.toBlob(), (contents: string) => {
var result: DockerfileInfoImpl | null = DockerfileInfoImpl.forData(contents, this.Config);
if (result == null) {
failure('Dockerfile inside archive is not a valid Dockerfile');
}
else {
success(result);
}
});
found = true;
}
});
if (!found) {
failure('No Dockerfile found in root of archive');
}
}
private processFiles1(files: any[]): Promise<DockerfileInfoImpl | string> {
private processFiles(files: any[]): Promise<DockerfileInfoImpl | string> {
return new Promise((resolve, reject) => {
var found: boolean = false;
files.forEach((file) => {

View file

@ -7,19 +7,9 @@ export abstract class DockerfileService {
/**
* Retrieve Dockerfile from given file.
* @param file Dockerfile or archive file containing Dockerfile.
* @param success Success callback with retrieved Dockerfile as parameter.
* @param failure Failure callback with failure message as parameter.
* @return promise Promise which resolves to new DockerfileInfo instance or rejects with error message.
*/
public abstract getDockerfile(file: any,
success: (dockerfile: DockerfileInfo) => void,
failure: (error: ErrorEvent | string) => void): void;
/**
* Retrieve Dockerfile from given file.
* @param file Dockerfile or archive file containing Dockerfile.
* @return promise Promise resolving to new DockerfileInfo instance or rejecting to error message.
*/
public abstract extractDockerfile(file: any): Promise<DockerfileInfo | string>;
public abstract getDockerfile(file: any): Promise<DockerfileInfo | string>;
}

View file

@ -208,10 +208,6 @@ angular.module('quay').factory('TriggerService', ['UtilService', '$sanitize', 'K
return '//Dockerfile';
}
if (subdirectory[subdirectory.length - 1] != '/') {
subdirectory = subdirectory + '/';
}
return '//' + subdirectory.replace(new RegExp('(^\/+|\/+$)'), '') + 'Dockerfile';
};

View file

@ -4059,12 +4059,19 @@ class TestBuildTriggers(ApiTestCase):
self.assertEquals('bar', py_json.loads(build_obj.job_config)['trigger_metadata']['foo'])
# Start another manual build, with a ref.
start_json = self.postJsonResponse(ActivateBuildTrigger,
self.postJsonResponse(ActivateBuildTrigger,
params=dict(repository=ADMIN_ACCESS_USER + '/simple',
trigger_uuid=trigger.uuid),
data=dict(refs={'kind': 'branch', 'name': 'foobar'}),
expected_code=201)
# Start another manual build with a null ref.
self.postJsonResponse(ActivateBuildTrigger,
params=dict(repository=ADMIN_ACCESS_USER + '/simple',
trigger_uuid=trigger.uuid),
data=dict(refs=None),
expected_code=201)
def test_invalid_robot_account(self):
self.login(ADMIN_ACCESS_USER)

View file

@ -6,8 +6,8 @@ import features
from app import app, secscan_api, prometheus
from workers.worker import Worker
from data.database import UseThenDisconnect
from data.model.image import (get_images_eligible_for_scan, get_max_id_for_sec_scan,
get_min_id_for_sec_scan, get_image_id)
from data.model.tag import (get_tags_images_eligible_for_scan, get_tag_pk_field,
get_max_id_for_sec_scan, get_min_id_for_sec_scan)
from util.secscan.api import SecurityConfigValidator
from util.secscan.analyzer import LayerAnalyzer, PreemptedException
from util.migrate.allocator import yield_random_entries
@ -43,7 +43,7 @@ class SecurityWorker(Worker):
def _index_images(self):
def batch_query():
return get_images_eligible_for_scan(self._target_version)
return get_tags_images_eligible_for_scan(self._target_version)
# Get the ID of the last image we can analyze. Will be None if there are no images in the
# database.
@ -56,14 +56,14 @@ class SecurityWorker(Worker):
with UseThenDisconnect(app.config):
to_scan_generator = yield_random_entries(
batch_query,
get_image_id(),
get_tag_pk_field(),
BATCH_SIZE,
max_id,
self._min_id,
)
for candidate, abt, num_remaining in to_scan_generator:
try:
self._analyzer.analyze_recursively(candidate)
self._analyzer.analyze_recursively(candidate.image)
except PreemptedException:
logger.info('Another worker pre-empted us for layer: %s', candidate.id)
abt.set()

View file

@ -20,9 +20,9 @@
dependencies:
"@types/angular" "*"
"@types/angular@*", "@types/angular@1.5.16":
version "1.5.16"
resolved "https://registry.yarnpkg.com/@types/angular/-/angular-1.5.16.tgz#02a56754b50dbf9209266b4339031a54317702d9"
"@types/angular@*", "@types/angular@1.6.2":
version "1.6.2"
resolved "https://registry.yarnpkg.com/@types/angular/-/angular-1.6.2.tgz#a5c323ea5d4426ad18984cc8167fa091f7c8201b"
dependencies:
"@types/jquery" "*"
@ -36,7 +36,7 @@
dependencies:
typescript ">=2.1.4"
"@types/jquery@*":
"@types/jquery@*", "@types/jquery@^2.0.40":
version "2.0.40"
resolved "https://registry.yarnpkg.com/@types/jquery/-/jquery-2.0.40.tgz#acdd69e29b74cdec15dc3c074bcb064bc1b87213"
@ -109,23 +109,23 @@ amdefine@>=0.0.4:
version "1.0.1"
resolved "https://registry.yarnpkg.com/amdefine/-/amdefine-1.0.1.tgz#4a5282ac164729e93619bcfd3ad151f817ce91f5"
angular-animate@^1.5.3:
angular-animate@1.6.2:
version "1.6.2"
resolved "https://registry.yarnpkg.com/angular-animate/-/angular-animate-1.6.2.tgz#def2a8b9ede53b4b6e234c25f5c64e4b4385df15"
angular-cookies@^1.5.3:
angular-cookies@1.6.2:
version "1.6.2"
resolved "https://registry.yarnpkg.com/angular-cookies/-/angular-cookies-1.6.2.tgz#ffb69f35f84d1efe71addac20a2905476b658884"
angular-mocks@^1.5.3:
angular-mocks@1.6.2:
version "1.6.2"
resolved "https://registry.yarnpkg.com/angular-mocks/-/angular-mocks-1.6.2.tgz#fbb28208e74d3512769afdb8771f5cc5a99f9128"
angular-route@^1.5.3:
angular-route@1.6.2:
version "1.6.2"
resolved "https://registry.yarnpkg.com/angular-route/-/angular-route-1.6.2.tgz#95a349de2e73674f3dd783bb21e8d7b3fc526312"
angular-sanitize@^1.5.3:
angular-sanitize@1.6.2:
version "1.6.2"
resolved "https://registry.yarnpkg.com/angular-sanitize/-/angular-sanitize-1.6.2.tgz#8a327c1acb2c14f50da5b5cad5ea452750a1a375"
@ -135,9 +135,9 @@ angular-ts-decorators@0.0.19:
dependencies:
reflect-metadata "^0.1.8"
angular@1.5.3:
version "1.5.3"
resolved "https://registry.yarnpkg.com/angular/-/angular-1.5.3.tgz#37c2f198ae76c2d6f3717a4ecef1cddcb048af79"
angular@1.6.2:
version "1.6.2"
resolved "https://registry.yarnpkg.com/angular/-/angular-1.6.2.tgz#d0b677242ac4bf9ae81424297c6320973af4bb5a"
ansi-align@^1.1.0:
version "1.1.0"