vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge remote-tracking branch 'origin/pullinprivate'

Browse source
This commit is contained in:
jakedt 2014-04-02 11:58:31 -04:00
commit d5634bc3c7
18 changed files with 418 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
data/database.py
View file

@ -176,6 +176,7 @@ class RepositoryBuildTrigger(BaseModel):
auth_token = CharField()
config = TextField(default='{}')
write_token = ForeignKeyField(AccessToken, null=True)
pull_robot = ForeignKeyField(User, null=True, related_name='triggerpullrobot')
class EmailConfirmation(BaseModel):
@ -244,6 +245,7 @@ class RepositoryBuild(BaseModel):
started = DateTimeField(default=datetime.now)
display_name = CharField()
trigger = ForeignKeyField(RepositoryBuildTrigger, null=True, index=True)
pull_robot = ForeignKeyField(User, null=True, related_name='buildpullrobot')
class QueueItem(BaseModel):

48
data/model/legacy.py
View file

@ -154,6 +154,16 @@ def create_robot(robot_shortname, parent):
raise DataModelException(ex.message)
def lookup_robot(robot_username):
joined = User.select().join(FederatedLogin).join(LoginService)
found = list(joined.where(LoginService.name == 'quayrobot',
User.username == robot_username))
if not found or len(found) < 1 or not found[0].robot:
return None
return found[0]
def verify_robot(robot_username, password):
joined = User.select().join(FederatedLogin).join(LoginService)
found = list(joined.where(FederatedLogin.service_ident == password,
@ -1443,11 +1453,40 @@ def get_recent_repository_build(namespace_name, repository_name):
def create_repository_build(repo, access_token, job_config_obj, dockerfile_id,
display_name, trigger=None):
display_name, trigger=None, pull_robot_name=None):
pull_robot = None
if pull_robot_name:
pull_robot = lookup_robot(pull_robot_name)
return RepositoryBuild.create(repository=repo, access_token=access_token,
job_config=json.dumps(job_config_obj),
display_name=display_name, trigger=trigger,
resource_key=dockerfile_id)
resource_key=dockerfile_id,
pull_robot=pull_robot)
def get_pull_robot_name(trigger):
if not trigger.pull_robot:
return None
return trigger.pull_robot.username
def get_pull_credentials(robotname):
robot = lookup_robot(robotname)
if not robot:
return None
try:
login_info = FederatedLogin.get(user=robot)
except FederatedLogin.DoesNotExist:
return None
return {
'username': robot.username,
'password': login_info.service_ident,
'registry': '%s://%s/v1/' % (app.config['URL_SCHEME'], app.config['URL_HOST']),
}
def create_webhook(repo, params_obj):
@ -1506,11 +1545,12 @@ def log_action(kind_name, user_or_organization_name, performer=None,
metadata_json=json.dumps(metadata), datetime=timestamp)
def create_build_trigger(repo, service_name, auth_token, user):
def create_build_trigger(repo, service_name, auth_token, user, pull_robot=None):
service = BuildTriggerService.get(name=service_name)
trigger = RepositoryBuildTrigger.create(repository=repo, service=service,
auth_token=auth_token,
connected_user=user)
connected_user=user,
pull_robot=pull_robot)
return trigger

39
endpoints/api/build.py
View file

@ -10,8 +10,10 @@ from endpoints.api import (RepositoryParamResource, parse_args, query_param, nic
from endpoints.common import start_build
from endpoints.trigger import BuildTrigger
from data import model
from auth.permissions import ModifyRepositoryPermission
from auth.auth_context import get_authenticated_user
from auth.permissions import ModifyRepositoryPermission, AdministerOrganizationPermission
from data.buildlogs import BuildStatusRetrievalError
from util.names import parse_robot_username
logger = logging.getLogger(__name__)
@ -33,7 +35,15 @@ def get_job_config(build_obj):
return None
def user_view(user):
return {
'name': user.username,
'kind': 'user',
'is_robot': user.robot,
}
def trigger_view(trigger):
if trigger and trigger.uuid:
config_dict = get_trigger_config(trigger)
build_trigger = BuildTrigger.get_trigger_for_service(trigger.service.name)
@ -42,7 +52,8 @@ def trigger_view(trigger):
'config': config_dict,
'id': trigger.uuid,
'connected_user': trigger.connected_user.username,
'is_active': build_trigger.is_active(config_dict)
'is_active': build_trigger.is_active(config_dict),
'pull_robot': user_view(trigger.pull_robot) if trigger.pull_robot else None
}
return None
@ -67,6 +78,7 @@ def build_status_view(build_obj, can_write=False):
'is_writer': can_write,
'trigger': trigger_view(build_obj.trigger),
'resource_key': build_obj.resource_key,
'pull_robot': user_view(build_obj.pull_robot) if build_obj.pull_robot else None,
}
if can_write:
@ -95,6 +107,10 @@ class RepositoryBuildList(RepositoryParamResource):
'type': 'string',
'description': 'Subdirectory in which the Dockerfile can be found',
},
'pull_robot': {
'type': 'string',
'description': 'Username of a Quay robot account to use as pull credentials',
}
},
},
}
@ -123,6 +139,22 @@ class RepositoryBuildList(RepositoryParamResource):
dockerfile_id = request_json['file_id']
subdir = request_json['subdirectory'] if 'subdirectory' in request_json else ''
pull_robot_name = request_json.get('pull_robot', None)
# Verify the security behind the pull robot.
if pull_robot_name:
result = parse_robot_username(pull_robot_name)
if result:
pull_robot = model.lookup_robot(pull_robot_name)
if not pull_robot:
raise NotFound()
# Make sure the user has administer permissions for the robot's namespace.
(robot_namespace, shortname) = result
if not AdministerOrganizationPermission(robot_namespace).can():
raise Unauthorized()
else:
raise Unauthorized()
# Check if the dockerfile resource has already been used. If so, then it
# can only be reused if the user has access to the repository for which it
@ -137,7 +169,8 @@ class RepositoryBuildList(RepositoryParamResource):
repo = model.get_repository(namespace, repository)
display_name = user_files.get_file_checksum(dockerfile_id)
build_request = start_build(repo, dockerfile_id, ['latest'], display_name, subdir, True)
build_request = start_build(repo, dockerfile_id, ['latest'], display_name, subdir, True,
pull_robot_name=pull_robot_name)
resp = build_status_view(build_request, True)
repo_string = '%s/%s' % (namespace, repository)

44
endpoints/api/trigger.py
View file

@ -16,7 +16,8 @@ from endpoints.trigger import (BuildTrigger as BuildTriggerBase, TriggerDeactiva
TriggerActivationException, EmptyRepositoryException,
RepositoryReadException)
from data import model
from auth.permissions import UserAdminPermission
from auth.permissions import UserAdminPermission, AdministerOrganizationPermission
from util.names import parse_robot_username
logger = logging.getLogger(__name__)
@ -139,7 +140,19 @@ class BuildTriggerActivate(RepositoryParamResource):
'BuildTriggerActivateRequest': {
'id': 'BuildTriggerActivateRequest',
'type': 'object',
'description': 'Arbitrary json.',
'required': [
'config'
],
'properties': {
'config': {
'type': 'object',
'description': 'Arbitrary json.',
},
'pull_robot': {
'type': 'string',
'description': 'The name of the robot that will be used to pull images.'
}
}
},
}
@ -160,7 +173,27 @@ class BuildTriggerActivate(RepositoryParamResource):
user_permission = UserAdminPermission(trigger.connected_user.username)
if user_permission.can():
new_config_dict = request.get_json()
# Update the pull robot (if any).
pull_robot_name = request.get_json().get('pull_robot', None)
if pull_robot_name:
pull_robot = model.lookup_robot(pull_robot_name)
if not pull_robot:
raise NotFound()
# Make sure the user has administer permissions for the robot's namespace.
(robot_namespace, shortname) = parse_robot_username(pull_robot_name)
if not AdministerOrganizationPermission(robot_namespace).can():
raise Unauthorized()
# Make sure the namespace matches that of the trigger.
if robot_namespace != namespace:
raise Unauthorized()
# Set the pull robot.
trigger.pull_robot = pull_robot
# Update the config.
new_config_dict = request.get_json()['config']
token_name = 'Build Trigger: %s' % trigger.service.name
token = model.create_delegate_token(namespace, repository, token_name,
@ -191,6 +224,7 @@ class BuildTriggerActivate(RepositoryParamResource):
log_action('setup_repo_trigger', namespace,
{'repo': repository, 'namespace': namespace,
'trigger_id': trigger.uuid, 'service': trigger.service.name,
'pull_robot': trigger.pull_robot.username if trigger.pull_robot else None,
'config': final_config}, repo=repo)
return trigger_view(trigger)
@ -220,8 +254,10 @@ class ActivateBuildTrigger(RepositoryParamResource):
dockerfile_id, tags, name, subdir = specs
repo = model.get_repository(namespace, repository)
pull_robot_name = model.get_pull_robot_name(trigger)
build_request = start_build(repo, dockerfile_id, tags, name, subdir, True)
build_request = start_build(repo, dockerfile_id, tags, name, subdir, True,
pull_robot_name=pull_robot_name)
resp = build_status_view(build_request, True)
repo_string = '%s/%s' % (namespace, repository)

8
endpoints/common.py
View file

@ -107,7 +107,7 @@ def check_repository_usage(user_or_org, plan_found):
def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
trigger=None):
trigger=None, pull_robot_name=None):
host = urlparse.urlparse(request.url).netloc
repo_path = '%s/%s/%s' % (host, repository.namespace, repository.name)
@ -118,16 +118,18 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
job_config = {
'docker_tags': tags,
'repository': repo_path,
'build_subdir': subdir,
'build_subdir': subdir
}
build_request = model.create_repository_build(repository, token, job_config,
dockerfile_id, build_name,
trigger)
trigger, pull_robot_name = pull_robot_name)
dockerfile_build_queue.put(json.dumps({
'build_uuid': build_request.uuid,
'namespace': repository.namespace,
'repository': repository.name,
'pull_credentials': model.get_pull_credentials(pull_robot_name) if pull_robot_name else None
}), retries_remaining=1)
metadata = {

4
endpoints/webhooks.py
View file

@ -73,8 +73,10 @@ def build_trigger_webhook(namespace, repository, trigger_uuid):
# This was just a validation request, we don't need to build anything
return make_response('Okay')
pull_robot_name = model.get_pull_robot_name(trigger)
repo = model.get_repository(namespace, repository)
start_build(repo, dockerfile_id, tags, name, subdir, False, trigger)
start_build(repo, dockerfile_id, tags, name, subdir, False, trigger,
pull_robot_name=pull_robot_name)
return make_response('Okay')

8
initdb.py
View file

@ -257,7 +257,7 @@ def populate_database():
new_user_1.stripe_id = TEST_STRIPE_ID
new_user_1.save()
model.create_robot('dtrobot', new_user_1)
dtrobot = model.create_robot('dtrobot', new_user_1)
new_user_2 = model.create_user('public', 'password',
'jacob.moshenko@gmail.com')
@ -268,6 +268,8 @@ def populate_database():
new_user_3.verified = True
new_user_3.save()
model.create_robot('anotherrobot', new_user_3)
new_user_4 = model.create_user('randomuser', 'password', 'no4@thanks.com')
new_user_4.verified = True
new_user_4.save()
@ -330,7 +332,7 @@ def populate_database():
token = model.create_access_token(building, 'write')
trigger = model.create_build_trigger(building, 'github', '123authtoken',
new_user_1)
new_user_1, pull_robot=dtrobot[0])
trigger.config = json.dumps({
'build_source': 'jakedt/testconnect',
'subdir': '',
@ -366,6 +368,8 @@ def populate_database():
org.stripe_id = TEST_STRIPE_ID
org.save()
model.create_robot('coolrobot', org)
oauth.create_application(org, 'Some Test App', 'http://localhost:8000', 'http://localhost:8000/o2c.html',
client_id='deadbeef')

2
requirements-nover.txt
View file

@ -21,7 +21,7 @@ xhtml2pdf
logstash_formatter
redis
hiredis
docker-py
git+https://github.com/DevTable/docker-py.git
loremipsum
pygithub
flask-restful

34
static/css/quay.css
View file

@ -3589,4 +3589,38 @@ pre.command:before {
.auth-info .scope {
cursor: pointer;
margin-right: 4px;
}
.trigger-pull-credentials {
margin-top: 4px;
padding-left: 26px;
font-size: 12px;
}
.trigger-pull-credentials .context-tooltip {
color: gray;
margin-right: 4px;
}
.trigger-description .trigger-description-subtitle {
display: inline-block;
margin-right: 34px;
}
.trigger-option-section:not(:last-child) {
border-bottom: 1px solid #eee;
padding-bottom: 16px;
margin-bottom: 16px;
}
.trigger-option-section .entity-search-element .twitter-typeahead {
width: 370px;
}
.trigger-option-section .entity-search-element input {
width: 100%;
}
.trigger-option-section table td {
padding: 6px;
}

2
static/directives/trigger-description.html
View file

@ -10,7 +10,7 @@
</span>
</div>
<div style="margin-top: 4px; margin-left: 26px; font-size: 12px; color: gray;" ng-if="!trigger.config.subdir && !short">
<span>Dockerfile:
<span><span class="trigger-description-subtitle">Dockerfile:</span>
<a href="https://github.com/{{ trigger.config.build_source }}/tree/{{ trigger.config.master_branch || 'master' }}/Dockerfile" target="_blank">
//Dockerfile
</a>

28
static/js/app.js
View file

@ -2644,7 +2644,8 @@ quayApp.directive('entitySearch', function () {
'isOrganization': '=isOrganization',
'isPersistent': '=isPersistent',
'currentEntity': '=currentEntity',
'clearNow': '=clearNow'
'clearNow': '=clearNow',
'filter': '=filter',
},
controller: function($scope, $element, Restangular, UserService, ApiService) {
$scope.lazyLoading = true;
@ -2727,7 +2728,7 @@ quayApp.directive('entitySearch', function () {
entity['is_org_member'] = true;
}
$scope.setEntityInternal(entity);
$scope.setEntityInternal(entity, false);
};
$scope.clearEntityInternal = function() {
@ -2737,8 +2738,12 @@ quayApp.directive('entitySearch', function () {
}
};
$scope.setEntityInternal = function(entity) {
$(input).typeahead('val', $scope.isPersistent ? entity.name : '');
$scope.setEntityInternal = function(entity, updateTypeahead) {
if (updateTypeahead) {
$(input).typeahead('val', $scope.isPersistent ? entity.name : '');
} else {
$(input).val($scope.isPersistent ? entity.name : '');
}
if ($scope.isPersistent) {
$scope.currentEntity = entity;
@ -2768,6 +2773,19 @@ quayApp.directive('entitySearch', function () {
var datums = [];
for (var i = 0; i < data.results.length; ++i) {
var entity = data.results[i];
if ($scope.filter) {
var allowed = $scope.filter;
var found = 'user';
if (entity.kind == 'user') {
found = entity.is_robot ? 'robot' : 'user';
} else if (entity.kind == 'team') {
found = 'team';
}
if (allowed.indexOf(found)) {
continue;
}
}
datums.push({
'value': entity.name,
'tokens': [entity.name],
@ -2840,7 +2858,7 @@ quayApp.directive('entitySearch', function () {
$(input).on('typeahead:selected', function(e, datum) {
$scope.$apply(function() {
$scope.setEntityInternal(datum.entity);
$scope.setEntityInternal(datum.entity, true);
});
});

29
static/js/controllers.js
View file

@ -962,9 +962,13 @@ function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope
var data = {
'file_id': build['resource_key'],
'subdirectory': subdirectory
'subdirectory': subdirectory,
};
if (build['pull_robot']) {
data['pull_robot'] = build['pull_robot']['name'];
}
var params = {
'repository': namespace + '/' + name
};
@ -1148,7 +1152,7 @@ function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope
fetchRepository();
}
function RepoAdminCtrl($scope, Restangular, ApiService, KeyService, $routeParams, $rootScope, $location) {
function RepoAdminCtrl($scope, Restangular, ApiService, KeyService, $routeParams, $rootScope, $location, UserService) {
var namespace = $routeParams.namespace;
var name = $routeParams.name;
@ -1452,6 +1456,10 @@ function RepoAdminCtrl($scope, Restangular, ApiService, KeyService, $routeParams
$scope.setupTrigger = function(trigger) {
$scope.triggerSetupReady = false;
$scope.currentSetupTrigger = trigger;
trigger['_pullEntity'] = null;
trigger['_publicPull'] = true;
$('#setupTriggerModal').modal({});
$('#setupTriggerModal').on('hidden.bs.modal', function () {
$scope.$apply(function() {
@ -1460,6 +1468,10 @@ function RepoAdminCtrl($scope, Restangular, ApiService, KeyService, $routeParams
});
};
$scope.isNamespaceAdmin = function(namespace) {
return UserService.isNamespaceAdmin(namespace);
};
$scope.finishSetupTrigger = function(trigger) {
$('#setupTriggerModal').modal('hide');
$scope.currentSetupTrigger = null;
@ -1469,8 +1481,17 @@ function RepoAdminCtrl($scope, Restangular, ApiService, KeyService, $routeParams
'trigger_uuid': trigger.id
};
ApiService.activateBuildTrigger(trigger['config'], params).then(function(resp) {
trigger['is_active'] = true;
var data = {
'config': trigger['config']
};
if (trigger['_pullEntity']) {
data['pull_robot'] = trigger['_pullEntity']['name'];
}
ApiService.activateBuildTrigger(data, params).then(function(resp) {
trigger['is_active'] = true;
trigger['pull_robot'] = resp['pull_robot'];
}, function(resp) {
$scope.triggers.splice($scope.triggers.indexOf(trigger), 1);
bootbox.dialog({

50
static/partials/repo-admin.html
View file

@ -270,6 +270,12 @@
Setting up trigger
</div>
<div ng-show="trigger.is_active" class="trigger-description" trigger="trigger"></div>
<div class="trigger-pull-credentials" ng-if="trigger.is_active && trigger.pull_robot">
<span class="context-tooltip" title="The credentials used by the builder when pulling images" bs-tooltip>
Pull Credentials:
</span>
<span class="entity-reference" entity="trigger.pull_robot"></span>
</div>
</td>
<td style="white-space: nowrap;">
<div class="dropdown" style="display: inline-block" ng-visible="trigger.is_active">
@ -387,14 +393,54 @@
<h4 class="modal-title">Setup new build trigger</h4>
</div>
<div class="modal-body">
<div class="trigger-description-element" ng-switch on="currentSetupTrigger.service">
<div class="trigger-option-section">
<table style="width: 100%;">
<tr>
<td style="width: 114px">
<div class="context-tooltip" title="The credentials used by the builder when pulling images" bs-tooltip>
Pull Credentials:
</div>
</td>
<td>
<div ng-if="!isNamespaceAdmin(repo.namespace)" style="color: #aaa;">
In order to set pull credentials for a build trigger, you must be an Administrator of the namespace <strong>{{ repo.namespace }}</strong>
</div>
<div class="btn-group btn-group-sm" ng-if="isNamespaceAdmin(repo.namespace)">
<button type="button" class="btn btn-default"
ng-class="currentSetupTrigger._publicPull ? 'active btn-info' : ''" ng-click="currentSetupTrigger._publicPull = true">Public</button>
<button type="button" class="btn btn-default"
ng-class="currentSetupTrigger._publicPull ? '' : 'active btn-info'" ng-click="currentSetupTrigger._publicPull = false">
<i class="fa fa-wrench"></i>
Robot account
</button>
</div>
</td>
</tr>
<tr ng-show="!currentSetupTrigger._publicPull">
<td>
</td>
<td>
<div class="entity-search" namespace="repo.namespace" include-teams="false"
input-title="'Select robot account for pulling...'"
is-organization="repo.is_organization"
is-persistent="true"
current-entity="currentSetupTrigger._pullEntity"
filter="['robot']"></div>
</td>
</tr>
</table>
</div>
<div class="trigger-description-element trigger-option-section" ng-switch on="currentSetupTrigger.service">
<div ng-switch-when="github">
<div class="trigger-setup-github" repository="repo" trigger="currentSetupTrigger"></div>
</div>
</div>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-primary" ng-disabled="!currentSetupTrigger.$ready" ng-click="finishSetupTrigger(currentSetupTrigger)">Finished</button>
<button type="button" class="btn btn-primary"
ng-disabled="!currentSetupTrigger.$ready || (!currentSetupTrigger._publicPull && !currentSetupTrigger._pullEntity)"
ng-click="finishSetupTrigger(currentSetupTrigger)">Finished</button>
<button type="button" class="btn btn-default" data-dismiss="modal">Cancel</button>
</div>
</div><!-- /.modal-content -->

BIN
test/data/test.db
View file

Binary file not shown.

4
test/test_api_security.py
View file

@ -962,7 +962,7 @@ class TestBuildTriggerActivateSwo1DevtableShared(ApiTestCase):
self._run_test('POST', 403, 'reader', {})
def test_post_devtable(self):
self._run_test('POST', 404, 'devtable', {})
self._run_test('POST', 404, 'devtable', {'config': {}})
class TestBuildTriggerActivateSwo1BuynlargeOrgrepo(ApiTestCase):
@ -980,7 +980,7 @@ class TestBuildTriggerActivateSwo1BuynlargeOrgrepo(ApiTestCase):
self._run_test('POST', 403, 'reader', {})
def test_post_devtable(self):
self._run_test('POST', 404, 'devtable', {})
self._run_test('POST', 404, 'devtable', {'config': {}})
class TestBuildTriggerSources831cPublicPublicrepo(ApiTestCase):

113
test/test_api_usage.py
View file

@ -964,7 +964,7 @@ class TestRequestRepoBuild(ApiTestCase):
def test_requestrepobuild(self):
self.login(ADMIN_ACCESS_USER)
# Ensure where not yet building.
# Ensure we are not yet building.
json = self.getJsonResponse(RepositoryBuildList,
params=dict(repository=ADMIN_ACCESS_USER + '/simple'))
@ -982,6 +982,50 @@ class TestRequestRepoBuild(ApiTestCase):
assert len(json['builds']) > 0
def test_requestrepobuild_with_robot(self):
self.login(ADMIN_ACCESS_USER)
# Ensure we are not yet building.
json = self.getJsonResponse(RepositoryBuildList,
params=dict(repository=ADMIN_ACCESS_USER + '/simple'))
assert len(json['builds']) == 0
# Request a (fake) build.
pull_robot = ADMIN_ACCESS_USER + '+dtrobot'
self.postResponse(RepositoryBuildList,
params=dict(repository=ADMIN_ACCESS_USER + '/simple'),
data=dict(file_id='foobarbaz', pull_robot=pull_robot),
expected_code=201)
# Check for the build.
json = self.getJsonResponse(RepositoryBuildList,
params=dict(repository=ADMIN_ACCESS_USER + '/building'))
assert len(json['builds']) > 0
def test_requestrepobuild_with_invalid_robot(self):
self.login(ADMIN_ACCESS_USER)
# Request a (fake) build.
pull_robot = ADMIN_ACCESS_USER + '+invalidrobot'
self.postResponse(RepositoryBuildList,
params=dict(repository=ADMIN_ACCESS_USER + '/simple'),
data=dict(file_id='foobarbaz', pull_robot=pull_robot),
expected_code=404)
def test_requestrepobuild_with_unauthorized_robot(self):
self.login(ADMIN_ACCESS_USER)
# Request a (fake) build.
pull_robot = 'freshuser+anotherrobot'
self.postResponse(RepositoryBuildList,
params=dict(repository=ADMIN_ACCESS_USER + '/simple'),
data=dict(file_id='foobarbaz', pull_robot=pull_robot),
expected_code=403)
class TestWebhooks(ApiTestCase):
def test_webhooks(self):
@ -1642,7 +1686,7 @@ class TestBuildTriggers(ApiTestCase):
trigger_config = {}
activate_json = self.postJsonResponse(BuildTriggerActivate,
params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid),
data=trigger_config)
data={'config': trigger_config})
self.assertEquals(True, activate_json['is_active'])
@ -1654,7 +1698,7 @@ class TestBuildTriggers(ApiTestCase):
# Make sure we cannot activate again.
self.postResponse(BuildTriggerActivate,
params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid),
data=trigger_config,
data={'config': trigger_config},
expected_code=400)
# Start a manual build.
@ -1667,6 +1711,69 @@ class TestBuildTriggers(ApiTestCase):
self.assertEquals(['bar'], start_json['job_config']['docker_tags'])
def test_invalid_robot_account(self):
self.login(ADMIN_ACCESS_USER)
database.BuildTriggerService.create(name='fakeservice')
# Add a new fake trigger.
repo = model.get_repository(ADMIN_ACCESS_USER, 'simple')
user = model.get_user(ADMIN_ACCESS_USER)
trigger = model.create_build_trigger(repo, 'fakeservice', 'sometoken', user)
# Try to activate it with an invalid robot account.
trigger_config = {}
activate_json = self.postJsonResponse(BuildTriggerActivate,
params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid),
data={'config': trigger_config, 'pull_robot': 'someinvalidrobot'},
expected_code=404)
def test_unauthorized_robot_account(self):
self.login(ADMIN_ACCESS_USER)
database.BuildTriggerService.create(name='fakeservice')
# Add a new fake trigger.
repo = model.get_repository(ADMIN_ACCESS_USER, 'simple')
user = model.get_user(ADMIN_ACCESS_USER)
trigger = model.create_build_trigger(repo, 'fakeservice', 'sometoken', user)
# Try to activate it with a robot account in the wrong namespace.
trigger_config = {}
activate_json = self.postJsonResponse(BuildTriggerActivate,
params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid),
data={'config': trigger_config, 'pull_robot': 'freshuser+anotherrobot'},
expected_code=403)
def test_robot_account(self):
self.login(ADMIN_ACCESS_USER)
database.BuildTriggerService.create(name='fakeservice')
# Add a new fake trigger.
repo = model.get_repository(ADMIN_ACCESS_USER, 'simple')
user = model.get_user(ADMIN_ACCESS_USER)
trigger = model.create_build_trigger(repo, 'fakeservice', 'sometoken', user)
# Try to activate it with a robot account.
trigger_config = {}
activate_json = self.postJsonResponse(BuildTriggerActivate,
params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid),
data={'config': trigger_config, 'pull_robot': ADMIN_ACCESS_USER + '+dtrobot'})
# Verify that the robot was saved.
self.assertEquals(True, activate_json['is_active'])
self.assertEquals(ADMIN_ACCESS_USER + '+dtrobot', activate_json['pull_robot']['name'])
# Start a manual build.
start_json = self.postJsonResponse(ActivateBuildTrigger,
params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid),
expected_code=201)
assert 'id' in start_json
self.assertEquals("build-name", start_json['display_name'])
self.assertEquals(['bar'], start_json['job_config']['docker_tags'])
class TestUserAuthorizations(ApiTestCase):
def test_list_get_delete_user_authorizations(self):

6
util/names.py
View file

@ -24,3 +24,9 @@ def parse_repository_name(f):
def format_robot_username(parent_username, robot_shortname):
return '%s+%s' % (parent_username, robot_shortname)
def parse_robot_username(robot_username):
if not '+' in robot_username:
return None
return robot_username.split('+', 2)

48
workers/dockerfilebuild.py
View file

@ -92,15 +92,21 @@ class StreamingDockerClient(Client):
class DockerfileBuildContext(object):
def __init__(self, build_context_dir, dockerfile_subdir, repo, tag_names,
push_token, build_uuid):
push_token, build_uuid, pull_credentials=None):
self._build_dir = build_context_dir
self._dockerfile_subdir = dockerfile_subdir
self._repo = repo
self._tag_names = tag_names
self._push_token = push_token
self._cl = StreamingDockerClient(timeout=1200)
self._status = StatusWrapper(build_uuid)
self._build_logger = partial(build_logs.append_log_message, build_uuid)
self._pull_credentials = pull_credentials
# Note: We have two different clients here because we (potentially) login
# with both, but with different credentials that we do not want shared between
# the build and push operations.
self._push_cl = StreamingDockerClient(timeout=1200)
self._build_cl = StreamingDockerClient(timeout=1200)
dockerfile_path = os.path.join(self._build_dir, dockerfile_subdir,
'Dockerfile')
@ -135,6 +141,14 @@ class DockerfileBuildContext(object):
return float(sent_bytes)/total_bytes*percentage_with_sizes
def build(self):
# Login with the specified credentials (if any).
if self._pull_credentials:
logger.debug('Logging in with pull credentials: %s@%s',
self._pull_credentials['username'], self._pull_credentials['registry'])
self._build_cl.login(self._pull_credentials['username'], self._pull_credentials['password'],
registry=self._pull_credentials['registry'], reauth=True)
# Start the build itself.
logger.debug('Starting build.')
with self._status as status:
@ -146,7 +160,7 @@ class DockerfileBuildContext(object):
logger.debug('Final context path: %s exists: %s' %
(context_path, os.path.exists(context_path)))
build_status = self._cl.build(path=context_path, stream=True)
build_status = self._build_cl.build(path=context_path, stream=True)
current_step = 0
built_image = None
@ -200,7 +214,7 @@ class DockerfileBuildContext(object):
logger.debug('Attempting login to registry: %s' % registry_endpoint)
try:
self._cl.login('$token', self._push_token, registry=registry_endpoint)
self._push_cl.login('$token', self._push_token, registry=registry_endpoint)
break
except APIError:
pass # Probably the wrong protocol
@ -208,15 +222,15 @@ class DockerfileBuildContext(object):
for tag in self._tag_names:
logger.debug('Tagging image %s as %s:%s' %
(built_image, self._repo, tag))
self._cl.tag(built_image, self._repo, tag)
self._push_cl.tag(built_image, self._repo, tag)
history = json.loads(self._cl.history(built_image))
history = json.loads(self._push_cl.history(built_image))
num_images = len(history)
with self._status as status:
status['total_images'] = num_images
logger.debug('Pushing to repo %s' % self._repo)
resp = self._cl.push(self._repo, stream=True)
resp = self._push_cl.push(self._repo, stream=True)
for status in resp:
logger.debug('Status: %s', status)
@ -245,20 +259,20 @@ class DockerfileBuildContext(object):
def __cleanup(self):
# First clean up any containers that might be holding the images
for running in self._cl.containers(quiet=True):
for running in self._build_cl.containers(quiet=True):
logger.debug('Killing container: %s' % running['Id'])
self._cl.kill(running['Id'])
self._build_cl.kill(running['Id'])
# Next, remove all of the containers (which should all now be killed)
for container in self._cl.containers(all=True, quiet=True):
for container in self._build_cl.containers(all=True, quiet=True):
logger.debug('Removing container: %s' % container['Id'])
self._cl.remove_container(container['Id'])
self._build_cl.remove_container(container['Id'])
# Iterate all of the images and remove the ones that the public registry
# doesn't know about, this should preserve base images.
images_to_remove = set()
repos = set()
for image in self._cl.images():
for image in self._build_cl.images():
images_to_remove.add(image['Id'])
for tag in image['RepoTags']:
@ -278,13 +292,13 @@ class DockerfileBuildContext(object):
for to_remove in images_to_remove:
logger.debug('Removing private image: %s' % to_remove)
try:
self._cl.remove_image(to_remove)
self._build_cl.remove_image(to_remove)
except APIError:
# Sometimes an upstream image removed this one
pass
# Verify that our images were actually removed
for image in self._cl.images():
for image in self._build_cl.images():
if image['Id'] in images_to_remove:
raise RuntimeError('Image was not removed: %s' % image['Id'])
@ -338,6 +352,8 @@ class DockerfileBuildWorker(Worker):
job_details['repository'],
job_details['build_uuid'])
pull_credentials = job_details.get('pull_credentials', None)
job_config = json.loads(repository_build.job_config)
resource_url = user_files.get_file_url(repository_build.resource_key)
@ -378,7 +394,7 @@ class DockerfileBuildWorker(Worker):
with DockerfileBuildContext(build_dir, build_subdir, repo, tag_names,
access_token,
repository_build.uuid) as build_ctxt:
repository_build.uuid, pull_credentials) as build_ctxt:
try:
built_image = build_ctxt.build()
@ -432,4 +448,4 @@ else:
handler = logging.StreamHandler()
handler.setFormatter(formatter)
root_logger.addHandler(handler)
worker.start()
worker.start()