vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add support for pull credentials on builds and build triggers

Browse source
This commit is contained in:
Joseph Schorr 2014-03-27 18:33:13 -04:00
parent 1fc3c922a9
commit 2006917e03
17 changed files with 355 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
data/database.py
View file

@ -176,6 +176,7 @@ class RepositoryBuildTrigger(BaseModel):
auth_token = CharField()
config = TextField(default='{}')
write_token = ForeignKeyField(AccessToken, null=True)
pull_user = ForeignKeyField(User, null=True, related_name='pulluser')
class EmailConfirmation(BaseModel):

29
data/model/legacy.py
View file

@ -154,6 +154,16 @@ def create_robot(robot_shortname, parent):
raise DataModelException(ex.message)
def lookup_robot(robot_username):
joined = User.select().join(FederatedLogin).join(LoginService)
found = list(joined.where(LoginService.name == 'quayrobot',
User.username == robot_username))
if not found or len(found) < 1 or not found[0].robot:
return None
return found[0]
def verify_robot(robot_username, password):
joined = User.select().join(FederatedLogin).join(LoginService)
found = list(joined.where(FederatedLogin.service_ident == password,
@ -1449,6 +1459,20 @@ def create_repository_build(repo, access_token, job_config_obj, dockerfile_id,
display_name=display_name, trigger=trigger,
resource_key=dockerfile_id)
def get_pull_credentials(trigger):
if not trigger.pull_user:
return None
try:
login_info = FederatedLogin.get(user=trigger.pull_user)
except FederatedLogin.DoesNotExist:
return None
return {
'username': trigger.pull_user.username,
'password': login_info.service_ident,
'registry': 'quay.io' # TODO: Is there a better way to do this?
}
def create_webhook(repo, params_obj):
return Webhook.create(repository=repo, parameters=json.dumps(params_obj))
@ -1506,11 +1530,12 @@ def log_action(kind_name, user_or_organization_name, performer=None,
metadata_json=json.dumps(metadata), datetime=timestamp)
def create_build_trigger(repo, service_name, auth_token, user):
def create_build_trigger(repo, service_name, auth_token, user, pull_user=None):
service = BuildTriggerService.get(name=service_name)
trigger = RepositoryBuildTrigger.create(repository=repo, service=service,
auth_token=auth_token,
connected_user=user)
connected_user=user,
pull_user=None)
return trigger

37
endpoints/api/build.py
View file

@ -33,6 +33,13 @@ def get_job_config(build_obj):
def trigger_view(trigger):
def user_view(user):
return {
'name': user.username,
'kind': 'user',
'is_robot': user.robot,
}
if trigger and trigger.uuid:
config_dict = get_trigger_config(trigger)
build_trigger = BuildTrigger.get_trigger_for_service(trigger.service.name)
@ -41,7 +48,8 @@ def trigger_view(trigger):
'config': config_dict,
'id': trigger.uuid,
'connected_user': trigger.connected_user.username,
'is_active': build_trigger.is_active(config_dict)
'is_active': build_trigger.is_active(config_dict),
'pull_user': user_view(trigger.pull_user) if trigger.pull_user else None
}
return None
@ -88,6 +96,29 @@ class RepositoryBuildList(RepositoryParamResource):
'type': 'string',
'description': 'Subdirectory in which the Dockerfile can be found',
},
'pull_credentials': {
'type': 'object',
'description': 'Credentials used by the builder when pulling images',
'required': [
'username',
'password',
'registry'
],
'properties': {
'username': {
'type': 'string',
'description': 'The username for the pull'
},
'password': {
'type': 'string',
'description': 'The password for the pull'
},
'registry': {
'type': 'string',
'description': 'The registry hostname for the pull'
},
}
}
},
},
}
@ -116,6 +147,7 @@ class RepositoryBuildList(RepositoryParamResource):
dockerfile_id = request_json['file_id']
subdir = request_json['subdirectory'] if 'subdirectory' in request_json else ''
pull_credentials = request_json.get('pull_credentials', None)
# Check if the dockerfile resource has already been used. If so, then it
# can only be reused if the user has access to the repository for which it
@ -130,7 +162,8 @@ class RepositoryBuildList(RepositoryParamResource):
repo = model.get_repository(namespace, repository)
display_name = user_files.get_file_checksum(dockerfile_id)
build_request = start_build(repo, dockerfile_id, ['latest'], display_name, subdir, True)
build_request = start_build(repo, dockerfile_id, ['latest'], display_name, subdir, True,
pull_credentials=pull_credentials)
resp = build_status_view(build_request, True)
repo_string = '%s/%s' % (namespace, repository)

44
endpoints/api/trigger.py
View file

@ -15,7 +15,8 @@ from endpoints.common import start_build
from endpoints.trigger import (BuildTrigger as BuildTriggerBase, TriggerDeactivationException,
TriggerActivationException, EmptyRepositoryException)
from data import model
from auth.permissions import UserAdminPermission
from auth.permissions import UserAdminPermission, AdministerOrganizationPermission
from util.names import parse_robot_username
logger = logging.getLogger(__name__)
@ -133,7 +134,19 @@ class BuildTriggerActivate(RepositoryParamResource):
'BuildTriggerActivateRequest': {
'id': 'BuildTriggerActivateRequest',
'type': 'object',
'description': 'Arbitrary json.',
'required': [
'config'
],
'properties': {
'config': {
'type': 'object',
'description': 'Arbitrary json.',
},
'pull_robot': {
'type': 'string',
'description': 'The name of the robot that will be used to pull images.'
}
}
},
}
@ -154,7 +167,27 @@ class BuildTriggerActivate(RepositoryParamResource):
user_permission = UserAdminPermission(trigger.connected_user.username)
if user_permission.can():
new_config_dict = request.get_json()
# Update the pull robot (if any).
pull_robot_name = request.get_json().get('pull_robot', None)
if pull_robot_name:
pull_robot = model.lookup_robot(pull_robot_name)
if not pull_robot:
raise NotFound()
# Make sure the user has administer permissions for the robot's namespace.
(robot_namespace, shortname) = parse_robot_username(pull_robot_name)
if not AdministerOrganizationPermission(robot_namespace).can():
raise Unauthorized()
# Make sure the namespace matches that of the trigger.
if robot_namespace != namespace:
raise Unauthorized()
# Set the pull robot.
trigger.pull_user = pull_robot
# Update the config.
new_config_dict = request.get_json()['config']
token_name = 'Build Trigger: %s' % trigger.service.name
token = model.create_delegate_token(namespace, repository, token_name,
@ -185,6 +218,7 @@ class BuildTriggerActivate(RepositoryParamResource):
log_action('setup_repo_trigger', namespace,
{'repo': repository, 'namespace': namespace,
'trigger_id': trigger.uuid, 'service': trigger.service.name,
'pull_user': trigger.pull_user.username if trigger.pull_user else None,
'config': final_config}, repo=repo)
return trigger_view(trigger)
@ -214,8 +248,10 @@ class ActivateBuildTrigger(RepositoryParamResource):
dockerfile_id, tags, name, subdir = specs
repo = model.get_repository(namespace, repository)
pull_credentials = model.get_pull_credentials(trigger)
build_request = start_build(repo, dockerfile_id, tags, name, subdir, True)
build_request = start_build(repo, dockerfile_id, tags, name, subdir, True,
pull_credentials=pull_credentials)
resp = build_status_view(build_request, True)
repo_string = '%s/%s' % (namespace, repository)

4
endpoints/common.py
View file

@ -100,7 +100,7 @@ def check_repository_usage(user_or_org, plan_found):
def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
trigger=None):
trigger=None, pull_credentials=None):
host = urlparse.urlparse(request.url).netloc
repo_path = '%s/%s/%s' % (host, repository.namespace, repository.name)
@ -112,7 +112,9 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
'docker_tags': tags,
'repository': repo_path,
'build_subdir': subdir,
'pull_credentials': pull_credentials,
}
build_request = model.create_repository_build(repository, token, job_config,
dockerfile_id, build_name,
trigger)

4
endpoints/webhooks.py
View file

@ -73,8 +73,10 @@ def build_trigger_webhook(namespace, repository, trigger_uuid):
# This was just a validation request, we don't need to build anything
return make_response('Okay')
pull_credentials = model.get_pull_credentials(trigger)
repo = model.get_repository(namespace, repository)
start_build(repo, dockerfile_id, tags, name, subdir, False, trigger)
start_build(repo, dockerfile_id, tags, name, subdir, False, trigger,
pull_credentials=pull_credentials)
return make_response('Okay')

8
initdb.py
View file

@ -257,7 +257,7 @@ def populate_database():
new_user_1.stripe_id = TEST_STRIPE_ID
new_user_1.save()
model.create_robot('dtrobot', new_user_1)
dtrobot = model.create_robot('dtrobot', new_user_1)
new_user_2 = model.create_user('public', 'password',
'jacob.moshenko@gmail.com')
@ -268,6 +268,8 @@ def populate_database():
new_user_3.verified = True
new_user_3.save()
model.create_robot('anotherrobot', new_user_3)
new_user_4 = model.create_user('randomuser', 'password', 'no4@thanks.com')
new_user_4.verified = True
new_user_4.save()
@ -330,7 +332,7 @@ def populate_database():
token = model.create_access_token(building, 'write')
trigger = model.create_build_trigger(building, 'github', '123authtoken',
new_user_1)
new_user_1, pull_user = dtrobot)
trigger.config = json.dumps({
'build_source': 'jakedt/testconnect',
'subdir': '',
@ -354,6 +356,8 @@ def populate_database():
org.stripe_id = TEST_STRIPE_ID
org.save()
model.create_robot('coolrobot', org)
oauth.create_application(org, 'Some Test App', 'http://localhost:8000', 'http://localhost:8000/o2c.html',
client_id='deadbeef')

34
static/css/quay.css
View file

@ -3589,4 +3589,38 @@ pre.command:before {
.auth-info .scope {
cursor: pointer;
margin-right: 4px;
}
.trigger-pull-credentials {
margin-top: 4px;
padding-left: 26px;
font-size: 12px;
}
.trigger-pull-credentials .context-tooltip {
color: gray;
margin-right: 4px;
}
.trigger-description .trigger-description-subtitle {
display: inline-block;
margin-right: 34px;
}
.trigger-option-section:not(:last-child) {
border-bottom: 1px solid #eee;
padding-bottom: 16px;
margin-bottom: 16px;
}
.trigger-option-section .entity-search-element .twitter-typeahead {
width: 370px;
}
.trigger-option-section .entity-search-element input {
width: 100%;
}
.trigger-option-section table td {
padding: 6px;
}

2
static/directives/trigger-description.html
View file

@ -10,7 +10,7 @@
</span>
</div>
<div style="margin-top: 4px; margin-left: 26px; font-size: 12px; color: gray;" ng-if="!trigger.config.subdir && !short">
<span>Dockerfile:
<span><span class="trigger-description-subtitle">Dockerfile:</span>
<a href="https://github.com/{{ trigger.config.build_source }}/tree/{{ trigger.config.master_branch || 'master' }}/Dockerfile" target="_blank">
//Dockerfile
</a>

16
static/js/app.js
View file

@ -2504,7 +2504,8 @@ quayApp.directive('entitySearch', function () {
'isOrganization': '=isOrganization',
'isPersistent': '=isPersistent',
'currentEntity': '=currentEntity',
'clearNow': '=clearNow'
'clearNow': '=clearNow',
'filter': '=filter',
},
controller: function($scope, $element, Restangular, UserService, ApiService) {
$scope.lazyLoading = true;
@ -2628,6 +2629,19 @@ quayApp.directive('entitySearch', function () {
var datums = [];
for (var i = 0; i < data.results.length; ++i) {
var entity = data.results[i];
if ($scope.filter) {
var allowed = $scope.filter;
var found = 'user';
if (entity.kind == 'user') {
found = entity.is_robot ? 'robot' : 'user';
} else if (entity.kind == 'team') {
found = 'team';
}
if (allowed.indexOf(found)) {
continue;
}
}
datums.push({
'value': entity.name,
'tokens': [entity.name],

20
static/js/controllers.js
View file

@ -1158,7 +1158,7 @@ function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope
fetchRepository();
}
function RepoAdminCtrl($scope, Restangular, ApiService, KeyService, $routeParams, $rootScope, $location) {
function RepoAdminCtrl($scope, Restangular, ApiService, KeyService, $routeParams, $rootScope, $location, UserService) {
var namespace = $routeParams.namespace;
var name = $routeParams.name;
@ -1462,6 +1462,10 @@ function RepoAdminCtrl($scope, Restangular, ApiService, KeyService, $routeParams
$scope.setupTrigger = function(trigger) {
$scope.triggerSetupReady = false;
$scope.currentSetupTrigger = trigger;
trigger['_pullEntity'] = null;
trigger['_publicPull'] = true;
$('#setupTriggerModal').modal({});
$('#setupTriggerModal').on('hidden.bs.modal', function () {
$scope.$apply(function() {
@ -1470,6 +1474,10 @@ function RepoAdminCtrl($scope, Restangular, ApiService, KeyService, $routeParams
});
};
$scope.isNamespaceAdmin = function(namespace) {
return UserService.isNamespaceAdmin(namespace);
};
$scope.finishSetupTrigger = function(trigger) {
$('#setupTriggerModal').modal('hide');
$scope.currentSetupTrigger = null;
@ -1479,7 +1487,15 @@ function RepoAdminCtrl($scope, Restangular, ApiService, KeyService, $routeParams
'trigger_uuid': trigger.id
};
ApiService.activateBuildTrigger(trigger['config'], params).then(function(resp) {
var data = {
'config': trigger['config']
};
if (trigger['_pullEntity']) {
data['pull_robot'] = trigger['_pullEntity']['name'];
}
ApiService.activateBuildTrigger(data, params).then(function(resp) {
trigger['is_active'] = true;
}, function(resp) {
$scope.triggers.splice($scope.triggers.indexOf(trigger), 1);

46
static/partials/repo-admin.html
View file

@ -270,6 +270,12 @@
Setting up trigger
</div>
<div ng-show="trigger.is_active" class="trigger-description" trigger="trigger"></div>
<div class="trigger-pull-credentials" ng-if="trigger.is_active && trigger.pull_user">
<span class="context-tooltip" title="The credentials used by the builder when pulling images" bs-tooltip>
Pull Credentials:
</span>
<span class="entity-reference" entity="trigger.pull_user"></span>
</div>
</td>
<td style="white-space: nowrap;">
<div class="dropdown" style="display: inline-block" ng-visible="trigger.is_active">
@ -387,7 +393,45 @@
<h4 class="modal-title">Setup new build trigger</h4>
</div>
<div class="modal-body">
<div class="trigger-description-element" ng-switch on="currentSetupTrigger.service">
<div class="trigger-option-section">
<table style="width: 100%;">
<tr>
<td style="width: 114px">
<div class="context-tooltip" title="The credentials used by the builder when pulling images" bs-tooltip>
Pull Credentials:
</div>
</td>
<td>
<div ng-if="!isNamespaceAdmin(repo.namespace)" style="color: #aaa;">
In order to set pull credentials for a build trigger, you must be an Administrator of the namespace <strong>{{ repo.namespace }}</strong>
</div>
<div class="btn-group btn-group-sm" ng-if="isNamespaceAdmin(repo.namespace)">
<button type="button" class="btn btn-default"
ng-class="currentSetupTrigger._publicPull ? 'active btn-info' : ''" ng-click="currentSetupTrigger._publicPull = true">Public</button>
<button type="button" class="btn btn-default"
ng-class="currentSetupTrigger._publicPull ? '' : 'active btn-info'" ng-click="currentSetupTrigger._publicPull = false">
<i class="fa fa-wrench"></i>
Robot account
</button>
</div>
</td>
</tr>
<tr ng-show="!currentSetupTrigger._publicPull">
<td>
</td>
<td>
<div class="entity-search" namespace="repo.namespace" include-teams="false"
input-title="'Select robot account for pulling...'"
is-organization="repo.is_organization"
is-persistent="true"
current-entity="currentSetupTrigger._pullEntity"
filter="['robot']"></div>
</td>
</tr>
</table>
</div>
<div class="trigger-description-element trigger-option-section" ng-switch on="currentSetupTrigger.service">
<div ng-switch-when="github">
<div class="trigger-setup-github" repository="repo" trigger="currentSetupTrigger"></div>
</div>

BIN
test/data/test.db
View file

Binary file not shown.

4
test/test_api_security.py
View file

@ -962,7 +962,7 @@ class TestBuildTriggerActivateSwo1DevtableShared(ApiTestCase):
self._run_test('POST', 403, 'reader', {})
def test_post_devtable(self):
self._run_test('POST', 404, 'devtable', {})
self._run_test('POST', 404, 'devtable', {'config': {}})
class TestBuildTriggerActivateSwo1BuynlargeOrgrepo(ApiTestCase):
@ -980,7 +980,7 @@ class TestBuildTriggerActivateSwo1BuynlargeOrgrepo(ApiTestCase):
self._run_test('POST', 403, 'reader', {})
def test_post_devtable(self):
self._run_test('POST', 404, 'devtable', {})
self._run_test('POST', 404, 'devtable', {'config': {}})
class TestBuildTriggerSources831cPublicPublicrepo(ApiTestCase):

94
test/test_api_usage.py
View file

@ -982,6 +982,33 @@ class TestRequestRepoBuild(ApiTestCase):
assert len(json['builds']) > 0
def test_requestrepobuild_with_credentials(self):
self.login(ADMIN_ACCESS_USER)
# Ensure where not yet building.
json = self.getJsonResponse(RepositoryBuildList,
params=dict(repository=ADMIN_ACCESS_USER + '/simple'))
assert len(json['builds']) == 0
# Request a (fake) build.
pull_creds = {
'username': 'foo',
'password': 'bar',
'registry': 'baz'
}
self.postResponse(RepositoryBuildList,
params=dict(repository=ADMIN_ACCESS_USER + '/simple'),
data=dict(file_id='foobarbaz', pull_credentials=pull_creds),
expected_code=201)
# Check for the build.
json = self.getJsonResponse(RepositoryBuildList,
params=dict(repository=ADMIN_ACCESS_USER + '/building'))
assert len(json['builds']) > 0
class TestWebhooks(ApiTestCase):
def test_webhooks(self):
@ -1642,7 +1669,7 @@ class TestBuildTriggers(ApiTestCase):
trigger_config = {}
activate_json = self.postJsonResponse(BuildTriggerActivate,
params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid),
data=trigger_config)
data={'config': trigger_config})
self.assertEquals(True, activate_json['is_active'])
@ -1654,7 +1681,7 @@ class TestBuildTriggers(ApiTestCase):
# Make sure we cannot activate again.
self.postResponse(BuildTriggerActivate,
params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid),
data=trigger_config,
data={'config': trigger_config},
expected_code=400)
# Start a manual build.
@ -1667,6 +1694,69 @@ class TestBuildTriggers(ApiTestCase):
self.assertEquals(['bar'], start_json['job_config']['docker_tags'])
def test_invalid_robot_account(self):
self.login(ADMIN_ACCESS_USER)
database.BuildTriggerService.create(name='fakeservice')
# Add a new fake trigger.
repo = model.get_repository(ADMIN_ACCESS_USER, 'simple')
user = model.get_user(ADMIN_ACCESS_USER)
trigger = model.create_build_trigger(repo, 'fakeservice', 'sometoken', user)
# Try to activate it with an invalid robot account.
trigger_config = {}
activate_json = self.postJsonResponse(BuildTriggerActivate,
params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid),
data={'config': trigger_config, 'pull_robot': 'someinvalidrobot'},
expected_code=404)
def test_unauthorized_robot_account(self):
self.login(ADMIN_ACCESS_USER)
database.BuildTriggerService.create(name='fakeservice')
# Add a new fake trigger.
repo = model.get_repository(ADMIN_ACCESS_USER, 'simple')
user = model.get_user(ADMIN_ACCESS_USER)
trigger = model.create_build_trigger(repo, 'fakeservice', 'sometoken', user)
# Try to activate it with a robot account in the wrong namespace.
trigger_config = {}
activate_json = self.postJsonResponse(BuildTriggerActivate,
params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid),
data={'config': trigger_config, 'pull_robot': 'freshuser+anotherrobot'},
expected_code=403)
def test_robot_account(self):
self.login(ADMIN_ACCESS_USER)
database.BuildTriggerService.create(name='fakeservice')
# Add a new fake trigger.
repo = model.get_repository(ADMIN_ACCESS_USER, 'simple')
user = model.get_user(ADMIN_ACCESS_USER)
trigger = model.create_build_trigger(repo, 'fakeservice', 'sometoken', user)
# Try to activate it with a robot account.
trigger_config = {}
activate_json = self.postJsonResponse(BuildTriggerActivate,
params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid),
data={'config': trigger_config, 'pull_robot': ADMIN_ACCESS_USER + '+dtrobot'})
# Verify that the robot was saved.
self.assertEquals(True, activate_json['is_active'])
self.assertEquals(ADMIN_ACCESS_USER + '+dtrobot', activate_json['pull_user']['name'])
# Start a manual build.
start_json = self.postJsonResponse(ActivateBuildTrigger,
params=dict(repository=ADMIN_ACCESS_USER + '/simple', trigger_uuid=trigger.uuid),
expected_code=201)
assert 'id' in start_json
self.assertEquals("build-name", start_json['display_name'])
self.assertEquals(['bar'], start_json['job_config']['docker_tags'])
class TestUserAuthorizations(ApiTestCase):
def test_list_get_delete_user_authorizations(self):

3
util/names.py
View file

@ -24,3 +24,6 @@ def parse_repository_name(f):
def format_robot_username(parent_username, robot_shortname):
return '%s+%s' % (parent_username, robot_shortname)
def parse_robot_username(robot_username):
return robot_username.split('+', 2)

46
workers/dockerfilebuild.py
View file

@ -56,15 +56,21 @@ def unwrap_stream(json_stream):
class DockerfileBuildContext(object):
def __init__(self, build_context_dir, dockerfile_subdir, repo, tag_names,
push_token, build_uuid):
push_token, build_uuid, pull_credentials=None):
self._build_dir = build_context_dir
self._dockerfile_subdir = dockerfile_subdir
self._repo = repo
self._tag_names = tag_names
self._push_token = push_token
self._cl = Client(timeout=1200)
self._status = StatusWrapper(build_uuid)
self._build_logger = partial(build_logs.append_log_message, build_uuid)
self._pull_credentials = pull_credentials
# Note: We have two different clients here because we (potentially) login
# with both, but with different credentials that we do not want shared between
# the build and push operations.
self._push_cl = Client(timeout=1200)
self._build_cl = Client(timeout=1200)
dockerfile_path = os.path.join(self._build_dir, dockerfile_subdir,
"Dockerfile")
@ -99,6 +105,13 @@ class DockerfileBuildContext(object):
return float(sent_bytes)/total_bytes*percentage_with_sizes
def build(self):
# Login with the specified credentials (if any).
if self._pull_credentials:
logger.debug('Logging in with pull credentials.')
self.build_cl_.login(self._pull_credentials['username'], self._pull_credentials['password'],
registry=self._pull_credentials['registry'], reauth=True)
# Start the build itself.
logger.debug('Starting build.')
with self._status as status:
@ -110,7 +123,7 @@ class DockerfileBuildContext(object):
logger.debug('Final context path: %s exists: %s' %
(context_path, os.path.exists(context_path)))
build_status = self._cl.build(path=context_path, stream=True)
build_status = self._build_cl.build(path=context_path, stream=True)
current_step = 0
built_image = None
@ -158,7 +171,7 @@ class DockerfileBuildContext(object):
logger.debug('Attempting login to registry: %s' % registry_endpoint)
try:
self._cl.login('$token', self._push_token, registry=registry_endpoint)
self._push_cl.login('$token', self._push_token, registry=registry_endpoint)
break
except APIError:
pass # Probably the wrong protocol
@ -166,15 +179,15 @@ class DockerfileBuildContext(object):
for tag in self._tag_names:
logger.debug('Tagging image %s as %s:%s' %
(built_image, self._repo, tag))
self._cl.tag(built_image, self._repo, tag)
self._push_cl.tag(built_image, self._repo, tag)
history = json.loads(self._cl.history(built_image))
history = json.loads(self._push_cl.history(built_image))
num_images = len(history)
with self._status as status:
status['total_images'] = num_images
logger.debug('Pushing to repo %s' % self._repo)
resp = self._cl.push(self._repo, stream=True)
resp = self._push_cl.push(self._repo, stream=True)
for status_str in resp:
status = json.loads(status_str)
@ -204,20 +217,20 @@ class DockerfileBuildContext(object):
def __cleanup(self):
# First clean up any containers that might be holding the images
for running in self._cl.containers(quiet=True):
for running in self._build_cl.containers(quiet=True):
logger.debug('Killing container: %s' % running['Id'])
self._cl.kill(running['Id'])
self._build_cl.kill(running['Id'])
# Next, remove all of the containers (which should all now be killed)
for container in self._cl.containers(all=True, quiet=True):
for container in self._build_cl.containers(all=True, quiet=True):
logger.debug('Removing container: %s' % container['Id'])
self._cl.remove_container(container['Id'])
self._build_cl.remove_container(container['Id'])
# Iterate all of the images and remove the ones that the public registry
# doesn't know about, this should preserve base images.
images_to_remove = set()
repos = set()
for image in self._cl.images():
for image in self._build_cl.images():
images_to_remove.add(image['Id'])
for tag in image['RepoTags']:
@ -237,13 +250,13 @@ class DockerfileBuildContext(object):
for to_remove in images_to_remove:
logger.debug('Removing private image: %s' % to_remove)
try:
self._cl.remove_image(to_remove)
self._build_cl.remove_image(to_remove)
except APIError:
# Sometimes an upstream image removed this one
pass
# Verify that our images were actually removed
for image in self._cl.images():
for image in self._build_cl.images():
if image['Id'] in images_to_remove:
raise RuntimeError('Image was not removed: %s' % image['Id'])
@ -291,6 +304,7 @@ class DockerfileBuildWorker(Worker):
tag_names = job_config['docker_tags']
build_subdir = job_config['build_subdir']
repo = job_config['repository']
pull_credentials = job_config.get('pull_credentials', None)
access_token = repository_build.access_token.code
@ -325,7 +339,7 @@ class DockerfileBuildWorker(Worker):
with DockerfileBuildContext(build_dir, build_subdir, repo, tag_names,
access_token,
repository_build.uuid) as build_ctxt:
repository_build.uuid, pull_credentials) as build_ctxt:
try:
built_image = build_ctxt.build()
@ -379,4 +393,4 @@ else:
handler = logging.StreamHandler()
handler.setFormatter(formatter)
root_logger.addHandler(handler)
worker.start()
worker.start()