vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Never load the full repo image list

Browse source 
Always make smaller queries per tag to ensure we scale better

Fixes #754
This commit is contained in:
Joseph Schorr 2015-11-03 18:15:23 -05:00
parent 43720b27e7
commit 4f41f79fa8
15 changed files with 268 additions and 254 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
endpoints/api/tag.py
View file

@ -4,7 +4,7 @@ from flask import request, abort
from endpoints.api import (resource, nickname, require_repo_read, require_repo_write,
RepositoryParamResource, log_action, NotFound, validate_json_request,
path_param, parse_args, query_param)
path_param, parse_args, query_param, truthy_bool)
from endpoints.api.image import image_view
from data import model
from auth.auth_context import get_authenticated_user
@ -135,7 +135,10 @@ class RepositoryTagImages(RepositoryParamResource):
""" Resource for listing the images in a specific repository tag. """
@require_repo_read
@nickname('listTagImages')
def get(self, namespace, repository, tag):
@parse_args
@query_param('owned', 'If specified, only images wholely owned by this tag are returned.',
type=truthy_bool, default=False)
def get(self, args, namespace, repository, tag):
""" List the images for the specified repository tag. """
try:
tag_image = model.tag.get_tag_image(namespace, repository, tag)
@ -144,15 +147,37 @@ class RepositoryTagImages(RepositoryParamResource):
parent_images = model.image.get_parent_images(namespace, repository, tag_image)
image_map = {}
image_map[str(tag_image.id)] = tag_image
for image in parent_images:
image_map[str(image.id)] = image
image_map_all = dict(image_map)
parents = list(parent_images)
parents.reverse()
all_images = [tag_image] + parents
# Filter the images returned to those not found in the ancestry of any of the other tags in
# the repository.
if args['owned']:
all_tags = model.tag.list_repository_tags(namespace, repository)
for current_tag in all_tags:
if current_tag.name == tag:
continue
# Remove the tag's image ID.
tag_image_id = str(current_tag.image_id)
image_map.pop(tag_image_id, None)
# Remove any ancestors:
for ancestor_id in current_tag.image.ancestors.split('/'):
image_map.pop(ancestor_id, None)
return {
'images': [image_view(image, image_map) for image in all_images]
'images': [image_view(image, image_map_all) for image in all_images
if not args['owned'] or (str(image.id) in image_map)]
}

8
static/directives/image-info-sidebar.html
View file

@ -20,14 +20,14 @@
<div class="image-section">
<i class="fa fa-tag section-icon" data-title="Current Tags" bs-tooltip></i>
<span class="section-info section-info-with-dropdown">
<a class="label tag label-default" ng-repeat="tag in imageData.tags"
<a class="label tag label-default" ng-repeat="tag in getTags(imageData)"
href="javascript:void(0)" ng-click="tagSelected({'tag': tag})">
{{ tag }}
</a>
<span style="color: #ccc;" ng-if="!imageData.tags.length">(No Tags)</span>
<span style="color: #ccc;" ng-if="!getTags(imageData).length">(No Tags)</span>
<div class="dropdown" data-placement="top"
ng-if="tracker.repository.can_write || imageData.tags">
ng-if="tracker.repository.can_write || getTags(imageData)">
<a href="javascript:void(0)" class="dropdown-button" data-toggle="dropdown"
bs-tooltip="tooltip.title" data-title="Manage Tags"
data-container="body">
@ -35,7 +35,7 @@
</a>
<ul class="dropdown-menu pull-right">
<li ng-repeat="tag in imageData.tags">
<li ng-repeat="tag in getTags(imageData)">
<a href="javascript:void(0)" ng-click="tagSelected({'tag': tag})">
<i class="fa fa-tag"></i>{{ tag }}
</a>

11
static/directives/repo-view/repo-panel-changes.html
View file

@ -1,6 +1,6 @@
<div class="repo-panel-changes-element">
<div class="resource-view" resource="imagesResource"
error-message="'Could not load repository images'">
<div class="cor-loader" ng-show="loading"></div>
<div ng-show="!loading">
<h3 class="tab-header">
Visualize Tags:
<span class="multiselect-dropdown" items="tagNames" selected-items="selectedTagsSlice"
@ -19,9 +19,6 @@
<!-- Tags Selected -->
<div ng-show="selectedTagsSlice.length > 0">
<div id="image-history row" class="resource-view" resource="imagesResource"
error-message="'Cannot load repository images'">
<!-- Tree View container -->
<div class="col-md-8">
<div class="panel panel-default">
@ -53,6 +50,7 @@
<div class="image-info-sidebar"
tracker="tracker"
image="currentImage"
image-loader="imageLoader"
tag-selected="setTag(tag)"
add-tag-requested="tagActionHandler.askAddTag(image)"
ng-if="currentImage">
@ -61,7 +59,6 @@
</div>
</div>
</div>
</div>
<div class="tag-operations-dialog" repository="repository" images="images"
<div class="tag-operations-dialog" repository="repository" image-loader="imageLoader"
action-handler="tagActionHandler" tag-changed="handleTagChanged(data)"></div>

2
static/directives/repo-view/repo-panel-tags.html
View file

@ -172,7 +172,7 @@
</div>
<div class="tag-operations-dialog" repository="repository"
get-images="getImages({'callback': callback})"
image-loader="imageLoader"
action-handler="tagActionHandler"></div>
<div class="fetch-tag-dialog" repository="repository" action-handler="fetchTagActionHandler"></div>

8
static/directives/tag-operations-dialog.html
View file

@ -49,11 +49,11 @@
<div class="tag-specific-images-view"
tag="tagToCreate"
repository="repo"
images="imagesInternal"
repository="repository"
image-cutoff="toTagImage"
style="margin: 10px; margin-top: 20px; margin-bottom: -10px;"
ng-show="isAnotherImageTag(toTagImage, tagToCreate)">
ng-show="isAnotherImageTag(toTagImage, tagToCreate)"
image-loader="imageLoader">
This will also delete any unattached images and delete the following images:
</div>
</div>
@ -100,7 +100,7 @@
<span class="label label-default tag">{{ deleteTagInfo.tag }}</span>?
<div class="tag-specific-images-view" tag="deleteTagInfo.tag" repository="repository"
images="imagesInternal" style="margin-top: 20px">
image-loader="imageLoader" style="margin-top: 20px">
The following images and any other images not referenced by a tag will be deleted:
</div>
</div>

6
static/directives/tag-specific-images-view.html
View file

@ -1,4 +1,8 @@
<div class="tag-specific-images-view-element" ng-show="tagSpecificImages.length">
<div class="tag-specific-images-view-element" ng-show="loading">
<div class="cor-loader-inline"></div>
</div>
<div class="tag-specific-images-view-element" ng-show="tagSpecificImages.length && !loading">
<div ng-transclude></div>
<div class="image-listings">
<div class="image-listing" ng-repeat="image in tagSpecificImages | limitTo:5"

39
static/js/directives/repo-view/repo-panel-changes.js
View file

@ -2,13 +2,15 @@
* An element which displays the changes visualization panel for a repository view.
*/
angular.module('quay').directive('repoPanelChanges', function () {
var RepositoryImageTracker = function(repository, images) {
var RepositoryImageTracker = function(repository, imageLoader) {
this.repository = repository;
this.images = images;
this.imageLoader = imageLoader;
// Build a map of image ID -> image.
var images = imageLoader.images;
var imageIDMap = {};
this.images.map(function(image) {
images.forEach(function(image) {
imageIDMap[image.id] = image;
});
@ -91,12 +93,13 @@ angular.module('quay').directive('repoPanelChanges', function () {
'selectedTags': '=selectedTags',
'imagesResource': '=imagesResource',
'images': '=images',
'imageLoader': '=imageLoader',
'isEnabled': '=isEnabled'
},
controller: function($scope, $element, $timeout, ApiService, UtilService, ImageMetadataService) {
$scope.tagNames = [];
$scope.loading = true;
$scope.$watch('selectedTags', function(selectedTags) {
if (!selectedTags) { return; }
@ -110,18 +113,17 @@ angular.module('quay').directive('repoPanelChanges', function () {
$scope.currentImage = null;
$scope.currentTag = null;
if ($scope.tracker) {
refreshTree();
} else {
$scope.loading = true;
$scope.imageLoader.loadImages($scope.selectedTagsSlice, function() {
$scope.loading = false;
updateImages();
}
});
};
var updateImages = function() {
if (!$scope.repository || !$scope.images || !$scope.isEnabled) { return; }
$scope.tracker = new RepositoryImageTracker($scope.repository, $scope.images);
if (!$scope.repository || !$scope.imageLoader || !$scope.isEnabled) { return; }
$scope.tracker = new RepositoryImageTracker($scope.repository, $scope.imageLoader);
if ($scope.selectedTagsSlice && $scope.selectedTagsSlice.length) {
refreshTree();
}
@ -131,22 +133,25 @@ angular.module('quay').directive('repoPanelChanges', function () {
$scope.$watch('repository', update);
$scope.$watch('isEnabled', update);
$scope.$watch('images', updateImages);
$scope.updateState = function() {
update();
};
var refreshTree = function() {
if (!$scope.repository || !$scope.images || !$scope.isEnabled) { return; }
if (!$scope.repository || !$scope.imageLoader || !$scope.isEnabled) { return; }
if ($scope.selectedTagsSlice.length < 1) { return; }
$('#image-history-container').empty();
var getTagsForImage = function(image) {
return $scope.imageLoader.getTagsForImage(image);
};
var tree = new ImageHistoryTree(
$scope.repository.namespace,
$scope.repository.name,
$scope.images,
$scope.imageLoader.images,
getTagsForImage,
UtilService.getFirstMarkdownLineAsText,
$scope.getTimeSince,
ImageMetadataService.getEscapedFormattedCommand,
@ -194,8 +199,6 @@ angular.module('quay').directive('repoPanelChanges', function () {
};
$scope.handleTagChanged = function(data) {
$scope.tracker = new RepositoryImageTracker($scope.repository, $scope.images);
data.removed.map(function(tag) {
$scope.currentImage = null;
$scope.currentTag = null;
@ -206,7 +209,7 @@ angular.module('quay').directive('repoPanelChanges', function () {
$scope.currentTag = tag;
});
refreshTree();
update();
};
}
};

2
static/js/directives/repo-view/repo-panel-tags.js
View file

@ -12,7 +12,7 @@ angular.module('quay').directive('repoPanelTags', function () {
'repository': '=repository',
'selectedTags': '=selectedTags',
'imagesResource': '=imagesResource',
'images': '=images',
'imageLoader': '=imageLoader',
'isEnabled': '=isEnabled',

5
static/js/directives/ui/image-info-sidebar.js
View file

@ -11,6 +11,7 @@ angular.module('quay').directive('imageInfoSidebar', function () {
scope: {
'tracker': '=tracker',
'image': '=image',
'imageLoader': '=imageLoader',
'tagSelected': '&tagSelected',
'addTagRequested': '&addTagRequested'
@ -25,6 +26,10 @@ angular.module('quay').directive('imageInfoSidebar', function () {
return Date.parse(dateString);
};
$scope.getTags = function(imageData) {
return $scope.imageLoader.getTagsForImage(imageData);
};
$scope.getFormattedCommand = ImageMetadataService.getFormattedCommand;
}
};

56
static/js/directives/ui/tag-operations-dialog.js
View file

@ -11,32 +11,18 @@ angular.module('quay').directive('tagOperationsDialog', function () {
restrict: 'C',
scope: {
'repository': '=repository',
'images': '=images',
'actionHandler': '=actionHandler',
'getImages': '&getImages',
'imageLoader': '=imageLoader',
'tagChanged': '&tagChanged'
},
controller: function($scope, $element, $timeout, ApiService) {
$scope.addingTag = false;
$scope.imagesInternal = [];
$scope.$watch('images', function(images) {
if (!images) { return; }
$scope.imagesInternal = images;
});
var markChanged = function(added, removed) {
// Reload the repository and the images.
// Reload the repository.
$scope.repository.get().then(function(resp) {
$scope.repository = resp;
var params = {
'repository': resp.namespace + '/' + resp.name
};
ApiService.listRepositoryImages(null, params).then(function(resp) {
$scope.images = resp.images;
$scope.imageLoader.reset()
// Note: We need the timeout here so that Angular can $digest the images change
// on the parent scope before the tagChanged callback occurs.
@ -45,12 +31,11 @@ angular.module('quay').directive('tagOperationsDialog', function () {
'data': { 'added': added, 'removed': removed }
});
}, 1);
})
});
};
$scope.isAnotherImageTag = function(image, tag) {
if (!$scope.repository || !$scope.imagesInternal) { return; }
if (!$scope.repository) { return; }
var found = $scope.repository.tags[tag];
if (found == null) { return false; }
@ -58,7 +43,7 @@ angular.module('quay').directive('tagOperationsDialog', function () {
};
$scope.isOwnedTag = function(image, tag) {
if (!$scope.repository || !$scope.imagesInternal) { return; }
if (!$scope.repository) { return; }
var found = $scope.repository.tags[tag];
if (found == null) { return false; }
@ -149,58 +134,28 @@ angular.module('quay').directive('tagOperationsDialog', function () {
}, errorHandler);
};
var lazyLoadImages = function(callback) {
if ($scope.imagesInternal.length) {
callback();
return;
}
var isLoading = true;
$timeout(function() {
if (isLoading) {
$('#loadingImagesModal').modal({});
}
}, 500);
var cb = function(images) {
isLoading = false;
$('#loadingImagesModal').modal('hide');
$scope.imagesInternal = images;
callback();
};
$scope.getImages({'callback': cb});
};
$scope.actionHandler = {
'askDeleteTag': function(tag) {
lazyLoadImages(function() {
$scope.deleteTagInfo = {
'tag': tag
};
});
},
'askDeleteMultipleTags': function(tags) {
lazyLoadImages(function() {
$scope.deleteMultipleTagsInfo = {
'tags': tags
};
});
},
'askAddTag': function(image) {
lazyLoadImages(function() {
$scope.tagToCreate = '';
$scope.toTagImage = image;
$scope.addingTag = false;
$scope.addTagForm.$setPristine();
$element.find('#createOrMoveTagModal').modal('show');
});
},
'askRevertTag': function(tag, image_id) {
lazyLoadImages(function() {
if (tag.image_id == image_id) {
bootbox.alert('This is the current image for the tag');
return;
@ -212,7 +167,6 @@ angular.module('quay').directive('tagOperationsDialog', function () {
};
$element.find('#revertTagModal').modal('show');
});
}
};
}

80
static/js/directives/ui/tag-specific-image-view.js
View file

@ -12,13 +12,12 @@ angular.module('quay').directive('tagSpecificImagesView', function () {
scope: {
'repository': '=repository',
'tag': '=tag',
'images': '=images',
'imageLoader': '=imageLoader',
'imageCutoff': '=imageCutoff'
},
controller: function($scope, $element, UtilService) {
$scope.getFirstTextLine = UtilService.getFirstMarkdownLineAsText;
$scope.hasImages = false;
$scope.loading = false;
$scope.tagSpecificImages = [];
$scope.getImageListingClasses = function(image) {
@ -35,39 +34,8 @@ angular.module('quay').directive('tagSpecificImagesView', function () {
return classes;
};
var forAllTagImages = function(tag, callback, opt_cutoff) {
if (!tag) { return; }
if (!$scope.imageByDockerId) {
$scope.imageByDockerId = [];
for (var i = 0; i < $scope.images.length; ++i) {
var currentImage = $scope.images[i];
$scope.imageByDockerId[currentImage.id] = currentImage;
}
}
var tag_image = $scope.imageByDockerId[tag.image_id];
if (!tag_image) {
return;
}
callback(tag_image);
var ancestors = tag_image.ancestors.split('/').reverse();
for (var i = 0; i < ancestors.length; ++i) {
var image = $scope.imageByDockerId[ancestors[i]];
if (image) {
if (image == opt_cutoff) {
return;
}
callback(image);
}
}
};
var refresh = function() {
if (!$scope.repository || !$scope.tag || !$scope.images) {
if (!$scope.repository || !$scope.tag || !$scope.imageLoader) {
$scope.tagSpecificImages = [];
return;
}
@ -78,49 +46,15 @@ angular.module('quay').directive('tagSpecificImagesView', function () {
return;
}
var getIdsForTag = function(currentTag) {
var ids = {};
forAllTagImages(currentTag, function(image) {
ids[image.id] = true;
}, $scope.imageCutoff);
return ids;
};
// Remove any IDs that match other tags.
var toDelete = getIdsForTag(tag);
for (var currentTagName in $scope.repository.tags) {
var currentTag = $scope.repository.tags[currentTagName];
if (currentTag != tag) {
for (var id in getIdsForTag(currentTag)) {
delete toDelete[id];
}
}
}
// Return the matching list of images.
var images = [];
for (var i = 0; i < $scope.images.length; ++i) {
var image = $scope.images[i];
if (toDelete[image.id]) {
images.push(image);
}
}
images.sort(function(a, b) {
var result = new Date(b.created) - new Date(a.created);
if (result != 0) {
return result;
}
return b.sort_index - a.sort_index;
});
$scope.loading = true;
$scope.imageLoader.getTagSpecificImages($scope.tag, function(images) {
$scope.loading = false;
$scope.tagSpecificImages = images;
});
};
$scope.$watch('repository', refresh);
$scope.$watch('tag', refresh);
$scope.$watch('images', refresh);
}
};
return directiveDefinitionObject;

14
static/js/graphing.js
View file

@ -31,8 +31,8 @@ var DEPTH_WIDTH = 140;
/**
* Based off of http://mbostock.github.io/d3/talk/20111018/tree.html by Mike Bostock (@mbostock)
*/
function ImageHistoryTree(namespace, name, images, formatComment, formatTime, formatCommand,
opt_tagFilter) {
function ImageHistoryTree(namespace, name, images, getTagsForImage, formatComment, formatTime,
formatCommand, opt_tagFilter) {
/**
* The namespace of the repo.
@ -49,6 +49,11 @@ function ImageHistoryTree(namespace, name, images, formatComment, formatTime, fo
*/
this.images_ = images;
/**
* Method to retrieve the tags for an image.
*/
this.getTagsForImage_ = getTagsForImage;
/**
* Method to invoke to format a comment for an image.
*/
@ -424,7 +429,7 @@ ImageHistoryTree.prototype.buildRoot_ = function() {
"name": image.id.substr(0, 12),
"children": [],
"image": image,
"tags": image.tags,
"tags": this.getTagsForImage_(image),
"level": null
};
imageByDockerId[image.id] = imageNode;
@ -663,8 +668,9 @@ ImageHistoryTree.prototype.setTag_ = function(tagName) {
this.currentImage_ = null;
// Update the path.
var that = this;
var tagImage = this.findImage_(function(image) {
return image.tags.indexOf(tagName || '(no tag specified)') >= 0;
return tagName && (that.getTagsForImage_(image).indexOf(tagName) >= 0);
});
if (tagImage) {

26
static/js/pages/repo-view.js
View file

@ -10,11 +10,11 @@
})
}]);
function RepoViewCtrl($scope, $routeParams, $location, $timeout, ApiService, UserService, AngularPollChannel) {
function RepoViewCtrl($scope, $routeParams, $location, $timeout, ApiService, UserService, AngularPollChannel, ImageLoaderService) {
$scope.namespace = $routeParams.namespace;
$scope.name = $routeParams.name;
$scope.imagesRequired = false;
var imageLoader = ImageLoaderService.getLoader($scope.namespace, $scope.name);
// Tab-enabled counters.
$scope.tagsShown = 0;
@ -25,8 +25,7 @@
$scope.viewScope = {
'selectedTags': [],
'repository': null,
'images': null,
'imagesResource': null,
'imageLoader': imageLoader,
'builds': null,
'changesVisible': false
};
@ -72,17 +71,6 @@
});
};
var loadImages = function(opt_callback) {
var params = {
'repository': $scope.namespace + '/' + $scope.name
};
$scope.viewScope.imagesResource = ApiService.listRepositoryImagesAsResource(params).get(function(resp) {
$scope.viewScope.images = resp.images;
opt_callback && opt_callback(resp.images);
});
};
var loadRepositoryBuilds = function(callback) {
var params = {
'repository': $scope.namespace + '/' + $scope.name,
@ -149,14 +137,6 @@
}, 10);
};
$scope.requireImages = function() {
// Lazily load the repo's images if this is the first call to a tab
// that needs the images.
if ($scope.viewScope.images == null) {
loadImages();
}
};
$scope.handleChangesState = function(value) {
$scope.viewScope.changesVisible = value;
};

110
static/js/services/image-loader-service.js Normal file
View file

@ -0,0 +1,110 @@
/**
* Helper service for tracking images needed by tags and caching them.
*/
angular.module('quay').factory('ImageLoaderService', ['ApiService', function(ApiService) {
var imageLoader = function(namespace, name) {
this.namespace = namespace;
this.name = name;
this.tagCache = {};
this.images = [];
this.imageMap = {};
this.imageTagMap = {};
};
imageLoader.prototype.getTagSpecificImages = function(tag, callback) {
var errorDisplay = ApiService.errorDisplay('Could not load tag specific images', function() {
callback([]);
});
var params = {
'repository': this.namespace + '/' + this.name,
'tag': tag,
'owned': true
};
ApiService.listTagImages(null, params).then(function(resp) {
callback(resp['images']);
}, errorDisplay);
};
imageLoader.prototype.getTagsForImage = function(image) {
return this.imageTagMap[image.id] || [];
};
imageLoader.prototype.registerTagImages_ = function(tag, images) {
this.tagCache[tag] = images;
if (!images.length) {
return;
}
var that = this;
images.forEach(function(image) {
if (!that.imageMap[image.id]) {
that.imageMap[image.id] = image;
that.images.push(image);
}
});
var rootImage = images[0];
if (!this.imageTagMap[rootImage.id]) {
this.imageTagMap[rootImage.id] = [];
}
this.imageTagMap[rootImage.id].push(tag);
}
imageLoader.prototype.loadImages = function(tags, callback) {
var toLoad = [];
var that = this;
tags.forEach(function(tag) {
if (that.tagCache[tag]) {
return;
}
toLoad.push(tag);
});
if (!toLoad.length) {
callback();
return;
}
var loadImages = function(index) {
if (index >= toLoad.length) {
callback();
return;
}
var tag = toLoad[index];
var params = {
'repository': that.namespace + '/' + that.name,
'tag': tag,
};
ApiService.listTagImages(null, params).then(function(resp) {
that.registerTagImages_(tag, resp['images']);
loadImages(index + 1);
}, function() {
loadImages(index + 1);
})
};
loadImages(0);
};
imageLoader.prototype.reset = function() {
this.tagCache = {};
this.images = [];
this.imageMap = {};
this.imageTagMap = {};
};
var imageLoaderService = {};
imageLoaderService.getLoader = function(namespace, name) {
return new imageLoader(namespace, name);
};
return imageLoaderService
}]);

10
static/partials/repo-view.html
View file

@ -34,8 +34,7 @@
<span class="cor-tab" tab-title="Visualize" tab-target="#changes"
tab-shown="handleChangesState(true)"
tab-hidden="handleChangesState(false)"
tab-init="requireImages()">
tab-hidden="handleChangesState(false)">
<i class="fa fa-code-fork"></i>
</span>
@ -64,10 +63,8 @@
<div id="tags" class="tab-pane">
<div class="repo-panel-tags"
repository="viewScope.repository"
images="viewScope.images"
images-resource="viewScope.imagesResource"
image-loader="viewScope.imageLoader"
selected-tags="viewScope.selectedTags"
get-images="getImages(callback)"
is-enabled="tagsShown"></div>
</div>
@ -83,8 +80,7 @@
<div id="changes" class="tab-pane">
<div class="repo-panel-changes"
repository="viewScope.repository"
images="viewScope.images"
images-resource="viewScope.imagesResource"
image-loader="viewScope.imageLoader"
selected-tags="viewScope.selectedTags"
is-enabled="viewScope.changesVisible"></div>
</div>