diff --git a/Dockerfile.web b/Dockerfile.web index b24694b42..1c1b60c69 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -38,6 +38,7 @@ ADD conf/init/gunicorn /etc/service/gunicorn ADD conf/init/nginx /etc/service/nginx ADD conf/init/diffsworker /etc/service/diffsworker ADD conf/init/notificationworker /etc/service/notificationworker +ADD conf/init/buildlogsarchiver /etc/service/buildlogsarchiver # Download any external libs. RUN mkdir static/fonts static/ldn diff --git a/app.py b/app.py index bcc4e86d7..8f0a57d62 100644 --- a/app.py +++ b/app.py @@ -20,6 +20,7 @@ from util.exceptionlog import Sentry from util.queuemetrics import QueueMetrics from data.billing import Billing from data.buildlogs import BuildLogs +from data.archivedlogs import LogArchive from data.queue import WorkQueue from data.userevent import UserEventsBuilderModule from datetime import datetime @@ -89,6 +90,7 @@ login_manager = LoginManager(app) mail = Mail(app) storage = Storage(app) userfiles = Userfiles(app, storage) +log_archive = LogArchive(app, storage) analytics = Analytics(app) billing = Billing(app) sentry = Sentry(app) diff --git a/conf/init/buildlogsarchiver/log/run b/conf/init/buildlogsarchiver/log/run new file mode 100755 index 000000000..c35fb1fb9 --- /dev/null +++ b/conf/init/buildlogsarchiver/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec svlogd /var/log/buildlogsarchiver/ \ No newline at end of file diff --git a/conf/init/buildlogsarchiver/run b/conf/init/buildlogsarchiver/run new file mode 100755 index 000000000..f263770af --- /dev/null +++ b/conf/init/buildlogsarchiver/run @@ -0,0 +1,8 @@ +#! /bin/bash + +echo 'Starting build logs archiver worker' + +cd / +venv/bin/python -m workers.buildlogsarchiver + +echo 'Diffs worker exited' \ No newline at end of file diff --git a/config.py b/config.py index ffcf7f79e..f810007e8 100644 --- a/config.py +++ b/config.py @@ -172,3 +172,7 @@ class DefaultConfig(object): # Userfiles USERFILES_LOCATION = 'local_us' USERFILES_PATH = 'userfiles/' + + # Build logs archive + LOG_ARCHIVE_LOCATION = 'local_us' + LOG_ARCHIVE_PATH = 'logarchive/' diff --git a/data/archivedlogs.py b/data/archivedlogs.py new file mode 100644 index 000000000..e190b9782 --- /dev/null +++ b/data/archivedlogs.py @@ -0,0 +1,56 @@ +import logging + +from gzip import GzipFile +from flask import send_file, abort +from cStringIO import StringIO + +from data.userfiles import DelegateUserfiles, UserfilesHandlers + + +JSON_MIMETYPE = 'application/json' + + +logger = logging.getLogger(__name__) + + +class LogArchiveHandlers(UserfilesHandlers): + def get(self, file_id): + path = self._files.get_file_id_path(file_id) + try: + with self._storage.stream_read_file(self._locations, path) as gzip_stream: + with GzipFile(fileobj=gzip_stream) as unzipped: + unzipped_buffer = StringIO(unzipped.read()) + return send_file(unzipped_buffer, mimetype=JSON_MIMETYPE) + except IOError: + abort(404) + + +class LogArchive(object): + def __init__(self, app=None, distributed_storage=None): + self.app = app + if app is not None: + self.state = self.init_app(app, distributed_storage) + else: + self.state = None + + def init_app(self, app, distributed_storage): + location = app.config.get('LOG_ARCHIVE_LOCATION') + path = app.config.get('LOG_ARCHIVE_PATH', None) + + handler_name = 'logarchive_handlers' + + log_archive = DelegateUserfiles(app, distributed_storage, location, path, handler_name) + + app.add_url_rule('/logarchive/', + view_func=LogArchiveHandlers.as_view(handler_name, + distributed_storage=distributed_storage, + location=location, + files=log_archive)) + + # register extension with app + app.extensions = getattr(app, 'extensions', {}) + app.extensions['log_archive'] = log_archive + return log_archive + + def __getattr__(self, name): + return getattr(self.state, name, None) diff --git a/data/billing.py b/data/billing.py index 8c604aac2..e1510c054 100644 --- a/data/billing.py +++ b/data/billing.py @@ -3,7 +3,7 @@ import stripe from datetime import datetime, timedelta from calendar import timegm -from util.collections import AttrDict +from util.morecollections import AttrDict PLANS = [ # Deprecated Plans diff --git a/data/buildlogs.py b/data/buildlogs.py index 2ccd03899..9128390af 100644 --- a/data/buildlogs.py +++ b/data/buildlogs.py @@ -2,6 +2,11 @@ import redis import json from util.dynamic import import_class +from datetime import timedelta + + +ONE_DAY = timedelta(days=1) + class BuildStatusRetrievalError(Exception): pass @@ -25,7 +30,7 @@ class RedisBuildLogs(object): """ return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj)) - def append_log_message(self, build_id, log_message, log_type=None): + def append_log_message(self, build_id, log_message, log_type=None, log_data=None): """ Wraps the message in an envelope and push it to the end of the log entry list and returns the index at which it was inserted. @@ -37,6 +42,9 @@ class RedisBuildLogs(object): if log_type: log_obj['type'] = log_type + if log_data: + log_obj['data'] = log_data + return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj)) - 1 def get_log_entries(self, build_id, start_index): @@ -51,6 +59,13 @@ class RedisBuildLogs(object): except redis.ConnectionError: raise BuildStatusRetrievalError('Cannot retrieve build logs') + def expire_log_entries(self, build_id): + """ + Sets the log entry to expire in 1 day. + """ + self._redis.expire(self._logs_key(build_id), ONE_DAY) + + @staticmethod def _status_key(build_id): return 'builds/%s/status' % build_id @@ -106,4 +121,4 @@ class BuildLogs(object): return buildlogs def __getattr__(self, name): - return getattr(self.state, name, None) \ No newline at end of file + return getattr(self.state, name, None) diff --git a/data/database.py b/data/database.py index 4731a06bb..96e85a7d2 100644 --- a/data/database.py +++ b/data/database.py @@ -289,6 +289,16 @@ class RepositoryTag(BaseModel): ) +class BUILD_PHASE(object): + """ Build phases enum """ + ERROR = 'error' + UNPACKING = 'unpacking' + PULLING = 'pulling' + BUILDING = 'building' + PUSHING = 'pushing' + COMPLETE = 'complete' + + class RepositoryBuild(BaseModel): uuid = CharField(default=uuid_generator, index=True) repository = ForeignKeyField(Repository, index=True) @@ -300,6 +310,7 @@ class RepositoryBuild(BaseModel): display_name = CharField() trigger = ForeignKeyField(RepositoryBuildTrigger, null=True, index=True) pull_robot = ForeignKeyField(User, null=True, related_name='buildpullrobot') + logs_archived = BooleanField(default=False) class QueueItem(BaseModel): diff --git a/data/migrations/env.py b/data/migrations/env.py index 863e3d98f..d64cf4ee7 100644 --- a/data/migrations/env.py +++ b/data/migrations/env.py @@ -8,7 +8,7 @@ from peewee import SqliteDatabase from data.database import all_models, db from app import app from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from util.collections import AttrDict +from util.morecollections import AttrDict # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/data/migrations/versions/34fd69f63809_add_support_for_build_log_migration.py b/data/migrations/versions/34fd69f63809_add_support_for_build_log_migration.py new file mode 100644 index 000000000..a731d0158 --- /dev/null +++ b/data/migrations/versions/34fd69f63809_add_support_for_build_log_migration.py @@ -0,0 +1,26 @@ +"""Add support for build log migration. + +Revision ID: 34fd69f63809 +Revises: 4a0c94399f38 +Create Date: 2014-09-12 11:50:09.217777 + +""" + +# revision identifiers, used by Alembic. +revision = '34fd69f63809' +down_revision = '4a0c94399f38' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('repositorybuild', sa.Column('logs_archived', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false())) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('repositorybuild', 'logs_archived') + ### end Alembic commands ### diff --git a/data/model/legacy.py b/data/model/legacy.py index 64bcdc860..92a130dca 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -12,6 +12,7 @@ from util.backoff import exponential_backoff EXPONENTIAL_BACKOFF_SCALE = timedelta(seconds=1) +PRESUMED_DEAD_BUILD_AGE = timedelta(days=15) logger = logging.getLogger(__name__) @@ -1877,3 +1878,11 @@ def confirm_email_authorization_for_repo(code): found.save() return found + + +def archivable_buildlogs_query(): + presumed_dead_date = datetime.utcnow() - PRESUMED_DEAD_BUILD_AGE + return (RepositoryBuild.select() + .where((RepositoryBuild.phase == BUILD_PHASE.COMPLETE) | + (RepositoryBuild.phase == BUILD_PHASE.ERROR) | + (RepositoryBuild.started < presumed_dead_date), RepositoryBuild.logs_archived == False)) diff --git a/data/userfiles.py b/data/userfiles.py index 950c4dd60..f4b786df5 100644 --- a/data/userfiles.py +++ b/data/userfiles.py @@ -81,10 +81,13 @@ class DelegateUserfiles(object): return (url, file_id) - def store_file(self, file_like_obj, content_type): - file_id = str(uuid4()) + def store_file(self, file_like_obj, content_type, content_encoding=None, file_id=None): + if file_id is None: + file_id = str(uuid4()) + path = self.get_file_id_path(file_id) - self._storage.stream_write(self._locations, path, file_like_obj, content_type) + self._storage.stream_write(self._locations, path, file_like_obj, content_type, + content_encoding) return file_id def get_file_url(self, file_id, expires_in=300, requires_cors=False): diff --git a/endpoints/api/build.py b/endpoints/api/build.py index 74677fadb..d792234dd 100644 --- a/endpoints/api/build.py +++ b/endpoints/api/build.py @@ -1,9 +1,9 @@ import logging import json -from flask import request +from flask import request, redirect -from app import app, userfiles as user_files, build_logs +from app import app, userfiles as user_files, build_logs, log_archive from endpoints.api import (RepositoryParamResource, parse_args, query_param, nickname, resource, require_repo_read, require_repo_write, validate_json_request, ApiResource, internal_only, format_date, api, Unauthorized, NotFound) @@ -215,6 +215,10 @@ class RepositoryBuildLogs(RepositoryParamResource): build = model.get_repository_build(namespace, repository, build_uuid) + # If the logs have been archived, just redirect to the completed archive + if build.logs_archived: + return redirect(log_archive.get_file_url(build.uuid)) + start = int(request.args.get('start', 0)) try: diff --git a/endpoints/trigger.py b/endpoints/trigger.py index ab7aa9065..ae0b4b2b7 100644 --- a/endpoints/trigger.py +++ b/endpoints/trigger.py @@ -291,6 +291,9 @@ class GithubBuildTrigger(BuildTrigger): with tarfile.open(fileobj=tarball) as archive: tarball_subdir = archive.getnames()[0] + # Seek to position 0 to make boto multipart happy + tarball.seek(0) + dockerfile_id = user_files.store_file(tarball, TARBALL_MIME) logger.debug('Successfully prepared job') diff --git a/static/css/quay.css b/static/css/quay.css index 38d03893f..c2652c8c1 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -2558,7 +2558,7 @@ p.editable:hover i { margin-top: 10px; } -.repo-build .build-log-error-element { +.repo-build .build-log-error-element .error-message-container { position: relative; display: inline-block; margin: 10px; @@ -2568,7 +2568,7 @@ p.editable:hover i { margin-left: 22px; } -.repo-build .build-log-error-element i.fa { +.repo-build .build-log-error-element .error-message-container i.fa { color: red; position: absolute; top: 13px; diff --git a/static/directives/build-log-error.html b/static/directives/build-log-error.html index 095f8edd0..13b399bb9 100644 --- a/static/directives/build-log-error.html +++ b/static/directives/build-log-error.html @@ -1,4 +1,23 @@ - - - - +
+ + + + + caused by attempting to pull private repository {{ getLocalPullInfo().repo }} + with inaccessible crdentials + without credentials + + + + +
+
+ Note: The credentials {{ getLocalPullInfo().login.username }} for registry {{ getLocalPullInfo().login.registry }} cannot + access repository {{ getLocalPullInfo().repo }}. +
+
+ Note: No robot account is specified for this build. Without such credentials, this pull will always fail. Please setup a new + build trigger with a robot account that has access to {{ getLocalPullInfo().repo }} or make that repository public. +
+
+
diff --git a/static/js/app.js b/static/js/app.js index 26b8a4be1..6a6bf27e3 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -153,6 +153,14 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading this.currentIndex_ = 0; } + _ViewArray.prototype.length = function() { + return this.entries.length; + }; + + _ViewArray.prototype.get = function(index) { + return this.entries[index]; + }; + _ViewArray.prototype.push = function(elem) { this.entries.push(elem); this.hasEntries = true; @@ -215,6 +223,78 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading return service; }]); + /** + * Specialized class for conducting an HTTP poll, while properly preventing multiple calls. + */ + $provide.factory('AngularPollChannel', ['ApiService', '$timeout', function(ApiService, $timeout) { + var _PollChannel = function(scope, requester, opt_sleeptime) { + this.scope_ = scope; + this.requester_ = requester; + this.sleeptime_ = opt_sleeptime || (60 * 1000 /* 60s */); + this.timer_ = null; + + this.working = false; + this.polling = false; + + var that = this; + scope.$on('$destroy', function() { + that.stop(); + }); + }; + + _PollChannel.prototype.stop = function() { + if (this.timer_) { + $timeout.cancel(this.timer_); + this.timer_ = null; + this.polling_ = false; + } + + this.working = false; + }; + + _PollChannel.prototype.start = function() { + // Make sure we invoke call outside the normal digest cycle, since + // we'll call $scope.$apply ourselves. + var that = this; + setTimeout(function() { that.call_(); }, 0); + }; + + _PollChannel.prototype.call_ = function() { + if (this.working) { return; } + + var that = this; + this.working = true; + this.scope_.$apply(function() { + that.requester_(function(status) { + if (status) { + that.working = false; + that.setupTimer_(); + } else { + that.stop(); + } + }); + }); + }; + + _PollChannel.prototype.setupTimer_ = function() { + if (this.timer_) { return; } + + var that = this; + this.polling = true; + this.timer_ = $timeout(function() { + that.timer_ = null; + that.call_(); + }, this.sleeptime_) + }; + + var service = { + 'create': function(scope, requester, opt_sleeptime) { + return new _PollChannel(scope, requester, opt_sleeptime); + } + }; + + return service; + }]); $provide.factory('DataFileService', [function() { var dataFileService = {}; @@ -4368,9 +4448,48 @@ quayApp.directive('buildLogError', function () { transclude: false, restrict: 'C', scope: { - 'error': '=error' + 'error': '=error', + 'entries': '=entries' }, - controller: function($scope, $element) { + controller: function($scope, $element, Config) { + $scope.getLocalPullInfo = function() { + if ($scope.entries.__localpull !== undefined) { + return $scope.entries.__localpull; + } + + var localInfo = { + 'isLocal': false + }; + + // Find the 'pulling' phase entry, and then extra any metadata found under + // it. + for (var i = 0; i < $scope.entries.length; ++i) { + var entry = $scope.entries[i]; + if (entry.type == 'phase' && entry.message == 'pulling') { + for (var j = 0; j < entry.logs.length(); ++j) { + var log = entry.logs.get(j); + if (log.data && log.data.phasestep == 'login') { + localInfo['login'] = log.data; + } + + if (log.data && log.data.phasestep == 'pull') { + var repo_url = log.data['repo_url']; + var repo_and_tag = repo_url.substring(Config.SERVER_HOSTNAME.length + 1); + var tagIndex = repo_and_tag.lastIndexOf(':'); + var repo = repo_and_tag.substring(0, tagIndex); + + localInfo['repo_url'] = repo_url; + localInfo['repo'] = repo; + + localInfo['isLocal'] = repo_url.indexOf(Config.SERVER_HOSTNAME + '/') == 0; + } + } + break; + } + } + + return $scope.entries.__localpull = localInfo; + }; } }; return directiveDefinitionObject; diff --git a/static/js/controllers.js b/static/js/controllers.js index 9131a0140..f259ead68 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -978,14 +978,9 @@ function BuildPackageCtrl($scope, Restangular, ApiService, DataFileService, $rou } function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope, $location, $interval, $sanitize, - ansi2html, AngularViewArray) { + ansi2html, AngularViewArray, AngularPollChannel) { var namespace = $routeParams.namespace; var name = $routeParams.name; - var pollTimerHandle = null; - - $scope.$on('$destroy', function() { - stopPollTimer(); - }); // Watch for changes to the current parameter. $scope.$on('$routeUpdate', function(){ @@ -995,8 +990,7 @@ function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope }); $scope.builds = null; - $scope.polling = false; - + $scope.pollChannel = null; $scope.buildDialogShowCounter = 0; $scope.showNewBuildDialog = function() { @@ -1081,8 +1075,6 @@ function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope $scope.setCurrentBuildInternal = function(index, build, opt_updateURL) { if (build == $scope.currentBuild) { return; } - stopPollTimer(); - $scope.logEntries = null; $scope.logStartIndex = null; $scope.currentParentEntry = null; @@ -1103,47 +1095,35 @@ function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope $scope.adjustLogHeight(); }, 1); - // Load the first set of logs. - getBuildStatusAndLogs(); - - // If the build is currently processing, start the build timer. - checkPollTimer(); - }; - - var checkPollTimer = function() { - var build = $scope.currentBuild; - if (!build) { - stopPollTimer(); - return; + // Stop any existing polling. + if ($scope.pollChannel) { + $scope.pollChannel.stop(); } + + // Create a new channel for polling the build status and logs. + var conductStatusAndLogRequest = function(callback) { + getBuildStatusAndLogs(build, callback); + }; - if (build['phase'] != 'complete' && build['phase'] != 'error') { - startPollTimer(); - return true; - } else { - stopPollTimer(); - return false; - } + $scope.pollChannel = AngularPollChannel.create($scope, conductStatusAndLogRequest, 5 * 1000 /* 5s */); + $scope.pollChannel.start(); }; - var stopPollTimer = function() { - $interval.cancel(pollTimerHandle); - }; - - var startPollTimer = function() { - stopPollTimer(); - pollTimerHandle = $interval(getBuildStatusAndLogs, 2000); - }; - - var processLogs = function(logs, startIndex) { + var processLogs = function(logs, startIndex, endIndex) { if (!$scope.logEntries) { $scope.logEntries = []; } + // If the start index given is less than that requested, then we've received a larger + // pool of logs, and we need to only consider the new ones. + if (startIndex < $scope.logStartIndex) { + logs = logs.slice($scope.logStartIndex - startIndex); + } + for (var i = 0; i < logs.length; ++i) { var entry = logs[i]; var type = entry['type'] || 'entry'; if (type == 'command' || type == 'phase' || type == 'error') { entry['logs'] = AngularViewArray.create(); - entry['index'] = startIndex + i; + entry['index'] = $scope.logStartIndex + i; $scope.logEntries.push(entry); $scope.currentParentEntry = entry; @@ -1151,18 +1131,19 @@ function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope $scope.currentParentEntry['logs'].push(entry); } } + + return endIndex; }; - var getBuildStatusAndLogs = function() { - if (!$scope.currentBuild || $scope.polling) { return; } - $scope.polling = true; - + var getBuildStatusAndLogs = function(build, callback) { var params = { 'repository': namespace + '/' + name, - 'build_uuid': $scope.currentBuild.id + 'build_uuid': build.id }; ApiService.getRepoBuildStatus(null, params, true).then(function(resp) { + if (build != $scope.currentBuild) { callback(false); return; } + // Note: We use extend here rather than replacing as Angular is depending on the // root build object to remain the same object. var matchingBuilds = $.grep($scope.builds, function(elem) { @@ -1177,22 +1158,16 @@ function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope $scope.builds.push(currentBuild); } - checkPollTimer(); - // Load the updated logs for the build. var options = { 'start': $scope.logStartIndex }; - ApiService.getRepoBuildLogsAsResource(params, true).withOptions(options).get(function(resp) { - if ($scope.logStartIndex != null && resp['start'] != $scope.logStartIndex) { - $scope.polling = false; - return; - } + ApiService.getRepoBuildLogsAsResource(params, true).withOptions(options).get(function(resp) { + if (build != $scope.currentBuild) { callback(false); return; } - processLogs(resp['logs'], resp['start']); - $scope.logStartIndex = resp['total']; - $scope.polling = false; + // Process the logs we've received. + $scope.logStartIndex = processLogs(resp['logs'], resp['start'], resp['total']); // If the build status is an error, open the last two log entries. if (currentBuild['phase'] == 'error' && $scope.logEntries.length > 1) { @@ -1205,9 +1180,15 @@ function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope openLogEntries($scope.logEntries[$scope.logEntries.length - 2]); openLogEntries($scope.logEntries[$scope.logEntries.length - 1]); } + + // If the build phase is an error or a complete, then we mark the channel + // as closed. + callback(currentBuild['phase'] != 'error' && currentBuild['phase'] != 'complete'); }, function() { - $scope.polling = false; + callback(false); }); + }, function() { + callback(false); }); }; diff --git a/static/partials/repo-build.html b/static/partials/repo-build.html index 3afe87508..c8a352b5f 100644 --- a/static/partials/repo-build.html +++ b/static/partials/repo-build.html @@ -77,7 +77,7 @@
- +
@@ -94,7 +94,7 @@
- +