diff --git a/data/model/log.py b/data/model/log.py index b5039f407..6f2aeecc8 100644 --- a/data/model/log.py +++ b/data/model/log.py @@ -5,7 +5,7 @@ from peewee import JOIN_LEFT_OUTER, SQL, fn from datetime import datetime, timedelta, date from cachetools import lru_cache -from data.database import LogEntry, LogEntryKind, User, db +from data.database import LogEntry, LogEntryKind, User, RepositoryActionCount, db from data.model import config def _logs_query(selections, start_time, end_time, performer=None, repository=None, namespace=None, @@ -95,62 +95,6 @@ def log_action(kind_name, user_or_organization_name, performer=None, repository= datetime=timestamp) -def _get_repository_events(repository, time_delta, time_delta_earlier, clause): - """ Returns a pair representing the count of the number of events for the given - repository in each of the specified time deltas. The date ranges are calculated by - taking the current time today and subtracting the time delta given. Since - we want to grab *two* ranges, we restrict the second range to be greater - than the first (i.e. referring to an earlier time), so we can conduct the - lookup in a single query. The clause is used to further filter the kind of - events being found. - """ - since = date.today() - time_delta - since_earlier = date.today() - time_delta_earlier - - if since_earlier >= since: - raise ValueError('time_delta_earlier must be greater than time_delta') - - # This uses a CASE WHEN inner clause to further filter the count. - formatted = since.strftime('%Y-%m-%d') - case_query = 'CASE WHEN datetime >= \'%s\' THEN 1 ELSE 0 END' % formatted - - result = (LogEntry - .select(fn.Sum(SQL(case_query)), fn.Count(SQL('*'))) - .where(LogEntry.repository == repository) - .where(clause) - .where(LogEntry.datetime >= since_earlier) - .tuples() - .get()) - - return (int(result[0]) if result[0] else 0, int(result[1]) if result[1] else 0) - - -def get_repository_pushes(repository, time_delta, time_delta_earlier): - push_repo = _get_log_entry_kind('push_repo') - clauses = (LogEntry.kind == push_repo) - return _get_repository_events(repository, time_delta, time_delta_earlier, clauses) - - -def get_repository_pulls(repository, time_delta, time_delta_earlier): - repo_pull = _get_log_entry_kind('pull_repo') - repo_verb = _get_log_entry_kind('repo_verb') - clauses = ((LogEntry.kind == repo_pull) | (LogEntry.kind == repo_verb)) - return _get_repository_events(repository, time_delta, time_delta_earlier, clauses) - - -def get_repository_usage(): - one_month_ago = date.today() - timedelta(weeks=4) - repo_pull = _get_log_entry_kind('pull_repo') - repo_verb = _get_log_entry_kind('repo_verb') - return (LogEntry - .select(LogEntry.ip, LogEntry.repository) - .where((LogEntry.kind == repo_pull) | (LogEntry.kind == repo_verb)) - .where(~(LogEntry.repository >> None)) - .where(LogEntry.datetime >= one_month_ago) - .group_by(LogEntry.ip, LogEntry.repository) - .count()) - - def get_stale_logs_start_id(): """ Gets the oldest log entry. """ try: @@ -182,3 +126,28 @@ def get_stale_logs(start_id, end_id): def delete_stale_logs(start_id, end_id): """ Deletes all the logs with IDs between start_id and end_id. """ LogEntry.delete().where((LogEntry.id >= start_id), (LogEntry.id <= end_id)).execute() + + +def get_repository_action_counts(repo, start_date): + return RepositoryActionCount.select().where(RepositoryActionCount.repository == repo, + RepositoryActionCount.date >= start_date) + + +def get_repositories_action_sums(repository_ids): + if not repository_ids: + return {} + + # Filter the join to recent entries only. + last_week = datetime.now() - timedelta(weeks=1) + tuples = (RepositoryActionCount + .select(RepositoryActionCount.repository, fn.Sum(RepositoryActionCount.count)) + .where(RepositoryActionCount.repository << repository_ids) + .where(RepositoryActionCount.date >= last_week) + .group_by(RepositoryActionCount.repository) + .tuples()) + + action_count_map = {} + for record in tuples: + action_count_map[record[0]] = record[1] + + return action_count_map diff --git a/data/model/repository.py b/data/model/repository.py index a5e0b2e89..b5b67db16 100644 --- a/data/model/repository.py +++ b/data/model/repository.py @@ -3,9 +3,9 @@ import logging from peewee import JOIN_LEFT_OUTER, fn from datetime import timedelta, datetime -from data.model import (DataModelException, tag, db_transaction, storage, image, permission, +from data.model import (DataModelException, tag, db_transaction, storage, permission, _basequery, config) -from data.database import (Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, +from data.database import (Repository, Namespace, RepositoryTag, Star, Image, User, Visibility, RepositoryPermission, TupleSelector, RepositoryActionCount, Role, RepositoryAuthorizedEmail, TagManifest, DerivedStorageForImage, db_for_update, get_epoch_timestamp, db_random_func) @@ -227,26 +227,6 @@ def get_when_last_modified(repository_ids): return last_modified_map -def get_action_counts(repository_ids): - if not repository_ids: - return {} - - # Filter the join to recent entries only. - last_week = datetime.now() - timedelta(weeks=1) - tuples = (RepositoryActionCount - .select(RepositoryActionCount.repository, fn.Sum(RepositoryActionCount.count)) - .where(RepositoryActionCount.repository << repository_ids) - .where(RepositoryActionCount.date >= last_week) - .group_by(RepositoryActionCount.repository) - .tuples()) - - action_count_map = {} - for record in tuples: - action_count_map[record[0]] = record[1] - - return action_count_map - - def get_visible_repositories(username, namespace=None, include_public=False): """ Returns the repositories visible to the given user (if any). """ diff --git a/endpoints/api/repository.py b/endpoints/api/repository.py index 8878c9f84..b974a1fcd 100644 --- a/endpoints/api/repository.py +++ b/endpoints/api/repository.py @@ -4,12 +4,11 @@ import logging import datetime import features -from datetime import timedelta +from datetime import timedelta, datetime from flask import request, abort from data import model -from data.database import Repository as RepositoryTable from endpoints.api import (truthy_bool, format_date, nickname, log_action, validate_json_request, require_repo_read, require_repo_write, require_repo_admin, RepositoryParamResource, resource, query_param, parse_args, ApiResource, @@ -29,6 +28,7 @@ from util.names import REPOSITORY_NAME_REGEX logger = logging.getLogger(__name__) REPOS_PER_PAGE = 100 +MAX_DAYS_IN_3_MONTHS = 92 def check_allowed_private_repos(namespace): """ Checks to see if the given namespace has reached its private repository limit. If so, @@ -180,7 +180,7 @@ class RepositoryList(ApiResource): last_modified_map = model.repository.get_when_last_modified(repository_ids) if parsed_args['popularity']: - action_count_map = model.repository.get_action_counts(repository_ids) + action_sum_map = model.log.get_repositories_action_sums(repository_ids) # Collect the IDs of the repositories that are starred for the user, so we can mark them # in the returned results. @@ -203,7 +203,7 @@ class RepositoryList(ApiResource): repo['last_modified'] = last_modified_map.get(repo_id) if parsed_args['popularity']: - repo['popularity'] = action_count_map.get(repo_id, 0) + repo['popularity'] = action_sum_map.get(repo_id, 0) if username: repo['is_starred'] = repo_id in star_set @@ -236,7 +236,7 @@ class Repository(RepositoryParamResource): } @parse_args() - @query_param('includeStats', 'Whether to include pull and push statistics', type=truthy_bool, + @query_param('includeStats', 'Whether to include action statistics', type=truthy_bool, default=False) @require_repo_read @nickname('getRepo') @@ -252,7 +252,7 @@ class Repository(RepositoryParamResource): } if tag.lifetime_start_ts > 0: - last_modified = format_date(datetime.datetime.fromtimestamp(tag.lifetime_start_ts)) + last_modified = format_date(datetime.fromtimestamp(tag.lifetime_start_ts)) tag_info['last_modified'] = last_modified return tag_info @@ -270,22 +270,28 @@ class Repository(RepositoryParamResource): is_public = model.repository.is_repository_public(repo) if parsed_args['includeStats']: - (pull_today, pull_thirty_day) = model.log.get_repository_pulls(repo, timedelta(days=1), - timedelta(days=30)) + stats = [] + found_dates = {} - (push_today, push_thirty_day) = model.log.get_repository_pushes(repo, timedelta(days=1), - timedelta(days=30)) + start_date = datetime.now() - timedelta(days=MAX_DAYS_IN_3_MONTHS) + counts = model.log.get_repository_action_counts(repo, start_date) + for count in counts: + stats.append({ + 'date': count.date.isoformat(), + 'count': count.count, + }) - stats = { - 'pulls': { - 'today': pull_today, - 'thirty_day': pull_thirty_day - }, - 'pushes': { - 'today': push_today, - 'thirty_day': push_thirty_day - } - } + found_dates['%s/%s' % (count.date.month, count.date.day)] = True + + # Fill in any missing stats with zeros. + for day in range(-31, MAX_DAYS_IN_3_MONTHS): + day_date = datetime.now() - timedelta(days=day) + key = '%s/%s' % (day_date.month, day_date.day) + if not key in found_dates: + stats.append({ + 'date': day_date.date().isoformat(), + 'count': 0, + }) repo_data = { 'namespace': namespace, diff --git a/external_libraries.py b/external_libraries.py index 64304d698..383f03cd6 100644 --- a/external_libraries.py +++ b/external_libraries.py @@ -17,6 +17,7 @@ EXTERNAL_JS = [ 'cdnjs.cloudflare.com/ajax/libs/bootstrap-datetimepicker/4.17.37/js/bootstrap-datetimepicker.min.js', 'cdn.jsdelivr.net/g/bootbox@4.1.0,underscorejs@1.5.2,restangular@1.2.0,d3js@3.3.3', 'cdn.ravenjs.com/3.1.0/angular/raven.min.js', + 'cdn.jsdelivr.net/cal-heatmap/3.3.10/cal-heatmap.min.js', ] EXTERNAL_CSS = [ @@ -25,6 +26,7 @@ EXTERNAL_CSS = [ 'fonts.googleapis.com/css?family=Source+Sans+Pro:300,400,700', 's3.amazonaws.com/cdn.core-os.net/icons/core-icons.css', 'cdnjs.cloudflare.com/ajax/libs/bootstrap-datetimepicker/4.17.37/css/bootstrap-datetimepicker.min.css', + 'cdn.jsdelivr.net/cal-heatmap/3.3.10/cal-heatmap.css', ] EXTERNAL_FONTS = [ diff --git a/static/css/directives/repo-view/repo-panel-info.css b/static/css/directives/repo-view/repo-panel-info.css index a17c53657..61def4469 100644 --- a/static/css/directives/repo-view/repo-panel-info.css +++ b/static/css/directives/repo-view/repo-panel-info.css @@ -56,6 +56,7 @@ .repo-panel-info-element .stat-col { border-right: 2px solid #eee; + padding-right: 26px; } .repo-panel-info-element .stat-title { @@ -65,11 +66,21 @@ margin-bottom: 10px; } -.repo-panel-info-element .stat { +.repo-panel-info-element .stat-row { text-align: center; margin-bottom: 20px; } +.repo-panel-info-element .stat-row .heatmap { + margin-top: 20px; +} + +.repo-panel-info-element .stat { + display: inline-block; + margin-left: 20px; + margin-right: 20px; +} + .repo-panel-info-element .stat .stat-value { font-size: 46px; } diff --git a/static/css/directives/ui/heatmap.css b/static/css/directives/ui/heatmap.css new file mode 100644 index 000000000..ab5e27960 --- /dev/null +++ b/static/css/directives/ui/heatmap.css @@ -0,0 +1,17 @@ +.heatmap-element { + display: inline-block; +} + +.heatmap-element svg { + overflow: visible; +} + +.heatmap-element .cal-heatmap-container { + display: inline-block !important; +} + +.heatmap-element .graph-label { + font-family: 'Source Sans Pro', sans-serif; + font-size: 12px; + text-transform: uppercase; +} \ No newline at end of file diff --git a/static/directives/heatmap.html b/static/directives/heatmap.html new file mode 100644 index 000000000..17c205e23 --- /dev/null +++ b/static/directives/heatmap.html @@ -0,0 +1 @@ +
\ No newline at end of file diff --git a/static/directives/repo-view/repo-panel-info.html b/static/directives/repo-view/repo-panel-info.html index aafa4de26..6605fecf7 100644 --- a/static/directives/repo-view/repo-panel-info.html +++ b/static/directives/repo-view/repo-panel-info.html @@ -1,38 +1,29 @@
- -
-
Repo Pulls
+ +
+
Repository Activity
-
-
{{ repository.stats.pulls.today | abbreviated }}
-
Last 24 hours
-
+
+ -
-
{{ repository.stats.pulls.thirty_day | abbreviated }}
-
Last 30 days
-
-
+
+
{{ getAggregatedUsage(repository.stats, 1) | abbreviated }}
+
Yesterday
+
- -
-
Repo Pushes
- -
-
{{ repository.stats.pushes.today | abbreviated }}
-
Last 24 hours
-
- -
-
{{ repository.stats.pushes.thirty_day | abbreviated }}
-
Last 30 days
+
+
{{ getAggregatedUsage(repository.stats, 30) | abbreviated }}
+
Last 30 days
+
-
+
Recent Repo Builds
diff --git a/static/js/directives/repo-view/repo-panel-info.js b/static/js/directives/repo-view/repo-panel-info.js index f11894732..26fc92781 100644 --- a/static/js/directives/repo-view/repo-panel-info.js +++ b/static/js/directives/repo-view/repo-panel-info.js @@ -27,6 +27,21 @@ angular.module('quay').directive('repoPanelInfo', function () { $scope.repository.description = content; $scope.repository.put(); }; + + $scope.getAggregatedUsage = function(stats, days) { + var count = 0; + var startDate = moment().subtract(days + 1, 'days'); + for (var i = 0; i < stats.length; ++i) { + var stat = stats[i]; + var statDate = moment(stat['date']); + if (statDate.isBefore(startDate)) { + continue; + } + + count += stat['count']; + } + return count; + }; } }; return directiveDefinitionObject; diff --git a/static/js/directives/ui/heatmap.js b/static/js/directives/ui/heatmap.js new file mode 100644 index 000000000..c035b25ba --- /dev/null +++ b/static/js/directives/ui/heatmap.js @@ -0,0 +1,67 @@ +/** + * An element which displays a date+count heatmap. + */ +angular.module('quay').directive('heatmap', function () { + var directiveDefinitionObject = { + priority: 0, + templateUrl: '/static/directives/heatmap.html', + replace: false, + transclude: true, + restrict: 'C', + scope: { + 'data': '=data', + 'startCount': '@startCount', + 'startDomain': '@startDomain', + + 'itemName': '@itemName', + 'domain': '@domain', + 'range': '@range' + }, + controller: function($scope, $element, $timeout) { + var cal = null; + + var refresh = function() { + var data = $scope.data; + if (!data) { return; } + + if (!cal) { + var start = moment().add($scope.startCount * 1, $scope.startDomain).toDate(); + + cal = new CalHeatMap(); + cal.init({ + itemName: $scope.itemName, + domain: $scope.domain, + range: $scope.range * 1, + + start: start, + itemSelector: $element.find('.heatmap-element')[0], + cellSize: 15, + domainMargin: [10, 10, 10, 10], + displayLegend: false, + tooltip: true, + legendColors: { + empty: "#f4f4f4", + min: "#c9e9fb", + max: "steelblue", + } + }); + } + + cal.update(formatData(data)); + }; + + var formatData = function(data) { + var timestamps = {}; + data.forEach(function(entry) { + timestamps[moment(entry.date).unix()] = entry.count; + }); + return timestamps; + }; + + $scope.$watch('data', function() { + $timeout(refresh, 500); + }); + } + }; + return directiveDefinitionObject; +}); \ No newline at end of file