Change repo stats to use the RAC table and a nice UI

data/model/log.py

@@ -5,7 +5,7 @@ from peewee import JOIN_LEFT_OUTER, SQL, fn
 from datetime import datetime, timedelta, date
 from cachetools import lru_cache
 
-from data.database import LogEntry, LogEntryKind, User, db
+from data.database import LogEntry, LogEntryKind, User, RepositoryActionCount, db
 from data.model import config
 
 def _logs_query(selections, start_time, end_time, performer=None, repository=None, namespace=None,
@@ -95,62 +95,6 @@ def log_action(kind_name, user_or_organization_name, performer=None, repository=
                   datetime=timestamp)
 
 
-def _get_repository_events(repository, time_delta, time_delta_earlier, clause):
-  """ Returns a pair representing the count of the number of events for the given
-      repository in each of the specified time deltas. The date ranges are calculated by
-      taking the current time today and subtracting the time delta given. Since
-      we want to grab *two* ranges, we restrict the second range to be greater
-      than the first (i.e. referring to an earlier time), so we can conduct the
-      lookup in a single query. The clause is used to further filter the kind of
-      events being found.
-  """
-  since = date.today() - time_delta
-  since_earlier = date.today() - time_delta_earlier
-
-  if since_earlier >= since:
-    raise ValueError('time_delta_earlier must be greater than time_delta')
-
-  # This uses a CASE WHEN inner clause to further filter the count.
-  formatted = since.strftime('%Y-%m-%d')
-  case_query = 'CASE WHEN datetime >= \'%s\' THEN 1 ELSE 0 END' % formatted
-
-  result = (LogEntry
-            .select(fn.Sum(SQL(case_query)), fn.Count(SQL('*')))
-            .where(LogEntry.repository == repository)
-            .where(clause)
-            .where(LogEntry.datetime >= since_earlier)
-            .tuples()
-            .get())
-
-  return (int(result[0]) if result[0] else 0, int(result[1]) if result[1] else 0)
-
-
-def get_repository_pushes(repository, time_delta, time_delta_earlier):
-  push_repo = _get_log_entry_kind('push_repo')
-  clauses = (LogEntry.kind == push_repo)
-  return _get_repository_events(repository, time_delta, time_delta_earlier, clauses)
-
-
-def get_repository_pulls(repository, time_delta, time_delta_earlier):
-  repo_pull = _get_log_entry_kind('pull_repo')
-  repo_verb = _get_log_entry_kind('repo_verb')
-  clauses = ((LogEntry.kind == repo_pull) | (LogEntry.kind == repo_verb))
-  return _get_repository_events(repository, time_delta, time_delta_earlier, clauses)
-
-
-def get_repository_usage():
-  one_month_ago = date.today() - timedelta(weeks=4)
-  repo_pull = _get_log_entry_kind('pull_repo')
-  repo_verb = _get_log_entry_kind('repo_verb')
-  return (LogEntry
-          .select(LogEntry.ip, LogEntry.repository)
-          .where((LogEntry.kind == repo_pull) | (LogEntry.kind == repo_verb))
-          .where(~(LogEntry.repository >> None))
-          .where(LogEntry.datetime >= one_month_ago)
-          .group_by(LogEntry.ip, LogEntry.repository)
-          .count())
-
-
 def get_stale_logs_start_id():
   """ Gets the oldest log entry. """
   try:
@@ -182,3 +126,28 @@ def get_stale_logs(start_id, end_id):
 def delete_stale_logs(start_id, end_id):
   """ Deletes all the logs with IDs between start_id and end_id. """
   LogEntry.delete().where((LogEntry.id >= start_id), (LogEntry.id <= end_id)).execute()
+
+
+def get_repository_action_counts(repo, start_date):
+  return RepositoryActionCount.select().where(RepositoryActionCount.repository == repo,
+                                              RepositoryActionCount.date >= start_date)
+
+
+def get_repositories_action_sums(repository_ids):
+  if not repository_ids:
+    return {}
+
+  # Filter the join to recent entries only.
+  last_week = datetime.now() - timedelta(weeks=1)
+  tuples = (RepositoryActionCount
+            .select(RepositoryActionCount.repository, fn.Sum(RepositoryActionCount.count))
+            .where(RepositoryActionCount.repository << repository_ids)
+            .where(RepositoryActionCount.date >= last_week)
+            .group_by(RepositoryActionCount.repository)
+            .tuples())
+
+  action_count_map = {}
+  for record in tuples:
+    action_count_map[record[0]] = record[1]
+
+  return action_count_map

data/model/repository.py

@@ -3,9 +3,9 @@ import logging
 from peewee import JOIN_LEFT_OUTER, fn
 from datetime import timedelta, datetime
 
-from data.model import (DataModelException, tag, db_transaction, storage, image, permission,
+from data.model import (DataModelException, tag, db_transaction, storage, permission,
                         _basequery, config)
-from data.database import (Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User,
+from data.database import (Repository, Namespace, RepositoryTag, Star, Image, User,
                            Visibility, RepositoryPermission, TupleSelector, RepositoryActionCount,
                            Role, RepositoryAuthorizedEmail, TagManifest, DerivedStorageForImage,
                            db_for_update, get_epoch_timestamp, db_random_func)
@@ -227,26 +227,6 @@ def get_when_last_modified(repository_ids):
   return last_modified_map
 
 
-def get_action_counts(repository_ids):
-  if not repository_ids:
-    return {}
-
-  # Filter the join to recent entries only.
-  last_week = datetime.now() - timedelta(weeks=1)
-  tuples = (RepositoryActionCount
-            .select(RepositoryActionCount.repository, fn.Sum(RepositoryActionCount.count))
-            .where(RepositoryActionCount.repository << repository_ids)
-            .where(RepositoryActionCount.date >= last_week)
-            .group_by(RepositoryActionCount.repository)
-            .tuples())
-
-  action_count_map = {}
-  for record in tuples:
-    action_count_map[record[0]] = record[1]
-
-  return action_count_map
-
-
 def get_visible_repositories(username, namespace=None, include_public=False):
   """ Returns the repositories visible to the given user (if any).
   """

endpoints/api/repository.py

@@ -4,12 +4,11 @@ import logging
 import datetime
 import features
 
-from datetime import timedelta
+from datetime import timedelta, datetime
 
 from flask import request, abort
 
 from data import model
-from data.database import Repository as RepositoryTable
 from endpoints.api import (truthy_bool, format_date, nickname, log_action, validate_json_request,
                            require_repo_read, require_repo_write, require_repo_admin,
                            RepositoryParamResource, resource, query_param, parse_args, ApiResource,
@@ -29,6 +28,7 @@ from util.names import REPOSITORY_NAME_REGEX
 logger = logging.getLogger(__name__)
 
 REPOS_PER_PAGE = 100
+MAX_DAYS_IN_3_MONTHS = 92
 
 def check_allowed_private_repos(namespace):
   """ Checks to see if the given namespace has reached its private repository limit. If so,
@@ -180,7 +180,7 @@ class RepositoryList(ApiResource):
         last_modified_map = model.repository.get_when_last_modified(repository_ids)
 
       if parsed_args['popularity']:
-        action_count_map = model.repository.get_action_counts(repository_ids)
+        action_sum_map = model.log.get_repositories_action_sums(repository_ids)
 
     # Collect the IDs of the repositories that are starred for the user, so we can mark them
     # in the returned results.
@@ -203,7 +203,7 @@ class RepositoryList(ApiResource):
         repo['last_modified'] = last_modified_map.get(repo_id)
 
       if parsed_args['popularity']:
-        repo['popularity'] = action_count_map.get(repo_id, 0)
+        repo['popularity'] = action_sum_map.get(repo_id, 0)
 
       if username:
         repo['is_starred'] = repo_id in star_set
@@ -236,7 +236,7 @@ class Repository(RepositoryParamResource):
   }
 
   @parse_args()
-  @query_param('includeStats', 'Whether to include pull and push statistics', type=truthy_bool,
+  @query_param('includeStats', 'Whether to include action statistics', type=truthy_bool,
                default=False)
   @require_repo_read
   @nickname('getRepo')
@@ -252,7 +252,7 @@ class Repository(RepositoryParamResource):
       }
 
       if tag.lifetime_start_ts > 0:
-        last_modified = format_date(datetime.datetime.fromtimestamp(tag.lifetime_start_ts))
+        last_modified = format_date(datetime.fromtimestamp(tag.lifetime_start_ts))
         tag_info['last_modified'] = last_modified
 
       return tag_info
@@ -270,22 +270,28 @@ class Repository(RepositoryParamResource):
       is_public = model.repository.is_repository_public(repo)
 
       if parsed_args['includeStats']:
-        (pull_today, pull_thirty_day) = model.log.get_repository_pulls(repo, timedelta(days=1),
+        stats = []
-                                                                       timedelta(days=30))
+        found_dates = {}
 
-        (push_today, push_thirty_day) = model.log.get_repository_pushes(repo, timedelta(days=1),
+        start_date = datetime.now() - timedelta(days=MAX_DAYS_IN_3_MONTHS)
-                                                                        timedelta(days=30))
+        counts = model.log.get_repository_action_counts(repo, start_date)
+        for count in counts:
+          stats.append({
+            'date': count.date.isoformat(),
+            'count': count.count,
+          })
 
-        stats = {
+          found_dates['%s/%s' % (count.date.month, count.date.day)] = True
-          'pulls': {
+
-            'today': pull_today,
+        # Fill in any missing stats with zeros.
-            'thirty_day': pull_thirty_day
+        for day in range(-31, MAX_DAYS_IN_3_MONTHS):
-          },
+          day_date = datetime.now() - timedelta(days=day)
-          'pushes': {
+          key = '%s/%s' % (day_date.month, day_date.day)
-            'today': push_today,
+          if not key in found_dates:
-            'thirty_day': push_thirty_day
+            stats.append({
-          }
+              'date': day_date.date().isoformat(),
-        }
+              'count': 0,
+            })
 
       repo_data = {
         'namespace': namespace,

external_libraries.py

@@ -17,6 +17,7 @@ EXTERNAL_JS = [
   'cdnjs.cloudflare.com/ajax/libs/bootstrap-datetimepicker/4.17.37/js/bootstrap-datetimepicker.min.js',
   'cdn.jsdelivr.net/g/bootbox@4.1.0,underscorejs@1.5.2,restangular@1.2.0,d3js@3.3.3',
   'cdn.ravenjs.com/3.1.0/angular/raven.min.js',
+  'cdn.jsdelivr.net/cal-heatmap/3.3.10/cal-heatmap.min.js',
 ]
 
 EXTERNAL_CSS = [
@@ -25,6 +26,7 @@ EXTERNAL_CSS = [
   'fonts.googleapis.com/css?family=Source+Sans+Pro:300,400,700',
   's3.amazonaws.com/cdn.core-os.net/icons/core-icons.css',
   'cdnjs.cloudflare.com/ajax/libs/bootstrap-datetimepicker/4.17.37/css/bootstrap-datetimepicker.min.css',
+  'cdn.jsdelivr.net/cal-heatmap/3.3.10/cal-heatmap.css',
 ]
 
 EXTERNAL_FONTS = [

static/css/directives/repo-view/repo-panel-info.css

@@ -56,6 +56,7 @@
 
 .repo-panel-info-element .stat-col {
   border-right: 2px solid #eee;
+  padding-right: 26px;
 }
 
 .repo-panel-info-element .stat-title {
@@ -65,11 +66,21 @@
   margin-bottom: 10px;
 }
 
-.repo-panel-info-element .stat {
+.repo-panel-info-element .stat-row {
   text-align: center;
   margin-bottom: 20px;
 }
 
+.repo-panel-info-element .stat-row .heatmap {
+  margin-top: 20px;
+}
+
+.repo-panel-info-element .stat {
+  display: inline-block;
+  margin-left: 20px;
+  margin-right: 20px;
+}
+
 .repo-panel-info-element .stat .stat-value {
   font-size: 46px;
 }

static/css/directives/ui/heatmap.css

@@ -0,0 +1,17 @@
+.heatmap-element {
+    display: inline-block;
+}
+
+.heatmap-element svg {
+    overflow: visible;
+}
+
+.heatmap-element .cal-heatmap-container {
+    display: inline-block !important;
+}
+
+.heatmap-element .graph-label {
+    font-family: 'Source Sans Pro', sans-serif;
+    font-size: 12px;
+    text-transform: uppercase;
+}

static/directives/heatmap.html

@@ -0,0 +1 @@
+<div class="heatmap-element"></div>

static/directives/repo-view/repo-panel-info.html

@@ -1,38 +1,29 @@
 <div class="repo-panel-info-element">
   <!-- Repository stats and builds summary -->
   <div class="repository-stats row">
-    <!-- Pull Stats -->
+    <!-- Stats -->
-    <div class="col-sm-3 stat-col">
+    <div class="col-sm-5 stat-col">
-      <div class="stat-title">Repo Pulls</div>
+      <div class="stat-title">Repository Activity</div>
 
-      <div class="stat">
+      <div class="stat-row">
-        <div class="stat-value">{{ repository.stats.pulls.today | abbreviated }}</div>
+        <div class="heatmap hidden-xs hidden-sm" data="repository.stats"
-        <div class="stat-subtitle">Last 24 hours</div>
+             item-name="action" domain="month" range="3"
-      </div>
+             start-count="-2" start-domain="months"></div>
 
-      <div class="stat">
+        <div class="stat visible-xs visible-sm">
-        <div class="stat-value">{{ repository.stats.pulls.thirty_day | abbreviated }}</div>
+          <div class="stat-value">{{ getAggregatedUsage(repository.stats, 1) | abbreviated }}</div>
-        <div class="stat-subtitle">Last 30 days</div>
+          <div class="stat-subtitle">Yesterday</div>
-      </div>
+        </div>
-    </div>
 
-    <!-- Push Stats -->
+        <div class="stat visible-xs visible-sm">
-    <div class="col-sm-3 stat-col">
+          <div class="stat-value">{{ getAggregatedUsage(repository.stats, 30) | abbreviated }}</div>
-      <div class="stat-title">Repo Pushes</div>
+          <div class="stat-subtitle">Last 30 days</div>
-
+        </div>
-      <div class="stat">
-        <div class="stat-value">{{ repository.stats.pushes.today | abbreviated }}</div>
-        <div class="stat-subtitle">Last 24 hours</div>
-      </div>
-
-      <div class="stat">
-        <div class="stat-value">{{ repository.stats.pushes.thirty_day | abbreviated }}</div>
-        <div class="stat-subtitle">Last 30 days</div>
       </div>
     </div>
 
     <!-- Builds -->
-    <div class="col-sm-6 builds-list">
+    <div class="col-sm-7 builds-list">
       <div class="stat-title">Recent Repo Builds</div>
 
       <!-- Loading -->

static/js/directives/repo-view/repo-panel-info.js

@@ -27,6 +27,21 @@ angular.module('quay').directive('repoPanelInfo', function () {
         $scope.repository.description = content;
         $scope.repository.put();
       };
+
+      $scope.getAggregatedUsage = function(stats, days) {
+        var count = 0;
+        var startDate = moment().subtract(days + 1, 'days');
+        for (var i = 0; i < stats.length; ++i) {
+          var stat = stats[i];
+          var statDate = moment(stat['date']);
+          if (statDate.isBefore(startDate)) {
+            continue;
+          }
+
+          count += stat['count'];
+        }
+        return count;
+      };
     }
   };
   return directiveDefinitionObject;

static/js/directives/ui/heatmap.js

@@ -0,0 +1,67 @@
+/**
+ * An element which displays a date+count heatmap.
+ */
+angular.module('quay').directive('heatmap', function () {
+  var directiveDefinitionObject = {
+    priority: 0,
+    templateUrl: '/static/directives/heatmap.html',
+    replace: false,
+    transclude: true,
+    restrict: 'C',
+    scope: {
+      'data': '=data',
+      'startCount': '@startCount',
+      'startDomain': '@startDomain',
+
+      'itemName': '@itemName',
+      'domain': '@domain',
+      'range': '@range'
+    },
+    controller: function($scope, $element, $timeout) {
+      var cal = null;
+
+      var refresh = function() {
+        var data = $scope.data;
+        if (!data) { return; }
+
+        if (!cal) {
+          var start = moment().add($scope.startCount * 1, $scope.startDomain).toDate();
+
+          cal = new CalHeatMap();
+          cal.init({
+            itemName: $scope.itemName,
+            domain: $scope.domain,
+            range: $scope.range * 1,
+
+            start: start,
+            itemSelector: $element.find('.heatmap-element')[0],
+            cellSize: 15,
+            domainMargin: [10, 10, 10, 10],
+            displayLegend: false,
+            tooltip: true,
+            legendColors: {
+              empty: "#f4f4f4",
+              min: "#c9e9fb",
+              max: "steelblue",
+            }
+          });
+        }
+
+        cal.update(formatData(data));
+      };
+
+      var formatData = function(data) {
+        var timestamps = {};
+        data.forEach(function(entry) {
+          timestamps[moment(entry.date).unix()] = entry.count;
+        });
+        return timestamps;
+      };
+
+      $scope.$watch('data', function() {
+        $timeout(refresh, 500);
+      });
+    }
+  };
+  return directiveDefinitionObject;
+});