Merge pull request #2082 from coreos-inc/moar-stats

Add new metrics as requested by some customers

endpoints/v1/index.py

@@ -234,7 +234,7 @@ def update_images(namespace_name, repo_name):
 
     track_and_log('push_repo', repo)
     spawn_notification(repo, 'repo_push', event_data)
-    metric_queue.repository_push.Inc(labelvalues=[namespace_name, repo_name, 'v1'])
+    metric_queue.repository_push.Inc(labelvalues=[namespace_name, repo_name, 'v1', True])
     return make_response('Updated', 204)
 
   abort(403)
@@ -261,7 +261,7 @@ def get_repository_images(namespace_name, repo_name):
     resp.mimetype = 'application/json'
 
     track_and_log('pull_repo', repo, analytics_name='pull_repo_100x', analytics_sample=0.01)
-    metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v1'])
+    metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v1', True])
     return resp
 
   abort(403)

endpoints/v1/registry.py

@@ -7,7 +7,7 @@ from time import time
 
 from flask import make_response, request, session, Response, redirect, abort as flask_abort
 
-from app import storage as store, app
+from app import storage as store, app, metric_queue
 from auth.auth_context import get_authenticated_user
 from auth.permissions import (ReadRepositoryPermission,
                               ModifyRepositoryPermission)
@@ -197,7 +197,10 @@ def put_image_layer(namespace, repository, image_id):
   locations, path = model.placement_locations_and_path_docker_v1(namespace, repository, image_id)
   with database.CloseForLongOperation(app.config):
     try:
+      start_time = time()
       store.stream_write(locations, path, sr)
+      metric_queue.chunk_upload_time.Observe(time() - start_time,
+                         labelvalues=[size_info.compressed_size, list(locations)[0]])
     except IOError:
       logger.exception('Exception when writing image data')
       abort(520, 'Image %(image_id)s could not be written. Please try again.', image_id=image_id)

endpoints/v2/blob.py

@@ -1,11 +1,12 @@
 import logging
 import re
+import time
 
 from flask import url_for, request, redirect, Response, abort as flask_abort
 
 import resumablehashlib
 
-from app import storage, app, get_app_url
+from app import storage, app, get_app_url, metric_queue
 from auth.registry_jwt_auth import process_registry_jwt_auth
 from data import database
 from data.interfaces.v2 import pre_oci_model as model
@@ -398,6 +399,7 @@ def _upload_chunk(blob_upload, range_header):
       size_info, fn = calculate_size_handler()
       input_fp = wrap_with_handler(input_fp, fn)
 
+    start_time = time.time()
     length_written, new_metadata, upload_error = storage.stream_upload_chunk(
       location_set,
       blob_upload.uuid,
@@ -412,6 +414,10 @@ def _upload_chunk(blob_upload, range_header):
       logger.error('storage.stream_upload_chunk returned error %s', upload_error)
       return None
 
+    # Update the chunk upload time metric.
+    metric_queue.chunk_upload_time.Observe(time.time() - start_time,
+                       labelvalues=[length_written, list(location_set)[0]])
+
   # If we determined an uncompressed size and this is the first chunk, add it to the blob.
   # Otherwise, we clear the size from the blob as it was uploaded in multiple chunks.
   if size_info is not None and blob_upload.chunk_count == 0 and size_info.is_valid:
@@ -428,6 +434,7 @@ def _upload_chunk(blob_upload, range_header):
   blob_upload.storage_metadata = new_metadata
   blob_upload.byte_count += length_written
   blob_upload.chunk_count += 1
+
   return blob_upload
 
 

endpoints/v2/manifest.py

@@ -51,7 +51,7 @@ def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
   repo = model.get_repository(namespace_name, repo_name)
   if repo is not None:
     track_and_log('pull_repo', repo, analytics_name='pull_repo_100x', analytics_sample=0.01)
-    metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2'])
+    metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2', True])
 
   return Response(
     manifest.json,
@@ -74,7 +74,7 @@ def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref):
   repo = model.get_repository(namespace_name, repo_name)
   if repo is not None:
     track_and_log('pull_repo', repo)
-    metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2'])
+    metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2', True])
 
   return Response(manifest.json, status=200, headers={'Content-Type': manifest.media_type,
                                                       'Docker-Content-Digest': manifest.digest})
@@ -204,7 +204,7 @@ def _write_manifest_and_log(namespace_name, repo_name, manifest):
 
   track_and_log('push_repo', repo, tag=manifest.tag)
   spawn_notification(repo, 'repo_push', {'updated_tags': [manifest.tag]})
-  metric_queue.repository_push.Inc(labelvalues=[namespace_name, repo_name, 'v2'])
+  metric_queue.repository_push.Inc(labelvalues=[namespace_name, repo_name, 'v2', True])
 
   return Response(
     'OK',

endpoints/verbs/__init__.py

@@ -212,12 +212,12 @@ def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=
   # Check for torrent. If found, we return a torrent for the repo verb image (if the derived
   # image already exists).
   if request.accept_mimetypes.best == 'application/x-bittorrent':
-    metric_queue.repository_pull.Inc(labelvalues=[namespace, repository, verb + '+torrent'])
+    metric_queue.repository_pull.Inc(labelvalues=[namespace, repository, verb + '+torrent', True])
     return _torrent_repo_verb(repo_image, tag, verb, **kwargs)
 
   # Log the action.
   track_and_log('repo_verb', repo_image.repository, tag=tag, verb=verb, **kwargs)
-  metric_queue.repository_pull.Inc(labelvalues=[namespace, repository, verb])
+  metric_queue.repository_pull.Inc(labelvalues=[namespace, repository, verb, True])
 
   # Lookup/create the derived image for the verb and repo image.
   derived_image = model.lookup_or_create_derived_image(repo_image, verb,
@@ -360,5 +360,5 @@ def get_tag_torrent(namespace_name, repo_name, digest):
   if blob is None:
     abort(404)
 
-  metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'torrent'])
+  metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'torrent', True])
   return _torrent_for_blob(blob, public_repo)