initial import for Open Source 🎉

data/model/test/test_appspecifictoken.py

@@ -0,0 +1,126 @@
+from datetime import datetime, timedelta
+from mock import patch
+
+import pytest
+
+from data.model import config as _config
+from data import model
+from data.model.appspecifictoken import create_token, revoke_token, access_valid_token
+from data.model.appspecifictoken import gc_expired_tokens, get_expiring_tokens
+from data.model.appspecifictoken import get_full_token_string
+from util.timedeltastring import convert_to_timedelta
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('expiration', [
+  (None),
+  ('-1m'),
+  ('-1d'),
+  ('-1w'),
+  ('10m'),
+  ('10d'),
+  ('10w'),
+])
+def test_gc(expiration, initialized_db):
+  user = model.user.get_user('devtable')
+
+  expiration_date = None
+  is_expired = False
+  if expiration:
+    if expiration[0] == '-':
+      is_expired = True
+      expiration_date = datetime.now() - convert_to_timedelta(expiration[1:])
+    else:
+      expiration_date = datetime.now() + convert_to_timedelta(expiration)
+
+  # Create a token.
+  token = create_token(user, 'Some token', expiration=expiration_date)
+
+  # GC tokens.
+  gc_expired_tokens(timedelta(seconds=0))
+
+  # Ensure the token was GCed if expired and not if it wasn't.
+  assert (access_valid_token(get_full_token_string(token)) is None) == is_expired
+
+
+def test_access_token(initialized_db):
+  user = model.user.get_user('devtable')
+
+  # Create a token.
+  token = create_token(user, 'Some token')
+  assert token.last_accessed is None
+
+  # Lookup the token.
+  token = access_valid_token(get_full_token_string(token))
+  assert token.last_accessed is not None
+
+  # Revoke the token.
+  revoke_token(token)
+
+  # Ensure it cannot be accessed
+  assert access_valid_token(get_full_token_string(token)) is None
+
+
+def test_expiring_soon(initialized_db):
+  user = model.user.get_user('devtable')
+
+  # Create some tokens.
+  create_token(user, 'Some token')
+  exp_token = create_token(user, 'Some expiring token', datetime.now() + convert_to_timedelta('1d'))
+  create_token(user, 'Some other token', expiration=datetime.now() + convert_to_timedelta('2d'))
+
+  # Get the token expiring soon.
+  expiring_soon = get_expiring_tokens(user, convert_to_timedelta('25h'))
+  assert expiring_soon
+  assert len(expiring_soon) == 1
+  assert expiring_soon[0].id == exp_token.id
+
+  expiring_soon = get_expiring_tokens(user, convert_to_timedelta('49h'))
+  assert expiring_soon
+  assert len(expiring_soon) == 2
+
+
+@pytest.fixture(scope='function')
+def app_config():
+  with patch.dict(_config.app_config, {}, clear=True):
+    yield _config.app_config
+
+@pytest.mark.parametrize('expiration', [
+  (None),
+  ('10m'),
+  ('10d'),
+  ('10w'),
+])
+@pytest.mark.parametrize('default_expiration', [
+  (None),
+  ('10m'),
+  ('10d'),
+  ('10w'),
+])
+def test_create_access_token(expiration, default_expiration, initialized_db, app_config):
+  user = model.user.get_user('devtable')
+  expiration_date = datetime.now() + convert_to_timedelta(expiration) if expiration else None
+  with patch.dict(_config.app_config, {}, clear=True):
+    app_config['APP_SPECIFIC_TOKEN_EXPIRATION'] = default_expiration
+    if expiration:
+      exp_token = create_token(user, 'Some token', expiration=expiration_date)
+      assert exp_token.expiration == expiration_date
+    else:
+      exp_token = create_token(user, 'Some token')
+      assert (exp_token.expiration is None) == (default_expiration is None)
+
+
+@pytest.mark.parametrize('invalid_token', [
+  '',
+  'foo',
+  'a' * 40,
+  'b' * 40,
+  '%s%s' % ('b' * 40, 'a' * 40),
+  '%s%s' % ('a' * 39, 'b' * 40),
+  '%s%s' % ('a' * 40, 'b' * 39),
+  '%s%s' % ('a' * 40, 'b' * 41),
+])
+def test_invalid_access_token(invalid_token, initialized_db):
+  user = model.user.get_user('devtable')
+  token = access_valid_token(invalid_token)
+  assert token is None

data/model/test/test_basequery.py

@@ -0,0 +1,107 @@
+import pytest
+
+from peewee import JOIN
+from playhouse.test_utils import assert_query_count
+
+from data.database import Repository, RepositoryPermission, TeamMember, Namespace
+from data.model._basequery import filter_to_repos_for_user
+from data.model.organization import get_admin_users
+from data.model.user import get_namespace_user
+from util.names import parse_robot_username
+
+from test.fixtures import *
+
+def _is_team_member(team, user):
+  return user.id in [member.user_id for member in
+                     TeamMember.select().where(TeamMember.team == team)]
+
+def _get_visible_repositories_for_user(user, repo_kind='image', include_public=False,
+                                       namespace=None):
+  """ Returns all repositories directly visible to the given user, by either repo permission,
+      or the user being the admin of a namespace.
+  """
+  for repo in Repository.select():
+    if repo_kind is not None and repo.kind.name != repo_kind:
+      continue
+
+    if namespace is not None and repo.namespace_user.username != namespace:
+      continue
+
+    if include_public and repo.visibility.name == 'public':
+      yield repo
+      continue
+
+    # Direct repo permission.
+    try:
+      RepositoryPermission.get(repository=repo, user=user).get()
+      yield repo
+      continue
+    except RepositoryPermission.DoesNotExist:
+      pass
+
+    # Team permission.
+    found_in_team = False
+    for perm in RepositoryPermission.select().where(RepositoryPermission.repository == repo):
+      if perm.team and _is_team_member(perm.team, user):
+        found_in_team = True
+        break
+
+    if found_in_team:
+      yield repo
+      continue
+
+    # Org namespace admin permission.
+    if user in get_admin_users(repo.namespace_user):
+      yield repo
+      continue
+
+
+@pytest.mark.parametrize('username', [
+  'devtable',
+  'devtable+dtrobot',
+  'public',
+  'reader',
+])
+@pytest.mark.parametrize('include_public', [
+  True,
+  False
+])
+@pytest.mark.parametrize('filter_to_namespace', [
+  True,
+  False
+])
+@pytest.mark.parametrize('repo_kind', [
+  None,
+  'image',
+  'application',
+])
+def test_filter_repositories(username, include_public, filter_to_namespace, repo_kind,
+                             initialized_db):
+  namespace = username if filter_to_namespace else None
+  if '+' in username and filter_to_namespace:
+    namespace, _ = parse_robot_username(username)
+
+  user = get_namespace_user(username)
+  query = (Repository
+           .select()
+           .distinct()
+           .join(Namespace, on=(Repository.namespace_user == Namespace.id))
+           .switch(Repository)
+           .join(RepositoryPermission, JOIN.LEFT_OUTER))
+
+  # Prime the cache.
+  Repository.kind.get_id('image')
+
+  with assert_query_count(1):
+    found = list(filter_to_repos_for_user(query, user.id,
+                                          namespace=namespace,
+                                          include_public=include_public,
+                                          repo_kind=repo_kind))
+
+  expected = list(_get_visible_repositories_for_user(user,
+                                                     repo_kind=repo_kind,
+                                                     namespace=namespace,
+                                                     include_public=include_public))
+
+  assert len(found) == len(expected)
+  assert {r.id for r in found} == {r.id for r in expected}

data/model/test/test_build.py

@@ -0,0 +1,107 @@
+import pytest
+
+from mock import patch
+
+from data.database import BUILD_PHASE, RepositoryBuildTrigger, RepositoryBuild
+from data.model.build import (update_trigger_disable_status, create_repository_build,
+                              get_repository_build, update_phase_then_close)
+from test.fixtures import *
+
+TEST_FAIL_THRESHOLD = 5
+TEST_INTERNAL_ERROR_THRESHOLD = 2
+
+@pytest.mark.parametrize('starting_failure_count, starting_error_count, status, expected_reason', [
+  (0, 0, BUILD_PHASE.COMPLETE, None),
+  (10, 10, BUILD_PHASE.COMPLETE, None),
+
+  (TEST_FAIL_THRESHOLD - 1, TEST_INTERNAL_ERROR_THRESHOLD - 1, BUILD_PHASE.COMPLETE, None),
+  (TEST_FAIL_THRESHOLD - 1, 0, BUILD_PHASE.ERROR, 'successive_build_failures'),
+  (0, TEST_INTERNAL_ERROR_THRESHOLD - 1, BUILD_PHASE.INTERNAL_ERROR,
+   'successive_build_internal_errors'),
+])
+def test_update_trigger_disable_status(starting_failure_count, starting_error_count, status,
+                                       expected_reason, initialized_db):
+  test_config = {
+    'SUCCESSIVE_TRIGGER_FAILURE_DISABLE_THRESHOLD': TEST_FAIL_THRESHOLD,
+    'SUCCESSIVE_TRIGGER_INTERNAL_ERROR_DISABLE_THRESHOLD': TEST_INTERNAL_ERROR_THRESHOLD,
+  }
+
+  trigger = model.build.list_build_triggers('devtable', 'building')[0]
+  trigger.successive_failure_count = starting_failure_count
+  trigger.successive_internal_error_count = starting_error_count
+  trigger.enabled = True
+  trigger.save()
+
+  with patch('data.model.config.app_config', test_config):
+    update_trigger_disable_status(trigger, status)
+    updated_trigger = RepositoryBuildTrigger.get(uuid=trigger.uuid)
+
+    assert updated_trigger.enabled == (expected_reason is None)
+
+    if expected_reason is not None:
+      assert updated_trigger.disabled_reason.name == expected_reason
+    else:
+      assert updated_trigger.disabled_reason is None
+      assert updated_trigger.successive_failure_count == 0
+      assert updated_trigger.successive_internal_error_count == 0
+
+
+def test_archivable_build_logs(initialized_db):
+  # Make sure there are no archivable logs.
+  result = model.build.get_archivable_build()
+  assert result is None
+
+  # Add a build that cannot (yet) be archived.
+  repo = model.repository.get_repository('devtable', 'simple')
+  token = model.token.create_access_token(repo, 'write')
+  created = RepositoryBuild.create(repository=repo, access_token=token,
+                                   phase=model.build.BUILD_PHASE.WAITING,
+                                   logs_archived=False, job_config='{}',
+                                   display_name='')
+
+  # Make sure there are no archivable logs.
+  result = model.build.get_archivable_build()
+  assert result is None
+
+  # Change the build to being complete.
+  created.phase = model.build.BUILD_PHASE.COMPLETE
+  created.save()
+
+  # Make sure we now find an archivable build.
+  result = model.build.get_archivable_build()
+  assert result.id == created.id
+
+
+def test_update_build_phase(initialized_db):
+  build = create_build(model.repository.get_repository("devtable", "building"))
+
+  repo_build = get_repository_build(build.uuid)
+
+  assert repo_build.phase == BUILD_PHASE.WAITING
+  assert update_phase_then_close(build.uuid, BUILD_PHASE.COMPLETE)
+
+  repo_build = get_repository_build(build.uuid)
+  assert repo_build.phase == BUILD_PHASE.COMPLETE
+
+  repo_build.delete_instance()
+  assert not update_phase_then_close(repo_build.uuid, BUILD_PHASE.PULLING)
+
+
+def create_build(repository):
+  new_token = model.token.create_access_token(repository, 'write', 'build-worker')
+  repo = 'ci.devtable.com:5000/%s/%s' % (repository.namespace_user.username, repository.name)
+  job_config = {
+    'repository': repo,
+    'docker_tags': ['latest'],
+    'build_subdir': '',
+    'trigger_metadata': {
+      'commit': '3482adc5822c498e8f7db2e361e8d57b3d77ddd9',
+      'ref': 'refs/heads/master',
+      'default_branch': 'master'
+    }
+  }
+  build = create_repository_build(repository, new_token, job_config,
+                                  '68daeebd-a5b9-457f-80a0-4363b882f8ea',
+                                  "build_name")
+  build.save()
+  return build

data/model/test/test_gc.py

@@ -0,0 +1,725 @@
+import hashlib
+import pytest
+
+from datetime import datetime, timedelta
+
+from mock import patch
+
+from app import storage, docker_v2_signing_key
+
+from contextlib import contextmanager
+from playhouse.test_utils import assert_query_count
+
+from freezegun import freeze_time
+
+from data import model, database
+from data.database import (Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel,
+                           ApprBlob, Manifest, TagManifestToManifest, ManifestBlob, Tag,
+                           TagToRepositoryTag)
+from data.model.oci.test.test_oci_manifest import create_manifest_for_testing
+from image.docker.schema1 import DockerSchema1ManifestBuilder
+from image.docker.schema2.manifest import DockerSchema2ManifestBuilder
+from image.docker.schemas import parse_manifest_from_bytes
+from util.bytes import Bytes
+
+from test.fixtures import *
+
+
+ADMIN_ACCESS_USER = 'devtable'
+PUBLIC_USER = 'public'
+
+REPO = 'somerepo'
+
+def _set_tag_expiration_policy(namespace, expiration_s):
+  namespace_user = model.user.get_user(namespace)
+  model.user.change_user_tag_expiration(namespace_user, expiration_s)
+
+
+@pytest.fixture()
+def default_tag_policy(initialized_db):
+  _set_tag_expiration_policy(ADMIN_ACCESS_USER, 0)
+  _set_tag_expiration_policy(PUBLIC_USER, 0)
+
+
+def create_image(docker_image_id, repository_obj, username):
+  preferred = storage.preferred_locations[0]
+  image = model.image.find_create_or_link_image(docker_image_id, repository_obj, username, {},
+                                                preferred)
+  image.storage.uploading = False
+  image.storage.save()
+
+  # Create derived images as well.
+  model.image.find_or_create_derived_storage(image, 'squash', preferred)
+  model.image.find_or_create_derived_storage(image, 'aci', preferred)
+
+  # Add some torrent info.
+  try:
+    database.TorrentInfo.get(storage=image.storage)
+  except database.TorrentInfo.DoesNotExist:
+    model.storage.save_torrent_info(image.storage, 1, 'helloworld')
+
+  # Add some additional placements to the image.
+  for location_name in ['local_eu']:
+    location = database.ImageStorageLocation.get(name=location_name)
+
+    try:
+      database.ImageStoragePlacement.get(location=location, storage=image.storage)
+    except:
+      continue
+
+    database.ImageStoragePlacement.create(location=location, storage=image.storage)
+
+  return image.storage
+
+
+def store_tag_manifest(namespace, repo_name, tag_name, image_id):
+  builder = DockerSchema1ManifestBuilder(namespace, repo_name, tag_name)
+  storage_id_map = {}
+  try:
+    image_storage = ImageStorage.select().where(~(ImageStorage.content_checksum >> None)).get()
+    builder.add_layer(image_storage.content_checksum, '{"id": "foo"}')
+    storage_id_map[image_storage.content_checksum] = image_storage.id
+  except ImageStorage.DoesNotExist:
+    pass
+
+  manifest = builder.build(docker_v2_signing_key)
+  manifest_row, _ = model.tag.store_tag_manifest_for_testing(namespace, repo_name, tag_name,
+                                                             manifest, image_id, storage_id_map)
+  return manifest_row
+
+
+def create_repository(namespace=ADMIN_ACCESS_USER, name=REPO, **kwargs):
+  user = model.user.get_user(namespace)
+  repo = model.repository.create_repository(namespace, name, user)
+
+  # Populate the repository with the tags.
+  image_map = {}
+  for tag_name in kwargs:
+    image_ids = kwargs[tag_name]
+    parent = None
+
+    for image_id in image_ids:
+      if not image_id in image_map:
+        image_map[image_id] = create_image(image_id, repo, namespace)
+
+      v1_metadata = {
+        'id': image_id,
+      }
+      if parent is not None:
+        v1_metadata['parent'] = parent.docker_image_id
+
+      # Set the ancestors for the image.
+      parent = model.image.set_image_metadata(image_id, namespace, name, '', '', '', v1_metadata,
+                                              parent=parent)
+
+    # Set the tag for the image.
+    tag_manifest = store_tag_manifest(namespace, name, tag_name, image_ids[-1])
+
+    # Add some labels to the tag.
+    model.label.create_manifest_label(tag_manifest, 'foo', 'bar', 'manifest')
+    model.label.create_manifest_label(tag_manifest, 'meh', 'grah', 'manifest')
+
+  return repo
+
+
+def gc_now(repository):
+  assert model.gc.garbage_collect_repo(repository)
+
+
+def delete_tag(repository, tag, perform_gc=True, expect_gc=True):
+  model.tag.delete_tag(repository.namespace_user.username, repository.name, tag)
+  if perform_gc:
+    assert model.gc.garbage_collect_repo(repository) == expect_gc
+
+
+def move_tag(repository, tag, docker_image_id, expect_gc=True):
+  model.tag.create_or_update_tag(repository.namespace_user.username, repository.name, tag,
+                                 docker_image_id)
+  assert model.gc.garbage_collect_repo(repository) == expect_gc
+
+
+def assert_not_deleted(repository, *args):
+  for docker_image_id in args:
+    assert model.image.get_image_by_id(repository.namespace_user.username, repository.name,
+                                       docker_image_id)
+
+
+def assert_deleted(repository, *args):
+  for docker_image_id in args:
+    try:
+      # Verify the image is missing when accessed by the repository.
+      model.image.get_image_by_id(repository.namespace_user.username, repository.name,
+                                  docker_image_id)
+    except model.DataModelException:
+      return
+
+    assert False, 'Expected image %s to be deleted' % docker_image_id
+
+
+def _get_dangling_storage_count():
+  storage_ids = set([current.id for current in ImageStorage.select()])
+  referenced_by_image = set([image.storage_id for image in Image.select()])
+  referenced_by_manifest = set([blob.blob_id for blob in ManifestBlob.select()])
+  referenced_by_derived = set([derived.derivative_id
+                               for derived in DerivedStorageForImage.select()])
+  return len(storage_ids - referenced_by_image - referenced_by_derived - referenced_by_manifest)
+
+
+def _get_dangling_label_count():
+  return len(_get_dangling_labels())
+
+
+def _get_dangling_labels():
+  label_ids = set([current.id for current in Label.select()])
+  referenced_by_manifest = set([mlabel.label_id for mlabel in TagManifestLabel.select()])
+  return label_ids - referenced_by_manifest
+
+
+def _get_dangling_manifest_count():
+  manifest_ids = set([current.id for current in Manifest.select()])
+  referenced_by_tag_manifest = set([tmt.manifest_id for tmt in TagManifestToManifest.select()])
+  return len(manifest_ids - referenced_by_tag_manifest)
+
+
+
+@contextmanager
+def assert_gc_integrity(expect_storage_removed=True, check_oci_tags=True):
+  """ Specialized assertion for ensuring that GC cleans up all dangling storages
+      and labels, invokes the callback for images removed and doesn't invoke the
+      callback for images *not* removed.
+  """
+  # Add a callback for when images are removed.
+  removed_image_storages = []
+  model.config.register_image_cleanup_callback(removed_image_storages.extend)
+
+  # Store the number of dangling storages and labels.
+  existing_storage_count = _get_dangling_storage_count()
+  existing_label_count = _get_dangling_label_count()
+  existing_manifest_count = _get_dangling_manifest_count()
+  yield
+
+  # Ensure the number of dangling storages, manifests and labels has not changed.
+  updated_storage_count = _get_dangling_storage_count()
+  assert updated_storage_count == existing_storage_count
+
+  updated_label_count = _get_dangling_label_count()
+  assert updated_label_count == existing_label_count, _get_dangling_labels()
+
+  updated_manifest_count = _get_dangling_manifest_count()
+  assert updated_manifest_count == existing_manifest_count
+
+  # Ensure that for each call to the image+storage cleanup callback, the image and its
+  # storage is not found *anywhere* in the database.
+  for removed_image_and_storage in removed_image_storages:
+    with pytest.raises(Image.DoesNotExist):
+      Image.get(id=removed_image_and_storage.id)
+
+    # Ensure that image storages are only removed if not shared.
+    shared = Image.select().where(Image.storage == removed_image_and_storage.storage_id).count()
+    if shared == 0:
+      shared = (ManifestBlob
+                .select()
+                .where(ManifestBlob.blob == removed_image_and_storage.storage_id)
+                .count())
+
+    if shared == 0:
+      with pytest.raises(ImageStorage.DoesNotExist):
+        ImageStorage.get(id=removed_image_and_storage.storage_id)
+
+      with pytest.raises(ImageStorage.DoesNotExist):
+        ImageStorage.get(uuid=removed_image_and_storage.storage.uuid)
+
+  # Ensure all CAS storage is in the storage engine.
+  preferred = storage.preferred_locations[0]
+  for storage_row in ImageStorage.select():
+    if storage_row.cas_path:
+      storage.get_content({preferred}, storage.blob_path(storage_row.content_checksum))
+
+  for blob_row in ApprBlob.select():
+    storage.get_content({preferred}, storage.blob_path(blob_row.digest))
+
+  # Ensure there are no danglings OCI tags.
+  if check_oci_tags:
+    oci_tags = {t.id for t in Tag.select()}
+    referenced_oci_tags = {t.tag_id for t in TagToRepositoryTag.select()}
+    assert not oci_tags - referenced_oci_tags
+
+  # Ensure all tags have valid manifests.
+  for manifest in {t.manifest for t in Tag.select()}:
+    # Ensure that the manifest's blobs all exist.
+    found_blobs = {b.blob.content_checksum
+                   for b in ManifestBlob.select().where(ManifestBlob.manifest == manifest)}
+
+    parsed = parse_manifest_from_bytes(Bytes.for_string_or_unicode(manifest.manifest_bytes),
+                                       manifest.media_type.name)
+    assert set(parsed.local_blob_digests) == found_blobs
+
+
+def test_has_garbage(default_tag_policy, initialized_db):
+  """ Remove all existing repositories, then add one without garbage, check, then add one with
+      garbage, and check again.
+  """
+  # Delete all existing repos.
+  for repo in database.Repository.select().order_by(database.Repository.id):
+    assert model.gc.purge_repository(repo.namespace_user.username, repo.name)
+
+  # Change the time machine expiration on the namespace.
+  (database.User
+   .update(removed_tag_expiration_s=1000000000)
+   .where(database.User.username == ADMIN_ACCESS_USER)
+   .execute())
+
+  # Create a repository without any garbage.
+  repository = create_repository(latest=['i1', 'i2', 'i3'])
+
+  # Ensure that no repositories are returned by the has garbage check.
+  assert model.repository.find_repository_with_garbage(1000000000) is None
+
+  # Delete a tag.
+  delete_tag(repository, 'latest', perform_gc=False)
+
+  # There should still not be any repositories with garbage, due to time machine.
+  assert model.repository.find_repository_with_garbage(1000000000) is None
+
+  # Change the time machine expiration on the namespace.
+  (database.User
+   .update(removed_tag_expiration_s=0)
+   .where(database.User.username == ADMIN_ACCESS_USER)
+   .execute())
+
+  # Now we should find the repository for GC.
+  repository = model.repository.find_repository_with_garbage(0)
+  assert repository is not None
+  assert repository.name == REPO
+
+  # GC the repository.
+  assert model.gc.garbage_collect_repo(repository)
+
+  # There should now be no repositories with garbage.
+  assert model.repository.find_repository_with_garbage(0) is None
+
+
+def test_find_garbage_policy_functions(default_tag_policy, initialized_db):
+  with assert_query_count(1):
+    one_policy = model.repository.get_random_gc_policy()
+    all_policies = model.repository._get_gc_expiration_policies()
+    assert one_policy in all_policies
+
+
+def test_one_tag(default_tag_policy, initialized_db):
+  """ Create a repository with a single tag, then remove that tag and verify that the repository
+      is now empty. """
+  with assert_gc_integrity():
+    repository = create_repository(latest=['i1', 'i2', 'i3'])
+    delete_tag(repository, 'latest')
+    assert_deleted(repository, 'i1', 'i2', 'i3')
+
+
+def test_two_tags_unshared_images(default_tag_policy, initialized_db):
+  """ Repository has two tags with no shared images between them. """
+  with assert_gc_integrity():
+    repository = create_repository(latest=['i1', 'i2', 'i3'], other=['f1', 'f2'])
+    delete_tag(repository, 'latest')
+    assert_deleted(repository, 'i1', 'i2', 'i3')
+    assert_not_deleted(repository, 'f1', 'f2')
+
+
+def test_two_tags_shared_images(default_tag_policy, initialized_db):
+  """ Repository has two tags with shared images. Deleting the tag should only remove the
+      unshared images.
+  """
+  with assert_gc_integrity():
+    repository = create_repository(latest=['i1', 'i2', 'i3'], other=['i1', 'f1'])
+    delete_tag(repository, 'latest')
+    assert_deleted(repository, 'i2', 'i3')
+    assert_not_deleted(repository, 'i1', 'f1')
+
+
+def test_unrelated_repositories(default_tag_policy, initialized_db):
+  """ Two repositories with different images. Removing the tag from one leaves the other's
+      images intact.
+  """
+  with assert_gc_integrity():
+    repository1 = create_repository(latest=['i1', 'i2', 'i3'], name='repo1')
+    repository2 = create_repository(latest=['j1', 'j2', 'j3'], name='repo2')
+
+    delete_tag(repository1, 'latest')
+
+    assert_deleted(repository1, 'i1', 'i2', 'i3')
+    assert_not_deleted(repository2, 'j1', 'j2', 'j3')
+
+
+def test_related_repositories(default_tag_policy, initialized_db):
+  """ Two repositories with shared images. Removing the tag from one leaves the other's
+      images intact.
+  """
+  with assert_gc_integrity():
+    repository1 = create_repository(latest=['i1', 'i2', 'i3'], name='repo1')
+    repository2 = create_repository(latest=['i1', 'i2', 'j1'], name='repo2')
+
+    delete_tag(repository1, 'latest')
+
+    assert_deleted(repository1, 'i3')
+    assert_not_deleted(repository2, 'i1', 'i2', 'j1')
+
+
+def test_inaccessible_repositories(default_tag_policy, initialized_db):
+  """ Two repositories under different namespaces should result in the images being deleted
+      but not completely removed from the database.
+  """
+  with assert_gc_integrity():
+    repository1 = create_repository(namespace=ADMIN_ACCESS_USER, latest=['i1', 'i2', 'i3'])
+    repository2 = create_repository(namespace=PUBLIC_USER, latest=['i1', 'i2', 'i3'])
+
+    delete_tag(repository1, 'latest')
+    assert_deleted(repository1, 'i1', 'i2', 'i3')
+    assert_not_deleted(repository2, 'i1', 'i2', 'i3')
+
+
+def test_many_multiple_shared_images(default_tag_policy, initialized_db):
+  """ Repository has multiple tags with shared images. Delete all but one tag.
+  """
+  with assert_gc_integrity():
+    repository = create_repository(latest=['i1', 'i2', 'i3', 'i4', 'i5', 'i6', 'i7', 'i8', 'j0'],
+                                   master=['i1', 'i2', 'i3', 'i4', 'i5', 'i6', 'i7', 'i8', 'j1'])
+
+    # Delete tag latest. Should only delete j0, since it is not shared.
+    delete_tag(repository, 'latest')
+
+    assert_deleted(repository, 'j0')
+    assert_not_deleted(repository, 'i1', 'i2', 'i3', 'i4', 'i5', 'i6', 'i7', 'i8', 'j1')
+
+    # Delete tag master. Should delete the rest of the images.
+    delete_tag(repository, 'master')
+
+    assert_deleted(repository, 'i1', 'i2', 'i3', 'i4', 'i5', 'i6', 'i7', 'i8', 'j1')
+
+
+def test_multiple_shared_images(default_tag_policy, initialized_db):
+  """ Repository has multiple tags with shared images. Selectively deleting the tags, and
+      verifying at each step.
+  """
+  with assert_gc_integrity():
+    repository = create_repository(latest=['i1', 'i2', 'i3'], other=['i1', 'f1', 'f2'],
+                                   third=['t1', 't2', 't3'], fourth=['i1', 'f1'])
+
+    # Current state:
+    # latest -> i3->i2->i1
+    # other -> f2->f1->i1
+    # third -> t3->t2->t1
+    # fourth -> f1->i1
+
+    # Delete tag other. Should delete f2, since it is not shared.
+    delete_tag(repository, 'other')
+    assert_deleted(repository, 'f2')
+    assert_not_deleted(repository, 'i1', 'i2', 'i3', 't1', 't2', 't3', 'f1')
+
+    # Current state:
+    # latest -> i3->i2->i1
+    # third -> t3->t2->t1
+    # fourth -> f1->i1
+
+    # Move tag fourth to i3. This should remove f1 since it is no longer referenced.
+    move_tag(repository, 'fourth', 'i3')
+    assert_deleted(repository, 'f1')
+    assert_not_deleted(repository, 'i1', 'i2', 'i3', 't1', 't2', 't3')
+
+    # Current state:
+    # latest -> i3->i2->i1
+    # third -> t3->t2->t1
+    # fourth -> i3->i2->i1
+
+    # Delete tag 'latest'. This should do nothing since fourth is on the same branch.
+    delete_tag(repository, 'latest')
+    assert_not_deleted(repository, 'i1', 'i2', 'i3', 't1', 't2', 't3')
+
+    # Current state:
+    # third -> t3->t2->t1
+    # fourth -> i3->i2->i1
+
+    # Delete tag 'third'. This should remove t1->t3.
+    delete_tag(repository, 'third')
+    assert_deleted(repository, 't1', 't2', 't3')
+    assert_not_deleted(repository, 'i1', 'i2', 'i3')
+
+    # Current state:
+    # fourth -> i3->i2->i1
+
+    # Add tag to i1.
+    move_tag(repository, 'newtag', 'i1', expect_gc=False)
+    assert_not_deleted(repository, 'i1', 'i2', 'i3')
+
+    # Current state:
+    # fourth -> i3->i2->i1
+    # newtag -> i1
+
+    # Delete tag 'fourth'. This should remove i2 and i3.
+    delete_tag(repository, 'fourth')
+    assert_deleted(repository, 'i2', 'i3')
+    assert_not_deleted(repository, 'i1')
+
+    # Current state:
+    # newtag -> i1
+
+    # Delete tag 'newtag'. This should remove the remaining image.
+    delete_tag(repository, 'newtag')
+    assert_deleted(repository, 'i1')
+
+    # Current state:
+    # (Empty)
+
+
+def test_empty_gc(default_tag_policy, initialized_db):
+  with assert_gc_integrity(expect_storage_removed=False):
+    repository = create_repository(latest=['i1', 'i2', 'i3'], other=['i1', 'f1', 'f2'],
+                                   third=['t1', 't2', 't3'], fourth=['i1', 'f1'])
+
+    assert not model.gc.garbage_collect_repo(repository)
+    assert_not_deleted(repository, 'i1', 'i2', 'i3', 't1', 't2', 't3', 'f1', 'f2')
+
+
+def test_time_machine_no_gc(default_tag_policy, initialized_db):
+  """ Repository has two tags with shared images. Deleting the tag should not remove any images
+  """
+  with assert_gc_integrity(expect_storage_removed=False):
+    repository = create_repository(latest=['i1', 'i2', 'i3'], other=['i1', 'f1'])
+    _set_tag_expiration_policy(repository.namespace_user.username, 60*60*24)
+
+    delete_tag(repository, 'latest', expect_gc=False)
+    assert_not_deleted(repository, 'i2', 'i3')
+    assert_not_deleted(repository, 'i1', 'f1')
+
+
+def test_time_machine_gc(default_tag_policy, initialized_db):
+  """ Repository has two tags with shared images. Deleting the second tag should cause the images
+      for the first deleted tag to gc.
+  """
+  now = datetime.utcnow()
+
+  with assert_gc_integrity():
+    with freeze_time(now):
+      repository = create_repository(latest=['i1', 'i2', 'i3'], other=['i1', 'f1'])
+
+      _set_tag_expiration_policy(repository.namespace_user.username, 1)
+
+      delete_tag(repository, 'latest', expect_gc=False)
+      assert_not_deleted(repository, 'i2', 'i3')
+      assert_not_deleted(repository, 'i1', 'f1')
+
+    with freeze_time(now + timedelta(seconds=2)):
+      # This will cause the images associated with latest to gc
+      delete_tag(repository, 'other')
+      assert_deleted(repository, 'i2', 'i3')
+      assert_not_deleted(repository, 'i1', 'f1')
+
+
+def test_images_shared_storage(default_tag_policy, initialized_db):
+  """ Repository with two tags, both with the same shared storage. Deleting the first
+      tag should delete the first image, but *not* its storage.
+  """
+  with assert_gc_integrity(expect_storage_removed=False):
+    repository = create_repository()
+
+    # Add two tags, each with their own image, but with the same storage.
+    image_storage = model.storage.create_v1_storage(storage.preferred_locations[0])
+
+    first_image = Image.create(docker_image_id='i1',
+                               repository=repository, storage=image_storage,
+                               ancestors='/')
+
+    second_image = Image.create(docker_image_id='i2',
+                                repository=repository, storage=image_storage,
+                                ancestors='/')
+
+    store_tag_manifest(repository.namespace_user.username, repository.name,
+                       'first', first_image.docker_image_id)
+
+    store_tag_manifest(repository.namespace_user.username, repository.name,
+                       'second', second_image.docker_image_id)
+
+    # Delete the first tag.
+    delete_tag(repository, 'first')
+    assert_deleted(repository, 'i1')
+    assert_not_deleted(repository, 'i2')
+
+
+def test_image_with_cas(default_tag_policy, initialized_db):
+  """ A repository with a tag pointing to an image backed by CAS. Deleting and GCing the tag
+      should result in the storage and its CAS data being removed.
+  """
+  with assert_gc_integrity(expect_storage_removed=True):
+    repository = create_repository()
+
+    # Create an image storage record under CAS.
+    content = 'hello world'
+    digest = 'sha256:' + hashlib.sha256(content).hexdigest()
+    preferred = storage.preferred_locations[0]
+    storage.put_content({preferred}, storage.blob_path(digest), content)
+
+    image_storage = database.ImageStorage.create(content_checksum=digest, uploading=False)
+    location = database.ImageStorageLocation.get(name=preferred)
+    database.ImageStoragePlacement.create(location=location, storage=image_storage)
+
+    # Ensure the CAS path exists.
+    assert storage.exists({preferred}, storage.blob_path(digest))
+
+    # Create the image and the tag.
+    first_image = Image.create(docker_image_id='i1',
+                               repository=repository, storage=image_storage,
+                               ancestors='/')
+
+    store_tag_manifest(repository.namespace_user.username, repository.name,
+                       'first', first_image.docker_image_id)
+
+    assert_not_deleted(repository, 'i1')
+
+    # Delete the tag.
+    delete_tag(repository, 'first')
+    assert_deleted(repository, 'i1')
+
+    # Ensure the CAS path is gone.
+    assert not storage.exists({preferred}, storage.blob_path(digest))
+
+
+def test_images_shared_cas(default_tag_policy, initialized_db):
+  """ A repository, each two tags, pointing to the same image, which has image storage
+      with the same *CAS path*, but *distinct records*. Deleting the first tag should delete the
+      first image, and its storage, but not the file in storage, as it shares its CAS path.
+  """
+  with assert_gc_integrity(expect_storage_removed=True):
+    repository = create_repository()
+
+    # Create two image storage records with the same content checksum.
+    content = 'hello world'
+    digest = 'sha256:' + hashlib.sha256(content).hexdigest()
+    preferred = storage.preferred_locations[0]
+    storage.put_content({preferred}, storage.blob_path(digest), content)
+
+    is1 = database.ImageStorage.create(content_checksum=digest, uploading=False)
+    is2 = database.ImageStorage.create(content_checksum=digest, uploading=False)
+
+    location = database.ImageStorageLocation.get(name=preferred)
+
+    database.ImageStoragePlacement.create(location=location, storage=is1)
+    database.ImageStoragePlacement.create(location=location, storage=is2)
+
+    # Ensure the CAS path exists.
+    assert storage.exists({preferred}, storage.blob_path(digest))
+
+    # Create two images in the repository, and two tags, each pointing to one of the storages.
+    first_image = Image.create(docker_image_id='i1',
+                               repository=repository, storage=is1,
+                               ancestors='/')
+
+    second_image = Image.create(docker_image_id='i2',
+                                repository=repository, storage=is2,
+                                ancestors='/')
+
+    store_tag_manifest(repository.namespace_user.username, repository.name,
+                       'first', first_image.docker_image_id)
+
+    store_tag_manifest(repository.namespace_user.username, repository.name,
+                       'second', second_image.docker_image_id)
+
+    assert_not_deleted(repository, 'i1', 'i2')
+
+    # Delete the first tag.
+    delete_tag(repository, 'first')
+    assert_deleted(repository, 'i1')
+    assert_not_deleted(repository, 'i2')
+
+    # Ensure the CAS path still exists.
+    assert storage.exists({preferred}, storage.blob_path(digest))
+
+
+def test_images_shared_cas_with_new_blob_table(default_tag_policy, initialized_db):
+  """ A repository with a tag and image that shares its CAS path with a record in the new Blob
+      table. Deleting the first tag should delete the first image, and its storage, but not the
+      file in storage, as it shares its CAS path with the blob row.
+  """
+  with assert_gc_integrity(expect_storage_removed=True):
+    repository = create_repository()
+
+    # Create two image storage records with the same content checksum.
+    content = 'hello world'
+    digest = 'sha256:' + hashlib.sha256(content).hexdigest()
+    preferred = storage.preferred_locations[0]
+    storage.put_content({preferred}, storage.blob_path(digest), content)
+
+    media_type = database.MediaType.get(name='text/plain')
+
+    is1 = database.ImageStorage.create(content_checksum=digest, uploading=False)
+    database.ApprBlob.create(digest=digest, size=0, media_type=media_type)
+
+    location = database.ImageStorageLocation.get(name=preferred)
+    database.ImageStoragePlacement.create(location=location, storage=is1)
+
+    # Ensure the CAS path exists.
+    assert storage.exists({preferred}, storage.blob_path(digest))
+
+    # Create the image in the repository, and the tag.
+    first_image = Image.create(docker_image_id='i1',
+                               repository=repository, storage=is1,
+                               ancestors='/')
+
+    store_tag_manifest(repository.namespace_user.username, repository.name,
+                       'first', first_image.docker_image_id)
+
+    assert_not_deleted(repository, 'i1')
+
+    # Delete the tag.
+    delete_tag(repository, 'first')
+    assert_deleted(repository, 'i1')
+
+    # Ensure the CAS path still exists, as it is referenced by the Blob table
+    assert storage.exists({preferred}, storage.blob_path(digest))
+
+
+def test_purge_repo(app):
+  """ Test that app registers delete_metadata function on repository deletions """
+  with assert_gc_integrity():
+    with patch('app.tuf_metadata_api') as mock_tuf:
+      model.gc.purge_repository("ns", "repo")
+      assert mock_tuf.delete_metadata.called_with("ns", "repo")
+
+
+def test_super_long_image_chain_gc(app, default_tag_policy):
+  """ Test that a super long chain of images all gets properly GCed. """
+  with assert_gc_integrity():
+    images = ['i%s' % i for i in range(0, 100)]
+    repository = create_repository(latest=images)
+    delete_tag(repository, 'latest')
+
+    # Ensure the repository is now empty.
+    assert_deleted(repository, *images)
+
+
+def test_manifest_v2_shared_config_and_blobs(app, default_tag_policy):
+  """ Test that GCing a tag that refers to a V2 manifest with the same config and some shared
+      blobs as another manifest ensures that the config blob and shared blob are NOT GCed.
+  """
+  repo = model.repository.create_repository('devtable', 'newrepo', None)
+  manifest1, built1 = create_manifest_for_testing(repo, differentiation_field='1',
+                                                  include_shared_blob=True)
+  manifest2, built2 = create_manifest_for_testing(repo, differentiation_field='2',
+                                                  include_shared_blob=True)
+
+  assert set(built1.local_blob_digests).intersection(built2.local_blob_digests)
+  assert built1.config.digest == built2.config.digest
+
+  # Create tags pointing to the manifests.
+  model.oci.tag.retarget_tag('tag1', manifest1)
+  model.oci.tag.retarget_tag('tag2', manifest2)
+
+  with assert_gc_integrity(expect_storage_removed=True, check_oci_tags=False):
+    # Delete tag2.
+    model.oci.tag.delete_tag(repo, 'tag2')
+    assert model.gc.garbage_collect_repo(repo)
+
+  # Ensure the blobs for manifest1 still all exist.
+  preferred = storage.preferred_locations[0]
+  for blob_digest in built1.local_blob_digests:
+    storage_row = ImageStorage.get(content_checksum=blob_digest)
+
+    assert storage_row.cas_path
+    storage.get_content({preferred}, storage.blob_path(storage_row.content_checksum))

data/model/test/test_image.py

@@ -0,0 +1,104 @@
+import pytest
+
+from collections import defaultdict
+from data.model import image, repository
+from playhouse.test_utils import assert_query_count
+
+from test.fixtures import *
+
+@pytest.fixture()
+def images(initialized_db):
+  images = image.get_repository_images('devtable', 'simple')
+  assert len(images)
+  return images
+
+
+def test_get_image_with_storage(images, initialized_db):
+  for current in images:
+    storage_uuid = current.storage.uuid
+
+    with assert_query_count(1):
+      retrieved = image.get_image_with_storage(current.docker_image_id, storage_uuid)
+      assert retrieved.id == current.id
+      assert retrieved.storage.uuid == storage_uuid
+
+
+def test_get_parent_images(images, initialized_db):
+  for current in images:
+    if not len(current.ancestor_id_list()):
+      continue
+
+    with assert_query_count(1):
+      parent_images = list(image.get_parent_images('devtable', 'simple', current))
+
+    assert len(parent_images) == len(current.ancestor_id_list())
+    assert set(current.ancestor_id_list()) == {i.id for i in parent_images}
+
+    for parent in parent_images:
+      with assert_query_count(0):
+        assert parent.storage.id
+
+
+def test_get_image(images, initialized_db):
+  for current in images:
+    repo = current.repository
+
+    with assert_query_count(1):
+      found = image.get_image(repo, current.docker_image_id)
+
+    assert found.id == current.id
+
+
+def test_placements(images, initialized_db):
+  with assert_query_count(1):
+    placements_map = image.get_placements_for_images(images)
+
+  for current in images:
+    assert current.storage.id in placements_map
+
+    with assert_query_count(2):
+      expected_image, expected_placements = image.get_image_and_placements('devtable', 'simple',
+                                                                           current.docker_image_id)
+
+    assert expected_image.id == current.id
+    assert len(expected_placements) == len(placements_map.get(current.storage.id))
+    assert ({p.id for p in expected_placements} == 
+            {p.id for p in placements_map.get(current.storage.id)})
+
+
+def test_get_repo_image(images, initialized_db):
+  for current in images:
+    with assert_query_count(1):
+      found = image.get_repo_image('devtable', 'simple', current.docker_image_id)
+
+    assert found.id == current.id
+    with assert_query_count(1):
+      assert found.storage.id
+
+
+def test_get_repo_image_and_storage(images, initialized_db):
+  for current in images:
+    with assert_query_count(1):
+      found = image.get_repo_image_and_storage('devtable', 'simple', current.docker_image_id)
+
+    assert found.id == current.id
+    with assert_query_count(0):
+      assert found.storage.id
+
+
+def test_get_repository_images_without_placements(images, initialized_db):
+  ancestors_map = defaultdict(list)
+  for img in images:
+    current = img.parent
+    while current is not None:
+      ancestors_map[current.id].append(img.id)
+      current = current.parent
+
+  for current in images:
+    repo = current.repository
+
+    with assert_query_count(1):
+      found = list(image.get_repository_images_without_placements(repo, with_ancestor=current))
+
+    assert len(found) == len(ancestors_map[current.id]) + 1
+    assert {i.id for i in found} == set(ancestors_map[current.id] + [current.id])

data/model/test/test_image_sharing.py

@@ -0,0 +1,215 @@
+import pytest
+
+from data import model
+
+from storage.distributedstorage import DistributedStorage
+from storage.fakestorage import FakeStorage
+from test.fixtures import *
+
+NO_ACCESS_USER = 'freshuser'
+READ_ACCESS_USER = 'reader'
+ADMIN_ACCESS_USER = 'devtable'
+PUBLIC_USER = 'public'
+RANDOM_USER = 'randomuser'
+OUTSIDE_ORG_USER = 'outsideorg'
+
+ADMIN_ROBOT_USER = 'devtable+dtrobot'
+
+ORGANIZATION = 'buynlarge'
+
+REPO = 'devtable/simple'
+PUBLIC_REPO = 'public/publicrepo'
+RANDOM_REPO = 'randomuser/randomrepo'
+
+OUTSIDE_ORG_REPO = 'outsideorg/coolrepo'
+
+ORG_REPO = 'buynlarge/orgrepo'
+ANOTHER_ORG_REPO = 'buynlarge/anotherorgrepo'
+
+# Note: The shared repo has devtable as admin, public as a writer and reader as a reader.
+SHARED_REPO = 'devtable/shared'
+
+
+@pytest.fixture()
+def storage(app):
+  return DistributedStorage({'local_us': FakeStorage(None)}, preferred_locations=['local_us'])
+
+
+def createStorage(storage, docker_image_id, repository=REPO, username=ADMIN_ACCESS_USER):
+  repository_obj = model.repository.get_repository(repository.split('/')[0],
+                                                   repository.split('/')[1])
+  preferred = storage.preferred_locations[0]
+  image = model.image.find_create_or_link_image(docker_image_id, repository_obj, username, {},
+                                                preferred)
+  image.storage.uploading = False
+  image.storage.save()
+  return image.storage
+
+
+def assertSameStorage(storage, docker_image_id, existing_storage, repository=REPO,
+                      username=ADMIN_ACCESS_USER):
+  new_storage = createStorage(storage, docker_image_id, repository, username)
+  assert existing_storage.id == new_storage.id
+
+
+def assertDifferentStorage(storage, docker_image_id, existing_storage, repository=REPO,
+                           username=ADMIN_ACCESS_USER):
+  new_storage = createStorage(storage, docker_image_id, repository, username)
+  assert existing_storage.id != new_storage.id
+
+
+def test_same_user(storage, initialized_db):
+  """ The same user creates two images, each which should be shared in the same repo. This is a
+    sanity check. """
+
+  # Create a reference to a new docker ID => new image.
+  first_storage_id = createStorage(storage, 'first-image')
+
+  # Create a reference to the same docker ID => same image.
+  assertSameStorage(storage, 'first-image', first_storage_id)
+
+  # Create a reference to another new docker ID => new image.
+  second_storage_id = createStorage(storage, 'second-image')
+
+  # Create a reference to that same docker ID => same image.
+  assertSameStorage(storage, 'second-image', second_storage_id)
+
+  # Make sure the images are different.
+  assert first_storage_id != second_storage_id
+
+
+def test_no_user_private_repo(storage, initialized_db):
+  """ If no user is specified (token case usually), then no sharing can occur on a private repo. """
+  # Create a reference to a new docker ID => new image.
+  first_storage = createStorage(storage, 'the-image', username=None, repository=SHARED_REPO)
+
+  # Create a areference to the same docker ID, but since no username => new image.
+  assertDifferentStorage(storage, 'the-image', first_storage, username=None, repository=RANDOM_REPO)
+
+
+def test_no_user_public_repo(storage, initialized_db):
+  """ If no user is specified (token case usually), then no sharing can occur on a private repo except when the image is first public. """
+  # Create a reference to a new docker ID => new image.
+  first_storage = createStorage(storage, 'the-image', username=None, repository=PUBLIC_REPO)
+
+  # Create a areference to the same docker ID. Since no username, we'd expect different but the first image is public so => shaed image.
+  assertSameStorage(storage, 'the-image', first_storage, username=None, repository=RANDOM_REPO)
+
+
+def test_different_user_same_repo(storage, initialized_db):
+  """ Two different users create the same image in the same repo. """
+
+  # Create a reference to a new docker ID under the first user => new image.
+  first_storage = createStorage(storage, 'the-image', username=PUBLIC_USER, repository=SHARED_REPO)
+
+  # Create a reference to the *same* docker ID under the second user => same image.
+  assertSameStorage(storage, 'the-image', first_storage, username=ADMIN_ACCESS_USER, repository=SHARED_REPO)
+
+
+def test_different_repo_no_shared_access(storage, initialized_db):
+  """ Neither user has access to the other user's repository. """
+
+  # Create a reference to a new docker ID under the first user => new image.
+  first_storage_id = createStorage(storage, 'the-image', username=RANDOM_USER, repository=RANDOM_REPO)
+
+  # Create a reference to the *same* docker ID under the second user => new image.
+  second_storage_id = createStorage(storage, 'the-image', username=ADMIN_ACCESS_USER, repository=REPO)
+
+  # Verify that the users do not share storage.
+  assert first_storage_id != second_storage_id
+
+
+def test_public_than_private(storage, initialized_db):
+  """ An image is created publicly then used privately, so it should be shared. """
+
+  # Create a reference to a new docker ID under the first user => new image.
+  first_storage = createStorage(storage, 'the-image', username=PUBLIC_USER, repository=PUBLIC_REPO)
+
+  # Create a reference to the *same* docker ID under the second user => same image, since the first was public.
+  assertSameStorage(storage, 'the-image', first_storage, username=ADMIN_ACCESS_USER, repository=REPO)
+
+
+def test_private_than_public(storage, initialized_db):
+  """ An image is created privately then used publicly, so it should *not* be shared. """
+
+  # Create a reference to a new docker ID under the first user => new image.
+  first_storage = createStorage(storage, 'the-image', username=ADMIN_ACCESS_USER, repository=REPO)
+
+  # Create a reference to the *same* docker ID under the second user => new image, since the first was private.
+  assertDifferentStorage(storage, 'the-image', first_storage, username=PUBLIC_USER, repository=PUBLIC_REPO)
+
+
+def test_different_repo_with_access(storage, initialized_db):
+  """ An image is created in one repo (SHARED_REPO) which the user (PUBLIC_USER) has access to. Later, the
+      image is created in another repo (PUBLIC_REPO) that the user also has access to. The image should
+      be shared since the user has access.
+  """
+  # Create the image in the shared repo => new image.
+  first_storage = createStorage(storage, 'the-image', username=ADMIN_ACCESS_USER, repository=SHARED_REPO)
+
+  # Create the image in the other user's repo, but since the user (PUBLIC) still has access to the shared
+  # repository, they should reuse the storage.
+  assertSameStorage(storage, 'the-image', first_storage, username=PUBLIC_USER, repository=PUBLIC_REPO)
+
+
+def test_org_access(storage, initialized_db):
+  """ An image is accessible by being a member of the organization. """
+
+  # Create the new image under the org's repo => new image.
+  first_storage = createStorage(storage, 'the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO)
+
+  # Create an image under the user's repo, but since the user has access to the organization => shared image.
+  assertSameStorage(storage, 'the-image', first_storage, username=ADMIN_ACCESS_USER, repository=REPO)
+
+  # Ensure that the user's robot does not have access, since it is not on the permissions list for the repo.
+  assertDifferentStorage(storage, 'the-image', first_storage, username=ADMIN_ROBOT_USER, repository=SHARED_REPO)
+
+
+def test_org_access_different_user(storage, initialized_db):
+  """ An image is accessible by being a member of the organization. """
+
+  # Create the new image under the org's repo => new image.
+  first_storage = createStorage(storage, 'the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO)
+
+  # Create an image under a user's repo, but since the user has access to the organization => shared image.
+  assertSameStorage(storage, 'the-image', first_storage, username=PUBLIC_USER, repository=PUBLIC_REPO)
+
+  # Also verify for reader.
+  assertSameStorage(storage, 'the-image', first_storage, username=READ_ACCESS_USER, repository=PUBLIC_REPO)
+
+
+def test_org_no_access(storage, initialized_db):
+  """ An image is not accessible if not a member of the organization. """
+
+  # Create the new image under the org's repo => new image.
+  first_storage = createStorage(storage, 'the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO)
+
+  # Create an image under a user's repo. Since the user is not a member of the organization => new image.
+  assertDifferentStorage(storage, 'the-image', first_storage, username=RANDOM_USER, repository=RANDOM_REPO)
+
+
+def test_org_not_team_member_with_access(storage, initialized_db):
+  """ An image is accessible to a user specifically listed as having permission on the org repo. """
+
+  # Create the new image under the org's repo => new image.
+  first_storage = createStorage(storage, 'the-image', username=ADMIN_ACCESS_USER, repository=ORG_REPO)
+
+  # Create an image under a user's repo. Since the user has read access on that repo, they can see the image => shared image.
+  assertSameStorage(storage, 'the-image', first_storage, username=OUTSIDE_ORG_USER, repository=OUTSIDE_ORG_REPO)
+
+
+def test_org_not_team_member_with_no_access(storage, initialized_db):
+  """ A user that has access to one org repo but not another and is not a team member. """
+
+  # Create the new image under the org's repo => new image.
+  first_storage = createStorage(storage, 'the-image', username=ADMIN_ACCESS_USER, repository=ANOTHER_ORG_REPO)
+
+  # Create an image under a user's repo. The user doesn't have access to the repo (ANOTHER_ORG_REPO) so => new image.
+  assertDifferentStorage(storage, 'the-image', first_storage, username=OUTSIDE_ORG_USER, repository=OUTSIDE_ORG_REPO)
+
+def test_no_link_to_uploading(storage, initialized_db):
+  still_uploading = createStorage(storage, 'an-image', repository=PUBLIC_REPO)
+  still_uploading.uploading = True
+  still_uploading.save()
+
+  assertDifferentStorage(storage, 'an-image', still_uploading)

data/model/test/test_log.py

@@ -0,0 +1,80 @@
+import pytest
+
+from data.database import LogEntry3, User
+from data.model import config as _config
+from data.model.log import log_action
+
+from mock import patch, Mock, DEFAULT, sentinel
+from peewee import PeeweeException
+
+
+@pytest.fixture(scope='function')
+def app_config():
+  with patch.dict(_config.app_config, {}, clear=True):
+    yield _config.app_config
+
+@pytest.fixture()
+def logentry_kind():
+  kinds = {'pull_repo': 'pull_repo_kind',  'push_repo': 'push_repo_kind'}
+  with patch('data.model.log.get_log_entry_kinds', return_value=kinds, spec=True):
+    yield kinds
+
+@pytest.fixture()
+def logentry(logentry_kind):
+  with patch('data.database.LogEntry3.create', spec=True):
+    yield LogEntry3
+
+@pytest.fixture()
+def user():
+  with patch.multiple('data.database.User', username=DEFAULT, get=DEFAULT, select=DEFAULT) as user:
+    user['get'].return_value = Mock(id='mock_user_id')
+    user['select'].return_value.tuples.return_value.get.return_value = ['default_user_id']
+    yield User
+
+@pytest.mark.parametrize('action_kind', [('pull'), ('oops')])
+def test_log_action_unknown_action(action_kind):
+  ''' test unknown action types throw an exception when logged '''
+  with pytest.raises(Exception):
+    log_action(action_kind, None)
+
+
+@pytest.mark.parametrize('user_or_org_name,account_id,account', [
+  ('my_test_org', 'N/A',             'mock_user_id'   ),
+  (None,          'test_account_id', 'test_account_id'),
+  (None,           None,             'default_user_id')
+])
+@pytest.mark.parametrize('unlogged_pulls_ok,action_kind,db_exception,throws', [
+  (False, 'pull_repo', None,            False),
+  (False, 'push_repo', None,            False),
+  (False, 'pull_repo', PeeweeException, True ),
+  (False, 'push_repo', PeeweeException, True ),
+
+  (True,  'pull_repo', PeeweeException, False),
+  (True,  'push_repo', PeeweeException, True ),
+  (True,  'pull_repo', Exception,       True ),
+  (True,  'push_repo', Exception,       True )
+])
+def test_log_action(user_or_org_name, account_id, account, unlogged_pulls_ok, action_kind,
+                      db_exception, throws, app_config, logentry, user):
+  log_args = {
+    'performer'  : Mock(id='TEST_PERFORMER_ID'),
+    'repository' : Mock(id='TEST_REPO'),
+    'ip'         : 'TEST_IP',
+    'metadata'   : { 'test_key' : 'test_value' },
+    'timestamp'  : 'TEST_TIMESTAMP'
+  }
+  app_config['SERVICE_LOG_ACCOUNT_ID'] = account_id
+  app_config['ALLOW_PULLS_WITHOUT_STRICT_LOGGING'] = unlogged_pulls_ok
+
+  logentry.create.side_effect = db_exception
+
+  if throws:
+    with pytest.raises(db_exception):
+      log_action(action_kind, user_or_org_name, **log_args)
+  else:
+    log_action(action_kind, user_or_org_name, **log_args)
+
+  logentry.create.assert_called_once_with(kind=action_kind+'_kind', account=account,
+                                          performer='TEST_PERFORMER_ID', repository='TEST_REPO',
+                                          ip='TEST_IP', metadata_json='{"test_key": "test_value"}',
+                                          datetime='TEST_TIMESTAMP')

data/model/test/test_model_blob.py

@@ -0,0 +1,51 @@
+from app import storage
+from data import model, database
+
+from test.fixtures import *
+
+ADMIN_ACCESS_USER = 'devtable'
+REPO = 'simple'
+
+def test_store_blob(initialized_db):
+  location = database.ImageStorageLocation.select().get()
+
+  # Create a new blob at a unique digest.
+  digest = 'somecooldigest'
+  blob_storage = model.blob.store_blob_record_and_temp_link(ADMIN_ACCESS_USER, REPO, digest,
+                                                            location, 1024, 0, 5000)
+  assert blob_storage.content_checksum == digest
+  assert blob_storage.image_size == 1024
+  assert blob_storage.uncompressed_size == 5000
+
+  # Link to the same digest.
+  blob_storage2 = model.blob.store_blob_record_and_temp_link(ADMIN_ACCESS_USER, REPO, digest,
+                                                             location, 2048, 0, 6000)
+  assert blob_storage2.id == blob_storage.id
+
+  # The sizes should be unchanged.
+  assert blob_storage2.image_size == 1024
+  assert blob_storage2.uncompressed_size == 5000
+
+  # Add a new digest, ensure it has a new record.
+  otherdigest = 'anotherdigest'
+  blob_storage3 = model.blob.store_blob_record_and_temp_link(ADMIN_ACCESS_USER, REPO, otherdigest,
+                                                             location, 1234, 0, 5678)
+  assert blob_storage3.id != blob_storage.id
+  assert blob_storage3.image_size == 1234
+  assert blob_storage3.uncompressed_size == 5678
+
+
+def test_get_or_create_shared_blob(initialized_db):
+  shared = model.blob.get_or_create_shared_blob('sha256:abcdef', 'somecontent', storage)
+  assert shared.content_checksum == 'sha256:abcdef'
+
+  again = model.blob.get_or_create_shared_blob('sha256:abcdef', 'somecontent', storage)
+  assert shared == again
+
+
+def test_lookup_repo_storages_by_content_checksum(initialized_db):
+  for image in database.Image.select():
+    found = model.storage.lookup_repo_storages_by_content_checksum(image.repository,
+                                                                   [image.storage.content_checksum])
+    assert len(found) == 1
+    assert found[0].content_checksum == image.storage.content_checksum

data/model/test/test_modelutil.py

@@ -0,0 +1,50 @@
+import pytest
+
+from data.database import Role
+from data.model.modelutil import paginate
+from test.fixtures import *
+
+@pytest.mark.parametrize('page_size', [
+  10,
+  20,
+  50,
+  100,
+  200,
+  500,
+  1000,
+])
+@pytest.mark.parametrize('descending', [
+  False,
+  True,
+])
+def test_paginate(page_size, descending, initialized_db):
+  # Add a bunch of rows into a test table (`Role`).
+  for i in range(0, 522):
+    Role.create(name='testrole%s' % i)
+
+  query = Role.select().where(Role.name ** 'testrole%')
+  all_matching_roles = list(query)
+  assert len(all_matching_roles) == 522
+
+  # Paginate a query to lookup roles.
+  collected = []
+  page_token = None
+  while True:
+    results, page_token = paginate(query, Role, limit=page_size, descending=descending,
+                                   page_token=page_token)
+    assert len(results) <= page_size
+    collected.extend(results)
+
+    if page_token is None:
+      break
+
+    assert len(results) == page_size
+
+    for index, result in enumerate(results[1:]):
+      if descending:
+        assert result.id < results[index].id
+      else:
+        assert result.id > results[index].id
+
+  assert len(collected) == len(all_matching_roles)
+  assert {c.id for c in collected} == {a.id for a in all_matching_roles}

data/model/test/test_organization.py

@@ -0,0 +1,22 @@
+import pytest
+
+from data.model.organization import get_organization, get_organizations
+from data.model.user import mark_namespace_for_deletion
+from data.queue import WorkQueue
+from test.fixtures import *
+
+@pytest.mark.parametrize('deleted', [
+  (True),
+  (False),
+])
+def test_get_organizations(deleted, initialized_db):
+  # Delete an org.
+  deleted_org = get_organization('sellnsmall')
+  queue = WorkQueue('testgcnamespace', lambda db: db.transaction())
+  mark_namespace_for_deletion(deleted_org, [], queue)
+
+  orgs = get_organizations(deleted=deleted)
+  assert orgs
+
+  deleted_found = [org for org in orgs if org.id == deleted_org.id]
+  assert bool(deleted_found) == deleted

data/model/test/test_repo_mirroring.py

@@ -0,0 +1,235 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from jsonschema import ValidationError
+
+from data.database import RepoMirrorConfig, RepoMirrorStatus, User
+from data import model
+from data.model.repo_mirror import (create_mirroring_rule, get_eligible_mirrors, update_sync_status_to_cancel,
+                                    MAX_SYNC_RETRIES, release_mirror)
+
+from test.fixtures import *
+
+
+def create_mirror_repo_robot(rules, repo_name="repo"):
+  try:
+    user = User.get(User.username == "mirror")
+  except User.DoesNotExist:
+    user = create_user_noverify("mirror", "mirror@example.com", email_required=False)
+
+  try:
+    robot = lookup_robot("mirror+robot")
+  except model.InvalidRobotException:
+    robot, _ = create_robot("robot", user)
+
+  repo = create_repository("mirror", repo_name, None, repo_kind="image", visibility="public")
+  repo.save()
+
+  rule = model.repo_mirror.create_mirroring_rule(repo, rules)
+
+  mirror_kwargs = {
+    "repository": repo,
+    "root_rule": rule,
+    "internal_robot": robot,
+    "external_reference": "registry.example.com/namespace/repository",
+    "sync_interval": timedelta(days=1).total_seconds()
+  }
+  mirror = enable_mirroring_for_repository(**mirror_kwargs)
+  mirror.sync_status = RepoMirrorStatus.NEVER_RUN
+  mirror.sync_start_date = datetime.utcnow() - timedelta(days=1)
+  mirror.sync_retries_remaining = 3
+  mirror.save()
+
+  return (mirror, repo)
+
+
+def disable_existing_mirrors():
+  mirrors = RepoMirrorConfig.select().execute()
+  for mirror in mirrors:
+    mirror.is_enabled = False
+    mirror.save()
+
+
+def test_eligible_oldest_first(initialized_db):
+  """
+  Eligible mirror candidates should be returned with the oldest (earliest created) first.
+  """
+
+  disable_existing_mirrors()
+  mirror_first, repo_first = create_mirror_repo_robot(["updated", "created"], repo_name="first")
+  mirror_second, repo_second = create_mirror_repo_robot(["updated", "created"], repo_name="second")
+  mirror_third, repo_third = create_mirror_repo_robot(["updated", "created"], repo_name="third")
+
+  candidates = get_eligible_mirrors()
+
+  assert len(candidates) == 3
+  assert candidates[0] == mirror_first
+  assert candidates[1] == mirror_second
+  assert candidates[2] == mirror_third
+
+
+def test_eligible_includes_expired_syncing(initialized_db):
+  """
+  Mirrors that have an end time in the past are eligible even if their state indicates still syncing.
+  """
+
+  disable_existing_mirrors()
+  mirror_first, repo_first = create_mirror_repo_robot(["updated", "created"], repo_name="first")
+  mirror_second, repo_second = create_mirror_repo_robot(["updated", "created"], repo_name="second")
+  mirror_third, repo_third = create_mirror_repo_robot(["updated", "created"], repo_name="third")
+  mirror_fourth, repo_third = create_mirror_repo_robot(["updated", "created"], repo_name="fourth")
+
+  mirror_second.sync_expiration_date = datetime.utcnow() - timedelta(hours=1)
+  mirror_second.sync_status = RepoMirrorStatus.SYNCING
+  mirror_second.save()
+
+  mirror_fourth.sync_expiration_date = datetime.utcnow() + timedelta(hours=1)
+  mirror_fourth.sync_status = RepoMirrorStatus.SYNCING
+  mirror_fourth.save()
+
+  candidates = get_eligible_mirrors()
+
+  assert len(candidates) == 3
+  assert candidates[0] == mirror_first
+  assert candidates[1] == mirror_second
+  assert candidates[2] == mirror_third
+
+
+def test_eligible_includes_immediate(initialized_db):
+  """
+  Mirrors that are SYNC_NOW, regardless of starting time
+  """
+
+  disable_existing_mirrors()
+  mirror_first, repo_first = create_mirror_repo_robot(["updated", "created"], repo_name="first")
+  mirror_second, repo_second = create_mirror_repo_robot(["updated", "created"], repo_name="second")
+  mirror_third, repo_third = create_mirror_repo_robot(["updated", "created"], repo_name="third")
+  mirror_fourth, repo_third = create_mirror_repo_robot(["updated", "created"], repo_name="fourth")
+  mirror_future, _ = create_mirror_repo_robot(["updated", "created"], repo_name="future")
+  mirror_past, _ = create_mirror_repo_robot(["updated", "created"], repo_name="past")
+
+  mirror_future.sync_start_date = datetime.utcnow() + timedelta(hours=6)
+  mirror_future.sync_status = RepoMirrorStatus.SYNC_NOW
+  mirror_future.save()
+
+  mirror_past.sync_start_date = datetime.utcnow() - timedelta(hours=6)
+  mirror_past.sync_status = RepoMirrorStatus.SYNC_NOW
+  mirror_past.save()
+
+  mirror_fourth.sync_expiration_date = datetime.utcnow() + timedelta(hours=1)
+  mirror_fourth.sync_status = RepoMirrorStatus.SYNCING
+  mirror_fourth.save()
+
+  candidates = get_eligible_mirrors()
+
+  assert len(candidates) == 5
+  assert candidates[0] == mirror_first
+  assert candidates[1] == mirror_second
+  assert candidates[2] == mirror_third
+  assert candidates[3] == mirror_past
+  assert candidates[4] == mirror_future
+
+
+def test_create_rule_validations(initialized_db):
+  mirror, repo = create_mirror_repo_robot(["updated", "created"], repo_name="first")
+
+  with pytest.raises(ValidationError):
+    create_mirroring_rule(repo, None)
+
+  with pytest.raises(ValidationError):
+    create_mirroring_rule(repo, "['tag1', 'tag2']")
+
+  with pytest.raises(ValidationError):
+    create_mirroring_rule(repo, ['tag1', 'tag2'], rule_type=None)
+
+
+def test_long_registry_passwords(initialized_db):
+  """
+  Verify that long passwords, such as Base64 JWT used by Redhat's Registry, work as expected.
+  """
+  MAX_PASSWORD_LENGTH = 1024
+
+  username = ''.join('a' for _ in range(MAX_PASSWORD_LENGTH))
+  password = ''.join('b' for _ in range(MAX_PASSWORD_LENGTH))
+  assert len(username) == MAX_PASSWORD_LENGTH
+  assert len(password) == MAX_PASSWORD_LENGTH
+
+  repo = model.repository.get_repository('devtable', 'mirrored')
+  assert repo
+
+  existing_mirror_conf = model.repo_mirror.get_mirror(repo)
+  assert existing_mirror_conf
+
+  assert model.repo_mirror.change_credentials(repo, username, password)
+
+  updated_mirror_conf = model.repo_mirror.get_mirror(repo)
+  assert updated_mirror_conf
+
+  assert updated_mirror_conf.external_registry_username.decrypt() == username
+  assert updated_mirror_conf.external_registry_password.decrypt() == password
+
+
+def test_sync_status_to_cancel(initialized_db):
+  """
+  SYNCING and SYNC_NOW mirrors may be canceled, ending in NEVER_RUN
+  """
+
+  disable_existing_mirrors()
+  mirror, repo = create_mirror_repo_robot(["updated", "created"], repo_name="cancel")
+
+  mirror.sync_status = RepoMirrorStatus.SYNCING
+  mirror.save()
+  updated = update_sync_status_to_cancel(mirror)
+  assert updated is not None
+  assert updated.sync_status == RepoMirrorStatus.NEVER_RUN
+
+  mirror.sync_status = RepoMirrorStatus.SYNC_NOW
+  mirror.save()
+  updated = update_sync_status_to_cancel(mirror)
+  assert updated is not None
+  assert updated.sync_status == RepoMirrorStatus.NEVER_RUN
+
+  mirror.sync_status = RepoMirrorStatus.FAIL
+  mirror.save()
+  updated = update_sync_status_to_cancel(mirror)
+  assert updated is None
+
+  mirror.sync_status = RepoMirrorStatus.NEVER_RUN
+  mirror.save()
+  updated = update_sync_status_to_cancel(mirror)
+  assert updated is None
+
+  mirror.sync_status = RepoMirrorStatus.SUCCESS
+  mirror.save()
+  updated = update_sync_status_to_cancel(mirror)
+  assert updated is None
+
+
+def test_release_mirror(initialized_db):
+  """
+  Mirrors that are SYNC_NOW, regardless of starting time
+  """
+
+  disable_existing_mirrors()
+  mirror, repo = create_mirror_repo_robot(["updated", "created"], repo_name="first")
+
+  # mysql rounds the milliseconds on update so force that to happen now
+  query = (RepoMirrorConfig
+           .update(sync_start_date=mirror.sync_start_date)
+           .where(RepoMirrorConfig.id == mirror.id))
+  query.execute()
+  mirror = RepoMirrorConfig.get_by_id(mirror.id)
+  original_sync_start_date = mirror.sync_start_date
+
+  assert mirror.sync_retries_remaining == 3
+
+  mirror = release_mirror(mirror, RepoMirrorStatus.FAIL)
+  assert mirror.sync_retries_remaining == 2
+  assert mirror.sync_start_date == original_sync_start_date
+
+  mirror = release_mirror(mirror, RepoMirrorStatus.FAIL)
+  assert mirror.sync_retries_remaining == 1
+  assert mirror.sync_start_date == original_sync_start_date
+
+  mirror = release_mirror(mirror, RepoMirrorStatus.FAIL)
+  assert mirror.sync_retries_remaining == 3
+  assert mirror.sync_start_date > original_sync_start_date

data/model/test/test_repository.py

@@ -0,0 +1,49 @@
+from datetime import timedelta
+
+import pytest
+
+from peewee import IntegrityError
+
+from data.model.gc import purge_repository
+from data.model.repository import create_repository, is_empty
+from data.model.repository import get_filtered_matching_repositories
+from test.fixtures import *
+
+
+def test_duplicate_repository_different_kinds(initialized_db):
+  # Create an image repo.
+  create_repository('devtable', 'somenewrepo', None, repo_kind='image')
+
+  # Try to create an app repo with the same name, which should fail.
+  with pytest.raises(IntegrityError):
+    create_repository('devtable', 'somenewrepo', None, repo_kind='application')
+
+
+def test_is_empty(initialized_db):
+  create_repository('devtable', 'somenewrepo', None, repo_kind='image')
+
+  assert is_empty('devtable', 'somenewrepo')
+  assert not is_empty('devtable', 'simple')
+
+@pytest.mark.skipif(os.environ.get('TEST_DATABASE_URI', '').find('mysql') >= 0, 
+                    reason='MySQL requires specialized indexing of newly created repos')
+@pytest.mark.parametrize('query', [
+  (''),
+  ('e'),
+])
+@pytest.mark.parametrize('authed_username', [
+  (None),
+  ('devtable'),
+])
+def test_search_pagination(query, authed_username, initialized_db):
+  # Create some public repos.
+  repo1 = create_repository('devtable', 'somenewrepo', None, repo_kind='image', visibility='public')
+  repo2 = create_repository('devtable', 'somenewrepo2', None, repo_kind='image', visibility='public')
+  repo3 = create_repository('devtable', 'somenewrepo3', None, repo_kind='image', visibility='public')
+
+  repositories = get_filtered_matching_repositories(query, filter_username=authed_username)
+  assert len(repositories) > 3
+
+  next_repos = get_filtered_matching_repositories(query, filter_username=authed_username, offset=1)
+  assert repositories[0].id != next_repos[0].id
+  assert repositories[1].id == next_repos[0].id

data/model/test/test_repositoryactioncount.py

@@ -0,0 +1,38 @@
+from datetime import date, timedelta
+
+import pytest
+
+from data.database import RepositoryActionCount, RepositorySearchScore
+from data.model.repository import create_repository
+from data.model.repositoryactioncount import update_repository_score, SEARCH_BUCKETS
+from test.fixtures import *
+
+@pytest.mark.parametrize('bucket_sums,expected_score', [
+  ((0, 0, 0, 0), 0),
+
+  ((1, 6, 24, 152), 100),
+  ((2, 6, 24, 152), 101),
+  ((1, 6, 24, 304), 171),
+
+  ((100, 480, 24, 152), 703),
+  ((1, 6, 24, 15200), 7131),
+
+  ((300, 500, 1000, 0), 1733),
+  ((5000, 0, 0, 0), 5434),
+])
+def test_update_repository_score(bucket_sums, expected_score, initialized_db):
+  # Create a new repository.
+  repo = create_repository('devtable', 'somenewrepo', None, repo_kind='image')
+
+  # Delete the RAC created in create_repository.
+  RepositoryActionCount.delete().where(RepositoryActionCount.repository == repo).execute()
+
+  # Add RAC rows for each of the buckets.
+  for index, bucket in enumerate(SEARCH_BUCKETS):
+    for day in range(0, bucket.days):
+      RepositoryActionCount.create(repository=repo,
+                                   count=(bucket_sums[index] / bucket.days * 1.0),
+                                   date=date.today() - bucket.delta + timedelta(days=day))
+
+  assert update_repository_score(repo)
+  assert RepositorySearchScore.get(repository=repo).score == expected_score

data/model/test/test_tag.py

@@ -0,0 +1,356 @@
+import json
+
+from datetime import datetime
+from time import time
+
+import pytest
+
+from mock import patch
+
+from app import docker_v2_signing_key
+from data.database import (Image, RepositoryTag, ImageStorage, Repository, Manifest, ManifestBlob,
+                           ManifestLegacyImage, TagManifestToManifest, Tag, TagToRepositoryTag)
+from data.model.repository import create_repository
+from data.model.tag import (list_active_repo_tags, create_or_update_tag, delete_tag,
+                            get_matching_tags, _tag_alive, get_matching_tags_for_images,
+                            change_tag_expiration, get_active_tag, store_tag_manifest_for_testing,
+                            get_most_recent_tag, get_active_tag_for_repo,
+                            create_or_update_tag_for_repo, set_tag_end_ts)
+from data.model.image import find_create_or_link_image
+from image.docker.schema1 import DockerSchema1ManifestBuilder
+from util.timedeltastring import convert_to_timedelta
+
+from test.fixtures import *
+
+
+def _get_expected_tags(image):
+  expected_query = (RepositoryTag
+                    .select()
+                    .join(Image)
+                    .where(RepositoryTag.hidden == False)
+                    .where((Image.id == image.id) | (Image.ancestors ** ('%%/%s/%%' % image.id))))
+  return set([tag.id for tag in _tag_alive(expected_query)])
+
+
+@pytest.mark.parametrize('max_subqueries,max_image_lookup_count', [
+  (1, 1),
+  (10, 10),
+  (100, 500),
+])
+def test_get_matching_tags(max_subqueries, max_image_lookup_count, initialized_db):
+  with patch('data.model.tag._MAX_SUB_QUERIES', max_subqueries):
+    with patch('data.model.tag._MAX_IMAGE_LOOKUP_COUNT', max_image_lookup_count):
+      # Test for every image in the test database.
+      for image in Image.select(Image, ImageStorage).join(ImageStorage):
+        matching_query = get_matching_tags(image.docker_image_id, image.storage.uuid)
+        matching_tags = set([tag.id for tag in matching_query])
+        expected_tags = _get_expected_tags(image)
+        assert matching_tags == expected_tags, "mismatch for image %s" % image.id
+
+        oci_tags = list(Tag
+                        .select()
+                        .join(TagToRepositoryTag)
+                        .where(TagToRepositoryTag.repository_tag << expected_tags))
+        assert len(oci_tags) == len(expected_tags)        
+
+
+@pytest.mark.parametrize('max_subqueries,max_image_lookup_count', [
+  (1, 1),
+  (10, 10),
+  (100, 500),
+])
+def test_get_matching_tag_ids_for_images(max_subqueries, max_image_lookup_count, initialized_db):
+  with patch('data.model.tag._MAX_SUB_QUERIES', max_subqueries):
+    with patch('data.model.tag._MAX_IMAGE_LOOKUP_COUNT', max_image_lookup_count):
+      # Try for various sets of the first N images.
+      for count in [5, 10, 15]:
+        pairs = []
+        expected_tags_ids = set()
+        for image in Image.select(Image, ImageStorage).join(ImageStorage):
+          if len(pairs) >= count:
+            break
+
+          pairs.append((image.docker_image_id, image.storage.uuid))
+          expected_tags_ids.update(_get_expected_tags(image))
+
+        matching_tags_ids = set([tag.id for tag in get_matching_tags_for_images(pairs)])
+        assert matching_tags_ids == expected_tags_ids
+
+
+@pytest.mark.parametrize('max_subqueries,max_image_lookup_count', [
+  (1, 1),
+  (10, 10),
+  (100, 500),
+])
+def test_get_matching_tag_ids_for_all_images(max_subqueries, max_image_lookup_count, initialized_db):
+  with patch('data.model.tag._MAX_SUB_QUERIES', max_subqueries):
+    with patch('data.model.tag._MAX_IMAGE_LOOKUP_COUNT', max_image_lookup_count):
+      pairs = []
+      for image in Image.select(Image, ImageStorage).join(ImageStorage):
+        pairs.append((image.docker_image_id, image.storage.uuid))
+
+      expected_tags_ids = set([tag.id for tag in _tag_alive(RepositoryTag.select())])
+      matching_tags_ids = set([tag.id for tag in get_matching_tags_for_images(pairs)])
+
+      # Ensure every alive tag was found.
+      assert matching_tags_ids == expected_tags_ids
+
+
+def test_get_matching_tag_ids_images_filtered(initialized_db):
+  def filter_query(query):
+    return query.join(Repository).where(Repository.name == 'simple')
+
+  filtered_images = filter_query(Image
+                                 .select(Image, ImageStorage)
+                                 .join(RepositoryTag)
+                                 .switch(Image)
+                                 .join(ImageStorage)
+                                 .switch(Image))
+
+  expected_tags_query = _tag_alive(filter_query(RepositoryTag
+                                                .select()))
+
+  pairs = []
+  for image in filtered_images:
+    pairs.append((image.docker_image_id, image.storage.uuid))
+
+  matching_tags = get_matching_tags_for_images(pairs, filter_images=filter_query,
+                                               filter_tags=filter_query)
+
+  expected_tag_ids = set([tag.id for tag in expected_tags_query])
+  matching_tags_ids = set([tag.id for tag in matching_tags])
+
+  # Ensure every alive tag was found.
+  assert matching_tags_ids == expected_tag_ids
+
+
+def _get_oci_tag(tag):
+  return (Tag
+          .select()
+          .join(TagToRepositoryTag)
+          .where(TagToRepositoryTag.repository_tag == tag)).get()
+
+
+def assert_tags(repository, *args):
+  tags = list(list_active_repo_tags(repository))
+  assert len(tags) == len(args)
+
+  tags_dict = {}
+  for tag in tags:
+    assert not tag.name in tags_dict
+    assert not tag.hidden
+    assert not tag.lifetime_end_ts or tag.lifetime_end_ts > time()
+
+    tags_dict[tag.name] = tag
+
+    oci_tag = _get_oci_tag(tag)
+    assert oci_tag.name == tag.name
+    assert not oci_tag.hidden
+    assert oci_tag.reversion == tag.reversion
+
+    if tag.lifetime_end_ts:
+      assert oci_tag.lifetime_end_ms == (tag.lifetime_end_ts * 1000)
+    else:
+      assert oci_tag.lifetime_end_ms is None
+
+  for expected in args:
+    assert expected in tags_dict
+
+
+def test_create_reversion_tag(initialized_db):
+  repository = create_repository('devtable', 'somenewrepo', None)
+  manifest = Manifest.get()
+  image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us')
+
+  footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id,
+                                         oci_manifest=manifest, reversion=True)
+  assert footag.reversion
+
+  oci_tag = _get_oci_tag(footag)
+  assert oci_tag.name == footag.name
+  assert not oci_tag.hidden
+  assert oci_tag.reversion == footag.reversion
+
+
+def test_list_active_tags(initialized_db):
+  # Create a new repository.
+  repository = create_repository('devtable', 'somenewrepo', None)
+  manifest = Manifest.get()
+
+  # Create some images.
+  image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us')
+  image2 = find_create_or_link_image('foobarimage2', repository, None, {}, 'local_us')
+
+  # Make sure its tags list is empty.
+  assert_tags(repository)
+
+  # Add some new tags.
+  footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id,
+                                         oci_manifest=manifest)
+  bartag = create_or_update_tag_for_repo(repository, 'bar', image1.docker_image_id,
+                                         oci_manifest=manifest)
+
+  # Since timestamps are stored on a second-granularity, we need to make the tags "start"
+  # before "now", so when we recreate them below, they don't conflict.
+  footag.lifetime_start_ts -= 5
+  footag.save()
+
+  bartag.lifetime_start_ts -= 5
+  bartag.save()
+
+  footag_oci = _get_oci_tag(footag)
+  footag_oci.lifetime_start_ms -= 5000
+  footag_oci.save()
+
+  bartag_oci = _get_oci_tag(bartag)
+  bartag_oci.lifetime_start_ms -= 5000
+  bartag_oci.save()
+
+  # Make sure they are returned.
+  assert_tags(repository, 'foo', 'bar')
+
+  # Set the expirations to be explicitly empty.
+  set_tag_end_ts(footag, None)
+  set_tag_end_ts(bartag, None)
+
+  # Make sure they are returned.
+  assert_tags(repository, 'foo', 'bar')
+
+  # Mark as a tag as expiring in the far future, and make sure it is still returned.
+  set_tag_end_ts(footag, footag.lifetime_start_ts + 10000000)
+
+  # Make sure they are returned.
+  assert_tags(repository, 'foo', 'bar')
+
+  # Delete a tag and make sure it isn't returned.
+  footag = delete_tag('devtable', 'somenewrepo', 'foo')
+  set_tag_end_ts(footag, footag.lifetime_end_ts - 4)
+
+  assert_tags(repository, 'bar')
+
+  # Add a new foo again.
+  footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id,
+                                         oci_manifest=manifest)
+  footag.lifetime_start_ts -= 3
+  footag.save()
+
+  footag_oci = _get_oci_tag(footag)
+  footag_oci.lifetime_start_ms -= 3000
+  footag_oci.save()
+
+  assert_tags(repository, 'foo', 'bar')
+
+  # Mark as a tag as expiring in the far future, and make sure it is still returned.
+  set_tag_end_ts(footag, footag.lifetime_start_ts + 10000000)
+
+  # Make sure they are returned.
+  assert_tags(repository, 'foo', 'bar')
+
+  # "Move" foo by updating it and make sure we don't get duplicates.
+  create_or_update_tag_for_repo(repository, 'foo', image2.docker_image_id, oci_manifest=manifest)
+  assert_tags(repository, 'foo', 'bar')
+
+
+@pytest.mark.parametrize('expiration_offset, expected_offset', [
+  (None, None),
+  ('0s', '1h'),
+  ('30m', '1h'),
+  ('2h', '2h'),
+  ('2w', '2w'),
+  ('200w', '104w'),
+])
+def test_change_tag_expiration(expiration_offset, expected_offset, initialized_db):
+  repository = create_repository('devtable', 'somenewrepo', None)
+  image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us')
+
+  manifest = Manifest.get()
+  footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id,
+                                         oci_manifest=manifest)
+
+  expiration_date = None
+  if expiration_offset is not None:
+    expiration_date = datetime.utcnow() + convert_to_timedelta(expiration_offset)
+
+  assert change_tag_expiration(footag, expiration_date)
+
+  # Lookup the tag again.
+  footag_updated = get_active_tag('devtable', 'somenewrepo', 'foo')
+  oci_tag = _get_oci_tag(footag_updated)
+
+  if expected_offset is None:
+    assert footag_updated.lifetime_end_ts is None
+    assert oci_tag.lifetime_end_ms is None
+  else:
+    start_date = datetime.utcfromtimestamp(footag_updated.lifetime_start_ts)
+    end_date = datetime.utcfromtimestamp(footag_updated.lifetime_end_ts)
+    expected_end_date = start_date + convert_to_timedelta(expected_offset)
+    assert (expected_end_date - end_date).total_seconds() < 5 # variance in test
+
+    assert oci_tag.lifetime_end_ms == (footag_updated.lifetime_end_ts * 1000)
+
+
+def random_storages():
+  return list(ImageStorage.select().where(~(ImageStorage.content_checksum >> None)).limit(10))
+
+
+def repeated_storages():
+  storages = list(ImageStorage.select().where(~(ImageStorage.content_checksum >> None)).limit(5))
+  return storages + storages
+
+
+@pytest.mark.parametrize('get_storages', [
+  random_storages,
+  repeated_storages,
+])
+def test_store_tag_manifest(get_storages, initialized_db):
+  # Create a manifest with some layers.
+  builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'sometag')
+
+  storages = get_storages()
+  assert storages
+
+  repo = model.repository.get_repository('devtable', 'simple')
+  storage_id_map = {}
+  for index, storage in enumerate(storages):
+    image_id = 'someimage%s' % index
+    builder.add_layer(storage.content_checksum, json.dumps({'id': image_id}))
+    find_create_or_link_image(image_id, repo, 'devtable', {}, 'local_us')
+    storage_id_map[storage.content_checksum] = storage.id
+
+  manifest = builder.build(docker_v2_signing_key)
+  tag_manifest, _ = store_tag_manifest_for_testing('devtable', 'simple', 'sometag', manifest,
+                                                   manifest.leaf_layer_v1_image_id, storage_id_map)
+
+  # Ensure we have the new-model expected rows.
+  mapping_row = TagManifestToManifest.get(tag_manifest=tag_manifest)
+
+  assert mapping_row.manifest is not None
+  assert mapping_row.manifest.manifest_bytes == manifest.bytes.as_encoded_str()
+  assert mapping_row.manifest.digest == str(manifest.digest)
+
+  blob_rows = {m.blob_id for m in
+               ManifestBlob.select().where(ManifestBlob.manifest == mapping_row.manifest)}
+  assert blob_rows == {s.id for s in storages}
+
+  assert ManifestLegacyImage.get(manifest=mapping_row.manifest).image == tag_manifest.tag.image
+
+
+def test_get_most_recent_tag(initialized_db):
+  # Create a hidden tag that is the most recent.
+  repo = model.repository.get_repository('devtable', 'simple')
+  image = model.tag.get_tag_image('devtable', 'simple', 'latest')
+  model.tag.create_temporary_hidden_tag(repo, image, 10000000)
+
+  # Ensure we find a non-hidden tag.
+  found = model.tag.get_most_recent_tag(repo)
+  assert not found.hidden
+
+
+def test_get_active_tag_for_repo(initialized_db):
+  repo = model.repository.get_repository('devtable', 'simple')
+  image = model.tag.get_tag_image('devtable', 'simple', 'latest')
+  hidden_tag = model.tag.create_temporary_hidden_tag(repo, image, 10000000)
+
+  # Ensure get active tag for repo cannot find it.
+  assert model.tag.get_active_tag_for_repo(repo, hidden_tag) is None
+  assert model.tag.get_active_tag_for_repo(repo, 'latest') is not None

data/model/test/test_team.py

@@ -0,0 +1,61 @@
+import pytest
+
+from data.model.team import (add_or_invite_to_team, create_team, confirm_team_invite,
+                             list_team_users, validate_team_name)
+from data.model.organization import create_organization
+from data.model.user import get_user, create_user_noverify
+
+from test.fixtures import *
+
+
+@pytest.mark.parametrize('name, is_valid', [
+  ('', False),
+  ('f', False),
+  ('fo', True),
+  ('f' * 255, True),
+  ('f' * 256, False),
+  (' ', False),
+  ('helloworld', True),
+  ('hello_world', True),
+  ('hello-world', True),
+  ('hello world', False),
+  ('HelloWorld', False),
+])
+def test_validate_team_name(name, is_valid):
+  result, _ = validate_team_name(name)
+  assert result == is_valid
+
+
+def is_in_team(team, user):
+  return user.username in {u.username for u in list_team_users(team)}
+
+
+def test_invite_to_team(initialized_db):
+  first_user = get_user('devtable')
+  second_user = create_user_noverify('newuser', 'foo@example.com')
+
+  def run_invite_flow(orgname):
+    # Create an org owned by `devtable`.
+    org = create_organization(orgname, orgname + '@example.com', first_user)
+
+    # Create another team and add `devtable` to it. Since `devtable` is already
+    # in the org, it should be done directly.
+    other_team = create_team('otherteam', org, 'admin')
+    invite = add_or_invite_to_team(first_user, other_team, user_obj=first_user)
+    assert invite is None
+    assert is_in_team(other_team, first_user)
+
+    # Try to add `newuser` to the team, which should require an invite.
+    invite = add_or_invite_to_team(first_user, other_team, user_obj=second_user)
+    assert invite is not None
+    assert not is_in_team(other_team, second_user)
+
+    # Accept the invite.
+    confirm_team_invite(invite.invite_token, second_user)
+    assert is_in_team(other_team, second_user)
+
+  # Run for a new org.
+  run_invite_flow('firstorg')
+
+  # Create another org and repeat, ensuring the same operations perform the same way.
+  run_invite_flow('secondorg')

data/model/test/test_user.py

@@ -0,0 +1,205 @@
+from datetime import datetime
+
+import pytest
+
+from mock import patch
+
+from data.database import EmailConfirmation, User, DeletedNamespace
+from data.model.organization import get_organization
+from data.model.notification import create_notification
+from data.model.team import create_team, add_user_to_team
+from data.model.user import create_user_noverify, validate_reset_code, get_active_users
+from data.model.user import mark_namespace_for_deletion, delete_namespace_via_marker
+from data.model.user import create_robot, lookup_robot, list_namespace_robots
+from data.model.user import get_pull_credentials, retrieve_robot_token, verify_robot
+from data.model.user import InvalidRobotException, delete_robot, get_matching_users
+from data.model.repository import create_repository
+from data.fields import Credential
+from data.queue import WorkQueue
+from util.timedeltastring import convert_to_timedelta
+from util.timedeltastring import convert_to_timedelta
+from util.security.token import encode_public_private_token
+from test.fixtures import *
+
+def test_create_user_with_expiration(initialized_db):
+  with patch('data.model.config.app_config', {'DEFAULT_TAG_EXPIRATION': '1h'}):
+    user = create_user_noverify('foobar', 'foo@example.com', email_required=False)
+    assert user.removed_tag_expiration_s == 60 * 60
+
+@pytest.mark.parametrize('token_lifetime, time_since', [
+  ('1m', '2m'),
+  ('2m', '1m'),
+  ('1h', '1m'),
+])
+def test_validation_code(token_lifetime, time_since, initialized_db):
+  user = create_user_noverify('foobar', 'foo@example.com', email_required=False)
+  created = datetime.now() - convert_to_timedelta(time_since)
+  verification_code, unhashed = Credential.generate()
+  confirmation = EmailConfirmation.create(user=user, pw_reset=True,
+                                          created=created, verification_code=verification_code)
+  encoded = encode_public_private_token(confirmation.code, unhashed)
+
+  with patch('data.model.config.app_config', {'USER_RECOVERY_TOKEN_LIFETIME': token_lifetime}):
+    result = validate_reset_code(encoded)
+    expect_success = convert_to_timedelta(token_lifetime) >= convert_to_timedelta(time_since)
+    assert expect_success == (result is not None)
+
+
+@pytest.mark.parametrize('disabled', [
+  (True),
+  (False),
+])
+@pytest.mark.parametrize('deleted', [
+  (True),
+  (False),
+])
+def test_get_active_users(disabled, deleted, initialized_db):
+  # Delete a user.
+  deleted_user = model.user.get_user('public')
+  queue = WorkQueue('testgcnamespace', lambda db: db.transaction())
+  mark_namespace_for_deletion(deleted_user, [], queue)
+
+  users = get_active_users(disabled=disabled, deleted=deleted)
+  deleted_found = [user for user in users if user.id == deleted_user.id]
+  assert bool(deleted_found) == (deleted and disabled)
+
+  for user in users:
+    if not disabled:
+      assert user.enabled
+
+
+def test_mark_namespace_for_deletion(initialized_db):
+  def create_transaction(db):
+    return db.transaction()
+
+  # Create a user and then mark it for deletion.
+  user = create_user_noverify('foobar', 'foo@example.com', email_required=False)
+
+  # Add some robots.
+  create_robot('foo', user)
+  create_robot('bar', user)
+
+  assert lookup_robot('foobar+foo') is not None
+  assert lookup_robot('foobar+bar') is not None
+  assert len(list(list_namespace_robots('foobar'))) == 2
+
+  # Mark the user for deletion.
+  queue = WorkQueue('testgcnamespace', create_transaction)
+  mark_namespace_for_deletion(user, [], queue)
+
+  # Ensure the older user is still in the DB.
+  older_user = User.get(id=user.id)
+  assert older_user.username != 'foobar'
+
+  # Ensure the robots are deleted.
+  with pytest.raises(InvalidRobotException):
+    assert lookup_robot('foobar+foo')
+
+  with pytest.raises(InvalidRobotException):
+    assert lookup_robot('foobar+bar')
+
+  assert len(list(list_namespace_robots(older_user.username))) == 0
+
+  # Ensure we can create a user with the same namespace again.
+  new_user = create_user_noverify('foobar', 'foo@example.com', email_required=False)
+  assert new_user.id != user.id
+
+  # Ensure the older user is still in the DB.
+  assert User.get(id=user.id).username != 'foobar'
+
+
+def test_delete_namespace_via_marker(initialized_db):
+  def create_transaction(db):
+    return db.transaction()
+
+  # Create a user and then mark it for deletion.
+  user = create_user_noverify('foobar', 'foo@example.com', email_required=False)
+
+  # Add some repositories.
+  create_repository('foobar', 'somerepo', user)
+  create_repository('foobar', 'anotherrepo', user)
+
+  # Mark the user for deletion.
+  queue = WorkQueue('testgcnamespace', create_transaction)
+  marker_id = mark_namespace_for_deletion(user, [], queue)
+
+  # Delete the user.
+  delete_namespace_via_marker(marker_id, [])
+
+  # Ensure the user was actually deleted.
+  with pytest.raises(User.DoesNotExist):
+    User.get(id=user.id)
+
+  with pytest.raises(DeletedNamespace.DoesNotExist):
+    DeletedNamespace.get(id=marker_id)
+
+
+def test_delete_robot(initialized_db):
+  # Create a robot account.
+  user = create_user_noverify('foobar', 'foo@example.com', email_required=False)
+  robot, _ = create_robot('foo', user)
+
+  # Add some notifications and other rows pointing to the robot.
+  create_notification('repo_push', robot)
+
+  team = create_team('someteam', get_organization('buynlarge'), 'member')
+  add_user_to_team(robot, team)
+
+  # Ensure the robot exists.
+  assert lookup_robot(robot.username).id == robot.id
+
+  # Delete the robot.
+  delete_robot(robot.username)
+
+  # Ensure it is gone.
+  with pytest.raises(InvalidRobotException):
+    lookup_robot(robot.username)
+
+
+def test_get_matching_users(initialized_db):
+  # Exact match.
+  for user in User.select().where(User.organization == False, User.robot == False):
+    assert list(get_matching_users(user.username))[0].username == user.username
+
+  # Prefix matching.
+  for user in User.select().where(User.organization == False, User.robot == False):
+    assert user.username in [r.username for r in get_matching_users(user.username[:2])]
+
+
+def test_get_matching_users_with_same_prefix(initialized_db):
+  # Create a bunch of users with the same prefix.
+  for index in range(0, 20):
+    create_user_noverify('foo%s' % index, 'foo%s@example.com' % index, email_required=False)
+
+  # For each user, ensure that lookup of the exact name is found first.
+  for index in range(0, 20):
+    username = 'foo%s' % index
+    assert list(get_matching_users(username))[0].username == username
+
+  # Prefix matching.
+  found = list(get_matching_users('foo', limit=50))
+  assert len(found) == 20
+
+
+def test_robot(initialized_db):
+  # Create a robot account.
+  user = create_user_noverify('foobar', 'foo@example.com', email_required=False)
+  robot, token = create_robot('foo', user)
+  assert retrieve_robot_token(robot) == token
+
+  # Ensure we can retrieve its information.
+  found = lookup_robot('foobar+foo')
+  assert found == robot
+
+  creds = get_pull_credentials('foobar+foo')
+  assert creds is not None
+  assert creds['username'] == 'foobar+foo'
+  assert creds['password'] == token
+
+  assert verify_robot('foobar+foo', token) == robot
+
+  with pytest.raises(InvalidRobotException):
+    assert verify_robot('foobar+foo', 'someothertoken')
+
+  with pytest.raises(InvalidRobotException):
+    assert verify_robot('foobar+unknownbot', token)

data/model/test/test_visible_repos.py

@@ -0,0 +1,89 @@
+from data import model
+
+from test.fixtures import *
+
+
+NO_ACCESS_USER = 'freshuser'
+READ_ACCESS_USER = 'reader'
+ADMIN_ACCESS_USER = 'devtable'
+PUBLIC_USER = 'public'
+RANDOM_USER = 'randomuser'
+OUTSIDE_ORG_USER = 'outsideorg'
+
+ADMIN_ROBOT_USER = 'devtable+dtrobot'
+
+ORGANIZATION = 'buynlarge'
+
+SIMPLE_REPO = 'simple'
+PUBLIC_REPO = 'publicrepo'
+RANDOM_REPO = 'randomrepo'
+
+OUTSIDE_ORG_REPO = 'coolrepo'
+
+ORG_REPO = 'orgrepo'
+ANOTHER_ORG_REPO = 'anotherorgrepo'
+
+# Note: The shared repo has devtable as admin, public as a writer and reader as a reader.
+SHARED_REPO = 'shared'
+
+
+def assertDoesNotHaveRepo(username, name):
+  repos = list(model.repository.get_visible_repositories(username))
+  names = [repo.name for repo in repos]
+  assert not name in names
+
+
+def assertHasRepo(username, name):
+  repos = list(model.repository.get_visible_repositories(username))
+  names = [repo.name for repo in repos]
+  assert name in names
+
+
+def test_noaccess(initialized_db):
+  repos = list(model.repository.get_visible_repositories(NO_ACCESS_USER))
+  names = [repo.name for repo in repos]
+  assert not names
+
+  # Try retrieving public repos now.
+  repos = list(model.repository.get_visible_repositories(NO_ACCESS_USER, include_public=True))
+  names = [repo.name for repo in repos]
+  assert PUBLIC_REPO in names
+
+
+def test_public(initialized_db):
+  assertHasRepo(PUBLIC_USER, PUBLIC_REPO)
+  assertHasRepo(PUBLIC_USER, SHARED_REPO)
+
+  assertDoesNotHaveRepo(PUBLIC_USER, SIMPLE_REPO)
+  assertDoesNotHaveRepo(PUBLIC_USER, RANDOM_REPO)
+  assertDoesNotHaveRepo(PUBLIC_USER, OUTSIDE_ORG_REPO)
+
+
+def test_reader(initialized_db):
+  assertHasRepo(READ_ACCESS_USER, SHARED_REPO)
+  assertHasRepo(READ_ACCESS_USER, ORG_REPO)
+
+  assertDoesNotHaveRepo(READ_ACCESS_USER, SIMPLE_REPO)
+  assertDoesNotHaveRepo(READ_ACCESS_USER, RANDOM_REPO)
+  assertDoesNotHaveRepo(READ_ACCESS_USER, OUTSIDE_ORG_REPO)
+  assertDoesNotHaveRepo(READ_ACCESS_USER, PUBLIC_REPO)
+
+
+def test_random(initialized_db):
+  assertHasRepo(RANDOM_USER, RANDOM_REPO)
+
+  assertDoesNotHaveRepo(RANDOM_USER, SIMPLE_REPO)
+  assertDoesNotHaveRepo(RANDOM_USER, SHARED_REPO)
+  assertDoesNotHaveRepo(RANDOM_USER, ORG_REPO)
+  assertDoesNotHaveRepo(RANDOM_USER, ANOTHER_ORG_REPO)
+  assertDoesNotHaveRepo(RANDOM_USER, PUBLIC_REPO)
+
+
+def test_admin(initialized_db):
+  assertHasRepo(ADMIN_ACCESS_USER, SIMPLE_REPO)
+  assertHasRepo(ADMIN_ACCESS_USER, SHARED_REPO)
+
+  assertHasRepo(ADMIN_ACCESS_USER, ORG_REPO)
+  assertHasRepo(ADMIN_ACCESS_USER, ANOTHER_ORG_REPO)
+
+  assertDoesNotHaveRepo(ADMIN_ACCESS_USER, OUTSIDE_ORG_REPO)