Merge pull request #191 from coreos-inc/carmen

Add automatic storage replication

data/database.py

@@ -539,6 +539,15 @@ class ImageStoragePlacement(BaseModel):
     )
 
 
+class UserRegion(BaseModel):
+  user = QuayUserField(index=True, allows_robots=False)
+  location = ForeignKeyField(ImageStorageLocation)
+
+  indexes = (
+    (('user', 'location'), True),
+  )
+
+
 class Image(BaseModel):
   # This class is intentionally denormalized. Even though images are supposed
   # to be globally unique we can't treat them as such for permissions and
@@ -751,4 +760,4 @@ all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission,
               ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification,
               RepositoryAuthorizedEmail, ImageStorageTransformation, DerivedImageStorage,
               TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind,
-              AccessTokenKind, Star, RepositoryActionCount, TagManifest]
+              AccessTokenKind, Star, RepositoryActionCount, TagManifest, UserRegion]

data/migrations/versions/9512773a4a2_add_userregion_table.py

@@ -0,0 +1,35 @@
+"""Add UserRegion table
+
+Revision ID: 9512773a4a2
+Revises: 499f6f08de3
+Create Date: 2015-09-01 14:17:08.628052
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '9512773a4a2'
+down_revision = '499f6f08de3'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('userregion',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('user_id', sa.Integer(), nullable=False),
+    sa.Column('location_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['location_id'], ['imagestoragelocation.id'], name=op.f('fk_userregion_location_id_imagestoragelocation')),
+    sa.ForeignKeyConstraint(['user_id'], ['user.id'], name=op.f('fk_userregion_user_id_user')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_userregion'))
+    )
+    op.create_index('userregion_location_id', 'userregion', ['location_id'], unique=False)
+    op.create_index('userregion_user_id', 'userregion', ['user_id'], unique=False)
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('userregion')
+    ### end Alembic commands ###

data/model/storage.py

@@ -11,6 +11,12 @@ from data.database import (ImageStorage, Image, DerivedImageStorage, ImageStorag
 logger = logging.getLogger(__name__)
 
 
+def add_storage_placement(storage, location_name):
+  """ Adds a storage placement for the given storage at the given location. """
+  location = ImageStorageLocation.get(name=location_name)
+  ImageStoragePlacement.create(location=location, storage=storage)
+
+
 def find_or_create_derived_storage(source, transformation_name, preferred_location):
   existing = find_derived_storage(source, transformation_name)
   if existing is not None:

data/model/user.py

@@ -8,7 +8,8 @@ from datetime import datetime, timedelta
 
 from data.database import (User, LoginService, FederatedLogin, RepositoryPermission, TeamMember,
                            Team, Repository, TupleSelector, TeamRole, Namespace, Visibility,
-                           EmailConfirmation, Role, db_for_update, random_string_generator)
+                           EmailConfirmation, Role, db_for_update, random_string_generator,
+                           UserRegion, ImageStorageLocation)
 from data.model import (DataModelException, InvalidPasswordException, InvalidRobotException,
                         InvalidUsernameException, InvalidEmailAddressException,
                         TooManyUsersException, TooManyLoginAttemptsException, db_transaction,
@@ -463,6 +464,13 @@ def get_user_by_id(user_db_id):
     return None
 
 
+def get_namespace_user_by_user_id(namespace_user_db_id):
+  try:
+    return User.get(User.id == namespace_user_db_id, User.robot == False)
+  except User.DoesNotExist:
+    raise InvalidUsernameException('User with id does not exist: %s' % namespace_user_db_id)
+
+
 def get_namespace_by_user_id(namespace_user_db_id):
   try:
     return User.get(User.id == namespace_user_db_id, User.robot == False).username
@@ -664,3 +672,8 @@ def get_pull_credentials(robotname):
       'registry': '%s://%s/v1/' % (config.app_config['PREFERRED_URL_SCHEME'],
                                    config.app_config['SERVER_HOSTNAME']),
   }
+
+def get_region_locations(user):
+  """ Returns the locations defined as preferred storage for the given user. """
+  query = UserRegion.select().join(ImageStorageLocation).where(UserRegion.user == user)
+  return set([region.location.name for region in query])