-
There was an error logging in with {{ service_name }}.
+
There was an error logging in with {{ service_name }}.
{% if error_message %}
{{ error_message }}
{% endif %}
+ {% if user_creation %}
Please register using the
registration form to continue.
You will be able to connect your account to your Quay.io account
in the user settings.
+ {% endif %}
diff --git a/test/data/test.db b/test/data/test.db
index 88e3bd7a3..fdbb5b861 100644
Binary files a/test/data/test.db and b/test/data/test.db differ
diff --git a/test/test_api_usage.py b/test/test_api_usage.py
index 2d73008f0..05fe525be 100644
--- a/test/test_api_usage.py
+++ b/test/test_api_usage.py
@@ -173,7 +173,7 @@ class ApiTestCase(unittest.TestCase):
if memberData['name'] == membername:
return
- self.fail(membername + ' not found in team: ' + json.dumps(data))
+ self.fail(membername + ' not found in team: ' + py_json.dumps(data))
def login(self, username, password='password'):
return self.postJsonResponse(Signin, data=dict(username=username, password=password))
@@ -405,7 +405,7 @@ class TestCreateNewUser(ApiTestCase):
invite = model.add_or_invite_to_team(inviter, team, None, 'foo@example.com')
details = {
- 'inviteCode': invite.invite_token
+ 'invite_code': invite.invite_token
}
details.update(NEW_USER_DETAILS);
diff --git a/tools/uncompressedsize.py b/tools/uncompressedsize.py
index 56a22756b..1b6c0e468 100644
--- a/tools/uncompressedsize.py
+++ b/tools/uncompressedsize.py
@@ -1,101 +1,60 @@
-import json
import logging
+import zlib
from data import model
from data.database import ImageStorage
from app import app, storage as store
from data.database import db
-from gzip import GzipFile
-from tempfile import SpooledTemporaryFile
+from util.gzipstream import ZLIB_GZIP_WINDOW
logger = logging.getLogger(__name__)
-def backfill_sizes_from_json():
- query = (ImageStorage
- .select()
- .where(ImageStorage.uncompressed_size == None, ImageStorage.uploading == False)
- .limit(100))
-
- total = 0
- missing = 0
- batch_processed = 1
-
- while batch_processed > 0:
- batch_processed = 0
- with app.config['DB_TRANSACTION_FACTORY'](db):
- for image_storage in query.clone():
- total += 1
- batch_processed += 1
-
- if (total - 1) % 100 == 0:
- logger.debug('Storing entry: %s', total)
-
- # Lookup the JSON for the image.
- uuid = image_storage.uuid
- with_locations = model.get_storage_by_uuid(uuid)
-
- try:
- json_string = store.get_content(with_locations.locations, store.image_json_path(uuid))
- json_data = json.loads(json_string)
- size = json_data.get('Size', json_data.get('size', -1))
- except IOError:
- logger.debug('Image storage with no json %s', uuid)
- size = -1
-
- if size == -1:
- missing += 1
-
- logger.debug('Missing entry %s (%s/%s)', uuid, missing, total)
-
- image_storage.uncompressed_size = size
- image_storage.save()
+CHUNK_SIZE = 512 * 1024 * 1024
def backfill_sizes_from_data():
- storage_ids = list(ImageStorage
- .select(ImageStorage.uuid)
- .where(ImageStorage.uncompressed_size == -1, ImageStorage.uploading == False))
-
- counter = 0
- for uuid in [s.uuid for s in storage_ids]:
- counter += 1
-
- # Load the storage with locations.
- logger.debug('Loading entry: %s (%s/%s)', uuid, counter, len(storage_ids))
- with_locations = model.get_storage_by_uuid(uuid)
- layer_size = -2
-
- # Read the layer from backing storage and calculate the uncompressed size.
+ while True:
+ # Load the record from the DB.
try:
- logger.debug('Loading data: %s (%s bytes)', uuid, with_locations.image_size)
- CHUNK_SIZE = 512 * 1024
- with SpooledTemporaryFile(CHUNK_SIZE) as tarball:
- layer_data = store.get_content(with_locations.locations, store.image_layer_path(uuid))
- tarball.write(layer_data)
- tarball.seek(0)
+ record = (ImageStorage
+ .select(ImageStorage.uuid)
+ .where(ImageStorage.uncompressed_size >> None, ImageStorage.uploading == False)
+ .get())
+ except ImageStorage.DoesNotExist:
+ # We're done!
+ return
- with GzipFile(fileobj=tarball, mode='rb') as gzip_file:
- gzip_file.read()
- layer_size = gzip_file.size
+ uuid = record.uuid
+
+ with_locations = model.get_storage_by_uuid(uuid)
+
+ # Read the layer from backing storage and calculate the uncompressed size.
+ logger.debug('Loading data: %s (%s bytes)', uuid, with_locations.image_size)
+ decompressor = zlib.decompressobj(ZLIB_GZIP_WINDOW)
+
+ uncompressed_size = 0
+ with store.stream_read_file(with_locations.locations, store.image_layer_path(uuid)) as stream:
+ while True:
+ current_data = stream.read(CHUNK_SIZE)
+ if len(current_data) == 0:
+ break
+
+ uncompressed_size += len(decompressor.decompress(current_data))
- except Exception as ex:
- logger.debug('Could not gunzip entry: %s. Reason: %s', uuid, ex)
- continue
-
# Write the size to the image storage. We do so under a transaction AFTER checking to
# make sure the image storage still exists and has not changed.
- logger.debug('Writing entry: %s. Size: %s', uuid, layer_size)
+ logger.debug('Writing entry: %s. Size: %s', uuid, uncompressed_size)
with app.config['DB_TRANSACTION_FACTORY'](db):
- try:
+ try:
current_record = model.get_storage_by_uuid(uuid)
- except:
- # Record no longer exists.
+ except model.InvalidImageException:
+ logger.warning('Storage with uuid no longer exists: %s', uuid)
continue
- if not current_record.uploading and current_record.uncompressed_size == -1:
- current_record.uncompressed_size = layer_size
+ if not current_record.uploading and current_record.uncompressed_size == None:
+ current_record.uncompressed_size = uncompressed_size
current_record.save()
@@ -103,5 +62,4 @@ if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
logging.getLogger('boto').setLevel(logging.CRITICAL)
- backfill_sizes_from_json()
backfill_sizes_from_data()
diff --git a/util/gzipstream.py b/util/gzipstream.py
index 7dbb2fc3e..8f5aa9f6d 100644
--- a/util/gzipstream.py
+++ b/util/gzipstream.py
@@ -9,17 +9,20 @@ import zlib
# http://stackoverflow.com/questions/3122145/zlib-error-error-3-while-decompressing-incorrect-header-check/22310760#22310760
ZLIB_GZIP_WINDOW = zlib.MAX_WBITS | 32
+class SizeInfo(object):
+ def __init__(self):
+ self.size = 0
+
def calculate_size_handler():
""" Returns an object and a SocketReader handler. The handler will gunzip the data it receives,
adding the size found to the object.
"""
- uncompressed_size_info = {
- 'size': 0
- }
-
+
+ size_info = SizeInfo()
+
decompressor = zlib.decompressobj(ZLIB_GZIP_WINDOW)
def fn(buf):
- uncompressed_size_info['size'] += len(decompressor.decompress(buf))
+ size_info.size += len(decompressor.decompress(buf))
- return uncompressed_size_info, fn
+ return size_info, fn
diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py
index 870eeff95..dc4699858 100644
--- a/workers/dockerfilebuild.py
+++ b/workers/dockerfilebuild.py
@@ -223,7 +223,6 @@ class DockerfileBuildContext(object):
raise RuntimeError(message)
def pull(self):
- # Pull the image, in case it was updated since the last build
image_and_tag_tuple = self._parsed_dockerfile.get_image_and_tag()
if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
self._build_logger('Missing FROM command in Dockerfile', build_logs.ERROR)