diff --git a/app.py b/app.py index 03920e5fb..4b2e71c66 100644 --- a/app.py +++ b/app.py @@ -21,12 +21,12 @@ from data.buildlogs import BuildLogs from data.archivedlogs import LogArchive from data.userevent import UserEventsBuilderModule from data.queue import WorkQueue -from util.analytics import Analytics -from util.exceptionlog import Sentry +from util.saas.analytics import Analytics +from util.saas.exceptionlog import Sentry from util.names import urn_generator -from util.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig -from util.signing import Signer -from util.queuemetrics import QueueMetrics +from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig +from util.security.signing import Signer +from util.saas.queuemetrics import QueueMetrics from util.config.provider import FileConfigProvider, TestConfigProvider from util.config.configutil import generate_secret_key from util.config.superusermanager import SuperUserManager diff --git a/buildman/jobutil/buildreporter.py b/buildman/jobutil/buildreporter.py index 738615f16..553f62ee7 100644 --- a/buildman/jobutil/buildreporter.py +++ b/buildman/jobutil/buildreporter.py @@ -1,5 +1,5 @@ from buildman.enums import BuildJobResult -from util.cloudwatch import get_queue +from util.saas.cloudwatch import get_queue class BuildReporter(object): diff --git a/data/migrations/versions/1c5b738283a5_backfill_user_uuids.py b/data/migrations/versions/1c5b738283a5_backfill_user_uuids.py index 6585d6f9a..44ea6f5ec 100644 --- a/data/migrations/versions/1c5b738283a5_backfill_user_uuids.py +++ b/data/migrations/versions/1c5b738283a5_backfill_user_uuids.py @@ -13,7 +13,7 @@ down_revision = '2fb36d4be80d' from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -from util.backfill_user_uuids import backfill_user_uuids +from util.migrate.backfill_user_uuids import backfill_user_uuids def upgrade(tables): backfill_user_uuids() diff --git a/data/migrations/versions/2430f55c41d5_calculate_uncompressed_sizes_for_all_.py b/data/migrations/versions/2430f55c41d5_calculate_uncompressed_sizes_for_all_.py index df2ed6f14..20d6fb094 100644 --- a/data/migrations/versions/2430f55c41d5_calculate_uncompressed_sizes_for_all_.py +++ b/data/migrations/versions/2430f55c41d5_calculate_uncompressed_sizes_for_all_.py @@ -12,7 +12,7 @@ down_revision = '3b4d3a4461dc' from alembic import op import sqlalchemy as sa -from util.uncompressedsize import backfill_sizes_from_data +from util.migrate.uncompressedsize import backfill_sizes_from_data def upgrade(tables): diff --git a/data/migrations/versions/437ee6269a9d_migrate_bitbucket_services_to_webhooks.py b/data/migrations/versions/437ee6269a9d_migrate_bitbucket_services_to_webhooks.py index 87e1558ca..9ebf6bcd2 100644 --- a/data/migrations/versions/437ee6269a9d_migrate_bitbucket_services_to_webhooks.py +++ b/data/migrations/versions/437ee6269a9d_migrate_bitbucket_services_to_webhooks.py @@ -6,7 +6,7 @@ Create Date: 2015-07-21 14:03:44.964200 """ -from util.migratebitbucketservices import run_bitbucket_migration +from util.migrate.migratebitbucketservices import run_bitbucket_migration # revision identifiers, used by Alembic. revision = '437ee6269a9d' diff --git a/data/migrations/versions/5b84373e5db_convert_slack_webhook_data.py b/data/migrations/versions/5b84373e5db_convert_slack_webhook_data.py index a117fd2b1..52f431b74 100644 --- a/data/migrations/versions/5b84373e5db_convert_slack_webhook_data.py +++ b/data/migrations/versions/5b84373e5db_convert_slack_webhook_data.py @@ -13,7 +13,7 @@ down_revision = '1c5b738283a5' from alembic import op import sqlalchemy as sa -from util.migrateslackwebhook import run_slackwebhook_migration +from util.migrate.migrateslackwebhook import run_slackwebhook_migration def upgrade(tables): diff --git a/data/users/__init__.py b/data/users/__init__.py index c9f7d41dd..220c0b6f8 100644 --- a/data/users/__init__.py +++ b/data/users/__init__.py @@ -10,8 +10,7 @@ from data.users.database import DatabaseUsers from data.users.externalldap import LDAPUsers from data.users.externaljwt import ExternalJWTAuthN from data.users.keystone import KeystoneUsers - -from util.aes import AESCipher +from util.security.aes import AESCipher logger = logging.getLogger(__name__) diff --git a/endpoints/trigger.py b/endpoints/trigger.py index f3c43c784..c4bc63912 100644 --- a/endpoints/trigger.py +++ b/endpoints/trigger.py @@ -17,8 +17,8 @@ from jsonschema import validate from data import model from app import app, userfiles as user_files, github_trigger, get_app_url -from util.tarfileappender import TarfileAppender -from util.ssh import generate_ssh_keypair +from util.registry.tarfileappender import TarfileAppender +from util.security.ssh import generate_ssh_keypair client = app.config['HTTPCLIENT'] diff --git a/endpoints/v1/registry.py b/endpoints/v1/registry.py index 4e6b7494d..2241a6089 100644 --- a/endpoints/v1/registry.py +++ b/endpoints/v1/registry.py @@ -10,12 +10,12 @@ from app import storage as store, image_diff_queue, app from auth.auth import process_auth, extract_namespace_repo_from_session from auth.auth_context import get_authenticated_user, get_grant_user_context from digest import checksums -from util import changes +from util.registry import changes from util.http import abort, exact_abort from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission) from data import model, database -from util import gzipstream +from util.registry import gzipstream from endpoints.v1 import v1_bp from endpoints.decorators import anon_protect @@ -578,8 +578,7 @@ def process_image_changes(namespace, repository, image_id): layer_files = changes.files_and_dirs_from_tar(layer_tar_stream, removed_files) - new_metadata = changes.compute_new_diffs_and_fs(parent_trie, layer_files, - removed_files) + new_metadata = changes.compute_new_diffs_and_fs(parent_trie, layer_files, removed_files) (new_trie, added, changed, removed) = new_metadata # Write out the new trie diff --git a/endpoints/verbs.py b/endpoints/verbs.py index fe8b37b99..be0067d1d 100644 --- a/endpoints/verbs.py +++ b/endpoints/verbs.py @@ -12,8 +12,8 @@ from endpoints.trackhelper import track_and_log from endpoints.decorators import anon_protect from storage import Storage -from util.queuefile import QueueFile -from util.queueprocess import QueueProcess +from util.registry.queuefile import QueueFile +from util.registry.queueprocess import QueueProcess from formats.squashed import SquashedDockerImage from formats.aci import ACIImage diff --git a/formats/aci.py b/formats/aci.py index 31a0d14a6..718c35445 100644 --- a/formats/aci.py +++ b/formats/aci.py @@ -1,5 +1,5 @@ from app import app -from util.streamlayerformat import StreamLayerMerger +from util.registry.streamlayerformat import StreamLayerMerger from formats.tarimageformatter import TarImageFormatter import json diff --git a/formats/squashed.py b/formats/squashed.py index e7c2fcac8..b26a069fd 100644 --- a/formats/squashed.py +++ b/formats/squashed.py @@ -1,6 +1,6 @@ from app import app -from util.gzipwrap import GZIP_BUFFER_SIZE -from util.streamlayerformat import StreamLayerMerger +from util.registry.gzipwrap import GZIP_BUFFER_SIZE +from util.registry.streamlayerformat import StreamLayerMerger from formats.tarimageformatter import TarImageFormatter import copy diff --git a/formats/tarimageformatter.py b/formats/tarimageformatter.py index 162c89b90..361f6256d 100644 --- a/formats/tarimageformatter.py +++ b/formats/tarimageformatter.py @@ -1,5 +1,5 @@ import tarfile -from util.gzipwrap import GzipWrap +from util.registry.gzipwrap import GzipWrap class TarImageFormatter(object): """ Base class for classes which produce a TAR containing image and layer data. """ diff --git a/storage/swift.py b/storage/swift.py index cfb0be0d5..a1cb32b07 100644 --- a/storage/swift.py +++ b/storage/swift.py @@ -1,7 +1,7 @@ """ Swift storage driver. Based on: github.com/bacongobbler/docker-registry-driver-swift/ """ from swiftclient.client import Connection, ClientException from storage.basestorage import BaseStorage -from util.generatorfile import GeneratorFile +from util.registry.generatorfile import GeneratorFile from random import SystemRandom import string diff --git a/test/test_changes.py b/test/test_changes.py index 051f1b486..b4ae716fa 100644 --- a/test/test_changes.py +++ b/test/test_changes.py @@ -2,8 +2,8 @@ import unittest import tarfile from StringIO import StringIO -from util.streamlayerformat import AUFS_WHITEOUT -from util.changes import files_and_dirs_from_tar +from util.registry.streamlayerformat import AUFS_WHITEOUT +from util.registry.changes import files_and_dirs_from_tar class TestChanges(unittest.TestCase): def create_layer(self, **kwargs): diff --git a/test/test_streamlayerformat.py b/test/test_streamlayerformat.py index 6e65e2b34..e2df30c92 100644 --- a/test/test_streamlayerformat.py +++ b/test/test_streamlayerformat.py @@ -2,8 +2,8 @@ import unittest import tarfile from StringIO import StringIO -from util.streamlayerformat import StreamLayerMerger, AUFS_WHITEOUT -from util.tarlayerformat import TarLayerReadException +from util.registry.streamlayerformat import StreamLayerMerger, AUFS_WHITEOUT +from util.registry.tarlayerformat import TarLayerReadException class TestStreamLayerMerger(unittest.TestCase): def create_layer(self, **kwargs): diff --git a/test/test_util.py b/test/test_util.py index 8afa611c4..7da9be2f5 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -3,7 +3,7 @@ import unittest from itertools import islice from util.validation import generate_valid_usernames -from util.generatorfile import GeneratorFile +from util.registry.generatorfile import GeneratorFile class TestGeneratorFile(unittest.TestCase): def sample_generator(self): diff --git a/util/oauth.py b/util/config/oauth.py similarity index 100% rename from util/oauth.py rename to util/config/oauth.py diff --git a/util/config/validator.py b/util/config/validator.py index e9f1b17e4..67def6a76 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -16,7 +16,7 @@ from flask.ext.mail import Mail, Message from data.database import validate_database_url, User from storage import get_storage_driver from auth.auth_context import get_authenticated_user -from util.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig +from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig from bitbucket import BitBucket from app import app, CONFIG_PROVIDER, get_app_url, OVERRIDE_CONFIG_DIRECTORY diff --git a/util/migrate/__init__.py b/util/migrate/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/util/backfill_aggregate_sizes.py b/util/migrate/backfill_aggregate_sizes.py similarity index 100% rename from util/backfill_aggregate_sizes.py rename to util/migrate/backfill_aggregate_sizes.py diff --git a/util/backfill_user_uuids.py b/util/migrate/backfill_user_uuids.py similarity index 100% rename from util/backfill_user_uuids.py rename to util/migrate/backfill_user_uuids.py diff --git a/util/delete_access_tokens.py b/util/migrate/delete_access_tokens.py similarity index 100% rename from util/delete_access_tokens.py rename to util/migrate/delete_access_tokens.py diff --git a/util/migratebitbucketservices.py b/util/migrate/migratebitbucketservices.py similarity index 100% rename from util/migratebitbucketservices.py rename to util/migrate/migratebitbucketservices.py diff --git a/util/migrateslackwebhook.py b/util/migrate/migrateslackwebhook.py similarity index 100% rename from util/migrateslackwebhook.py rename to util/migrate/migrateslackwebhook.py diff --git a/util/uncompressedsize.py b/util/migrate/uncompressedsize.py similarity index 98% rename from util/uncompressedsize.py rename to util/migrate/uncompressedsize.py index 97284826c..ff7de52e0 100644 --- a/util/uncompressedsize.py +++ b/util/migrate/uncompressedsize.py @@ -5,7 +5,7 @@ from data import model from data.database import ImageStorage from app import app, storage as store from data.database import db, db_random_func -from util.gzipstream import ZLIB_GZIP_WINDOW +from util.registry.gzipstream import ZLIB_GZIP_WINDOW logger = logging.getLogger(__name__) diff --git a/util/registry/__init__.py b/util/registry/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/util/aufs.py b/util/registry/aufs.py similarity index 100% rename from util/aufs.py rename to util/registry/aufs.py diff --git a/util/changes.py b/util/registry/changes.py similarity index 96% rename from util/changes.py rename to util/registry/changes.py index 38b931f03..9ffb3bddc 100644 --- a/util/changes.py +++ b/util/registry/changes.py @@ -1,7 +1,7 @@ import marisa_trie import os import tarfile -from aufs import is_aufs_metadata, get_deleted_prefix +from util.registry.aufs import is_aufs_metadata, get_deleted_prefix ALLOWED_TYPES = {tarfile.REGTYPE, tarfile.AREGTYPE} diff --git a/util/generatorfile.py b/util/registry/generatorfile.py similarity index 100% rename from util/generatorfile.py rename to util/registry/generatorfile.py diff --git a/util/gzipstream.py b/util/registry/gzipstream.py similarity index 100% rename from util/gzipstream.py rename to util/registry/gzipstream.py diff --git a/util/gzipwrap.py b/util/registry/gzipwrap.py similarity index 100% rename from util/gzipwrap.py rename to util/registry/gzipwrap.py diff --git a/util/queuefile.py b/util/registry/queuefile.py similarity index 100% rename from util/queuefile.py rename to util/registry/queuefile.py diff --git a/util/queueprocess.py b/util/registry/queueprocess.py similarity index 100% rename from util/queueprocess.py rename to util/registry/queueprocess.py diff --git a/util/streamlayerformat.py b/util/registry/streamlayerformat.py similarity index 92% rename from util/streamlayerformat.py rename to util/registry/streamlayerformat.py index 686c16204..15db6a3a4 100644 --- a/util/streamlayerformat.py +++ b/util/registry/streamlayerformat.py @@ -1,8 +1,7 @@ import marisa_trie import os -import tarfile -from aufs import is_aufs_metadata, get_deleted_prefix -from util.tarlayerformat import TarLayerFormat +from util.registry.aufs import is_aufs_metadata, get_deleted_prefix +from util.registry.tarlayerformat import TarLayerFormat AUFS_METADATA = u'.wh..wh.' diff --git a/util/tarfileappender.py b/util/registry/tarfileappender.py similarity index 94% rename from util/tarfileappender.py rename to util/registry/tarfileappender.py index d552f268c..1dcf9b05d 100644 --- a/util/tarfileappender.py +++ b/util/registry/tarfileappender.py @@ -2,8 +2,8 @@ import tarfile from cStringIO import StringIO -from util.tarlayerformat import TarLayerFormat -from util.gzipwrap import GzipWrap +from util.registry.tarlayerformat import TarLayerFormat +from util.registry.gzipwrap import GzipWrap class TarfileAppender(TarLayerFormat): """ Helper class which allows for appending entries to a gzipped-tarfile and doing so diff --git a/util/tarlayerformat.py b/util/registry/tarlayerformat.py similarity index 100% rename from util/tarlayerformat.py rename to util/registry/tarlayerformat.py diff --git a/util/saas/__init__.py b/util/saas/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/util/analytics.py b/util/saas/analytics.py similarity index 100% rename from util/analytics.py rename to util/saas/analytics.py diff --git a/util/cloudwatch.py b/util/saas/cloudwatch.py similarity index 100% rename from util/cloudwatch.py rename to util/saas/cloudwatch.py diff --git a/util/exceptionlog.py b/util/saas/exceptionlog.py similarity index 100% rename from util/exceptionlog.py rename to util/saas/exceptionlog.py diff --git a/util/queuemetrics.py b/util/saas/queuemetrics.py similarity index 97% rename from util/queuemetrics.py rename to util/saas/queuemetrics.py index 9e0a549f4..62ebd082f 100644 --- a/util/queuemetrics.py +++ b/util/saas/queuemetrics.py @@ -1,6 +1,6 @@ import logging -from util.cloudwatch import get_queue +from util.saas.cloudwatch import get_queue logger = logging.getLogger(__name__) diff --git a/util/security/__init__.py b/util/security/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/util/aes.py b/util/security/aes.py similarity index 100% rename from util/aes.py rename to util/security/aes.py diff --git a/util/signing.py b/util/security/signing.py similarity index 100% rename from util/signing.py rename to util/security/signing.py diff --git a/util/ssh.py b/util/security/ssh.py similarity index 100% rename from util/ssh.py rename to util/security/ssh.py