From 18100be481500e2f10ea5a042ebddeb83c47b3fc Mon Sep 17 00:00:00 2001
From: Jake Moshenko <jake.moshenko@coreos.com>
Date: Mon, 3 Aug 2015 15:49:10 -0400
Subject: [PATCH] Refactor the util directory to use subpackages.

---
 app.py                                                 | 10 +++++-----
 buildman/jobutil/buildreporter.py                      |  2 +-
 .../versions/1c5b738283a5_backfill_user_uuids.py       |  2 +-
 ...30f55c41d5_calculate_uncompressed_sizes_for_all_.py |  2 +-
 ...ee6269a9d_migrate_bitbucket_services_to_webhooks.py |  2 +-
 .../versions/5b84373e5db_convert_slack_webhook_data.py |  2 +-
 data/users/__init__.py                                 |  3 +--
 endpoints/trigger.py                                   |  4 ++--
 endpoints/v1/registry.py                               |  7 +++----
 endpoints/verbs.py                                     |  4 ++--
 formats/aci.py                                         |  2 +-
 formats/squashed.py                                    |  4 ++--
 formats/tarimageformatter.py                           |  2 +-
 storage/swift.py                                       |  2 +-
 test/test_changes.py                                   |  4 ++--
 test/test_streamlayerformat.py                         |  4 ++--
 test/test_util.py                                      |  2 +-
 util/{ => config}/oauth.py                             |  0
 util/config/validator.py                               |  2 +-
 util/migrate/__init__.py                               |  0
 util/{ => migrate}/backfill_aggregate_sizes.py         |  0
 util/{ => migrate}/backfill_user_uuids.py              |  0
 util/{ => migrate}/delete_access_tokens.py             |  0
 util/{ => migrate}/migratebitbucketservices.py         |  0
 util/{ => migrate}/migrateslackwebhook.py              |  0
 util/{ => migrate}/uncompressedsize.py                 |  2 +-
 util/registry/__init__.py                              |  0
 util/{ => registry}/aufs.py                            |  0
 util/{ => registry}/changes.py                         |  2 +-
 util/{ => registry}/generatorfile.py                   |  0
 util/{ => registry}/gzipstream.py                      |  0
 util/{ => registry}/gzipwrap.py                        |  0
 util/{ => registry}/queuefile.py                       |  0
 util/{ => registry}/queueprocess.py                    |  0
 util/{ => registry}/streamlayerformat.py               |  5 ++---
 util/{ => registry}/tarfileappender.py                 |  4 ++--
 util/{ => registry}/tarlayerformat.py                  |  0
 util/saas/__init__.py                                  |  0
 util/{ => saas}/analytics.py                           |  0
 util/{ => saas}/cloudwatch.py                          |  0
 util/{ => saas}/exceptionlog.py                        |  0
 util/{ => saas}/queuemetrics.py                        |  2 +-
 util/security/__init__.py                              |  0
 util/{ => security}/aes.py                             |  0
 util/{ => security}/signing.py                         |  0
 util/{ => security}/ssh.py                             |  0
 46 files changed, 36 insertions(+), 39 deletions(-)
 rename util/{ => config}/oauth.py (100%)
 create mode 100644 util/migrate/__init__.py
 rename util/{ => migrate}/backfill_aggregate_sizes.py (100%)
 rename util/{ => migrate}/backfill_user_uuids.py (100%)
 rename util/{ => migrate}/delete_access_tokens.py (100%)
 rename util/{ => migrate}/migratebitbucketservices.py (100%)
 rename util/{ => migrate}/migrateslackwebhook.py (100%)
 rename util/{ => migrate}/uncompressedsize.py (98%)
 create mode 100644 util/registry/__init__.py
 rename util/{ => registry}/aufs.py (100%)
 rename util/{ => registry}/changes.py (96%)
 rename util/{ => registry}/generatorfile.py (100%)
 rename util/{ => registry}/gzipstream.py (100%)
 rename util/{ => registry}/gzipwrap.py (100%)
 rename util/{ => registry}/queuefile.py (100%)
 rename util/{ => registry}/queueprocess.py (100%)
 rename util/{ => registry}/streamlayerformat.py (92%)
 rename util/{ => registry}/tarfileappender.py (94%)
 rename util/{ => registry}/tarlayerformat.py (100%)
 create mode 100644 util/saas/__init__.py
 rename util/{ => saas}/analytics.py (100%)
 rename util/{ => saas}/cloudwatch.py (100%)
 rename util/{ => saas}/exceptionlog.py (100%)
 rename util/{ => saas}/queuemetrics.py (97%)
 create mode 100644 util/security/__init__.py
 rename util/{ => security}/aes.py (100%)
 rename util/{ => security}/signing.py (100%)
 rename util/{ => security}/ssh.py (100%)

diff --git a/app.py b/app.py
index 03920e5fb..4b2e71c66 100644
--- a/app.py
+++ b/app.py
@@ -21,12 +21,12 @@ from data.buildlogs import BuildLogs
 from data.archivedlogs import LogArchive
 from data.userevent import UserEventsBuilderModule
 from data.queue import WorkQueue
-from util.analytics import Analytics
-from util.exceptionlog import Sentry
+from util.saas.analytics import Analytics
+from util.saas.exceptionlog import Sentry
 from util.names import urn_generator
-from util.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig
-from util.signing import Signer
-from util.queuemetrics import QueueMetrics
+from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig
+from util.security.signing import Signer
+from util.saas.queuemetrics import QueueMetrics
 from util.config.provider import FileConfigProvider, TestConfigProvider
 from util.config.configutil import generate_secret_key
 from util.config.superusermanager import SuperUserManager
diff --git a/buildman/jobutil/buildreporter.py b/buildman/jobutil/buildreporter.py
index 738615f16..553f62ee7 100644
--- a/buildman/jobutil/buildreporter.py
+++ b/buildman/jobutil/buildreporter.py
@@ -1,5 +1,5 @@
 from buildman.enums import BuildJobResult
-from util.cloudwatch import get_queue
+from util.saas.cloudwatch import get_queue
 
 
 class BuildReporter(object):
diff --git a/data/migrations/versions/1c5b738283a5_backfill_user_uuids.py b/data/migrations/versions/1c5b738283a5_backfill_user_uuids.py
index 6585d6f9a..44ea6f5ec 100644
--- a/data/migrations/versions/1c5b738283a5_backfill_user_uuids.py
+++ b/data/migrations/versions/1c5b738283a5_backfill_user_uuids.py
@@ -13,7 +13,7 @@ down_revision = '2fb36d4be80d'
 from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
-from util.backfill_user_uuids import backfill_user_uuids
+from util.migrate.backfill_user_uuids import backfill_user_uuids
 
 def upgrade(tables):
   backfill_user_uuids()
diff --git a/data/migrations/versions/2430f55c41d5_calculate_uncompressed_sizes_for_all_.py b/data/migrations/versions/2430f55c41d5_calculate_uncompressed_sizes_for_all_.py
index df2ed6f14..20d6fb094 100644
--- a/data/migrations/versions/2430f55c41d5_calculate_uncompressed_sizes_for_all_.py
+++ b/data/migrations/versions/2430f55c41d5_calculate_uncompressed_sizes_for_all_.py
@@ -12,7 +12,7 @@ down_revision = '3b4d3a4461dc'
 
 from alembic import op
 import sqlalchemy as sa
-from util.uncompressedsize import backfill_sizes_from_data
+from util.migrate.uncompressedsize import backfill_sizes_from_data
 
 
 def upgrade(tables):
diff --git a/data/migrations/versions/437ee6269a9d_migrate_bitbucket_services_to_webhooks.py b/data/migrations/versions/437ee6269a9d_migrate_bitbucket_services_to_webhooks.py
index 87e1558ca..9ebf6bcd2 100644
--- a/data/migrations/versions/437ee6269a9d_migrate_bitbucket_services_to_webhooks.py
+++ b/data/migrations/versions/437ee6269a9d_migrate_bitbucket_services_to_webhooks.py
@@ -6,7 +6,7 @@ Create Date: 2015-07-21 14:03:44.964200
 
 """
 
-from util.migratebitbucketservices import run_bitbucket_migration
+from util.migrate.migratebitbucketservices import run_bitbucket_migration
 
 # revision identifiers, used by Alembic.
 revision = '437ee6269a9d'
diff --git a/data/migrations/versions/5b84373e5db_convert_slack_webhook_data.py b/data/migrations/versions/5b84373e5db_convert_slack_webhook_data.py
index a117fd2b1..52f431b74 100644
--- a/data/migrations/versions/5b84373e5db_convert_slack_webhook_data.py
+++ b/data/migrations/versions/5b84373e5db_convert_slack_webhook_data.py
@@ -13,7 +13,7 @@ down_revision = '1c5b738283a5'
 from alembic import op
 import sqlalchemy as sa
 
-from util.migrateslackwebhook import run_slackwebhook_migration
+from util.migrate.migrateslackwebhook import run_slackwebhook_migration
 
 
 def upgrade(tables):
diff --git a/data/users/__init__.py b/data/users/__init__.py
index c9f7d41dd..220c0b6f8 100644
--- a/data/users/__init__.py
+++ b/data/users/__init__.py
@@ -10,8 +10,7 @@ from data.users.database import DatabaseUsers
 from data.users.externalldap import LDAPUsers
 from data.users.externaljwt import ExternalJWTAuthN
 from data.users.keystone import KeystoneUsers
-
-from util.aes import AESCipher
+from util.security.aes import AESCipher
 
 logger = logging.getLogger(__name__)
 
diff --git a/endpoints/trigger.py b/endpoints/trigger.py
index f3c43c784..c4bc63912 100644
--- a/endpoints/trigger.py
+++ b/endpoints/trigger.py
@@ -17,8 +17,8 @@ from jsonschema import validate
 from data import model
 
 from app import app, userfiles as user_files, github_trigger, get_app_url
-from util.tarfileappender import TarfileAppender
-from util.ssh import generate_ssh_keypair
+from util.registry.tarfileappender import TarfileAppender
+from util.security.ssh import generate_ssh_keypair
 
 
 client = app.config['HTTPCLIENT']
diff --git a/endpoints/v1/registry.py b/endpoints/v1/registry.py
index 4e6b7494d..2241a6089 100644
--- a/endpoints/v1/registry.py
+++ b/endpoints/v1/registry.py
@@ -10,12 +10,12 @@ from app import storage as store, image_diff_queue, app
 from auth.auth import process_auth, extract_namespace_repo_from_session
 from auth.auth_context import get_authenticated_user, get_grant_user_context
 from digest import checksums
-from util import changes
+from util.registry import changes
 from util.http import abort, exact_abort
 from auth.permissions import (ReadRepositoryPermission,
                               ModifyRepositoryPermission)
 from data import model, database
-from util import gzipstream
+from util.registry import gzipstream
 from endpoints.v1 import v1_bp
 from endpoints.decorators import anon_protect
 
@@ -578,8 +578,7 @@ def process_image_changes(namespace, repository, image_id):
     layer_files = changes.files_and_dirs_from_tar(layer_tar_stream,
                                                   removed_files)
 
-    new_metadata = changes.compute_new_diffs_and_fs(parent_trie, layer_files,
-                                                    removed_files)
+    new_metadata = changes.compute_new_diffs_and_fs(parent_trie, layer_files, removed_files)
   (new_trie, added, changed, removed) = new_metadata
 
   # Write out the new trie
diff --git a/endpoints/verbs.py b/endpoints/verbs.py
index fe8b37b99..be0067d1d 100644
--- a/endpoints/verbs.py
+++ b/endpoints/verbs.py
@@ -12,8 +12,8 @@ from endpoints.trackhelper import track_and_log
 from endpoints.decorators import anon_protect
 from storage import Storage
 
-from util.queuefile import QueueFile
-from util.queueprocess import QueueProcess
+from util.registry.queuefile import QueueFile
+from util.registry.queueprocess import QueueProcess
 from formats.squashed import SquashedDockerImage
 from formats.aci import ACIImage
 
diff --git a/formats/aci.py b/formats/aci.py
index 31a0d14a6..718c35445 100644
--- a/formats/aci.py
+++ b/formats/aci.py
@@ -1,5 +1,5 @@
 from app import app
-from util.streamlayerformat import StreamLayerMerger
+from util.registry.streamlayerformat import StreamLayerMerger
 from formats.tarimageformatter import TarImageFormatter
 
 import json
diff --git a/formats/squashed.py b/formats/squashed.py
index e7c2fcac8..b26a069fd 100644
--- a/formats/squashed.py
+++ b/formats/squashed.py
@@ -1,6 +1,6 @@
 from app import app
-from util.gzipwrap import GZIP_BUFFER_SIZE
-from util.streamlayerformat import StreamLayerMerger
+from util.registry.gzipwrap import GZIP_BUFFER_SIZE
+from util.registry.streamlayerformat import StreamLayerMerger
 from formats.tarimageformatter import TarImageFormatter
 
 import copy
diff --git a/formats/tarimageformatter.py b/formats/tarimageformatter.py
index 162c89b90..361f6256d 100644
--- a/formats/tarimageformatter.py
+++ b/formats/tarimageformatter.py
@@ -1,5 +1,5 @@
 import tarfile
-from util.gzipwrap import GzipWrap
+from util.registry.gzipwrap import GzipWrap
 
 class TarImageFormatter(object):
   """ Base class for classes which produce a TAR containing image and layer data. """
diff --git a/storage/swift.py b/storage/swift.py
index cfb0be0d5..a1cb32b07 100644
--- a/storage/swift.py
+++ b/storage/swift.py
@@ -1,7 +1,7 @@
 """ Swift storage driver. Based on: github.com/bacongobbler/docker-registry-driver-swift/ """
 from swiftclient.client import Connection, ClientException
 from storage.basestorage import BaseStorage
-from util.generatorfile import GeneratorFile
+from util.registry.generatorfile import GeneratorFile
 
 from random import SystemRandom
 import string
diff --git a/test/test_changes.py b/test/test_changes.py
index 051f1b486..b4ae716fa 100644
--- a/test/test_changes.py
+++ b/test/test_changes.py
@@ -2,8 +2,8 @@ import unittest
 import tarfile
 
 from StringIO import StringIO
-from util.streamlayerformat import AUFS_WHITEOUT
-from util.changes import files_and_dirs_from_tar
+from util.registry.streamlayerformat import AUFS_WHITEOUT
+from util.registry.changes import files_and_dirs_from_tar
 
 class TestChanges(unittest.TestCase):
   def create_layer(self, **kwargs):
diff --git a/test/test_streamlayerformat.py b/test/test_streamlayerformat.py
index 6e65e2b34..e2df30c92 100644
--- a/test/test_streamlayerformat.py
+++ b/test/test_streamlayerformat.py
@@ -2,8 +2,8 @@ import unittest
 import tarfile
 
 from StringIO import StringIO
-from util.streamlayerformat import StreamLayerMerger, AUFS_WHITEOUT
-from util.tarlayerformat import TarLayerReadException
+from util.registry.streamlayerformat import StreamLayerMerger, AUFS_WHITEOUT
+from util.registry.tarlayerformat import TarLayerReadException
 
 class TestStreamLayerMerger(unittest.TestCase):
   def create_layer(self, **kwargs):
diff --git a/test/test_util.py b/test/test_util.py
index 8afa611c4..7da9be2f5 100644
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -3,7 +3,7 @@ import unittest
 from itertools import islice
 
 from util.validation import generate_valid_usernames
-from util.generatorfile import GeneratorFile
+from util.registry.generatorfile import GeneratorFile
 
 class TestGeneratorFile(unittest.TestCase):
   def sample_generator(self):
diff --git a/util/oauth.py b/util/config/oauth.py
similarity index 100%
rename from util/oauth.py
rename to util/config/oauth.py
diff --git a/util/config/validator.py b/util/config/validator.py
index e9f1b17e4..67def6a76 100644
--- a/util/config/validator.py
+++ b/util/config/validator.py
@@ -16,7 +16,7 @@ from flask.ext.mail import Mail, Message
 from data.database import validate_database_url, User
 from storage import get_storage_driver
 from auth.auth_context import get_authenticated_user
-from util.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig
+from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig
 from bitbucket import BitBucket
 
 from app import app, CONFIG_PROVIDER, get_app_url, OVERRIDE_CONFIG_DIRECTORY
diff --git a/util/migrate/__init__.py b/util/migrate/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/util/backfill_aggregate_sizes.py b/util/migrate/backfill_aggregate_sizes.py
similarity index 100%
rename from util/backfill_aggregate_sizes.py
rename to util/migrate/backfill_aggregate_sizes.py
diff --git a/util/backfill_user_uuids.py b/util/migrate/backfill_user_uuids.py
similarity index 100%
rename from util/backfill_user_uuids.py
rename to util/migrate/backfill_user_uuids.py
diff --git a/util/delete_access_tokens.py b/util/migrate/delete_access_tokens.py
similarity index 100%
rename from util/delete_access_tokens.py
rename to util/migrate/delete_access_tokens.py
diff --git a/util/migratebitbucketservices.py b/util/migrate/migratebitbucketservices.py
similarity index 100%
rename from util/migratebitbucketservices.py
rename to util/migrate/migratebitbucketservices.py
diff --git a/util/migrateslackwebhook.py b/util/migrate/migrateslackwebhook.py
similarity index 100%
rename from util/migrateslackwebhook.py
rename to util/migrate/migrateslackwebhook.py
diff --git a/util/uncompressedsize.py b/util/migrate/uncompressedsize.py
similarity index 98%
rename from util/uncompressedsize.py
rename to util/migrate/uncompressedsize.py
index 97284826c..ff7de52e0 100644
--- a/util/uncompressedsize.py
+++ b/util/migrate/uncompressedsize.py
@@ -5,7 +5,7 @@ from data import model
 from data.database import ImageStorage
 from app import app, storage as store
 from data.database import db, db_random_func
-from util.gzipstream import ZLIB_GZIP_WINDOW
+from util.registry.gzipstream import ZLIB_GZIP_WINDOW
 
 
 logger = logging.getLogger(__name__)
diff --git a/util/registry/__init__.py b/util/registry/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/util/aufs.py b/util/registry/aufs.py
similarity index 100%
rename from util/aufs.py
rename to util/registry/aufs.py
diff --git a/util/changes.py b/util/registry/changes.py
similarity index 96%
rename from util/changes.py
rename to util/registry/changes.py
index 38b931f03..9ffb3bddc 100644
--- a/util/changes.py
+++ b/util/registry/changes.py
@@ -1,7 +1,7 @@
 import marisa_trie
 import os
 import tarfile
-from aufs import is_aufs_metadata, get_deleted_prefix
+from util.registry.aufs import is_aufs_metadata, get_deleted_prefix
 
 ALLOWED_TYPES = {tarfile.REGTYPE, tarfile.AREGTYPE}
 
diff --git a/util/generatorfile.py b/util/registry/generatorfile.py
similarity index 100%
rename from util/generatorfile.py
rename to util/registry/generatorfile.py
diff --git a/util/gzipstream.py b/util/registry/gzipstream.py
similarity index 100%
rename from util/gzipstream.py
rename to util/registry/gzipstream.py
diff --git a/util/gzipwrap.py b/util/registry/gzipwrap.py
similarity index 100%
rename from util/gzipwrap.py
rename to util/registry/gzipwrap.py
diff --git a/util/queuefile.py b/util/registry/queuefile.py
similarity index 100%
rename from util/queuefile.py
rename to util/registry/queuefile.py
diff --git a/util/queueprocess.py b/util/registry/queueprocess.py
similarity index 100%
rename from util/queueprocess.py
rename to util/registry/queueprocess.py
diff --git a/util/streamlayerformat.py b/util/registry/streamlayerformat.py
similarity index 92%
rename from util/streamlayerformat.py
rename to util/registry/streamlayerformat.py
index 686c16204..15db6a3a4 100644
--- a/util/streamlayerformat.py
+++ b/util/registry/streamlayerformat.py
@@ -1,8 +1,7 @@
 import marisa_trie
 import os
-import tarfile
-from aufs import is_aufs_metadata, get_deleted_prefix
-from util.tarlayerformat import TarLayerFormat
+from util.registry.aufs import is_aufs_metadata, get_deleted_prefix
+from util.registry.tarlayerformat import TarLayerFormat
 
 AUFS_METADATA = u'.wh..wh.'
 
diff --git a/util/tarfileappender.py b/util/registry/tarfileappender.py
similarity index 94%
rename from util/tarfileappender.py
rename to util/registry/tarfileappender.py
index d552f268c..1dcf9b05d 100644
--- a/util/tarfileappender.py
+++ b/util/registry/tarfileappender.py
@@ -2,8 +2,8 @@ import tarfile
 
 from cStringIO import StringIO
 
-from util.tarlayerformat import TarLayerFormat
-from util.gzipwrap import GzipWrap
+from util.registry.tarlayerformat import TarLayerFormat
+from util.registry.gzipwrap import GzipWrap
 
 class TarfileAppender(TarLayerFormat):
   """ Helper class which allows for appending entries to a gzipped-tarfile and doing so
diff --git a/util/tarlayerformat.py b/util/registry/tarlayerformat.py
similarity index 100%
rename from util/tarlayerformat.py
rename to util/registry/tarlayerformat.py
diff --git a/util/saas/__init__.py b/util/saas/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/util/analytics.py b/util/saas/analytics.py
similarity index 100%
rename from util/analytics.py
rename to util/saas/analytics.py
diff --git a/util/cloudwatch.py b/util/saas/cloudwatch.py
similarity index 100%
rename from util/cloudwatch.py
rename to util/saas/cloudwatch.py
diff --git a/util/exceptionlog.py b/util/saas/exceptionlog.py
similarity index 100%
rename from util/exceptionlog.py
rename to util/saas/exceptionlog.py
diff --git a/util/queuemetrics.py b/util/saas/queuemetrics.py
similarity index 97%
rename from util/queuemetrics.py
rename to util/saas/queuemetrics.py
index 9e0a549f4..62ebd082f 100644
--- a/util/queuemetrics.py
+++ b/util/saas/queuemetrics.py
@@ -1,6 +1,6 @@
 import logging
 
-from util.cloudwatch import get_queue
+from util.saas.cloudwatch import get_queue
 
 
 logger = logging.getLogger(__name__)
diff --git a/util/security/__init__.py b/util/security/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/util/aes.py b/util/security/aes.py
similarity index 100%
rename from util/aes.py
rename to util/security/aes.py
diff --git a/util/signing.py b/util/security/signing.py
similarity index 100%
rename from util/signing.py
rename to util/security/signing.py
diff --git a/util/ssh.py b/util/security/ssh.py
similarity index 100%
rename from util/ssh.py
rename to util/security/ssh.py