import logging import string import shutil import os import hashlib import random from datetime import datetime, timedelta from flask import url_for from peewee import SqliteDatabase, create_model_tables, drop_model_tables from data.database import * from data import model from app import app logger = logging.getLogger(__name__) store = app.config['STORAGE'] SAMPLE_DIFFS = ['test/data/sample/diffs/diffs%s.json' % i for i in range(1, 10)] REFERENCE_DATE = datetime(2013, 6, 23) TEST_STRIPE_ID = 'cus_2tmnh3PkXQS8NG' def __gen_checksum(image_id): h = hashlib.md5(image_id) return 'tarsum+sha256:' + h.hexdigest() + h.hexdigest() def __gen_image_id(repo, image_num): str_to_hash = "%s/%s/%s" % (repo.namespace, repo.name, image_num) h = hashlib.md5(str_to_hash) return h.hexdigest() + h.hexdigest() global_image_num = [0] def __create_subtree(repo, structure, parent): num_nodes, subtrees, last_node_tags = structure # create the nodes for i in range(num_nodes): image_num = global_image_num[0] global_image_num[0] += 1 docker_image_id = __gen_image_id(repo, image_num) logger.debug('new docker id: %s' % docker_image_id) checksum = __gen_checksum(docker_image_id) new_image = model.create_image(docker_image_id, repo) model.set_image_checksum(docker_image_id, repo, checksum) creation_time = REFERENCE_DATE + timedelta(days=image_num) new_image = model.set_image_metadata(docker_image_id, repo.namespace, repo.name, str(creation_time), 'no comment', parent) model.set_image_size(docker_image_id, repo.namespace, repo.name, random.randrange(1, 1024 * 1024 * 1024)) # Populate the diff file diff_path = store.image_file_diffs_path(repo.namespace, repo.name, docker_image_id) source_diff = SAMPLE_DIFFS[image_num % len(SAMPLE_DIFFS)] with open(source_diff, 'r') as source_file: store.stream_write(diff_path, source_file) parent = new_image if last_node_tags: if not isinstance(last_node_tags, list): last_node_tags = [last_node_tags] for tag_name in last_node_tags: model.create_or_update_tag(repo.namespace, repo.name, tag_name, new_image.docker_image_id) for subtree in subtrees: __create_subtree(repo, subtree, new_image) def __generate_repository(user, name, description, is_public, permissions, structure): repo = model.create_repository(user.username, name, user) if is_public: model.set_repository_visibility(repo, 'public') if description: repo.description = description repo.save() for delegate, role in permissions: model.set_user_repo_permission(delegate.username, user.username, name, role) __create_subtree(repo, structure, None) return repo def initialize_database(): create_model_tables(all_models) Role.create(name='admin') Role.create(name='write') Role.create(name='read') TeamRole.create(name='admin') TeamRole.create(name='creator') TeamRole.create(name='member') Visibility.create(name='public') Visibility.create(name='private') LoginService.create(name='github') LoginService.create(name='quayrobot') LogEntryKind.create(name='account_change_plan') LogEntryKind.create(name='account_change_cc') LogEntryKind.create(name='account_change_password') LogEntryKind.create(name='account_convert') LogEntryKind.create(name='create_robot') LogEntryKind.create(name='delete_robot') LogEntryKind.create(name='create_repo') LogEntryKind.create(name='push_repo') LogEntryKind.create(name='pull_repo') LogEntryKind.create(name='delete_repo') LogEntryKind.create(name='delete_tag') LogEntryKind.create(name='add_repo_permission') LogEntryKind.create(name='change_repo_permission') LogEntryKind.create(name='delete_repo_permission') LogEntryKind.create(name='change_repo_visibility') LogEntryKind.create(name='add_repo_accesstoken') LogEntryKind.create(name='delete_repo_accesstoken') LogEntryKind.create(name='add_repo_webhook') LogEntryKind.create(name='delete_repo_webhook') LogEntryKind.create(name='set_repo_description') LogEntryKind.create(name='build_dockerfile') LogEntryKind.create(name='org_create_team') LogEntryKind.create(name='org_delete_team') LogEntryKind.create(name='org_add_team_member') LogEntryKind.create(name='org_remove_team_member') LogEntryKind.create(name='org_set_team_description') LogEntryKind.create(name='org_set_team_role') def wipe_database(): logger.debug('Wiping all data from the DB.') # Sanity check to make sure we're not killing our prod db db = model.db if (not isinstance(model.db, SqliteDatabase) or app.config['DB_DRIVER'] is not SqliteDatabase): raise RuntimeError('Attempted to wipe production database!') drop_model_tables(all_models, fail_silently=True) def populate_database(): logger.debug('Populating the DB with test data.') new_user_1 = model.create_user('devtable', 'password', 'jschorr@devtable.com') new_user_1.verified = True new_user_1.save() model.create_robot('dtrobot', new_user_1) new_user_2 = model.create_user('public', 'password', 'jacob.moshenko@gmail.com') new_user_2.verified = True new_user_2.save() new_user_3 = model.create_user('freshuser', 'password', 'no@thanks.com') new_user_3.verified = True new_user_3.save() reader = model.create_user('reader', 'password', 'no1@thanks.com') reader.verified = True reader.save() outside_org = model.create_user('outsideorg', 'password', 'no2@thanks.com') outside_org.verified = True outside_org.save() __generate_repository(new_user_1, 'simple', 'Simple repository.', False, [], (4, [], ['latest', 'prod'])) __generate_repository(new_user_1, 'complex', 'Complex repository with many branches and tags.', False, [(new_user_2, 'read')], (2, [(3, [], 'v2.0'), (1, [(1, [(1, [], ['prod'])], 'staging'), (1, [], None)], None)], None)) __generate_repository(new_user_1, 'gargantuan', None, False, [], (2, [(3, [], 'v2.0'), (1, [(1, [(1, [], ['latest', 'prod'])], 'staging'), (1, [], None)], None), (20, [], 'v3.0'), (5, [], 'v4.0'), (1, [(1, [], 'v5.0'), (1, [], 'v6.0')], None)], None)) __generate_repository(new_user_2, 'publicrepo', 'Public repository pullable by the world.', True, [], (10, [], 'latest')) __generate_repository(new_user_1, 'shared', 'Shared repository, another user can write.', False, [(new_user_2, 'write'), (reader, 'read')], (5, [], 'latest')) building = __generate_repository(new_user_1, 'building', 'Empty repository which is building.', False, [], (0, [], None)) org = model.create_organization('buynlarge', 'quay@devtable.com', new_user_1) org.stripe_id = TEST_STRIPE_ID org.save() model.create_robot('neworgrobot', org) owners = model.get_organization_team('buynlarge', 'owners') owners.description = 'Owners have unfetterd access across the entire org.' owners.save() org_repo = __generate_repository(org, 'orgrepo', 'Repository owned by an org.', False, [(outside_org, 'read')], (4, [], ['latest', 'prod'])) reader_team = model.create_team('readers', org, 'member', 'Readers of orgrepo.') model.set_team_repo_permission(reader_team.name, org_repo.namespace, org_repo.name, 'read') model.add_user_to_team(new_user_2, reader_team) model.add_user_to_team(reader, reader_team) token = model.create_access_token(building, 'write') tag = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name) build = model.create_repository_build(building, token, '123-45-6789', tag) build.build_node_id = 1 build.phase = 'building' build.status_url = 'http://localhost:5000/test/build/status' build.save() today = datetime.today() week_ago = today - timedelta(6) six_ago = today - timedelta(5) four_ago = today - timedelta(4) model.log_action('org_create_team', org.username, performer=new_user_1, timestamp=week_ago, metadata={'team': 'readers'}) model.log_action('org_set_team_role', org.username, performer=new_user_1, timestamp=week_ago, metadata={'team': 'readers', 'role': 'read'}) model.log_action('create_repo', org.username, performer=new_user_1, repository=org_repo, timestamp=week_ago, metadata={'namespace': org.username, 'repo': 'orgrepo'}) model.log_action('change_repo_permission', org.username, performer=new_user_2, repository=org_repo, timestamp=six_ago, metadata={'username': new_user_1.username, 'repo': 'orgrepo', 'role': 'admin'}) model.log_action('change_repo_permission', org.username, performer=new_user_1, repository=org_repo, timestamp=six_ago, metadata={'username': new_user_2.username, 'repo': 'orgrepo', 'role': 'read'}) model.log_action('add_repo_accesstoken', org.username, performer=new_user_1, repository=org_repo, timestamp=four_ago, metadata={'repo': 'orgrepo', 'token': 'deploytoken'}) model.log_action('push_repo', org.username, performer=new_user_2, repository=org_repo, timestamp=today, metadata={'username': new_user_2.username, 'repo': 'orgrepo'}) model.log_action('pull_repo', org.username, performer=new_user_2, repository=org_repo, timestamp=today, metadata={'username': new_user_2.username, 'repo': 'orgrepo'}) model.log_action('pull_repo', org.username, repository=org_repo, timestamp=today, metadata={'token': 'sometoken', 'token_code': 'somecode', 'repo': 'orgrepo'}) model.log_action('delete_tag', org.username, performer=new_user_2, repository=org_repo, timestamp=today, metadata={'username': new_user_2.username, 'repo': 'orgrepo', 'tag': 'sometag'}) model.log_action('pull_repo', org.username, repository=org_repo, timestamp=today, metadata={'token_code': 'somecode', 'repo': 'orgrepo'}) if __name__ == '__main__': logging.basicConfig(**app.config['LOGGING_CONFIG']) initialize_database() if app.config.get('POPULATE_DB_TEST_DATA', False): populate_database()