import json import os from datetime import datetime, timedelta import boto from httmock import urlmatch, HTTMock from moto import mock_s3_deprecated as mock_s3 from app import storage as test_storage from data import model, database from data.logs_model import logs_model from storage import S3Storage, StorageContext, DistributedStorage from workers.exportactionlogsworker import ExportActionLogsWorker, POLL_PERIOD_SECONDS from test.fixtures import * _TEST_CONTENT = os.urandom(1024) _TEST_BUCKET = 'some_bucket' _TEST_USER = 'someuser' _TEST_PASSWORD = 'somepassword' _TEST_PATH = 'some/cool/path' _TEST_CONTEXT = StorageContext('nyc', None, None, None, None) @pytest.fixture(params=['test', 'mock_s3']) def storage_engine(request): if request.param == 'test': yield test_storage else: with mock_s3(): # Create a test bucket and put some test content. boto.connect_s3().create_bucket(_TEST_BUCKET) engine = DistributedStorage( {'foo': S3Storage(_TEST_CONTEXT, 'some/path', _TEST_BUCKET, _TEST_USER, _TEST_PASSWORD)}, ['foo']) yield engine def test_export_logs_failure(initialized_db): # Make all uploads fail. test_storage.put_content('local_us', 'except_upload', 'true') repo = model.repository.get_repository('devtable', 'simple') user = model.user.get_user('devtable') worker = ExportActionLogsWorker(None) called = [{}] @urlmatch(netloc=r'testcallback') def handle_request(url, request): called[0] = json.loads(request.body) return {'status_code': 200, 'content': '{}'} def format_date(datetime): return datetime.strftime("%m/%d/%Y") now = datetime.now() with HTTMock(handle_request): with pytest.raises(IOError): worker._process_queue_item({ 'export_id': 'someid', 'repository_id': repo.id, 'namespace_id': repo.namespace_user.id, 'namespace_name': 'devtable', 'repository_name': 'simple', 'start_time': format_date(now + timedelta(days=-10)), 'end_time': format_date(now + timedelta(days=10)), 'callback_url': 'http://testcallback/', 'callback_email': None, }, test_storage) test_storage.remove('local_us', 'except_upload') assert called[0] assert called[0][u'export_id'] == 'someid' assert called[0][u'status'] == 'failed' @pytest.mark.parametrize('has_logs', [ True, False, ]) def test_export_logs(initialized_db, storage_engine, has_logs): # Delete all existing logs. database.LogEntry3.delete().execute() repo = model.repository.get_repository('devtable', 'simple') user = model.user.get_user('devtable') now = datetime.now() if has_logs: # Add new logs over a multi-day period. for index in range(-10, 10): logs_model.log_action('push_repo', 'devtable', user, '0.0.0.0', {'index': index}, repo, timestamp=now + timedelta(days=index)) worker = ExportActionLogsWorker(None) called = [{}] @urlmatch(netloc=r'testcallback') def handle_request(url, request): called[0] = json.loads(request.body) return {'status_code': 200, 'content': '{}'} def format_date(datetime): return datetime.strftime("%m/%d/%Y") with HTTMock(handle_request): worker._process_queue_item({ 'export_id': 'someid', 'repository_id': repo.id, 'namespace_id': repo.namespace_user.id, 'namespace_name': 'devtable', 'repository_name': 'simple', 'start_time': format_date(now + timedelta(days=-10)), 'end_time': format_date(now + timedelta(days=10)), 'callback_url': 'http://testcallback/', 'callback_email': None, }, storage_engine) assert called[0] assert called[0][u'export_id'] == 'someid' assert called[0][u'status'] == 'success' url = called[0][u'exported_data_url'] if url.find('http://localhost:5000/exportedlogs/') == 0: storage_id = url[len('http://localhost:5000/exportedlogs/'):] else: assert url.find('https://some_bucket.s3.amazonaws.com/some/path/exportedactionlogs/') == 0 storage_id, _ = url[len('https://some_bucket.s3.amazonaws.com/some/path/exportedactionlogs/'):].split('?') created = storage_engine.get_content(storage_engine.preferred_locations, 'exportedactionlogs/' + storage_id) created_json = json.loads(created) if has_logs: found = set() for log in created_json['logs']: if log.get('terminator'): continue found.add(log['metadata']['index']) for index in range(-10, 10): assert index in found else: assert created_json['logs'] == [{'terminator': True}]