initial import for Open Source 🎉

This commit is contained in:
Jimmy Zelinskie 2019-11-12 11:09:47 -05:00
parent 1898c361f3
commit 9c0dd3b722
2048 changed files with 218743 additions and 0 deletions

View file

@ -0,0 +1,59 @@
# -*- coding: utf-8 -*-
from util.dockerfileparse import parse_dockerfile
def test_basic_parse():
parsed = parse_dockerfile("""
FROM someimage:latest
RUN dosomething
""")
assert parsed.get_image_and_tag() == ("someimage", "latest")
assert parsed.get_base_image() == "someimage"
def test_basic_parse_notag():
parsed = parse_dockerfile("""
FROM someimage
RUN dosomething
""")
assert parsed.get_image_and_tag() == ("someimage", "latest")
assert parsed.get_base_image() == "someimage"
def test_two_from_lines():
parsed = parse_dockerfile("""
FROM someimage:latest
FROM secondimage:second
""")
assert parsed.get_image_and_tag() == ("secondimage", "second")
assert parsed.get_base_image() == "secondimage"
def test_parse_comments():
parsed = parse_dockerfile("""
# FROM someimage:latest
FROM anotherimage:foobar # This is a comment
RUN dosomething
""")
assert parsed.get_image_and_tag() == ("anotherimage", "foobar")
assert parsed.get_base_image() == "anotherimage"
def test_unicode_parse_as_ascii():
parsed = parse_dockerfile("""
FROM someimage:latest
MAINTAINER José Schorr <jschorr@whatever.com>
""")
assert parsed.get_image_and_tag() == ("someimage", "latest")
assert parsed.get_base_image() == "someimage"
def test_unicode_parse_as_unicode():
parsed = parse_dockerfile("""
FROM someimage:latest
MAINTAINER José Schorr <jschorr@whatever.com>
""".decode('utf-8'))
assert parsed.get_image_and_tag() == ("someimage", "latest")
assert parsed.get_base_image() == "someimage"

View file

@ -0,0 +1,48 @@
import pytest
from util.failover import failover, FailoverException
class FinishedException(Exception):
""" Exception raised at the end of every iteration to force failover. """
class Counter(object):
""" Wraps a counter in an object so that it'll be passed by reference. """
def __init__(self):
self.calls = 0
def increment(self):
self.calls += 1
@failover
def my_failover_func(i, should_raise=None):
""" Increments a counter and raises an exception when told. """
i.increment()
if should_raise is not None:
raise should_raise()
raise FailoverException(FinishedException())
@pytest.mark.parametrize('stop_on,exception', [
(10, None),
(5, IndexError),
])
def test_readonly_failover(stop_on, exception):
""" Generates failover arguments and checks against a counter to ensure that
the failover function has been called the proper amount of times and stops
at unhandled exceptions.
"""
counter = Counter()
arg_sets = []
for i in xrange(stop_on):
should_raise = exception if exception is not None and i == stop_on-1 else None
arg_sets.append(((counter,), {'should_raise': should_raise}))
if exception is not None:
with pytest.raises(exception):
my_failover_func(*arg_sets)
else:
with pytest.raises(FinishedException):
my_failover_func(*arg_sets)
assert counter.calls == stop_on

13
util/test/test_html.py Normal file
View file

@ -0,0 +1,13 @@
import pytest
from util.html import html2text
@pytest.mark.parametrize('input, expected', [
('hello world', 'hello world'),
('hello <strong>world</strong>', 'hello *world*'),
('<ul><li>foo</li><li>bar</li><li>baz</li></ul>', '* foo\n* bar\n* baz'),
('<hr>', ('-' * 80)),
('<a href="foo">bar</a>', '[bar](foo)'),
])
def test_html2text(input, expected):
assert html2text(input) == expected

View file

@ -0,0 +1,42 @@
import pytest
import os
from util.log import logfile_path, filter_logs
from app import FILTERED_VALUES
from _init import CONF_DIR
def test_filter_logs():
values = {
'user': {
'password': "toto"
},
'blob': '1234567890asdfewkqresfdsfewfdsfd',
'unfiltered': 'foo'
}
filter_logs(values, FILTERED_VALUES)
assert values == {'user': {'password': '[FILTERED]'}, 'blob': '12345678', 'unfiltered': "foo"}
@pytest.mark.parametrize('debug,jsonfmt,expected', [
(False, False, os.path.join(CONF_DIR, "logging.conf")),
(False, True, os.path.join(CONF_DIR, "logging_json.conf")),
(True, False, os.path.join(CONF_DIR, "logging_debug.conf")),
(True, True, os.path.join(CONF_DIR, "logging_debug_json.conf"))
])
def test_logfile_path(debug, jsonfmt, expected, monkeypatch):
assert logfile_path(jsonfmt=jsonfmt, debug=debug) == expected
@pytest.mark.parametrize('debug,jsonfmt,expected', [
("false", "false", os.path.join(CONF_DIR, "logging.conf")),
("false", "true", os.path.join(CONF_DIR, "logging_json.conf")),
("true", "false", os.path.join(CONF_DIR, "logging_debug.conf")),
("true", "true", os.path.join(CONF_DIR, "logging_debug_json.conf"))
])
def test_logfile_path_env(debug, jsonfmt, expected, monkeypatch):
monkeypatch.setenv("DEBUGLOG", debug)
monkeypatch.setenv("JSONLOG", jsonfmt)
assert logfile_path() == expected
def test_logfile_path_default():
assert logfile_path() == os.path.join(CONF_DIR, "logging.conf")

View file

@ -0,0 +1,300 @@
from util.morecollections import (FastIndexList, StreamingDiffTracker,
IndexedStreamingDiffTracker)
def test_fastindexlist_basic_usage():
indexlist = FastIndexList()
# Add 1
indexlist.add(1)
assert indexlist.values() == [1]
assert indexlist.index(1) == 0
# Add 2
indexlist.add(2)
assert indexlist.values() == [1, 2]
assert indexlist.index(1) == 0
assert indexlist.index(2) == 1
# Pop nothing.
indexlist.pop_until(-1)
assert indexlist.values() == [1, 2]
assert indexlist.index(1) == 0
assert indexlist.index(2) == 1
# Pop 1.
assert indexlist.pop_until(0) == [1]
assert indexlist.values() == [2]
assert indexlist.index(1) is None
assert indexlist.index(2) == 0
# Add 3.
indexlist.add(3)
assert indexlist.values() == [2, 3]
assert indexlist.index(2) == 0
assert indexlist.index(3) == 1
# Pop 2, 3.
assert indexlist.pop_until(1) == [2, 3]
assert indexlist.values() == []
assert indexlist.index(1) is None
assert indexlist.index(2) is None
assert indexlist.index(3) is None
def test_fastindexlist_popping():
indexlist = FastIndexList()
indexlist.add('hello')
indexlist.add('world')
indexlist.add('you')
indexlist.add('rock')
assert indexlist.index('hello') == 0
assert indexlist.index('world') == 1
assert indexlist.index('you') == 2
assert indexlist.index('rock') == 3
indexlist.pop_until(1)
assert indexlist.index('you') == 0
assert indexlist.index('rock') == 1
def test_indexedstreamingdifftracker_basic():
added = []
tracker = IndexedStreamingDiffTracker(added.append, 3)
tracker.push_new([('a', 0), ('b', 1), ('c', 2)])
tracker.push_old([('b', 1)])
tracker.done()
assert added == ['a', 'c']
def test_indexedstreamingdifftracker_multiple_done():
added = []
tracker = IndexedStreamingDiffTracker(added.append, 3)
tracker.push_new([('a', 0), ('b', 1), ('c', 2)])
tracker.push_old([('b', 1)])
tracker.done()
tracker.done()
assert added == ['a', 'c']
def test_indexedstreamingdifftracker_same_streams():
added = []
tracker = IndexedStreamingDiffTracker(added.append, 3)
tracker.push_new([('a', 0), ('b', 1), ('c', 2)])
tracker.push_old([('a', 0), ('b', 1), ('c', 2)])
tracker.done()
assert added == []
def test_indexedstreamingdifftracker_only_new():
added = []
tracker = IndexedStreamingDiffTracker(added.append, 3)
tracker.push_new([('a', 0), ('b', 1), ('c', 2)])
tracker.push_old([])
tracker.done()
assert added == ['a', 'b', 'c']
def test_indexedstreamingdifftracker_pagination():
added = []
tracker = IndexedStreamingDiffTracker(added.append, 2)
tracker.push_new([('a', 0), ('b', 1)])
tracker.push_old([])
tracker.push_new([('c', 2)])
tracker.push_old([])
tracker.done()
assert added == ['a', 'b', 'c']
def test_indexedstreamingdifftracker_old_pagination_no_repeat():
added = []
tracker = IndexedStreamingDiffTracker(added.append, 2)
tracker.push_new([('new1', 3), ('new2', 4)])
tracker.push_old([('old1', 1), ('old2', 2)])
tracker.push_new([])
tracker.push_old([('new1', 3)])
tracker.done()
assert added == ['new2']
def test_indexedstreamingdifftracker_old_pagination():
added = []
tracker = IndexedStreamingDiffTracker(added.append, 2)
tracker.push_new([('a', 10), ('b', 11)])
tracker.push_old([('z', 1), ('y', 2)])
tracker.push_new([('c', 12)])
tracker.push_old([('a', 10)])
tracker.done()
assert added == ['b', 'c']
def test_indexedstreamingdifftracker_very_offset():
added = []
tracker = IndexedStreamingDiffTracker(added.append, 2)
tracker.push_new([('a', 10), ('b', 11)])
tracker.push_old([('z', 1), ('y', 2)])
tracker.push_new([('c', 12), ('d', 13)])
tracker.push_old([('x', 3), ('w', 4)])
tracker.push_new([('e', 14)])
tracker.push_old([('a', 10), ('d', 13)])
tracker.done()
assert added == ['b', 'c', 'e']
def test_indexedstreamingdifftracker_many_old():
added = []
tracker = IndexedStreamingDiffTracker(added.append, 2)
tracker.push_new([('z', 26), ('hello', 100)])
tracker.push_old([('a', 1), ('b', 2)])
tracker.push_new([])
tracker.push_old([('c', 1), ('d', 2)])
tracker.push_new([])
tracker.push_old([('e', 3), ('f', 4)])
tracker.push_new([])
tracker.push_old([('g', 5), ('z', 26)])
tracker.done()
assert added == ['hello']
def test_indexedstreamingdifftracker_high_old_bound():
added = []
tracker = IndexedStreamingDiffTracker(added.append, 2)
tracker.push_new([('z', 26), ('hello', 100)])
tracker.push_old([('end1', 999), ('end2', 1000)])
tracker.push_new([])
tracker.push_old([])
tracker.done()
assert added == ['z', 'hello']
def test_streamingdifftracker_basic():
added = []
tracker = StreamingDiffTracker(added.append, 3)
tracker.push_new(['a', 'b', 'c'])
tracker.push_old(['b'])
tracker.done()
assert added == ['a', 'c']
def test_streamingdifftracker_same_streams():
added = []
tracker = StreamingDiffTracker(added.append, 3)
tracker.push_new(['a', 'b', 'c'])
tracker.push_old(['a', 'b', 'c'])
tracker.done()
assert added == []
def test_streamingdifftracker_some_new():
added = []
tracker = StreamingDiffTracker(added.append, 5)
tracker.push_new(['a', 'b', 'c', 'd', 'e'])
tracker.push_old(['a', 'b', 'c'])
tracker.done()
assert added == ['d', 'e']
def test_streamingdifftracker_offset_new():
added = []
tracker = StreamingDiffTracker(added.append, 5)
tracker.push_new(['b', 'c', 'd', 'e'])
tracker.push_old(['a', 'b', 'c'])
tracker.done()
assert added == ['d', 'e']
def test_streamingdifftracker_multiple_calls():
added = []
tracker = StreamingDiffTracker(added.append, 3)
tracker.push_new(['a', 'b', 'c'])
tracker.push_old(['b', 'd', 'e'])
tracker.push_new(['f', 'g', 'h'])
tracker.push_old(['g', 'h'])
tracker.done()
assert added == ['a', 'c', 'f']
def test_streamingdifftracker_empty_old():
added = []
tracker = StreamingDiffTracker(added.append, 3)
tracker.push_new(['a', 'b', 'c'])
tracker.push_old([])
tracker.push_new(['f', 'g', 'h'])
tracker.push_old([])
tracker.done()
assert added == ['a', 'b', 'c', 'f', 'g', 'h']
def test_streamingdifftracker_more_old():
added = []
tracker = StreamingDiffTracker(added.append, 2)
tracker.push_new(['c', 'd'])
tracker.push_old(['a', 'b'])
tracker.push_new([])
tracker.push_old(['c'])
tracker.done()
assert added == ['d']
def test_streamingdifftracker_more_new():
added = []
tracker = StreamingDiffTracker(added.append, 4)
tracker.push_new(['a', 'b', 'c', 'd'])
tracker.push_old(['r'])
tracker.push_new(['e', 'f', 'r', 'z'])
tracker.push_old([])
tracker.done()
assert added == ['a', 'b', 'c', 'd', 'e', 'f', 'z']
def test_streamingdifftracker_more_new2():
added = []
tracker = StreamingDiffTracker(added.append, 4)
tracker.push_new(['a', 'b', 'c', 'd'])
tracker.push_old(['r'])
tracker.push_new(['e', 'f', 'g', 'h'])
tracker.push_old([])
tracker.push_new(['i', 'j', 'r', 'z'])
tracker.push_old([])
tracker.done()
assert added == ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'z']

14
util/test/test_names.py Normal file
View file

@ -0,0 +1,14 @@
import pytest
from util.names import escape_tag
@pytest.mark.parametrize('input_tag, expected', [
('latest', 'latest'),
('latest124', 'latest124'),
('5de1e98d', '5de1e98d'),
('detailed_view#61', 'detailed_view_61'),
('-detailed_view#61', '_detailed_view_61')
])
def test_escape_tag(input_tag, expected):
assert escape_tag(input_tag) == expected

View file

@ -0,0 +1,58 @@
import pytest
from util.useremails import render_email
from test.fixtures import *
def test_render_email():
params = {
'username': 'someusername',
'new_email': 'new@example.com',
}
html, plain = render_email('Test App', 'test.quay', 'foo@example.com', 'Hello There!',
'emailchanged', params)
assert 'https://quay.io/contact/' in html
assert 'https://quay.io/contact/' in plain
@pytest.mark.parametrize('template_name, params', [
('passwordchanged', {
'username': 'someusername',
}),
('emailchanged', {
'username': 'someusername',
'new_email': 'new@example.com',
}),
('changeemail', {
'username': 'someusername',
'token': 'sometoken',
}),
('confirmemail', {
'username': 'someusername',
'token': 'sometoken',
}),
('repoauthorizeemail', {
'namespace': 'someusername',
'repository': 'somerepo',
'token': 'sometoken',
}),
('orgrecovery', {
'organization': 'someusername',
'admin_usernames': ['foo', 'bar', 'baz'],
}),
('recovery', {
'email': 'foo@example.com',
'token': 'sometoken',
}),
('paymentfailure', {
'username': 'someusername',
}),
('teaminvite', {
'inviter': 'someusername',
'token': 'sometoken',
'organization': 'someorg',
'teamname': 'someteam',
}),
])
def test_emails(template_name, params, initialized_db):
render_email('Test App', 'test.quay', 'foo@example.com', 'Hello There!', template_name, params)

20
util/test/test_util.py Normal file
View file

@ -0,0 +1,20 @@
import pytest
from util import slash_join
@pytest.mark.parametrize('pieces, expected', [
(['https://github.com', '/coreos-inc/' 'quay/pull/1092/files'],
'https://github.com/coreos-inc/quay/pull/1092/files'),
(['https://', 'github.com/', '/coreos-inc', '/quay/pull/1092/files/'],
'https://github.com/coreos-inc/quay/pull/1092/files'),
(['https://somegithub.com/', '/api/v3/'],
'https://somegithub.com/api/v3'),
(['https://github.somedomain.com/', '/api/v3/'],
'https://github.somedomain.com/api/v3'),
])
def test_slash_join(pieces, expected):
joined_url = slash_join(*pieces)
assert joined_url == expected

View file

@ -0,0 +1,132 @@
# -*- coding: utf-8 -*-
from itertools import islice
import pytest
from util.validation import is_json, validate_label_key, generate_valid_usernames, validate_username
@pytest.mark.parametrize('username, is_valid', [
('ja', True),
('jak', True),
('jake', True),
('ja_ke', True),
('te-st', True),
('te.st', True),
('z' * 30, True),
('z' * 255, True),
('j', False),
('z' * 256, False),
('_test', False),
('Test', False),
('hello world', False),
(u'hello→world', False),
('te---st', False),
])
def test_validate_username(username, is_valid):
valid, _ = validate_username(username)
assert valid == is_valid
@pytest.mark.parametrize('string_value,expected', [
('{}', True),
('[]', True),
('[hello world]', False),
('{hello world}', False),
('[test] this is a test', False),
('hello world', False),
('{"hi": "there"}', True),
('[1, 2, 3, 4]', True),
('[1, 2, {"num": 3}, 4]', True),
])
def test_is_json(string_value, expected):
assert is_json(string_value) == expected
@pytest.mark.parametrize('key, is_valid', [
('foo', True),
('bar', True),
('foo1', True),
('bar2', True),
('1', True),
('12', True),
('123', True),
('1234', True),
('git-sha', True),
('com.coreos.something', True),
('io.quay.git-sha', True),
('', False),
('git_sha', False),
('-125', False),
('-foo', False),
('foo-', False),
('123-', False),
('foo--bar', False),
('foo..bar', False),
])
def test_validate_label_key(key, is_valid):
assert validate_label_key(key) == is_valid
@pytest.mark.parametrize('input_username, expected_output', [
('jake', 'jake'),
('frank', 'frank'),
('fra-nk', 'fra_nk'),
('Jake', 'jake'),
('FranK', 'frank'),
('ja__ke', 'ja_ke'),
('ja___ke', 'ja_ke'),
('ja__', 'ja'),
('jake__', 'jake'),
('_jake', 'jake'),
('a', 'a0'),
('ab', 'ab'),
('abc', 'abc'),
('abcdefghijklmnopqrstuvwxyz1234567890', 'abcdefghijklmnopqrstuvwxyz1234567890'),
('c' * 256, 'c' * 255),
(u'\xc6neid', 'aeneid'),
(u'\xe9tude', 'etude'),
(u'\u5317\u4eb0', 'bei_jing'),
(u'\u1515\u14c7\u14c7', 'shanana'),
(u'\u13d4\u13b5\u13c6', 'taliqua'),
(u'\u0726\u071b\u073d\u0710\u073a', 'ptu_i'),
(u'\u0905\u092d\u093f\u091c\u0940\u0924', 'abhijiit'),
(u'\u0985\u09ad\u09bf\u099c\u09c0\u09a4', 'abhijiit'),
(u'\u0d05\u0d2d\u0d3f\u0d1c\u0d40\u0d24', 'abhijiit'),
(u'\u0d2e\u0d32\u0d2f\u0d3e\u0d32\u0d2e\u0d4d', 'mlyaalm'),
(u'\ue000', '00'),
(u'\u03ff', '00'),
(u'\u0d2e\u0d32\u03ff\u03ff\u0d2e\u0d32', 'mlml'),
])
def test_generate_valid_usernames(input_username, expected_output):
name_gen = generate_valid_usernames(input_username)
generated_output = list(islice(name_gen, 1))[0]
assert generated_output == expected_output
def test_multiple_suggestions():
name_gen = generate_valid_usernames('a')
generated_output = list(islice(name_gen, 4))
assert generated_output[0] == 'a0'
assert generated_output[1] == 'a1'
assert generated_output[2] == 'a2'
assert generated_output[3] == 'a3'

74
util/test/test_workers.py Normal file
View file

@ -0,0 +1,74 @@
from mock import patch
import pytest
from util.workers import get_worker_count
@pytest.mark.parametrize('kind_name,env_vars,cpu_affinity,multiplier,minimum,maximum,expected', [
# No override and CPU affinity * multiplier is between min and max => cpu affinity * multiplier.
('registry', {}, [0, 1], 10, 8, 64, 20),
# No override and CPU affinity * multiplier is below min => min.
('registry', {}, [0], 1, 8, 64, 8),
# No override and CPU affinity * multiplier is above max => max.
('registry', {}, [0, 1, 2, 3], 20, 8, 64, 64),
# Override based on specific env var.
('registry', {
'WORKER_COUNT_REGISTRY': 12,
}, [0, 1], 10, 8, 64, 12),
# Override based on specific env var (ignores maximum).
('registry', {
'WORKER_COUNT_REGISTRY': 120,
}, [0, 1], 10, 8, 64, 120),
# Override based on specific env var (respects minimum).
('registry', {
'WORKER_COUNT_REGISTRY': 1,
}, [0, 1], 10, 8, 64, 8),
# Override based on generic env var.
('registry', {
'WORKER_COUNT': 12,
}, [0, 1], 10, 8, 64, 12),
# Override based on generic env var (ignores maximum).
('registry', {
'WORKER_COUNT': 120,
}, [0, 1], 10, 8, 64, 120),
# Override based on generic env var (respects minimum).
('registry', {
'WORKER_COUNT': 1,
}, [0, 1], 10, 8, 64, 8),
# Override always uses specific first.
('registry', {
'WORKER_COUNT_REGISTRY': 120,
'WORKER_COUNT': 12,
}, [0, 1], 10, 8, 64, 120),
# Non-matching override.
('verbs', {
'WORKER_COUNT_REGISTRY': 120,
}, [0, 1], 10, 8, 64, 20),
# Zero worker count (use defaults).
('verbs', {
'WORKER_COUNT': 0,
}, [0, 1], 10, 8, 64, 8),
])
def test_get_worker_count(kind_name, env_vars, cpu_affinity, multiplier, minimum, maximum,
expected):
class FakeProcess(object):
def __init__(self, pid):
pass
def cpu_affinity(self):
return cpu_affinity
with patch('os.environ.get', env_vars.get):
with patch('psutil.Process', FakeProcess):
assert get_worker_count(kind_name, multiplier, minimum, maximum) == expected