Merge remote-tracking branch 'upstream/master' into python-registry-v2
This commit is contained in:
Jake Moshenko
commit
26cea9a07c
96 changed files with 2044 additions and 626 deletions
|
|
@ -1,5 +1,13 @@
|
|||
import urlparse
|
||||
import github
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
|
||||
from cachetools.func import TTLCache
|
||||
from jwkest.jwk import KEYS, keyrep
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class OAuthConfig(object):
|
||||
def __init__(self, config, key_name):
|
||||
|
|
@ -38,10 +46,8 @@ class OAuthConfig(object):
|
|||
|
||||
|
||||
def exchange_code_for_token(self, app_config, http_client, code, form_encode=False,
|
||||
redirect_suffix=''):
|
||||
redirect_suffix='', client_auth=False):
|
||||
payload = {
|
||||
'client_id': self.client_id(),
|
||||
'client_secret': self.client_secret(),
|
||||
'code': code,
|
||||
'grant_type': 'authorization_code',
|
||||
'redirect_uri': self.get_redirect_uri(app_config, redirect_suffix)
|
||||
|
|
@ -51,11 +57,18 @@ class OAuthConfig(object):
|
|||
'Accept': 'application/json'
|
||||
}
|
||||
|
||||
auth = None
|
||||
if client_auth:
|
||||
auth = (self.client_id(), self.client_secret())
|
||||
else:
|
||||
payload['client_id'] = self.client_id()
|
||||
payload['client_secret'] = self.client_secret()
|
||||
|
||||
token_url = self.token_endpoint()
|
||||
if form_encode:
|
||||
get_access_token = http_client.post(token_url, data=payload, headers=headers)
|
||||
get_access_token = http_client.post(token_url, data=payload, headers=headers, auth=auth)
|
||||
else:
|
||||
get_access_token = http_client.post(token_url, params=payload, headers=headers)
|
||||
get_access_token = http_client.post(token_url, params=payload, headers=headers, auth=auth)
|
||||
|
||||
json_data = get_access_token.json()
|
||||
if not json_data:
|
||||
|
|
@ -248,3 +261,102 @@ class GitLabOAuthConfig(OAuthConfig):
|
|||
'AUTHORIZE_ENDPOINT': self.authorize_endpoint(),
|
||||
'GITLAB_ENDPOINT': self._endpoint(),
|
||||
}
|
||||
|
||||
|
||||
OIDC_WELLKNOWN = ".well-known/openid-configuration"
|
||||
PUBLIC_KEY_CACHE_TTL = 3600 # 1 hour
|
||||
|
||||
class OIDCConfig(OAuthConfig):
|
||||
def __init__(self, config, key_name):
|
||||
super(OIDCConfig, self).__init__(config, key_name)
|
||||
|
||||
self._public_key_cache = TTLCache(1, PUBLIC_KEY_CACHE_TTL, missing=self._get_public_key)
|
||||
self._oidc_config = {}
|
||||
self._http_client = config['HTTPCLIENT']
|
||||
|
||||
if self.config.get('OIDC_SERVER'):
|
||||
self._load_via_discovery(config['DEBUGGING'])
|
||||
|
||||
def _load_via_discovery(self, is_debugging):
|
||||
oidc_server = self.config['OIDC_SERVER']
|
||||
if not oidc_server.startswith('https://') and not is_debugging:
|
||||
raise Exception('OIDC server must be accessed over SSL')
|
||||
|
||||
discovery_url = urlparse.urljoin(oidc_server, OIDC_WELLKNOWN)
|
||||
discovery = self._http_client.get(discovery_url, timeout=5)
|
||||
|
||||
if discovery.status_code / 100 != 2:
|
||||
raise Exception("Could not load OIDC discovery information")
|
||||
|
||||
try:
|
||||
self._oidc_config = json.loads(discovery.text)
|
||||
except ValueError:
|
||||
logger.exception('Could not parse OIDC discovery for url: %s', discovery_url)
|
||||
raise Exception("Could not parse OIDC discovery information")
|
||||
|
||||
def authorize_endpoint(self):
|
||||
return self._oidc_config.get('authorization_endpoint', '') + '?'
|
||||
|
||||
def token_endpoint(self):
|
||||
return self._oidc_config.get('token_endpoint')
|
||||
|
||||
def user_endpoint(self):
|
||||
return None
|
||||
|
||||
def validate_client_id_and_secret(self, http_client, app_config):
|
||||
pass
|
||||
|
||||
def get_public_config(self):
|
||||
return {
|
||||
'CLIENT_ID': self.client_id(),
|
||||
'AUTHORIZE_ENDPOINT': self.authorize_endpoint()
|
||||
}
|
||||
|
||||
@property
|
||||
def issuer(self):
|
||||
return self.config.get('OIDC_ISSUER', self.config['OIDC_SERVER'])
|
||||
|
||||
def get_public_key(self, force_refresh=False):
|
||||
""" Retrieves the public key for this handler. """
|
||||
# If force_refresh is true, we expire all the items in the cache by setting the time to
|
||||
# the current time + the expiration TTL.
|
||||
if force_refresh:
|
||||
self._public_key_cache.expire(time=time.time() + PUBLIC_KEY_CACHE_TTL)
|
||||
|
||||
# Retrieve the public key from the cache. If the cache does not contain the public key,
|
||||
# it will internally call _get_public_key to retrieve it and then save it. The None is
|
||||
# a random key chose to be stored in the cache, and could be anything.
|
||||
return self._public_key_cache[None]
|
||||
|
||||
def _get_public_key(self):
|
||||
""" Retrieves the public key for this handler. """
|
||||
keys_url = self._oidc_config['jwks_uri']
|
||||
|
||||
keys = KEYS()
|
||||
keys.load_from_url(keys_url)
|
||||
|
||||
if not list(keys):
|
||||
raise Exception('No keys provided by OIDC provider')
|
||||
|
||||
rsa_key = list(keys)[0]
|
||||
rsa_key.deserialize()
|
||||
return rsa_key.key.exportKey('PEM')
|
||||
|
||||
|
||||
class DexOAuthConfig(OIDCConfig):
|
||||
def service_name(self):
|
||||
return 'Dex'
|
||||
|
||||
@property
|
||||
def public_title(self):
|
||||
return self.get_public_config()['OIDC_TITLE']
|
||||
|
||||
def get_public_config(self):
|
||||
return {
|
||||
'CLIENT_ID': self.client_id(),
|
||||
'AUTHORIZE_ENDPOINT': self.authorize_endpoint(),
|
||||
|
||||
# TODO(jschorr): This should ideally come from the Dex side.
|
||||
'OIDC_TITLE': 'Dex',
|
||||
'OIDC_LOGO': 'https://tectonic.com/assets/ico/favicon-96x96.png'
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,181 +0,0 @@
|
|||
import os
|
||||
import yaml
|
||||
import logging
|
||||
import json
|
||||
from StringIO import StringIO
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class CannotWriteConfigException(Exception):
|
||||
""" Exception raised when the config cannot be written. """
|
||||
pass
|
||||
|
||||
def _import_yaml(config_obj, config_file):
|
||||
with open(config_file) as f:
|
||||
c = yaml.safe_load(f)
|
||||
if not c:
|
||||
logger.debug('Empty YAML config file')
|
||||
return
|
||||
|
||||
if isinstance(c, str):
|
||||
raise Exception('Invalid YAML config file: ' + str(c))
|
||||
|
||||
for key in c.iterkeys():
|
||||
if key.isupper():
|
||||
config_obj[key] = c[key]
|
||||
|
||||
return config_obj
|
||||
|
||||
|
||||
def _export_yaml(config_obj, config_file):
|
||||
try:
|
||||
with open(config_file, 'w') as f:
|
||||
f.write(yaml.safe_dump(config_obj, encoding='utf-8', allow_unicode=True))
|
||||
except IOError as ioe:
|
||||
raise CannotWriteConfigException(str(ioe))
|
||||
|
||||
|
||||
class BaseProvider(object):
|
||||
""" A configuration provider helps to load, save, and handle config override in the application.
|
||||
"""
|
||||
|
||||
def update_app_config(self, app_config):
|
||||
""" Updates the given application config object with the loaded override config. """
|
||||
raise NotImplementedError
|
||||
|
||||
def get_yaml(self):
|
||||
""" Returns the contents of the YAML config override file, or None if none. """
|
||||
raise NotImplementedError
|
||||
|
||||
def save_yaml(self, config_object):
|
||||
""" Updates the contents of the YAML config override file to those given. """
|
||||
raise NotImplementedError
|
||||
|
||||
def yaml_exists(self):
|
||||
""" Returns true if a YAML config override file exists in the config volume. """
|
||||
raise NotImplementedError
|
||||
|
||||
def volume_exists(self):
|
||||
""" Returns whether the config override volume exists. """
|
||||
raise NotImplementedError
|
||||
|
||||
def volume_file_exists(self, filename):
|
||||
""" Returns whether the file with the given name exists under the config override volume. """
|
||||
raise NotImplementedError
|
||||
|
||||
def get_volume_file(self, filename, mode='r'):
|
||||
""" Returns a Python file referring to the given name under the config override volumne. """
|
||||
raise NotImplementedError
|
||||
|
||||
def save_volume_file(self, filename, flask_file):
|
||||
""" Saves the given flask file to the config override volume, with the given
|
||||
filename.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def requires_restart(self, app_config):
|
||||
""" If true, the configuration loaded into memory for the app does not match that on disk,
|
||||
indicating that this container requires a restart.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class FileConfigProvider(BaseProvider):
|
||||
""" Implementation of the config provider that reads the data from the file system. """
|
||||
def __init__(self, config_volume, yaml_filename, py_filename):
|
||||
self.config_volume = config_volume
|
||||
self.yaml_filename = yaml_filename
|
||||
self.py_filename = py_filename
|
||||
|
||||
self.yaml_path = os.path.join(config_volume, yaml_filename)
|
||||
self.py_path = os.path.join(config_volume, py_filename)
|
||||
|
||||
def update_app_config(self, app_config):
|
||||
if os.path.exists(self.py_path):
|
||||
logger.debug('Applying config file: %s', self.py_path)
|
||||
app_config.from_pyfile(self.py_path)
|
||||
|
||||
if os.path.exists(self.yaml_path):
|
||||
logger.debug('Applying config file: %s', self.yaml_path)
|
||||
_import_yaml(app_config, self.yaml_path)
|
||||
|
||||
def get_yaml(self):
|
||||
if not os.path.exists(self.yaml_path):
|
||||
return None
|
||||
|
||||
config_obj = {}
|
||||
_import_yaml(config_obj, self.yaml_path)
|
||||
return config_obj
|
||||
|
||||
def save_yaml(self, config_obj):
|
||||
_export_yaml(config_obj, self.yaml_path)
|
||||
|
||||
def yaml_exists(self):
|
||||
return self.volume_file_exists(self.yaml_filename)
|
||||
|
||||
def volume_exists(self):
|
||||
return os.path.exists(self.config_volume)
|
||||
|
||||
def volume_file_exists(self, filename):
|
||||
return os.path.exists(os.path.join(self.config_volume, filename))
|
||||
|
||||
def get_volume_file(self, filename, mode='r'):
|
||||
return open(os.path.join(self.config_volume, filename), mode)
|
||||
|
||||
def save_volume_file(self, filename, flask_file):
|
||||
try:
|
||||
flask_file.save(os.path.join(self.config_volume, filename))
|
||||
except IOError as ioe:
|
||||
raise CannotWriteConfigException(str(ioe))
|
||||
|
||||
def requires_restart(self, app_config):
|
||||
file_config = self.get_yaml()
|
||||
if not file_config:
|
||||
return False
|
||||
|
||||
for key in file_config:
|
||||
if app_config.get(key) != file_config[key]:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
class TestConfigProvider(BaseProvider):
|
||||
""" Implementation of the config provider for testing. Everything is kept in-memory instead on
|
||||
the real file system. """
|
||||
def __init__(self):
|
||||
self.files = {}
|
||||
self._config = None
|
||||
|
||||
def update_app_config(self, app_config):
|
||||
self._config = app_config
|
||||
|
||||
def get_yaml(self):
|
||||
if not 'config.yaml' in self.files:
|
||||
return None
|
||||
|
||||
return json.loads(self.files.get('config.yaml', '{}'))
|
||||
|
||||
def save_yaml(self, config_obj):
|
||||
self.files['config.yaml'] = json.dumps(config_obj)
|
||||
|
||||
def yaml_exists(self):
|
||||
return 'config.yaml' in self.files
|
||||
|
||||
def volume_exists(self):
|
||||
return True
|
||||
|
||||
def volume_file_exists(self, filename):
|
||||
return filename in self.files
|
||||
|
||||
def save_volume_file(self, filename, flask_file):
|
||||
self.files[filename] = ''
|
||||
|
||||
def get_volume_file(self, filename, mode='r'):
|
||||
return StringIO(self.files[filename])
|
||||
|
||||
def requires_restart(self, app_config):
|
||||
return False
|
||||
|
||||
def reset_for_test(self):
|
||||
self._config['SUPER_USERS'] = ['devtable']
|
||||
self.files = {}
|
||||
16
util/config/provider/__init__.py
Normal file
16
util/config/provider/__init__.py
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
from util.config.provider.fileprovider import FileConfigProvider
|
||||
from util.config.provider.testprovider import TestConfigProvider
|
||||
from util.config.provider.k8sprovider import KubernetesConfigProvider
|
||||
|
||||
import os
|
||||
|
||||
def get_config_provider(config_volume, yaml_filename, py_filename, testing=False, kubernetes=False):
|
||||
""" Loads and returns the config provider for the current environment. """
|
||||
if testing:
|
||||
return TestConfigProvider()
|
||||
|
||||
if kubernetes:
|
||||
return KubernetesConfigProvider(config_volume, yaml_filename, py_filename)
|
||||
|
||||
return FileConfigProvider(config_volume, yaml_filename, py_filename)
|
||||
|
||||
84
util/config/provider/baseprovider.py
Normal file
84
util/config/provider/baseprovider.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
import yaml
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class CannotWriteConfigException(Exception):
|
||||
""" Exception raised when the config cannot be written. """
|
||||
pass
|
||||
|
||||
def import_yaml(config_obj, config_file):
|
||||
with open(config_file) as f:
|
||||
c = yaml.safe_load(f)
|
||||
if not c:
|
||||
logger.debug('Empty YAML config file')
|
||||
return
|
||||
|
||||
if isinstance(c, str):
|
||||
raise Exception('Invalid YAML config file: ' + str(c))
|
||||
|
||||
for key in c.iterkeys():
|
||||
if key.isupper():
|
||||
config_obj[key] = c[key]
|
||||
|
||||
return config_obj
|
||||
|
||||
|
||||
def get_yaml(config_obj):
|
||||
return yaml.safe_dump(config_obj, encoding='utf-8', allow_unicode=True)
|
||||
|
||||
def export_yaml(config_obj, config_file):
|
||||
try:
|
||||
with open(config_file, 'w') as f:
|
||||
f.write(get_yaml(config_obj))
|
||||
except IOError as ioe:
|
||||
raise CannotWriteConfigException(str(ioe))
|
||||
|
||||
|
||||
class BaseProvider(object):
|
||||
""" A configuration provider helps to load, save, and handle config override in the application.
|
||||
"""
|
||||
|
||||
@property
|
||||
def provider_id(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def update_app_config(self, app_config):
|
||||
""" Updates the given application config object with the loaded override config. """
|
||||
raise NotImplementedError
|
||||
|
||||
def get_config(self):
|
||||
""" Returns the contents of the config override file, or None if none. """
|
||||
raise NotImplementedError
|
||||
|
||||
def save_config(self, config_object):
|
||||
""" Updates the contents of the config override file to those given. """
|
||||
raise NotImplementedError
|
||||
|
||||
def config_exists(self):
|
||||
""" Returns true if a config override file exists in the config volume. """
|
||||
raise NotImplementedError
|
||||
|
||||
def volume_exists(self):
|
||||
""" Returns whether the config override volume exists. """
|
||||
raise NotImplementedError
|
||||
|
||||
def volume_file_exists(self, filename):
|
||||
""" Returns whether the file with the given name exists under the config override volume. """
|
||||
raise NotImplementedError
|
||||
|
||||
def get_volume_file(self, filename, mode='r'):
|
||||
""" Returns a Python file referring to the given name under the config override volumne. """
|
||||
raise NotImplementedError
|
||||
|
||||
def save_volume_file(self, filename, flask_file):
|
||||
""" Saves the given flask file to the config override volume, with the given
|
||||
filename.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def requires_restart(self, app_config):
|
||||
""" If true, the configuration loaded into memory for the app does not match that on disk,
|
||||
indicating that this container requires a restart.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
73
util/config/provider/fileprovider.py
Normal file
73
util/config/provider/fileprovider.py
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
import os
|
||||
import logging
|
||||
|
||||
from util.config.provider.baseprovider import (BaseProvider, import_yaml, export_yaml,
|
||||
CannotWriteConfigException)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class FileConfigProvider(BaseProvider):
|
||||
""" Implementation of the config provider that reads the data from the file system. """
|
||||
def __init__(self, config_volume, yaml_filename, py_filename):
|
||||
self.config_volume = config_volume
|
||||
self.yaml_filename = yaml_filename
|
||||
self.py_filename = py_filename
|
||||
|
||||
self.yaml_path = os.path.join(config_volume, yaml_filename)
|
||||
self.py_path = os.path.join(config_volume, py_filename)
|
||||
|
||||
@property
|
||||
def provider_id(self):
|
||||
return 'file'
|
||||
|
||||
def update_app_config(self, app_config):
|
||||
if os.path.exists(self.py_path):
|
||||
logger.debug('Applying config file: %s', self.py_path)
|
||||
app_config.from_pyfile(self.py_path)
|
||||
|
||||
if os.path.exists(self.yaml_path):
|
||||
logger.debug('Applying config file: %s', self.yaml_path)
|
||||
import_yaml(app_config, self.yaml_path)
|
||||
|
||||
def get_config(self):
|
||||
if not os.path.exists(self.yaml_path):
|
||||
return None
|
||||
|
||||
config_obj = {}
|
||||
import_yaml(config_obj, self.yaml_path)
|
||||
return config_obj
|
||||
|
||||
def save_config(self, config_obj):
|
||||
export_yaml(config_obj, self.yaml_path)
|
||||
|
||||
def config_exists(self):
|
||||
return self.volume_file_exists(self.yaml_filename)
|
||||
|
||||
def volume_exists(self):
|
||||
return os.path.exists(self.config_volume)
|
||||
|
||||
def volume_file_exists(self, filename):
|
||||
return os.path.exists(os.path.join(self.config_volume, filename))
|
||||
|
||||
def get_volume_file(self, filename, mode='r'):
|
||||
return open(os.path.join(self.config_volume, filename), mode)
|
||||
|
||||
def save_volume_file(self, filename, flask_file):
|
||||
filepath = os.path.join(self.config_volume, filename)
|
||||
try:
|
||||
flask_file.save(filepath)
|
||||
except IOError as ioe:
|
||||
raise CannotWriteConfigException(str(ioe))
|
||||
|
||||
return filepath
|
||||
|
||||
def requires_restart(self, app_config):
|
||||
file_config = self.get_config()
|
||||
if not file_config:
|
||||
return False
|
||||
|
||||
for key in file_config:
|
||||
if app_config.get(key) != file_config[key]:
|
||||
return True
|
||||
|
||||
return False
|
||||
109
util/config/provider/k8sprovider.py
Normal file
109
util/config/provider/k8sprovider.py
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
import os
|
||||
import logging
|
||||
import json
|
||||
import base64
|
||||
|
||||
from requests import Request, Session
|
||||
|
||||
from util.config.provider.baseprovider import get_yaml, CannotWriteConfigException
|
||||
from util.config.provider.fileprovider import FileConfigProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
KUBERNETES_API_HOST = 'kubernetes.default.svc.cluster.local'
|
||||
|
||||
SERVICE_ACCOUNT_TOKEN_PATH = '/var/run/secrets/kubernetes.io/serviceaccount/token'
|
||||
|
||||
ER_NAMESPACE = 'quay'
|
||||
ER_CONFIG_SECRET = 'quay-config-secret'
|
||||
|
||||
class KubernetesConfigProvider(FileConfigProvider):
|
||||
""" Implementation of the config provider that reads and writes configuration
|
||||
data from a Kubernetes Secret. """
|
||||
def __init__(self, config_volume, yaml_filename, py_filename):
|
||||
super(KubernetesConfigProvider, self).__init__(config_volume, yaml_filename, py_filename)
|
||||
|
||||
self.yaml_filename = yaml_filename
|
||||
|
||||
# Load the service account token from the local store.
|
||||
if not os.path.exists(SERVICE_ACCOUNT_TOKEN_PATH):
|
||||
raise Exception('Cannot load Kubernetes service account token')
|
||||
|
||||
with open(SERVICE_ACCOUNT_TOKEN_PATH, 'r') as f:
|
||||
self._service_token = f.read()
|
||||
|
||||
# Make sure the configuration volume exists.
|
||||
if not self.volume_exists():
|
||||
os.makedirs(config_volume)
|
||||
|
||||
@property
|
||||
def provider_id(self):
|
||||
return 'k8s'
|
||||
|
||||
def save_config(self, config_obj):
|
||||
self._update_secret_file(self.yaml_filename, get_yaml(config_obj))
|
||||
super(KubernetesConfigProvider, self).save_config(config_obj)
|
||||
|
||||
def save_volume_file(self, filename, flask_file):
|
||||
filepath = super(KubernetesConfigProvider, self).save_volume_file(filename, flask_file)
|
||||
|
||||
try:
|
||||
with open(filepath, 'r') as f:
|
||||
self._update_secret_file(filename, f.read())
|
||||
except IOError as ioe:
|
||||
raise CannotWriteConfigException(str(ioe))
|
||||
|
||||
def _assert_success(self, response):
|
||||
if response.status_code != 200:
|
||||
logger.error('K8s API call failed with response: %s => %s', response.status_code,
|
||||
response.text)
|
||||
raise CannotWriteConfigException('K8s API call failed. Please report this to support')
|
||||
|
||||
def _update_secret_file(self, filename, value):
|
||||
secret_data = {}
|
||||
secret_data[filename] = base64.b64encode(value)
|
||||
|
||||
data = {
|
||||
"kind": "Secret",
|
||||
"apiVersion": "v1",
|
||||
"metadata": {
|
||||
"name": ER_CONFIG_SECRET
|
||||
},
|
||||
"data": secret_data
|
||||
}
|
||||
|
||||
secret_url = 'namespaces/%s/secrets/%s' % (ER_NAMESPACE, ER_CONFIG_SECRET)
|
||||
secret = self._lookup_secret()
|
||||
if not secret:
|
||||
self._assert_success(self._execute_k8s_api('POST', secret_url, data))
|
||||
return
|
||||
|
||||
if not 'data' in secret:
|
||||
secret['data'] = {}
|
||||
|
||||
secret['data'][filename] = base64.b64encode(value)
|
||||
self._assert_success(self._execute_k8s_api('PUT', secret_url, secret))
|
||||
|
||||
|
||||
def _lookup_secret(self):
|
||||
secret_url = 'namespaces/%s/secrets/%s' % (ER_NAMESPACE, ER_CONFIG_SECRET)
|
||||
response = self._execute_k8s_api('GET', secret_url)
|
||||
if response.status_code != 200:
|
||||
return None
|
||||
|
||||
return json.loads(response.text)
|
||||
|
||||
def _execute_k8s_api(self, method, relative_url, data=None):
|
||||
headers = {
|
||||
'Authorization': 'Bearer ' + self._service_token
|
||||
}
|
||||
|
||||
if data:
|
||||
headers['Content-Type'] = 'application/json'
|
||||
|
||||
data = json.dumps(data) if data else None
|
||||
session = Session()
|
||||
url = 'https://%s/api/v1/%s' % (KUBERNETES_API_HOST, relative_url)
|
||||
|
||||
request = Request(method, url, data=data, headers=headers)
|
||||
return session.send(request.prepare(), verify=False, timeout=2)
|
||||
49
util/config/provider/testprovider.py
Normal file
49
util/config/provider/testprovider.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
import json
|
||||
from StringIO import StringIO
|
||||
|
||||
from util.config.provider.baseprovider import BaseProvider
|
||||
|
||||
class TestConfigProvider(BaseProvider):
|
||||
""" Implementation of the config provider for testing. Everything is kept in-memory instead on
|
||||
the real file system. """
|
||||
def __init__(self):
|
||||
self.files = {}
|
||||
self._config = None
|
||||
|
||||
@property
|
||||
def provider_id(self):
|
||||
return 'test'
|
||||
|
||||
def update_app_config(self, app_config):
|
||||
self._config = app_config
|
||||
|
||||
def get_config(self):
|
||||
if not 'config.yaml' in self.files:
|
||||
return None
|
||||
|
||||
return json.loads(self.files.get('config.yaml', '{}'))
|
||||
|
||||
def save_config(self, config_obj):
|
||||
self.files['config.yaml'] = json.dumps(config_obj)
|
||||
|
||||
def config_exists(self):
|
||||
return 'config.yaml' in self.files
|
||||
|
||||
def volume_exists(self):
|
||||
return True
|
||||
|
||||
def volume_file_exists(self, filename):
|
||||
return filename in self.files
|
||||
|
||||
def save_volume_file(self, filename, flask_file):
|
||||
self.files[filename] = ''
|
||||
|
||||
def get_volume_file(self, filename, mode='r'):
|
||||
return StringIO(self.files[filename])
|
||||
|
||||
def requires_restart(self, app_config):
|
||||
return False
|
||||
|
||||
def reset_for_test(self):
|
||||
self._config['SUPER_USERS'] = ['devtable']
|
||||
self.files = {}
|
||||
|
|
@ -19,7 +19,7 @@ from auth.auth_context import get_authenticated_user
|
|||
from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig
|
||||
from bitbucket import BitBucket
|
||||
|
||||
from app import app, CONFIG_PROVIDER, get_app_url, OVERRIDE_CONFIG_DIRECTORY
|
||||
from app import app, config_provider, get_app_url, OVERRIDE_CONFIG_DIRECTORY
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -83,7 +83,7 @@ def _validate_registry_storage(config, _):
|
|||
driver = get_storage_provider(config)
|
||||
|
||||
# Run custom validation on the driver.
|
||||
driver.validate()
|
||||
driver.validate(app.config['HTTPCLIENT'])
|
||||
|
||||
# Put and remove a temporary file to make sure the normal storage paths work.
|
||||
driver.put_content('_verify', 'testing 123')
|
||||
|
|
@ -223,10 +223,10 @@ def _validate_ssl(config, _):
|
|||
return
|
||||
|
||||
for filename in SSL_FILENAMES:
|
||||
if not CONFIG_PROVIDER.volume_file_exists(filename):
|
||||
if not config_provider.volume_file_exists(filename):
|
||||
raise Exception('Missing required SSL file: %s' % filename)
|
||||
|
||||
with CONFIG_PROVIDER.get_volume_file(SSL_FILENAMES[0]) as f:
|
||||
with config_provider.get_volume_file(SSL_FILENAMES[0]) as f:
|
||||
cert_contents = f.read()
|
||||
|
||||
# Validate the certificate.
|
||||
|
|
@ -239,7 +239,7 @@ def _validate_ssl(config, _):
|
|||
raise Exception('The specified SSL certificate has expired.')
|
||||
|
||||
private_key_path = None
|
||||
with CONFIG_PROVIDER.get_volume_file(SSL_FILENAMES[1]) as f:
|
||||
with config_provider.get_volume_file(SSL_FILENAMES[1]) as f:
|
||||
private_key_path = f.name
|
||||
|
||||
if not private_key_path:
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@ import logging
|
|||
import json
|
||||
|
||||
from app import app
|
||||
from data.database import configure, RepositoryBuildTrigger, BuildTriggerService
|
||||
from data.database import configure, BaseModel, uuid_generator
|
||||
from peewee import *
|
||||
from bitbucket import BitBucket
|
||||
from endpoints.trigger import BitbucketBuildTrigger
|
||||
|
||||
|
|
@ -10,6 +11,31 @@ configure(app.config)
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Note: We vendor the RepositoryBuildTrigger and its dependencies here
|
||||
class Repository(BaseModel):
|
||||
pass
|
||||
|
||||
class BuildTriggerService(BaseModel):
|
||||
name = CharField(index=True, unique=True)
|
||||
|
||||
class AccessToken(BaseModel):
|
||||
pass
|
||||
|
||||
class User(BaseModel):
|
||||
pass
|
||||
|
||||
class RepositoryBuildTrigger(BaseModel):
|
||||
uuid = CharField(default=uuid_generator)
|
||||
service = ForeignKeyField(BuildTriggerService, index=True)
|
||||
repository = ForeignKeyField(Repository, index=True)
|
||||
connected_user = ForeignKeyField(User)
|
||||
auth_token = CharField(null=True)
|
||||
private_key = TextField(null=True)
|
||||
config = TextField(default='{}')
|
||||
write_token = ForeignKeyField(AccessToken, null=True)
|
||||
pull_robot = ForeignKeyField(User, related_name='triggerpullrobot')
|
||||
|
||||
|
||||
def run_bitbucket_migration():
|
||||
bitbucket_trigger = BuildTriggerService.get(BuildTriggerService.name == "bitbucket")
|
||||
|
||||
|
|
|
|||
87
util/migrate/migrategithubdeploykeys.py
Normal file
87
util/migrate/migrategithubdeploykeys.py
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
import logging
|
||||
import logging.config
|
||||
import json
|
||||
|
||||
from data.database import RepositoryBuildTrigger, BuildTriggerService, db, db_for_update
|
||||
from app import app
|
||||
from endpoints.trigger import BuildTriggerHandler
|
||||
from util.security.ssh import generate_ssh_keypair
|
||||
from github import GithubException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def backfill_github_deploykeys():
|
||||
""" Generates and saves private deploy keys for any GitHub build triggers still relying on
|
||||
the old buildpack behavior. """
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.debug('GitHub deploy key backfill: Began execution')
|
||||
|
||||
encountered = set()
|
||||
github_service = BuildTriggerService.get(name='github')
|
||||
|
||||
while True:
|
||||
build_trigger_ids = list(RepositoryBuildTrigger
|
||||
.select(RepositoryBuildTrigger.id)
|
||||
.where(RepositoryBuildTrigger.private_key >> None)
|
||||
.where(RepositoryBuildTrigger.service == github_service)
|
||||
.limit(10))
|
||||
|
||||
filtered_ids = [trigger.id for trigger in build_trigger_ids if trigger.id not in encountered]
|
||||
if len(filtered_ids) == 0:
|
||||
# We're done!
|
||||
logger.debug('GitHub deploy key backfill: Backfill completed')
|
||||
return
|
||||
|
||||
logger.debug('GitHub deploy key backfill: Found %s records to update', len(filtered_ids))
|
||||
for trigger_id in filtered_ids:
|
||||
encountered.add(trigger_id)
|
||||
logger.debug('Updating build trigger: %s', trigger_id)
|
||||
|
||||
with app.config['DB_TRANSACTION_FACTORY'](db):
|
||||
try:
|
||||
query = RepositoryBuildTrigger.select(RepositoryBuildTrigger.id == trigger_id)
|
||||
trigger = db_for_update(query).get()
|
||||
except RepositoryBuildTrigger.DoesNotExist:
|
||||
logger.debug('Could not find build trigger %s', trigger_id)
|
||||
continue
|
||||
|
||||
handler = BuildTriggerHandler.get_handler(trigger)
|
||||
|
||||
config = handler.config
|
||||
build_source = config['build_source']
|
||||
gh_client = handler._get_client()
|
||||
|
||||
# Find the GitHub repository.
|
||||
try:
|
||||
gh_repo = gh_client.get_repo(build_source)
|
||||
except GithubException:
|
||||
logger.exception('Cannot find repository %s for trigger %s', build_source, trigger.id)
|
||||
continue
|
||||
|
||||
# Add a deploy key to the GitHub repository.
|
||||
public_key, private_key = generate_ssh_keypair()
|
||||
config['credentials'] = [
|
||||
{
|
||||
'name': 'SSH Public Key',
|
||||
'value': public_key,
|
||||
},
|
||||
]
|
||||
|
||||
logger.debug('Adding deploy key to build trigger %s', trigger.id)
|
||||
try:
|
||||
deploy_key = gh_repo.create_key('%s Builder' % app.config['REGISTRY_TITLE'], public_key)
|
||||
config['deploy_key_id'] = deploy_key.id
|
||||
except GithubException:
|
||||
logger.exception('Cannot add deploy key to repository %s for trigger %s', build_source, trigger.id)
|
||||
continue
|
||||
|
||||
logger.debug('Saving deploy key for trigger %s', trigger.id)
|
||||
trigger.used_legacy_github = True
|
||||
trigger.private_key = private_key
|
||||
trigger.config = json.dumps(config)
|
||||
trigger.save()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False)
|
||||
backfill_github_deploykeys()
|
||||
14
util/seo.py
14
util/seo.py
|
|
@ -32,11 +32,13 @@ def render_snapshot(url):
|
|||
# Remove script tags
|
||||
logger.info('Removing script tags: %s' % url)
|
||||
|
||||
soup = BeautifulSoup(out_html.decode('utf8'))
|
||||
to_extract = soup.findAll('script')
|
||||
for item in to_extract:
|
||||
item.extract()
|
||||
|
||||
logger.info('Snapshotted url: %s' % url)
|
||||
try:
|
||||
soup = BeautifulSoup(out_html.decode('utf8'), 'html.parser')
|
||||
to_extract = soup.findAll('script')
|
||||
for item in to_extract:
|
||||
item.extract()
|
||||
except:
|
||||
logger.exception('Exception when trying to parse served HTML')
|
||||
return out_html
|
||||
|
||||
return str(soup)
|
||||
|
|
|
|||
Reference in a new issue