Add cycling deployments and updating config

Add kube config with refactor to kube accessor

Add tests for k8s accessor, some styling changes
This commit is contained in:
Sam Chow 2018-08-09 16:43:11 -04:00
parent d387ba171f
commit eea5fe3391
23 changed files with 830 additions and 560 deletions

View file

@ -18,11 +18,11 @@ CROSS_DOMAIN_HEADERS = ['Authorization', 'Content-Type', 'X-Requested-With']
class ApiExceptionHandlingApi(Api):
pass
pass
@crossdomain(origin='*', headers=CROSS_DOMAIN_HEADERS)
def handle_error(self, error):
return super(ApiExceptionHandlingApi, self).handle_error(error)
@crossdomain(origin='*', headers=CROSS_DOMAIN_HEADERS)
def handle_error(self, error):
return super(ApiExceptionHandlingApi, self).handle_error(error)
api = ApiExceptionHandlingApi()
@ -31,111 +31,112 @@ api.init_app(api_bp)
def format_date(date):
""" Output an RFC822 date format. """
if date is None:
return None
return formatdate(timegm(date.utctimetuple()))
""" Output an RFC822 date format. """
if date is None:
return None
return formatdate(timegm(date.utctimetuple()))
def resource(*urls, **kwargs):
def wrapper(api_resource):
if not api_resource:
return None
def wrapper(api_resource):
if not api_resource:
return None
api_resource.registered = True
api.add_resource(api_resource, *urls, **kwargs)
return api_resource
api_resource.registered = True
api.add_resource(api_resource, *urls, **kwargs)
return api_resource
return wrapper
return wrapper
class ApiResource(Resource):
registered = False
method_decorators = []
registered = False
method_decorators = []
def options(self):
return None, 200
def options(self):
return None, 200
def add_method_metadata(name, value):
def modifier(func):
if func is None:
return None
def modifier(func):
if func is None:
return None
if '__api_metadata' not in dir(func):
func.__api_metadata = {}
func.__api_metadata[name] = value
return func
if '__api_metadata' not in dir(func):
func.__api_metadata = {}
func.__api_metadata[name] = value
return func
return modifier
return modifier
def method_metadata(func, name):
if func is None:
return None
if '__api_metadata' in dir(func):
return func.__api_metadata.get(name, None)
if func is None:
return None
if '__api_metadata' in dir(func):
return func.__api_metadata.get(name, None)
return None
def no_cache(f):
@wraps(f)
def add_no_cache(*args, **kwargs):
response = f(*args, **kwargs)
if response is not None:
response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
return response
return add_no_cache
@wraps(f)
def add_no_cache(*args, **kwargs):
response = f(*args, **kwargs)
if response is not None:
response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
return response
return add_no_cache
def define_json_response(schema_name):
def wrapper(func):
@add_method_metadata('response_schema', schema_name)
@wraps(func)
def wrapped(self, *args, **kwargs):
schema = self.schemas[schema_name]
resp = func(self, *args, **kwargs)
def wrapper(func):
@add_method_metadata('response_schema', schema_name)
@wraps(func)
def wrapped(self, *args, **kwargs):
schema = self.schemas[schema_name]
resp = func(self, *args, **kwargs)
if app.config['TESTING']:
try:
validate(resp, schema)
except ValidationError as ex:
raise InvalidResponse(ex.message)
if app.config['TESTING']:
try:
validate(resp, schema)
except ValidationError as ex:
raise InvalidResponse(ex.message)
return resp
return wrapped
return wrapper
return resp
return wrapped
return wrapper
def validate_json_request(schema_name, optional=False):
def wrapper(func):
@add_method_metadata('request_schema', schema_name)
@wraps(func)
def wrapped(self, *args, **kwargs):
schema = self.schemas[schema_name]
try:
json_data = request.get_json()
if json_data is None:
if not optional:
raise InvalidRequest('Missing JSON body')
else:
validate(json_data, schema)
return func(self, *args, **kwargs)
except ValidationError as ex:
raise InvalidRequest(ex.message)
return wrapped
return wrapper
def wrapper(func):
@add_method_metadata('request_schema', schema_name)
@wraps(func)
def wrapped(self, *args, **kwargs):
schema = self.schemas[schema_name]
try:
json_data = request.get_json()
if json_data is None:
if not optional:
raise InvalidRequest('Missing JSON body')
else:
validate(json_data, schema)
return func(self, *args, **kwargs)
except ValidationError as ex:
raise InvalidRequest(ex.message)
return wrapped
return wrapper
def kubernetes_only(f):
@wraps(f)
def abort_if_not_kube(*args, **kwargs):
if not IS_KUBERNETES:
abort(400)
""" Aborts the request with a 400 if the app is not running on kubernetes """
@wraps(f)
def abort_if_not_kube(*args, **kwargs):
if not IS_KUBERNETES:
abort(400)
return f(*args, **kwargs)
return abort_if_not_kube
return f(*args, **kwargs)
return abort_if_not_kube
nickname = partial(add_method_metadata, 'nickname')

View file

@ -5,8 +5,8 @@ from flask import abort, request
from config_app.config_endpoints.api.suconfig_models_pre_oci import pre_oci_model as model
from config_app.config_endpoints.api import resource, ApiResource, nickname, validate_json_request, kubernetes_only
from config_app.c_app import (app, config_provider, superusers, ip_resolver,
instance_keys, INIT_SCRIPTS_LOCATION, IS_KUBERNETES)
from config_app.config_util.k8sinterface import KubernetesAccessInterface, kubernetes_access_instance
instance_keys, INIT_SCRIPTS_LOCATION)
from config_app.config_util.k8saccessor import KubernetesAccessorSingleton
from data.database import configure
from data.runmigration import run_alembic_migration
@ -17,293 +17,316 @@ logger = logging.getLogger(__name__)
def database_is_valid():
""" Returns whether the database, as configured, is valid. """
return model.is_valid()
""" Returns whether the database, as configured, is valid. """
return model.is_valid()
def database_has_users():
""" Returns whether the database has any users defined. """
return model.has_users()
""" Returns whether the database has any users defined. """
return model.has_users()
@resource('/v1/superuser/config')
class SuperUserConfig(ApiResource):
""" Resource for fetching and updating the current configuration, if any. """
schemas = {
'UpdateConfig': {
'type': 'object',
'description': 'Updates the YAML config file',
'required': [
'config',
'hostname'
],
'properties': {
'config': {
'type': 'object'
},
'hostname': {
'type': 'string'
},
'password': {
'type': 'string'
},
},
""" Resource for fetching and updating the current configuration, if any. """
schemas = {
'UpdateConfig': {
'type': 'object',
'description': 'Updates the YAML config file',
'required': [
'config',
'hostname'
],
'properties': {
'config': {
'type': 'object'
},
'hostname': {
'type': 'string'
},
'password': {
'type': 'string'
},
},
},
}
@nickname('scGetConfig')
def get(self):
""" Returns the currently defined configuration, if any. """
config_object = config_provider.get_config()
return {
'config': config_object
}
@nickname('scGetConfig')
def get(self):
""" Returns the currently defined configuration, if any. """
config_object = config_provider.get_config()
return {
'config': config_object
}
@nickname('scUpdateConfig')
@validate_json_request('UpdateConfig')
def put(self):
""" Updates the config override file. """
# Note: This method is called to set the database configuration before super users exists,
# so we also allow it to be called if there is no valid registry configuration setup.
config_object = request.get_json()['config']
hostname = request.get_json()['hostname']
@nickname('scUpdateConfig')
@validate_json_request('UpdateConfig')
def put(self):
""" Updates the config override file. """
# Note: This method is called to set the database configuration before super users exists,
# so we also allow it to be called if there is no valid registry configuration setup.
config_object = request.get_json()['config']
hostname = request.get_json()['hostname']
# Add any enterprise defaults missing from the config.
add_enterprise_config_defaults(config_object, app.config['SECRET_KEY'], hostname)
# Add any enterprise defaults missing from the config.
add_enterprise_config_defaults(config_object, app.config['SECRET_KEY'], hostname)
# Write the configuration changes to the config override file.
config_provider.save_config(config_object)
# Write the configuration changes to the config override file.
config_provider.save_config(config_object)
return {
'exists': True,
'config': config_object
}
return {
'exists': True,
'config': config_object
}
@resource('/v1/superuser/registrystatus')
class SuperUserRegistryStatus(ApiResource):
""" Resource for determining the status of the registry, such as if config exists,
if a database is configured, and if it has any defined users.
"""
@nickname('scRegistryStatus')
def get(self):
""" Returns the status of the registry. """
""" Resource for determining the status of the registry, such as if config exists,
if a database is configured, and if it has any defined users.
"""
@nickname('scRegistryStatus')
def get(self):
""" Returns the status of the registry. """
# If there is no config file, we need to setup the database.
if not config_provider.config_exists():
return {
'status': 'config-db'
}
# If there is no config file, we need to setup the database.
if not config_provider.config_exists():
return {
'status': 'config-db'
}
# If the database isn't yet valid, then we need to set it up.
if not database_is_valid():
return {
'status': 'setup-db'
}
# If the database isn't yet valid, then we need to set it up.
if not database_is_valid():
return {
'status': 'setup-db'
}
return {
'status': 'create-superuser' if not database_has_users() else 'config'
}
return {
'status': 'create-superuser' if not database_has_users() else 'config'
}
class _AlembicLogHandler(logging.Handler):
def __init__(self):
super(_AlembicLogHandler, self).__init__()
self.records = []
def __init__(self):
super(_AlembicLogHandler, self).__init__()
self.records = []
def emit(self, record):
self.records.append({
'level': record.levelname,
'message': record.getMessage()
})
def emit(self, record):
self.records.append({
'level': record.levelname,
'message': record.getMessage()
})
@resource('/v1/superuser/setupdb')
class SuperUserSetupDatabase(ApiResource):
""" Resource for invoking alembic to setup the database. """
@nickname('scSetupDatabase')
def get(self):
""" Invokes the alembic upgrade process. """
# Note: This method is called after the database configured is saved, but before the
# database has any tables. Therefore, we only allow it to be run in that unique case.
if config_provider.config_exists() and not database_is_valid():
# Note: We need to reconfigure the database here as the config has changed.
combined = dict(**app.config)
combined.update(config_provider.get_config())
""" Resource for invoking alembic to setup the database. """
@nickname('scSetupDatabase')
def get(self):
""" Invokes the alembic upgrade process. """
# Note: This method is called after the database configured is saved, but before the
# database has any tables. Therefore, we only allow it to be run in that unique case.
if config_provider.config_exists() and not database_is_valid():
# Note: We need to reconfigure the database here as the config has changed.
combined = dict(**app.config)
combined.update(config_provider.get_config())
configure(combined)
app.config['DB_URI'] = combined['DB_URI']
db_uri = app.config['DB_URI']
configure(combined)
app.config['DB_URI'] = combined['DB_URI']
db_uri = app.config['DB_URI']
log_handler = _AlembicLogHandler()
log_handler = _AlembicLogHandler()
try:
run_alembic_migration(db_uri, log_handler, setup_app=False)
except Exception as ex:
return {
'error': str(ex)
}
try:
run_alembic_migration(db_uri, log_handler, setup_app=False)
except Exception as ex:
return {
'error': str(ex)
}
return {
'logs': log_handler.records
}
return {
'logs': log_handler.records
}
abort(403)
abort(403)
@resource('/v1/superuser/config/createsuperuser')
class SuperUserCreateInitialSuperUser(ApiResource):
""" Resource for creating the initial super user. """
schemas = {
'CreateSuperUser': {
'type': 'object',
'description': 'Information for creating the initial super user',
'required': [
'username',
'password',
'email'
],
'properties': {
'username': {
'type': 'string',
'description': 'The username for the superuser'
},
'password': {
'type': 'string',
'description': 'The password for the superuser'
},
'email': {
'type': 'string',
'description': 'The e-mail address for the superuser'
},
},
""" Resource for creating the initial super user. """
schemas = {
'CreateSuperUser': {
'type': 'object',
'description': 'Information for creating the initial super user',
'required': [
'username',
'password',
'email'
],
'properties': {
'username': {
'type': 'string',
'description': 'The username for the superuser'
},
}
'password': {
'type': 'string',
'description': 'The password for the superuser'
},
'email': {
'type': 'string',
'description': 'The e-mail address for the superuser'
},
},
},
}
@nickname('scCreateInitialSuperuser')
@validate_json_request('CreateSuperUser')
def post(self):
""" Creates the initial super user, updates the underlying configuration and
sets the current session to have that super user. """
@nickname('scCreateInitialSuperuser')
@validate_json_request('CreateSuperUser')
def post(self):
""" Creates the initial super user, updates the underlying configuration and
sets the current session to have that super user. """
# Special security check: This method is only accessible when:
# - There is a valid config YAML file.
# - There are currently no users in the database (clean install)
#
# We do this special security check because at the point this method is called, the database
# is clean but does not (yet) have any super users for our permissions code to check against.
if config_provider.config_exists() and not database_has_users():
data = request.get_json()
username = data['username']
password = data['password']
email = data['email']
# Special security check: This method is only accessible when:
# - There is a valid config YAML file.
# - There are currently no users in the database (clean install)
#
# We do this special security check because at the point this method is called, the database
# is clean but does not (yet) have any super users for our permissions code to check against.
if config_provider.config_exists() and not database_has_users():
data = request.get_json()
username = data['username']
password = data['password']
email = data['email']
# Create the user in the database.
superuser_uuid = model.create_superuser(username, password, email)
# Create the user in the database.
superuser_uuid = model.create_superuser(username, password, email)
# Add the user to the config.
config_object = config_provider.get_config()
config_object['SUPER_USERS'] = [username]
config_provider.save_config(config_object)
# Add the user to the config.
config_object = config_provider.get_config()
config_object['SUPER_USERS'] = [username]
config_provider.save_config(config_object)
# Update the in-memory config for the new superuser.
# TODO(config): do we need to register a list of the superusers? If so, we can take out the entire superuser in c_app
superusers.register_superuser(username)
# Update the in-memory config for the new superuser.
# TODO(config): do we need to register a list of the superusers? If so, we can take out the entire superuser in c_app
superusers.register_superuser(username)
# Conduct login with that user.
# TODO(config): figure out if we need validation for checking logged in user stuff
# common_login(superuser_uuid)
# Conduct login with that user.
# TODO(config): figure out if we need validation for checking logged in user stuff
# common_login(superuser_uuid)
return {
'status': True
}
return {
'status': True
}
abort(403)
abort(403)
@resource('/v1/superuser/config/validate/<service>')
class SuperUserConfigValidate(ApiResource):
""" Resource for validating a block of configuration against an external service. """
schemas = {
'ValidateConfig': {
'type': 'object',
'description': 'Validates configuration',
'required': [
'config'
],
'properties': {
'config': {
'type': 'object'
},
'password': {
'type': 'string',
'description': 'The users password, used for auth validation'
}
},
""" Resource for validating a block of configuration against an external service. """
schemas = {
'ValidateConfig': {
'type': 'object',
'description': 'Validates configuration',
'required': [
'config'
],
'properties': {
'config': {
'type': 'object'
},
}
'password': {
'type': 'string',
'description': 'The users password, used for auth validation'
}
},
},
}
@nickname('scValidateConfig')
@validate_json_request('ValidateConfig')
def post(self, service):
""" Validates the given config for the given service. """
# Note: This method is called to validate the database configuration before super users exists,
# so we also allow it to be called if there is no valid registry configuration setup. Note that
# this is also safe since this method does not access any information not given in the request.
config = request.get_json()['config']
validator_context = ValidatorContext.from_app(app, config, request.get_json().get('password', ''),
instance_keys=instance_keys,
ip_resolver=ip_resolver,
config_provider=config_provider,
init_scripts_location=INIT_SCRIPTS_LOCATION)
@nickname('scValidateConfig')
@validate_json_request('ValidateConfig')
def post(self, service):
""" Validates the given config for the given service. """
# Note: This method is called to validate the database configuration before super users exists,
# so we also allow it to be called if there is no valid registry configuration setup. Note that
# this is also safe since this method does not access any information not given in the request.
config = request.get_json()['config']
validator_context = ValidatorContext.from_app(app, config, request.get_json().get('password', ''),
instance_keys=instance_keys,
ip_resolver=ip_resolver,
config_provider=config_provider,
init_scripts_location=INIT_SCRIPTS_LOCATION)
return validate_service_for_config(service, validator_context)
return validate_service_for_config(service, validator_context)
@resource('/v1/kubernetes/deployments/')
class SuperUserKubernetesDeployment(ApiResource):
""" Resource for fetching the status of config files and overriding them. """
@kubernetes_only
@nickname('scGetNumDeployments')
def get(self):
return kubernetes_access_instance.get_qe_deployments()
""" Resource for the getting the status of Quay Enterprise deployments and cycling them """
schemas = {
'ValidateDeploymentNames': {
'type': 'object',
'description': 'Validates deployment names for cycling',
'required': [
'deploymentNames'
],
'properties': {
'deploymentNames': {
'type': 'array',
'description': 'The names of the deployments to cycle'
},
},
}
}
@kubernetes_only
@nickname('scGetNumDeployments')
def get(self):
return KubernetesAccessorSingleton.get_instance().get_qe_deployments()
@kubernetes_only
@nickname('scCycleQEDeployments')
def put(self):
deployment_names = request.get_json()['deploymentNames']
return KubernetesAccessorSingleton.get_instance().cycle_qe_deployments(deployment_names)
@resource('/v1/superuser/config/kubernetes')
class SuperUserKubernetesConfiguration(ApiResource):
""" Resource for fetching the status of config files and overriding them. """
@kubernetes_only
@nickname('scDeployConfiguration')
def post(self):
return config_provider.save_configuration_to_kubernetes()
""" Resource for saving the config files to kubernetes secrets. """
@kubernetes_only
@nickname('scDeployConfiguration')
def post(self):
return config_provider.save_configuration_to_kubernetes()
@resource('/v1/superuser/config/file/<filename>')
class SuperUserConfigFile(ApiResource):
""" Resource for fetching the status of config files and overriding them. """
@nickname('scConfigFileExists')
def get(self, filename):
""" Returns whether the configuration file with the given name exists. """
if not is_valid_config_upload_filename(filename):
abort(404)
""" Resource for fetching the status of config files and overriding them. """
@nickname('scConfigFileExists')
def get(self, filename):
""" Returns whether the configuration file with the given name exists. """
if not is_valid_config_upload_filename(filename):
abort(404)
return {
'exists': config_provider.volume_file_exists(filename)
}
return {
'exists': config_provider.volume_file_exists(filename)
}
@nickname('scUpdateConfigFile')
def post(self, filename):
""" Updates the configuration file with the given name. """
if not is_valid_config_upload_filename(filename):
abort(404)
@nickname('scUpdateConfigFile')
def post(self, filename):
""" Updates the configuration file with the given name. """
if not is_valid_config_upload_filename(filename):
abort(404)
# Note: This method can be called before the configuration exists
# to upload the database SSL cert.
uploaded_file = request.files['file']
if not uploaded_file:
abort(400)
# Note: This method can be called before the configuration exists
# to upload the database SSL cert.
uploaded_file = request.files['file']
if not uploaded_file:
abort(400)
config_provider.save_volume_file(filename, uploaded_file)
return {
'status': True
}
config_provider.save_volume_file(filename, uploaded_file)
return {
'status': True
}

View file

@ -9,6 +9,7 @@ from config import frontend_visible_config
from config_app.c_app import app, IS_KUBERNETES
from config_app._init_config import ROOT_DIR
from config_app.config_util.k8sconfig import get_k8s_namespace
def truthy_bool(param):
@ -49,7 +50,7 @@ def render_page_template(name, route_data=None, js_bundle_name=DEFAULT_JS_BUNDLE
route_data=route_data,
main_scripts=main_scripts,
config_set=frontend_visible_config(app.config),
is_kubernetes=IS_KUBERNETES,
kubernetes_namespace=IS_KUBERNETES and get_k8s_namespace(),
**kwargs)
resp = make_response(contents)