From db757edcd2ece75a501970c6e1d7db959cc61f8c Mon Sep 17 00:00:00 2001 From: Sam Chow Date: Thu, 28 Jun 2018 13:45:26 -0400 Subject: [PATCH] Create transient config provider, temp dir logic Allows us to have a new config provider for each setup, with no overlap of the directories used, and automatic cleanup of those directories. --- config_app/config_endpoints/api/suconfig.py | 14 -------- .../config_endpoints/api/tar_config_loader.py | 32 +++++++++++++------ .../config/TransientDirectoryProvider.py | 30 +++++++++++++++++ config_app/config_util/config/__init__.py | 3 +- .../config-setup-app.component.ts | 12 +++++-- .../download-tarball-modal.component.ts | 2 +- .../js/core-config-setup/core-config-setup.js | 5 --- config_app/js/services/api-service.js | 8 ++--- requirements.txt | 1 + 9 files changed, 70 insertions(+), 37 deletions(-) create mode 100644 config_app/config_util/config/TransientDirectoryProvider.py diff --git a/config_app/config_endpoints/api/suconfig.py b/config_app/config_endpoints/api/suconfig.py index fa8233a0b..9e17701ab 100644 --- a/config_app/config_endpoints/api/suconfig.py +++ b/config_app/config_endpoints/api/suconfig.py @@ -110,20 +110,6 @@ class SuperUserRegistryStatus(ApiResource): def get(self): """ Returns the status of the registry. """ - # If we have SETUP_COMPLETE, then we're ready to go! - if app.config.get('SETUP_COMPLETE', False): - return { - 'provider_id': config_provider.provider_id, - 'requires_restart': config_provider.requires_restart(app.config), - 'status': 'ready' - } - - # If there is no conf/stack volume, then report that status. - if not config_provider.volume_exists(): - return { - 'status': 'missing-config-dir' - } - # If there is no config file, we need to setup the database. if not config_provider.config_exists(): return { diff --git a/config_app/config_endpoints/api/tar_config_loader.py b/config_app/config_endpoints/api/tar_config_loader.py index 033a7a5f2..011de5531 100644 --- a/config_app/config_endpoints/api/tar_config_loader.py +++ b/config_app/config_endpoints/api/tar_config_loader.py @@ -1,4 +1,5 @@ import os +import tempfile import tarfile from flask import request, make_response, send_file @@ -9,6 +10,19 @@ from util.config.validator import EXTRA_CA_DIRECTORY from config_app.c_app import app, config_provider from config_app.config_endpoints.api import resource, ApiResource, nickname +@resource('/v1/configapp/initialization') +class ConfigInitialization(ApiResource): + """ + Resource for dealing with any initialization logic for the config app + """ + + @nickname('scStartNewConfig') + def get(self): + config_provider.new_config_dir() + + return make_response('OK') + + @resource('/v1/configapp/tarconfig') class TarConfigLoader(ApiResource): """ @@ -18,7 +32,7 @@ class TarConfigLoader(ApiResource): @nickname('scGetConfigTarball') def get(self): - config_path = config_provider.config_volume + config_path = config_provider.get_config_dir_path() # remove the initial trailing / from the prefix path, and add the last dir one tar_dir_prefix = config_path[1:] + '/' @@ -33,28 +47,28 @@ class TarConfigLoader(ApiResource): return tarinfo - # Remove the tar if it already exists so we don't write on top of existing tarball - if os.path.isfile('quay-config.tar.gz'): - os.remove('quay-config.tar.gz') + temp = tempfile.NamedTemporaryFile() - tar = tarfile.open('quay-config.tar.gz', mode="w|gz") + tar = tarfile.open(temp.name, mode="w|gz") for name in os.listdir(config_path): tar.add(os.path.join(config_path, name), filter=tarinfo_filter) tar.close() - return send_file('quay-config.tar.gz', mimetype='application/gzip') + return send_file(temp.name, mimetype='application/gzip') @nickname('scUploadTarballConfig') def put(self): """ Loads tarball config into the config provider """ + # Generate a new empty dir to load the config into + config_provider.new_config_dir() + input_stream = request.stream with tarfile.open(mode="r|gz", fileobj=input_stream) as tar_stream: - # TODO: find a way to remove the contents of the directory on shutdown? - tar_stream.extractall(config_provider.config_volume) + tar_stream.extractall(config_provider.get_config_dir_path()) - # now try to connect to the db provided in their config + # now try to connect to the db provided in their config to validate it works combined = dict(**app.config) combined.update(config_provider.get_config()) diff --git a/config_app/config_util/config/TransientDirectoryProvider.py b/config_app/config_util/config/TransientDirectoryProvider.py new file mode 100644 index 000000000..2e0509e74 --- /dev/null +++ b/config_app/config_util/config/TransientDirectoryProvider.py @@ -0,0 +1,30 @@ +import os +from backports.tempfile import TemporaryDirectory + +from config_app.config_util.config.fileprovider import FileConfigProvider + +class TransientDirectoryProvider(FileConfigProvider): + """ Implementation of the config provider that reads and writes the data + from/to the file system, only using temporary directories, + deleting old dirs and creating new ones as requested. + """ + def __init__(self, config_volume, yaml_filename, py_filename): + # Create a temp directory that will be cleaned up when we change the config path + # This should ensure we have no "pollution" of different configs: + # no uploaded config should ever affect subsequent config modifications/creations + temp_dir = TemporaryDirectory() + self.temp_dir = temp_dir + super(TransientDirectoryProvider, self).__init__(temp_dir.name, yaml_filename, py_filename) + + def new_config_dir(self): + """ + Update the path with a new temporary directory, deleting the old one in the process + """ + temp_dir = TemporaryDirectory() + + self.config_volume = temp_dir.name + self.temp_dir = temp_dir + self.yaml_path = os.path.join(temp_dir.name, self.yaml_filename) + + def get_config_dir_path(self): + return self.config_volume diff --git a/config_app/config_util/config/__init__.py b/config_app/config_util/config/__init__.py index 3735e4f66..b9edeba3a 100644 --- a/config_app/config_util/config/__init__.py +++ b/config_app/config_util/config/__init__.py @@ -1,5 +1,6 @@ from config_app.config_util.config.fileprovider import FileConfigProvider from config_app.config_util.config.testprovider import TestConfigProvider +from config_app.config_util.config.TransientDirectoryProvider import TransientDirectoryProvider def get_config_provider(config_volume, yaml_filename, py_filename, testing=False): @@ -8,4 +9,4 @@ def get_config_provider(config_volume, yaml_filename, py_filename, testing=False if testing: return TestConfigProvider() - return FileConfigProvider(config_volume, yaml_filename, py_filename) + return TransientDirectoryProvider(config_volume, yaml_filename, py_filename) diff --git a/config_app/js/components/config-setup-app/config-setup-app.component.ts b/config_app/js/components/config-setup-app/config-setup-app.component.ts index b453d7ef4..550741d60 100644 --- a/config_app/js/components/config-setup-app/config-setup-app.component.ts +++ b/config_app/js/components/config-setup-app/config-setup-app.component.ts @@ -1,4 +1,4 @@ -import { Component } from 'ng-metadata/core'; +import { Component, Inject } from 'ng-metadata/core'; const templateUrl = require('./config-setup-app.component.html'); /** @@ -17,12 +17,18 @@ export class ConfigSetupAppComponent { private loadedConfig = false; - constructor() { + constructor(@Inject('ApiService') private apiService) { this.state = 'choice'; } private chooseSetup(): void { - this.state = 'setup'; + this.apiService.scStartNewConfig() + .then(() => { + this.state = 'setup'; + }) + .catch(this.apiService.errorDisplay( + 'Could not initialize new setup. Please report this error' + )); } private chooseLoad(): void { diff --git a/config_app/js/components/download-tarball-modal/download-tarball-modal.component.ts b/config_app/js/components/download-tarball-modal/download-tarball-modal.component.ts index 0d586a92f..943c0c015 100644 --- a/config_app/js/components/download-tarball-modal/download-tarball-modal.component.ts +++ b/config_app/js/components/download-tarball-modal/download-tarball-modal.component.ts @@ -27,7 +27,7 @@ export class DownloadTarballModalComponent { // We need to set the response type to 'blob', to ensure it's never encoded as a string // (string encoded binary data can be difficult to transform with js) // and to make it easier to save (FileSaver expects a blob) - this.ApiService.scGetConfigTarball(null, null, null, null, true).then(function(resp) { + this.ApiService.scGetConfigTarball(null, null, null, null, 'blob').then(function(resp) { FileSaver.saveAs(resp, 'quay-config.tar.gz'); }, errorDisplay); } diff --git a/config_app/js/core-config-setup/core-config-setup.js b/config_app/js/core-config-setup/core-config-setup.js index d8e7c0edc..21fc8aa38 100644 --- a/config_app/js/core-config-setup/core-config-setup.js +++ b/config_app/js/core-config-setup/core-config-setup.js @@ -417,10 +417,6 @@ angular.module("quay-config") $scope.saveConfiguration = function() { $scope.savingConfiguration = true; - // Make sure to note that fully verified setup is completed. We use this as a signal - // in the setup tool. - // $scope.config['SETUP_COMPLETE'] = true; - var data = { 'config': $scope.config, 'hostname': window.location.host, @@ -441,7 +437,6 @@ angular.module("quay-config") $('#validateAndSaveModal').modal('hide'); - // $scope.configurationSaved({'config': $scope.config}); $scope.setupCompleted(); }, errorDisplay); }; diff --git a/config_app/js/services/api-service.js b/config_app/js/services/api-service.js index 2a36bf35b..814e25a45 100644 --- a/config_app/js/services/api-service.js +++ b/config_app/js/services/api-service.js @@ -212,17 +212,17 @@ angular.module('quay-config').factory('ApiService', ['Restangular', '$q', 'UtilS var urlPath = path['x-path']; // Add the operation itself. - apiService[operationName] = function(opt_options, opt_parameters, opt_background, opt_forceget, opt_blobresp) { + apiService[operationName] = function(opt_options, opt_parameters, opt_background, opt_forceget, opt_responseType) { var one = Restangular.one(buildUrl(urlPath, opt_parameters)); - if (opt_background || opt_blobresp) { + if (opt_background || opt_responseType) { let httpConfig = {}; if (opt_background) { httpConfig['ignoreLoadingBar'] = true; } - if (opt_blobresp) { - httpConfig['responseType'] = 'blob'; + if (opt_responseType) { + httpConfig['responseType'] = opt_responseType; } one.withHttpConfig(httpConfig); diff --git a/requirements.txt b/requirements.txt index 44ccab610..8c3c4e236 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,6 +20,7 @@ azure-storage-blob==1.1.0 azure-storage-common==1.1.0 azure-storage-nspkg==3.0.0 Babel==2.5.3 +backports.tempfile==1.0 beautifulsoup4==4.6.0 bencode==1.0 bintrees==2.0.7