Merge pull request #652 from BrianBland/configuration

NG: Configuration parser
This commit is contained in:
Olivier Gambier 2014-11-05 15:44:10 -08:00
commit f02cfee950
3 changed files with 534 additions and 0 deletions

73
configuration/README.md Normal file
View file

@ -0,0 +1,73 @@
Docker-Registry Configuration
=============================
This document describes the registry configuration model and how to specify a custom configuration with a configuration file and/or environment variables.
Semantic-ish Versioning
-----------------------
The configuration file is designed with versioning in mind, such that most upgrades will not require a change in configuration files, and such that configuration files can be "upgraded" from one version to another.
The version is specified as a string of the form `MajorVersion.MinorVersion`, where MajorVersion and MinorVersion are both non-negative integer values. Much like [semantic versioning](http://semver.org/), minor version increases denote inherently backwards-compatible changes, such as the addition of optional fields, whereas major version increases denote a restructuring, such as renaming fields or adding required fields. Because of the explicit version definition in the configuration file, it should be possible to parse old configuration files and port them to the current configuration version, although this is not guaranteed for all future versions.
File Structure (as of Version 0.1)
------------------------------------
The configuration structure is defined by the `Configuration` struct in `configuration.go`, and is best described by the following two examples:
```yaml
version: 0.1
loglevel: info
storage:
s3:
region: us-east-1
bucket: my-bucket
rootpath: /registry
encrypt: true
secure: false
accesskey: SAMPLEACCESSKEY
secretkey: SUPERSECRET
host: ~
port: ~
```
```yaml
version: 0.1
loglevel: debug
storage: inmemory
```
### version
The version is expected to remain a top-level field, as to allow for a consistent version check before parsing the remainder of the configuration file.
### loglevel
This specifies the log level of the registry.
Supported values:
* `error`
* `warn`
* `info`
* `debug`
### storage
This specifies the storage driver, and may be provided either as a string (only the driver type) or as a driver name with a parameters map, as seen in the first example above.
The parameters map will be passed into the factory constructor of the given storage driver type.
### Notes
All keys in the configuration file **must** be provided as a string of lowercase letters and numbers only, and values must be string-like (booleans and numerical values are fine to parse as strings).
Environment Variables
---------------------
To support the workflow of running a docker registry from a standard container without having to modify configuration files, the registry configuration also supports environment variables for overriding fields.
Any configuration field other than version can be replaced by providing an environment variable of the following form: `REGISTRY_<uppercase key>[_<uppercase key>]...`.
For example, to change the loglevel to `error`, one can provide `REGISTRY_LOGLEVEL=error`, and to change the s3 storage driver's region parameter to `us-west-1`, one can provide `REGISTRY_STORAGE_S3_LOGLEVEL=us-west-1`.
### Notes
If an environment variable changes a map value into a string, such as replacing the storage driver type with `REGISTRY_STORAGE=filesystem`, then all sub-fields will be erased. As such, changing the storage type will remove all parameters related to the old storage type.
By restricting all keys in the configuration file to lowercase letters and numbers, we can avoid any potential environment variable mapping ambiguity.

View file

@ -0,0 +1,266 @@
package configuration
import (
"fmt"
"os"
"regexp"
"strconv"
"strings"
"gopkg.in/BrianBland/yaml.v2"
)
// Configuration is a versioned registry configuration, intended to be provided by a yaml file, and
// optionally modified by environment variables
type Configuration struct {
// Version is the version which defines the format of the rest of the configuration
Version Version `yaml:"version"`
// Loglevel is the level at which registry operations are logged
Loglevel Loglevel `yaml:"loglevel"`
// Storage is the configuration for the registry's storage driver
Storage Storage `yaml:"storage"`
}
// v_0_1_Configuration is a Version 0.1 Configuration struct
// This is currently aliased to Configuration, as it is the current version
type v_0_1_Configuration Configuration
// Version is a major/minor version pair of the form Major.Minor
// Major version upgrades indicate structure or type changes
// Minor version upgrades should be strictly additive
type Version string
// MajorMinorVersion constructs a Version from its Major and Minor components
func MajorMinorVersion(major, minor uint) Version {
return Version(fmt.Sprintf("%d.%d", major, minor))
}
func (version Version) major() (uint, error) {
majorPart := strings.Split(string(version), ".")[0]
major, err := strconv.ParseUint(majorPart, 10, 0)
return uint(major), err
}
// Major returns the major version portion of a Version
func (version Version) Major() uint {
major, _ := version.major()
return major
}
func (version Version) minor() (uint, error) {
minorPart := strings.Split(string(version), ".")[1]
minor, err := strconv.ParseUint(minorPart, 10, 0)
return uint(minor), err
}
// Minor returns the minor version portion of a Version
func (version Version) Minor() uint {
minor, _ := version.minor()
return minor
}
// UnmarshalYAML implements the yaml.Unmarshaler interface
// Unmarshals a string of the form X.Y into a Version, validating that X and Y can represent uints
func (version *Version) UnmarshalYAML(unmarshal func(interface{}) error) error {
var versionString string
err := unmarshal(&versionString)
if err != nil {
return err
}
newVersion := Version(versionString)
if _, err := newVersion.major(); err != nil {
return err
}
if _, err := newVersion.minor(); err != nil {
return err
}
*version = newVersion
return nil
}
// CurrentVersion is the most recent Version that can be parsed
var CurrentVersion = MajorMinorVersion(0, 1)
// Loglevel is the level at which operations are logged
// This can be error, warn, info, or debug
type Loglevel string
// UnmarshalYAML implements the yaml.Umarshaler interface
// Unmarshals a string into a Loglevel, lowercasing the string and validating that it represents a
// valid loglevel
func (loglevel *Loglevel) UnmarshalYAML(unmarshal func(interface{}) error) error {
var loglevelString string
err := unmarshal(&loglevelString)
if err != nil {
return err
}
loglevelString = strings.ToLower(loglevelString)
switch loglevelString {
case "error", "warn", "info", "debug":
default:
return fmt.Errorf("Invalid loglevel %s Must be one of [error, warn, info, debug]", loglevelString)
}
*loglevel = Loglevel(loglevelString)
return nil
}
// Storage defines the configuration for registry object storage
type Storage map[string]Parameters
// Type returns the storage driver type, such as filesystem or s3
func (storage Storage) Type() string {
// Return only key in this map
for k := range storage {
return k
}
return ""
}
// Parameters returns the Parameters map for a Storage configuration
func (storage Storage) Parameters() Parameters {
return storage[storage.Type()]
}
// setParameter changes the parameter at the provided key to the new value
func (storage Storage) setParameter(key, value string) {
storage[storage.Type()][key] = value
}
// UnmarshalYAML implements the yaml.Unmarshaler interface
// Unmarshals a single item map into a Storage or a string into a Storage type with no parameters
func (storage *Storage) UnmarshalYAML(unmarshal func(interface{}) error) error {
var storageMap map[string]Parameters
err := unmarshal(&storageMap)
if err == nil {
if len(storageMap) > 1 {
types := make([]string, 0, len(storageMap))
for k := range storageMap {
types = append(types, k)
}
return fmt.Errorf("Must provide exactly one storage type. Provided: %v", types)
}
*storage = storageMap
return nil
}
var storageType string
err = unmarshal(&storageType)
if err == nil {
*storage = Storage{storageType: Parameters{}}
return nil
}
return err
}
// MarshalYAML implements the yaml.Marshaler interface
func (storage Storage) MarshalYAML() (interface{}, error) {
if storage.Parameters == nil {
return storage.Type, nil
}
return map[string]Parameters(storage), nil
}
// Parameters defines a key-value parameters mapping
type Parameters map[string]string
// Parse parses an input configuration yaml document into a Configuration struct
// This should generally be capable of handling old configuration format versions
//
// Environment variables may be used to override configuration parameters other than version,
// following the scheme below:
// Configuration.Abc may be replaced by the value of REGISTRY_ABC,
// Configuration.Abc.Xyz may be replaced by the value of REGISTRY_ABC_XYZ, and so forth
func Parse(in []byte) (*Configuration, error) {
var untypedConfig struct {
Version Version
}
var config *Configuration
err := yaml.Unmarshal(in, &untypedConfig)
if err != nil {
return nil, err
}
if untypedConfig.Version == "" {
return nil, fmt.Errorf("Please specify a configuration version. Current version is %s", CurrentVersion)
}
// Parse the remainder of the configuration depending on the provided version
switch untypedConfig.Version {
case "0.1":
config, err = parseV_0_1_Registry(in)
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("Unsupported configuration version %s Current version is %s", untypedConfig.Version, CurrentVersion)
}
return config, nil
}
// parseV_0_1_Registry parses a registry Configuration for Version 0.1
func parseV_0_1_Registry(in []byte) (*Configuration, error) {
envMap := getEnvMap()
var config v_0_1_Configuration
err := yaml.Unmarshal(in, &config)
if err != nil {
return nil, err
}
// Override config.Loglevel if environment variable is provided
if loglevel, ok := envMap["REGISTRY_LOGLEVEL"]; ok {
var newLoglevel Loglevel
err := yaml.Unmarshal([]byte(loglevel), &newLoglevel)
if err != nil {
return nil, err
}
config.Loglevel = newLoglevel
}
// Override config.Storage if environment variable is provided
if storageType, ok := envMap["REGISTRY_STORAGE"]; ok {
if storageType != config.Storage.Type() {
// Reset the storage parameters because we're using a different storage type
config.Storage = Storage{storageType: Parameters{}}
}
}
if config.Storage.Type() == "" {
return nil, fmt.Errorf("Must provide exactly one storage type, optionally with parameters. Provided: %v", config.Storage)
}
// Override storage parameters with all environment variables of the format:
// REGISTRY_STORAGE_<storage driver type>_<parameter name>
storageParamsRegexp, err := regexp.Compile(fmt.Sprintf("^REGISTRY_STORAGE_%s_([A-Z0-9]+)$", strings.ToUpper(config.Storage.Type())))
if err != nil {
return nil, err
}
for k, v := range envMap {
if submatches := storageParamsRegexp.FindStringSubmatch(k); submatches != nil {
config.Storage.setParameter(strings.ToLower(submatches[1]), v)
}
}
return (*Configuration)(&config), nil
}
// getEnvMap reads the current environment variables and converts these into a key/value map
// This is used to distinguish between empty strings returned by os.GetEnv(key) because of undefined
// environment variables and explicitly empty ones
func getEnvMap() map[string]string {
envMap := make(map[string]string)
for _, env := range os.Environ() {
envParts := strings.SplitN(env, "=", 2)
envMap[envParts[0]] = envParts[1]
}
return envMap
}

View file

@ -0,0 +1,195 @@
package configuration
import (
"os"
"testing"
"gopkg.in/yaml.v2"
. "gopkg.in/check.v1"
)
// Hook up gocheck into the "go test" runner
func Test(t *testing.T) { TestingT(t) }
// configStruct is a canonical example configuration, which should map to configYamlV_0_1
var configStruct = Configuration{
Version: "0.1",
Loglevel: "info",
Storage: Storage{
"s3": Parameters{
"region": "us-east-1",
"bucket": "my-bucket",
"rootpath": "/registry",
"encrypt": "true",
"secure": "false",
"accesskey": "SAMPLEACCESSKEY",
"secretkey": "SUPERSECRET",
"host": "",
"port": "",
},
},
}
// configYamlV_0_1 is a Version 0.1 yaml document representing configStruct
var configYamlV_0_1 = `
version: 0.1
loglevel: info
storage:
s3:
region: us-east-1
bucket: my-bucket
rootpath: /registry
encrypt: true
secure: false
accesskey: SAMPLEACCESSKEY
secretkey: SUPERSECRET
host: ~
port: ~
`
// inmemoryConfigYamlV_0_1 is a Version 0.1 yaml document specifying an inmemory storage driver with
// no parameters
var inmemoryConfigYamlV_0_1 = `
version: 0.1
loglevel: info
storage: inmemory
`
type ConfigSuite struct {
expectedConfig *Configuration
}
var _ = Suite(new(ConfigSuite))
func (suite *ConfigSuite) SetUpTest(c *C) {
os.Clearenv()
suite.expectedConfig = copyConfig(configStruct)
}
// TestMarshalRoundtrip validates that configStruct can be marshaled and unmarshaled without
// changing any parameters
func (suite *ConfigSuite) TestMarshalRoundtrip(c *C) {
configBytes, err := yaml.Marshal(suite.expectedConfig)
c.Assert(err, IsNil)
config, err := Parse(configBytes)
c.Assert(err, IsNil)
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseSimple validates that configYamlV_0_1 can be parsed into a struct matching configStruct
func (suite *ConfigSuite) TestParseSimple(c *C) {
config, err := Parse([]byte(configYamlV_0_1))
c.Assert(err, IsNil)
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseInmemory validates that configuration yaml with storage provided as a string can be
// parsed into a Configuration struct with no storage parameters
func (suite *ConfigSuite) TestParseInmemory(c *C) {
suite.expectedConfig.Storage = Storage{"inmemory": Parameters{}}
config, err := Parse([]byte(inmemoryConfigYamlV_0_1))
c.Assert(err, IsNil)
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseWithSameEnvStorage validates that providing environment variables that match the given
// storage type and parameters will not alter the parsed Configuration struct
func (suite *ConfigSuite) TestParseWithSameEnvStorage(c *C) {
os.Setenv("REGISTRY_STORAGE", "s3")
os.Setenv("REGISTRY_STORAGE_S3_REGION", "us-east-1")
config, err := Parse([]byte(configYamlV_0_1))
c.Assert(err, IsNil)
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseWithDifferentEnvStorageParams validates that providing environment variables that change
// and add to the given storage parameters will change and add parameters to the parsed
// Configuration struct
func (suite *ConfigSuite) TestParseWithDifferentEnvStorageParams(c *C) {
suite.expectedConfig.Storage.setParameter("region", "us-west-1")
suite.expectedConfig.Storage.setParameter("secure", "true")
suite.expectedConfig.Storage.setParameter("newparam", "some Value")
os.Setenv("REGISTRY_STORAGE_S3_REGION", "us-west-1")
os.Setenv("REGISTRY_STORAGE_S3_SECURE", "true")
os.Setenv("REGISTRY_STORAGE_S3_NEWPARAM", "some Value")
config, err := Parse([]byte(configYamlV_0_1))
c.Assert(err, IsNil)
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseWithDifferentEnvStorageType validates that providing an environment variable that
// changes the storage type will be reflected in the parsed Configuration struct
func (suite *ConfigSuite) TestParseWithDifferentEnvStorageType(c *C) {
suite.expectedConfig.Storage = Storage{"inmemory": Parameters{}}
os.Setenv("REGISTRY_STORAGE", "inmemory")
config, err := Parse([]byte(configYamlV_0_1))
c.Assert(err, IsNil)
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseWithDifferentEnvStorageTypeAndParams validates that providing an environment variable
// that changes the storage type will be reflected in the parsed Configuration struct and that
// environment storage parameters will also be included
func (suite *ConfigSuite) TestParseWithDifferentEnvStorageTypeAndParams(c *C) {
suite.expectedConfig.Storage = Storage{"filesystem": Parameters{}}
suite.expectedConfig.Storage.setParameter("rootdirectory", "/tmp/testroot")
os.Setenv("REGISTRY_STORAGE", "filesystem")
os.Setenv("REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY", "/tmp/testroot")
config, err := Parse([]byte(configYamlV_0_1))
c.Assert(err, IsNil)
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseWithSameEnvLoglevel validates that providing an environment variable defining the log
// level to the same as the one provided in the yaml will not change the parsed Configuration struct
func (suite *ConfigSuite) TestParseWithSameEnvLoglevel(c *C) {
os.Setenv("REGISTRY_LOGLEVEL", "info")
config, err := Parse([]byte(configYamlV_0_1))
c.Assert(err, IsNil)
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseWithDifferentEnvLoglevel validates that providing an environment variable defining the
// log level will override the value provided in the yaml document
func (suite *ConfigSuite) TestParseWithDifferentEnvLoglevel(c *C) {
suite.expectedConfig.Loglevel = "error"
os.Setenv("REGISTRY_LOGLEVEL", "error")
config, err := Parse([]byte(configYamlV_0_1))
c.Assert(err, IsNil)
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseInvalidVersion validates that the parser will fail to parse a newer configuration
// version than the CurrentVersion
func (suite *ConfigSuite) TestParseInvalidVersion(c *C) {
suite.expectedConfig.Version = MajorMinorVersion(CurrentVersion.Major(), CurrentVersion.Minor()+1)
configBytes, err := yaml.Marshal(suite.expectedConfig)
c.Assert(err, IsNil)
_, err = Parse(configBytes)
c.Assert(err, NotNil)
}
func copyConfig(config Configuration) *Configuration {
configCopy := new(Configuration)
configCopy.Version = MajorMinorVersion(config.Version.Major(), config.Version.Minor())
configCopy.Loglevel = config.Loglevel
configCopy.Storage = Storage{config.Storage.Type(): Parameters{}}
for k, v := range config.Storage.Parameters() {
configCopy.Storage.setParameter(k, v)
}
return configCopy
}