Genericizes the yaml+environment versioned configuration parser

Registry configuration parsing uses the new parser with a single version
declaration and an environment prefix of "REGISTRY"
This commit is contained in:
Brian Bland 2014-12-17 14:18:58 -08:00
parent 454d4e918a
commit f9b119974d
4 changed files with 274 additions and 149 deletions

View file

@ -68,6 +68,6 @@ Any configuration field other than version can be replaced by providing an envir
For example, to change the loglevel to `error`, one can provide `REGISTRY_LOGLEVEL=error`, and to change the s3 storage driver's region parameter to `us-west-1`, one can provide `REGISTRY_STORAGE_S3_LOGLEVEL=us-west-1`.
### Notes
If an environment variable changes a map value into a string, such as replacing the storage driver type with `REGISTRY_STORAGE=filesystem`, then all sub-fields will be erased. As such, changing the storage type will remove all parameters related to the old storage type.
If an environment variable changes a map value into a string, such as replacing the storage driver type with `REGISTRY_STORAGE=filesystem`, then all sub-fields will be erased. As such, specifying the storage type in the environment will remove all parameters related to the old storage configuration.
By restricting all keys in the configuration file to lowercase letters and numbers, we can avoid any potential environment variable mapping ambiguity.

View file

@ -4,12 +4,8 @@ import (
"fmt"
"io"
"io/ioutil"
"os"
"regexp"
"strconv"
"reflect"
"strings"
"gopkg.in/BrianBland/yaml.v2"
)
// Configuration is a versioned registry configuration, intended to be provided by a yaml file, and
@ -39,40 +35,6 @@ type Configuration struct {
// This is currently aliased to Configuration, as it is the current version
type v0_1Configuration Configuration
// Version is a major/minor version pair of the form Major.Minor
// Major version upgrades indicate structure or type changes
// Minor version upgrades should be strictly additive
type Version string
// MajorMinorVersion constructs a Version from its Major and Minor components
func MajorMinorVersion(major, minor uint) Version {
return Version(fmt.Sprintf("%d.%d", major, minor))
}
func (version Version) major() (uint, error) {
majorPart := strings.Split(string(version), ".")[0]
major, err := strconv.ParseUint(majorPart, 10, 0)
return uint(major), err
}
// Major returns the major version portion of a Version
func (version Version) Major() uint {
major, _ := version.major()
return major
}
func (version Version) minor() (uint, error) {
minorPart := strings.Split(string(version), ".")[1]
minor, err := strconv.ParseUint(minorPart, 10, 0)
return uint(minor), err
}
// Minor returns the minor version portion of a Version
func (version Version) Minor() uint {
minor, _ := version.minor()
return minor
}
// UnmarshalYAML implements the yaml.Unmarshaler interface
// Unmarshals a string of the form X.Y into a Version, validating that X and Y can represent uints
func (version *Version) UnmarshalYAML(unmarshal func(interface{}) error) error {
@ -223,109 +185,30 @@ func Parse(rd io.Reader) (*Configuration, error) {
return nil, err
}
var untypedConfig struct {
Version Version
}
var config *Configuration
p := NewParser("registry", []VersionedParseInfo{
{
Version: MajorMinorVersion(0, 1),
ParseAs: reflect.TypeOf(v0_1Configuration{}),
ConversionFunc: func(c interface{}) (interface{}, error) {
if v0_1, ok := c.(*v0_1Configuration); ok {
if v0_1.Loglevel == Loglevel("") {
v0_1.Loglevel = Loglevel("info")
}
if v0_1.Storage.Type() == "" {
return nil, fmt.Errorf("No storage configuration provided")
}
return (*Configuration)(v0_1), nil
}
return nil, fmt.Errorf("Expected *v0_1Configuration, received %#v", c)
},
},
})
if err := yaml.Unmarshal(in, &untypedConfig); err != nil {
config := new(Configuration)
err = p.Parse(in, config)
if err != nil {
return nil, err
}
if untypedConfig.Version == "" {
return nil, fmt.Errorf("Please specify a configuration version. Current version is %s", CurrentVersion)
}
// Parse the remainder of the configuration depending on the provided version
switch untypedConfig.Version {
case "0.1":
config, err = parseV0_1Registry(in)
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("Unsupported configuration version %s Current version is %s", untypedConfig.Version, CurrentVersion)
}
return config, nil
}
// parseV0_1Registry parses a registry Configuration for Version 0.1
func parseV0_1Registry(in []byte) (*Configuration, error) {
envMap := getEnvMap()
var config v0_1Configuration
err := yaml.Unmarshal(in, &config)
if err != nil {
return nil, err
}
// Override config.Loglevel if environment variable is provided
if loglevel, ok := envMap["REGISTRY_LOGLEVEL"]; ok {
var newLoglevel Loglevel
err := yaml.Unmarshal([]byte(loglevel), &newLoglevel)
if err != nil {
return nil, err
}
config.Loglevel = newLoglevel
}
// Override config.Storage if environment variable is provided
if storageType, ok := envMap["REGISTRY_STORAGE"]; ok {
if storageType != config.Storage.Type() {
// Reset the storage parameters because we're using a different storage type
config.Storage = Storage{storageType: Parameters{}}
}
}
if config.Storage.Type() == "" {
return nil, fmt.Errorf("Must provide exactly one storage type, optionally with parameters. Provided: %v", config.Storage)
}
// Override storage parameters with all environment variables of the format:
// REGISTRY_STORAGE_<storage driver type>_<parameter name>
storageParamsRegexp, err := regexp.Compile(fmt.Sprintf("^REGISTRY_STORAGE_%s_([A-Z0-9]+)$", strings.ToUpper(config.Storage.Type())))
if err != nil {
return nil, err
}
for k, v := range envMap {
if submatches := storageParamsRegexp.FindStringSubmatch(k); submatches != nil {
config.Storage.setParameter(strings.ToLower(submatches[1]), v)
}
}
if bugsnagAPIKey, ok := envMap["REGISTRY_REPORTING_BUGSNAG_APIKEY"]; ok {
config.Reporting.Bugsnag.APIKey = bugsnagAPIKey
}
if bugsnagReleaseStage, ok := envMap["REGISTRY_REPORTING_BUGSNAG_RELEASESTAGE"]; ok {
config.Reporting.Bugsnag.ReleaseStage = bugsnagReleaseStage
}
if bugsnagEndpoint, ok := envMap["REGISTRY_REPORTING_BUGSNAG_ENDPOINT"]; ok {
config.Reporting.Bugsnag.Endpoint = bugsnagEndpoint
}
if newRelicLicenseKey, ok := envMap["REGISTRY_REPORTING_NEWRELIC_LICENSEKEY"]; ok {
config.Reporting.NewRelic.LicenseKey = newRelicLicenseKey
}
if newRelicName, ok := envMap["REGISTRY_REPORTING_NEWRELIC_NAME"]; ok {
config.Reporting.NewRelic.Name = newRelicName
}
if httpAddr, ok := envMap["REGISTRY_HTTP_ADDR"]; ok {
config.HTTP.Addr = httpAddr
}
return (*Configuration)(&config), nil
}
// getEnvMap reads the current environment variables and converts these into a key/value map
// This is used to distinguish between empty strings returned by os.GetEnv(key) because of undefined
// environment variables and explicitly empty ones
func getEnvMap() map[string]string {
envMap := make(map[string]string)
for _, env := range os.Environ() {
envParts := strings.SplitN(env, "=", 2)
envMap[envParts[0]] = envParts[1]
}
return envMap
}

View file

@ -56,8 +56,8 @@ reporting:
apikey: BugsnagApiKey
`
// inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory storage driver with
// no parameters
// inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory
// storage driver with no parameters
var inmemoryConfigYamlV0_1 = `
version: 0.1
loglevel: info
@ -75,8 +75,8 @@ func (suite *ConfigSuite) SetUpTest(c *C) {
suite.expectedConfig = copyConfig(configStruct)
}
// TestMarshalRoundtrip validates that configStruct can be marshaled and unmarshaled without
// changing any parameters
// TestMarshalRoundtrip validates that configStruct can be marshaled and
// unmarshaled without changing any parameters
func (suite *ConfigSuite) TestMarshalRoundtrip(c *C) {
configBytes, err := yaml.Marshal(suite.expectedConfig)
c.Assert(err, IsNil)
@ -85,15 +85,16 @@ func (suite *ConfigSuite) TestMarshalRoundtrip(c *C) {
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseSimple validates that configYamlV0_1 can be parsed into a struct matching configStruct
// TestParseSimple validates that configYamlV0_1 can be parsed into a struct
// matching configStruct
func (suite *ConfigSuite) TestParseSimple(c *C) {
config, err := Parse(bytes.NewReader([]byte(configYamlV0_1)))
c.Assert(err, IsNil)
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseInmemory validates that configuration yaml with storage provided as a string can be
// parsed into a Configuration struct with no storage parameters
// TestParseInmemory validates that configuration yaml with storage provided as
// a string can be parsed into a Configuration struct with no storage parameters
func (suite *ConfigSuite) TestParseInmemory(c *C) {
suite.expectedConfig.Storage = Storage{"inmemory": Parameters{}}
suite.expectedConfig.Reporting = Reporting{}
@ -103,9 +104,31 @@ func (suite *ConfigSuite) TestParseInmemory(c *C) {
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseWithSameEnvStorage validates that providing environment variables that match the given
// storage type and parameters will not alter the parsed Configuration struct
// TestParseIncomplete validates that an incomplete yaml configuration cannot
// be parsed without providing environment variables to fill in the missing
// components.
func (suite *ConfigSuite) TestParseIncomplete(c *C) {
incompleteConfigYaml := "version: 0.1"
_, err := Parse(bytes.NewReader([]byte(incompleteConfigYaml)))
c.Assert(err, NotNil)
suite.expectedConfig.Storage = Storage{"filesystem": Parameters{"rootdirectory": "/tmp/testroot"}}
suite.expectedConfig.Reporting = Reporting{}
os.Setenv("REGISTRY_STORAGE", "filesystem")
os.Setenv("REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY", "/tmp/testroot")
config, err := Parse(bytes.NewReader([]byte(incompleteConfigYaml)))
c.Assert(err, IsNil)
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseWithSameEnvStorage validates that providing environment variables
// that match the given storage type will only include environment-defined
// parameters and remove yaml-defined parameters
func (suite *ConfigSuite) TestParseWithSameEnvStorage(c *C) {
suite.expectedConfig.Storage = Storage{"s3": Parameters{"region": "us-east-1"}}
os.Setenv("REGISTRY_STORAGE", "s3")
os.Setenv("REGISTRY_STORAGE_S3_REGION", "us-east-1")
@ -180,6 +203,22 @@ func (suite *ConfigSuite) TestParseWithDifferentEnvLoglevel(c *C) {
c.Assert(config, DeepEquals, suite.expectedConfig)
}
// TestParseInvalidLoglevel validates that the parser will fail to parse a
// configuration if the loglevel is malformed
func (suite *ConfigSuite) TestParseInvalidLoglevel(c *C) {
invalidConfigYaml := "version: 0.1\nloglevel: derp\nstorage: inmemory"
_, err := Parse(bytes.NewReader([]byte(invalidConfigYaml)))
c.Assert(err, NotNil)
os.Setenv("REGISTRY_LOGLEVEL", "derp")
_, err = Parse(bytes.NewReader([]byte(configYamlV0_1)))
c.Assert(err, NotNil)
}
// TestParseWithDifferentEnvReporting validates that environment variables
// properly override reporting parameters
func (suite *ConfigSuite) TestParseWithDifferentEnvReporting(c *C) {
suite.expectedConfig.Reporting.Bugsnag.APIKey = "anotherBugsnagApiKey"
suite.expectedConfig.Reporting.Bugsnag.Endpoint = "localhost:8080"

203
configuration/parser.go Normal file
View file

@ -0,0 +1,203 @@
package configuration
import (
"fmt"
"os"
"reflect"
"regexp"
"strconv"
"strings"
"gopkg.in/BrianBland/yaml.v2"
)
// Version is a major/minor version pair of the form Major.Minor
// Major version upgrades indicate structure or type changes
// Minor version upgrades should be strictly additive
type Version string
// MajorMinorVersion constructs a Version from its Major and Minor components
func MajorMinorVersion(major, minor uint) Version {
return Version(fmt.Sprintf("%d.%d", major, minor))
}
func (version Version) major() (uint, error) {
majorPart := strings.Split(string(version), ".")[0]
major, err := strconv.ParseUint(majorPart, 10, 0)
return uint(major), err
}
// Major returns the major version portion of a Version
func (version Version) Major() uint {
major, _ := version.major()
return major
}
func (version Version) minor() (uint, error) {
minorPart := strings.Split(string(version), ".")[1]
minor, err := strconv.ParseUint(minorPart, 10, 0)
return uint(minor), err
}
// Minor returns the minor version portion of a Version
func (version Version) Minor() uint {
minor, _ := version.minor()
return minor
}
// VersionedParseInfo defines how a specific version of a configuration should
// be parsed into the current version
type VersionedParseInfo struct {
// Version is the version which this parsing information relates to
Version Version
// ParseAs defines the type which a configuration file of this version
// should be parsed into
ParseAs reflect.Type
// ConversionFunc defines a method for converting the parsed configuration
// (of type ParseAs) into the current configuration version
// Note: this method signature is very unclear with the absence of generics
ConversionFunc func(interface{}) (interface{}, error)
}
// Parser can be used to parse a configuration file and environment of a defined
// version into a unified output structure
type Parser struct {
prefix string
mapping map[Version]VersionedParseInfo
env map[string]string
}
// NewParser returns a *Parser with the given environment prefix which handles
// versioned configurations which match the given parseInfos
func NewParser(prefix string, parseInfos []VersionedParseInfo) *Parser {
p := Parser{prefix: prefix, mapping: make(map[Version]VersionedParseInfo), env: make(map[string]string)}
for _, parseInfo := range parseInfos {
p.mapping[parseInfo.Version] = parseInfo
}
for _, env := range os.Environ() {
envParts := strings.SplitN(env, "=", 2)
p.env[envParts[0]] = envParts[1]
}
return &p
}
// Parse reads in the given []byte and environment and writes the resulting
// configuration into the input v
//
// Environment variables may be used to override configuration parameters other
// than version, following the scheme below:
// v.Abc may be replaced by the value of PREFIX_ABC,
// v.Abc.Xyz may be replaced by the value of PREFIX_ABC_XYZ, and so forth
func (p *Parser) Parse(in []byte, v interface{}) error {
var versionedStruct struct {
Version Version
}
if err := yaml.Unmarshal(in, &versionedStruct); err != nil {
return err
}
parseInfo, ok := p.mapping[versionedStruct.Version]
if !ok {
return fmt.Errorf("Unsupported version: %q", versionedStruct.Version)
}
parseAs := reflect.New(parseInfo.ParseAs)
err := yaml.Unmarshal(in, parseAs.Interface())
if err != nil {
return err
}
err = p.overwriteFields(parseAs, p.prefix)
if err != nil {
return err
}
c, err := parseInfo.ConversionFunc(parseAs.Interface())
if err != nil {
return err
}
reflect.ValueOf(v).Elem().Set(reflect.Indirect(reflect.ValueOf(c)))
return nil
}
func (p *Parser) overwriteFields(v reflect.Value, prefix string) error {
for v.Kind() == reflect.Ptr {
v = reflect.Indirect(v)
}
switch v.Kind() {
case reflect.Struct:
for i := 0; i < v.NumField(); i++ {
sf := v.Type().Field(i)
fieldPrefix := strings.ToUpper(prefix + "_" + sf.Name)
if e, ok := p.env[fieldPrefix]; ok {
fieldVal := reflect.New(sf.Type)
err := yaml.Unmarshal([]byte(e), fieldVal.Interface())
if err != nil {
return err
}
v.Field(i).Set(reflect.Indirect(fieldVal))
}
err := p.overwriteFields(v.Field(i), fieldPrefix)
if err != nil {
return err
}
}
case reflect.Map:
p.overwriteMap(v, prefix)
}
return nil
}
func (p *Parser) overwriteMap(m reflect.Value, prefix string) error {
switch m.Type().Elem().Kind() {
case reflect.Struct:
for _, k := range m.MapKeys() {
err := p.overwriteFields(m.MapIndex(k), strings.ToUpper(fmt.Sprintf("%s_%s", prefix, k)))
if err != nil {
return err
}
}
envMapRegexp, err := regexp.Compile(fmt.Sprintf("^%s_([A-Z0-9]+)$", strings.ToUpper(prefix)))
if err != nil {
return err
}
for key, val := range p.env {
if submatches := envMapRegexp.FindStringSubmatch(key); submatches != nil {
mapValue := reflect.New(m.Type().Elem())
err := yaml.Unmarshal([]byte(val), mapValue.Interface())
if err != nil {
return err
}
m.SetMapIndex(reflect.ValueOf(strings.ToLower(submatches[1])), reflect.Indirect(mapValue))
}
}
case reflect.Map:
for _, k := range m.MapKeys() {
err := p.overwriteMap(m.MapIndex(k), strings.ToUpper(fmt.Sprintf("%s_%s", prefix, k)))
if err != nil {
return err
}
}
default:
envMapRegexp, err := regexp.Compile(fmt.Sprintf("^%s_([A-Z0-9]+)$", strings.ToUpper(prefix)))
if err != nil {
return err
}
for key, val := range p.env {
if submatches := envMapRegexp.FindStringSubmatch(key); submatches != nil {
mapValue := reflect.New(m.Type().Elem())
err := yaml.Unmarshal([]byte(val), mapValue.Interface())
if err != nil {
return err
}
m.SetMapIndex(reflect.ValueOf(strings.ToLower(submatches[1])), reflect.Indirect(mapValue))
}
}
}
return nil
}