More flexible environment variable overrides

Overriding configuration parameters with environment variables used to
work by walking the configuration structure and checking for a
corresponding environment variable for each item. This was very limiting
because only variables corresponding to items that already existed in
the configuration structure would be checked. For example, an
environment variable corresponding to nested maps would only be noticed
if the outer map's key already existed.

This commit changes environment variable overriding to iterate over the
environment instead. For environment variables beginning with the
REGISTRY_ prefix, it splits the rest of their names on "_", and
interprets that as a path to the variable to unmarshal into. Map keys
are created as necessary. If we encounter an empty interface partway
through following the path, it becomes an implicit
map[string]interface{}.

With the new unit tests added here, parser.go now has 89.2% test
coverage.

TestParseWithExtraneousEnvStorageParams was removed, because the limit
of one storage driver is no longer enforced while parsing environment
variables. Now, Storage.Type will panic if multiple drivers are
specified.

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>

configuration/parser.go

@@ -4,10 +4,11 @@ import (
 	"fmt"
 	"os"
 	"reflect"
-	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 
+	"github.com/Sirupsen/logrus"
 	"gopkg.in/yaml.v2"
 )
 
@@ -59,18 +60,29 @@ type VersionedParseInfo struct {
 	ConversionFunc func(interface{}) (interface{}, error)
 }
 
+type envVar struct {
+	name  string
+	value string
+}
+
+type envVars []envVar
+
+func (a envVars) Len() int           { return len(a) }
+func (a envVars) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a envVars) Less(i, j int) bool { return a[i].name < a[j].name }
+
 // Parser can be used to parse a configuration file and environment of a defined
 // version into a unified output structure
 type Parser struct {
 	prefix  string
 	mapping map[Version]VersionedParseInfo
-	env     map[string]string
+	env     envVars
 }
 
 // NewParser returns a *Parser with the given environment prefix which handles
 // versioned configurations which match the given parseInfos
 func NewParser(prefix string, parseInfos []VersionedParseInfo) *Parser {
-	p := Parser{prefix: prefix, mapping: make(map[Version]VersionedParseInfo), env: make(map[string]string)}
+	p := Parser{prefix: prefix, mapping: make(map[Version]VersionedParseInfo)}
 
 	for _, parseInfo := range parseInfos {
 		p.mapping[parseInfo.Version] = parseInfo
@@ -78,9 +90,17 @@ func NewParser(prefix string, parseInfos []VersionedParseInfo) *Parser {
 
 	for _, env := range os.Environ() {
 		envParts := strings.SplitN(env, "=", 2)
-		p.env[envParts[0]] = envParts[1]
+		p.env = append(p.env, envVar{envParts[0], envParts[1]})
 	}
 
+	// We must sort the environment variables lexically by name so that
+	// more specific variables are applied before less specific ones
+	// (i.e. REGISTRY_STORAGE before
+	// REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY). This sucks, but it's a
+	// lot simpler and easier to get right than unmarshalling map entries
+	// into temporaries and merging with the existing entry.
+	sort.Sort(p.env)
+
 	return &p
 }
 
@@ -111,9 +131,16 @@ func (p *Parser) Parse(in []byte, v interface{}) error {
 		return err
 	}
 
-	err = p.overwriteFields(parseAs, p.prefix)
-	if err != nil {
-		return err
+	for _, envVar := range p.env {
+		pathStr := envVar.name
+		if strings.HasPrefix(pathStr, strings.ToUpper(p.prefix)+"_") {
+			path := strings.Split(pathStr, "_")
+
+			err = p.overwriteFields(parseAs, pathStr, path[1:], envVar.value)
+			if err != nil {
+				return err
+			}
+		}
 	}
 
 	c, err := parseInfo.ConversionFunc(parseAs.Interface())
@@ -124,80 +151,133 @@ func (p *Parser) Parse(in []byte, v interface{}) error {
 	return nil
 }
 
-func (p *Parser) overwriteFields(v reflect.Value, prefix string) error {
+// overwriteFields replaces configuration values with alternate values specified
+// through the environment. Precondition: an empty path slice must never be
+// passed in.
+func (p *Parser) overwriteFields(v reflect.Value, fullpath string, path []string, payload string) error {
 	for v.Kind() == reflect.Ptr {
+		if v.IsNil() {
+			panic("encountered nil pointer while handling environment variable " + fullpath)
+		}
 		v = reflect.Indirect(v)
 	}
 	switch v.Kind() {
 	case reflect.Struct:
-		for i := 0; i < v.NumField(); i++ {
-			sf := v.Type().Field(i)
-			fieldPrefix := strings.ToUpper(prefix + "_" + sf.Name)
-			if e, ok := p.env[fieldPrefix]; ok {
-				fieldVal := reflect.New(sf.Type)
-				err := yaml.Unmarshal([]byte(e), fieldVal.Interface())
-				if err != nil {
-					return err
-				}
-				v.Field(i).Set(reflect.Indirect(fieldVal))
-			}
-			err := p.overwriteFields(v.Field(i), fieldPrefix)
-			if err != nil {
-				return err
-			}
-		}
+		return p.overwriteStruct(v, fullpath, path, payload)
 	case reflect.Map:
-		p.overwriteMap(v, prefix)
+		return p.overwriteMap(v, fullpath, path, payload)
+	case reflect.Interface:
+		if v.NumMethod() == 0 {
+			if !v.IsNil() {
+				return p.overwriteFields(v.Elem(), fullpath, path, payload)
+			}
+			// Interface was empty; create an implicit map
+			var template map[string]interface{}
+			wrappedV := reflect.MakeMap(reflect.TypeOf(template))
+			v.Set(wrappedV)
+			return p.overwriteMap(wrappedV, fullpath, path, payload)
+		}
 	}
 	return nil
 }
 
-func (p *Parser) overwriteMap(m reflect.Value, prefix string) error {
-	switch m.Type().Elem().Kind() {
-	case reflect.Struct:
-		for _, k := range m.MapKeys() {
-			err := p.overwriteFields(m.MapIndex(k), strings.ToUpper(fmt.Sprintf("%s_%s", prefix, k)))
-			if err != nil {
-				return err
-			}
-		}
-		envMapRegexp, err := regexp.Compile(fmt.Sprintf("^%s_([A-Z0-9]+)$", strings.ToUpper(prefix)))
-		if err != nil {
-			return err
-		}
-		for key, val := range p.env {
-			if submatches := envMapRegexp.FindStringSubmatch(key); submatches != nil {
-				mapValue := reflect.New(m.Type().Elem())
-				err := yaml.Unmarshal([]byte(val), mapValue.Interface())
-				if err != nil {
-					return err
-				}
-				m.SetMapIndex(reflect.ValueOf(strings.ToLower(submatches[1])), reflect.Indirect(mapValue))
-			}
-		}
-	case reflect.Map:
-		for _, k := range m.MapKeys() {
-			err := p.overwriteMap(m.MapIndex(k), strings.ToUpper(fmt.Sprintf("%s_%s", prefix, k)))
-			if err != nil {
-				return err
-			}
-		}
-	default:
-		envMapRegexp, err := regexp.Compile(fmt.Sprintf("^%s_([A-Z0-9]+)$", strings.ToUpper(prefix)))
-		if err != nil {
-			return err
+func (p *Parser) overwriteStruct(v reflect.Value, fullpath string, path []string, payload string) error {
+	// Generate case-insensitive map of struct fields
+	byUpperCase := make(map[string]int)
+	for i := 0; i < v.NumField(); i++ {
+		sf := v.Type().Field(i)
+		upper := strings.ToUpper(sf.Name)
+		if _, present := byUpperCase[upper]; present {
+			panic(fmt.Sprintf("field name collision in configuration object: %s", sf.Name))
 		}
+		byUpperCase[upper] = i
+	}
 
-		for key, val := range p.env {
-			if submatches := envMapRegexp.FindStringSubmatch(key); submatches != nil {
-				mapValue := reflect.New(m.Type().Elem())
-				err := yaml.Unmarshal([]byte(val), mapValue.Interface())
-				if err != nil {
-					return err
+	fieldIndex, present := byUpperCase[path[0]]
+	if !present {
+		logrus.Warnf("Ignoring unrecognized environment variable %s", fullpath)
+		return nil
+	}
+	field := v.Field(fieldIndex)
+	sf := v.Type().Field(fieldIndex)
+
+	if len(path) == 1 {
+		// Env var specifies this field directly
+		fieldVal := reflect.New(sf.Type)
+		err := yaml.Unmarshal([]byte(payload), fieldVal.Interface())
+		if err != nil {
+			return err
+		}
+		field.Set(reflect.Indirect(fieldVal))
+		return nil
+	}
+
+	// If the field is nil, must create an object
+	switch sf.Type.Kind() {
+	case reflect.Map:
+		if field.IsNil() {
+			field.Set(reflect.MakeMap(sf.Type))
+		}
+	case reflect.Ptr:
+		if field.IsNil() {
+			field.Set(reflect.New(sf.Type))
+		}
+	}
+
+	err := p.overwriteFields(field, fullpath, path[1:], payload)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (p *Parser) overwriteMap(m reflect.Value, fullpath string, path []string, payload string) error {
+	if m.Type().Key().Kind() != reflect.String {
+		// non-string keys unsupported
+		logrus.Warnf("Ignoring environment variable %s involving map with non-string keys", fullpath)
+		return nil
+	}
+
+	if len(path) > 1 {
+		// If a matching key exists, get its value and continue the
+		// overwriting process.
+		for _, k := range m.MapKeys() {
+			if strings.ToUpper(k.String()) == path[0] {
+				mapValue := m.MapIndex(k)
+				// If the existing value is nil, we want to
+				// recreate it instead of using this value.
+				if (mapValue.Kind() == reflect.Ptr ||
+					mapValue.Kind() == reflect.Interface ||
+					mapValue.Kind() == reflect.Map) &&
+					mapValue.IsNil() {
+					break
 				}
-				m.SetMapIndex(reflect.ValueOf(strings.ToLower(submatches[1])), reflect.Indirect(mapValue))
+				return p.overwriteFields(mapValue, fullpath, path[1:], payload)
 			}
 		}
 	}
+
+	// (Re)create this key
+	var mapValue reflect.Value
+	if m.Type().Elem().Kind() == reflect.Map {
+		mapValue = reflect.MakeMap(m.Type().Elem())
+	} else {
+		mapValue = reflect.New(m.Type().Elem())
+	}
+	if len(path) > 1 {
+		err := p.overwriteFields(mapValue, fullpath, path[1:], payload)
+		if err != nil {
+			return err
+		}
+	} else {
+		err := yaml.Unmarshal([]byte(payload), mapValue.Interface())
+		if err != nil {
+			return err
+		}
+	}
+
+	m.SetMapIndex(reflect.ValueOf(strings.ToLower(path[0])), reflect.Indirect(mapValue))
+
 	return nil
 }