From 9b69e40c93856ff51f694303c0696de405635470 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Thu, 20 Aug 2015 13:56:36 -0700 Subject: [PATCH] Move initialization code from main.go to the registry package This makes it easier to embed a registry instance inside another application. Signed-off-by: Aaron Lehmann --- cmd/registry/main.go | 250 +------------------------- registry/doc.go | 3 +- registry/handlers/api_test.go | 2 +- registry/handlers/app.go | 12 +- registry/handlers/app_test.go | 4 +- registry/handlers/health_test.go | 6 +- registry/registry.go | 294 +++++++++++++++++++++++++++++++ 7 files changed, 313 insertions(+), 258 deletions(-) create mode 100644 registry/registry.go diff --git a/cmd/registry/main.go b/cmd/registry/main.go index ec4dedd6..6b156298 100644 --- a/cmd/registry/main.go +++ b/cmd/registry/main.go @@ -1,28 +1,17 @@ package main import ( - "crypto/tls" - "crypto/x509" - _ "expvar" "flag" "fmt" - "io/ioutil" - "net/http" _ "net/http/pprof" "os" - "time" log "github.com/Sirupsen/logrus" - "github.com/Sirupsen/logrus/formatters/logstash" - "github.com/bugsnag/bugsnag-go" "github.com/docker/distribution/configuration" - "github.com/docker/distribution/context" - "github.com/docker/distribution/health" + "github.com/docker/distribution/registry" _ "github.com/docker/distribution/registry/auth/htpasswd" _ "github.com/docker/distribution/registry/auth/silly" _ "github.com/docker/distribution/registry/auth/token" - "github.com/docker/distribution/registry/handlers" - "github.com/docker/distribution/registry/listener" _ "github.com/docker/distribution/registry/proxy" _ "github.com/docker/distribution/registry/storage/driver/azure" _ "github.com/docker/distribution/registry/storage/driver/filesystem" @@ -31,10 +20,8 @@ import ( _ "github.com/docker/distribution/registry/storage/driver/oss" _ "github.com/docker/distribution/registry/storage/driver/s3" _ "github.com/docker/distribution/registry/storage/driver/swift" - "github.com/docker/distribution/uuid" "github.com/docker/distribution/version" - gorhandlers "github.com/gorilla/handlers" - "github.com/yvasiyarov/gorelic" + "golang.org/x/net/context" ) var showVersion bool @@ -52,99 +39,19 @@ func main() { return } - ctx := context.Background() - ctx = context.WithValue(ctx, "version", version.Version) - config, err := resolveConfiguration() if err != nil { fatalf("configuration error: %v", err) } - ctx, err = configureLogging(ctx, config) + registry, err := registry.NewRegistry(context.Background(), config) if err != nil { - fatalf("error configuring logger: %v", err) + log.Fatalln(err) } - // inject a logger into the uuid library. warns us if there is a problem - // with uuid generation under low entropy. - uuid.Loggerf = context.GetLogger(ctx).Warnf - - app := handlers.NewApp(ctx, *config) - app.RegisterHealthChecks() - handler := configureReporting(app) - handler = alive("/", handler) - handler = health.Handler(handler) - handler = panicHandler(handler) - handler = gorhandlers.CombinedLoggingHandler(os.Stdout, handler) - - if config.HTTP.Debug.Addr != "" { - go debugServer(config.HTTP.Debug.Addr) - } - - server := &http.Server{ - Handler: handler, - } - - ln, err := listener.NewListener(config.HTTP.Net, config.HTTP.Addr) + err = registry.Serve() if err != nil { - context.GetLogger(app).Fatalln(err) - } - defer ln.Close() - - if config.HTTP.TLS.Certificate != "" { - tlsConf := &tls.Config{ - ClientAuth: tls.NoClientCert, - NextProtos: []string{"http/1.1"}, - Certificates: make([]tls.Certificate, 1), - MinVersion: tls.VersionTLS10, - PreferServerCipherSuites: true, - CipherSuites: []uint16{ - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, - tls.TLS_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_RSA_WITH_AES_256_CBC_SHA, - }, - } - - tlsConf.Certificates[0], err = tls.LoadX509KeyPair(config.HTTP.TLS.Certificate, config.HTTP.TLS.Key) - if err != nil { - context.GetLogger(app).Fatalln(err) - } - - if len(config.HTTP.TLS.ClientCAs) != 0 { - pool := x509.NewCertPool() - - for _, ca := range config.HTTP.TLS.ClientCAs { - caPem, err := ioutil.ReadFile(ca) - if err != nil { - context.GetLogger(app).Fatalln(err) - } - - if ok := pool.AppendCertsFromPEM(caPem); !ok { - context.GetLogger(app).Fatalln(fmt.Errorf("Could not add CA to pool")) - } - } - - for _, subj := range pool.Subjects() { - context.GetLogger(app).Debugf("CA Subject: %s", string(subj)) - } - - tlsConf.ClientAuth = tls.RequireAndVerifyClientCert - tlsConf.ClientCAs = pool - } - - ln = tls.NewListener(ln, tlsConf) - context.GetLogger(app).Infof("listening on %v, tls", ln.Addr()) - } else { - context.GetLogger(app).Infof("listening on %v", ln.Addr()) - } - - if err := server.Serve(ln); err != nil { - context.GetLogger(app).Fatalln(err) + log.Fatalln(err) } } @@ -186,148 +93,3 @@ func resolveConfiguration() (*configuration.Configuration, error) { return config, nil } - -func configureReporting(app *handlers.App) http.Handler { - var handler http.Handler = app - - if app.Config.Reporting.Bugsnag.APIKey != "" { - bugsnagConfig := bugsnag.Configuration{ - APIKey: app.Config.Reporting.Bugsnag.APIKey, - // TODO(brianbland): provide the registry version here - // AppVersion: "2.0", - } - if app.Config.Reporting.Bugsnag.ReleaseStage != "" { - bugsnagConfig.ReleaseStage = app.Config.Reporting.Bugsnag.ReleaseStage - } - if app.Config.Reporting.Bugsnag.Endpoint != "" { - bugsnagConfig.Endpoint = app.Config.Reporting.Bugsnag.Endpoint - } - bugsnag.Configure(bugsnagConfig) - - handler = bugsnag.Handler(handler) - } - - if app.Config.Reporting.NewRelic.LicenseKey != "" { - agent := gorelic.NewAgent() - agent.NewrelicLicense = app.Config.Reporting.NewRelic.LicenseKey - if app.Config.Reporting.NewRelic.Name != "" { - agent.NewrelicName = app.Config.Reporting.NewRelic.Name - } - agent.CollectHTTPStat = true - agent.Verbose = app.Config.Reporting.NewRelic.Verbose - agent.Run() - - handler = agent.WrapHTTPHandler(handler) - } - - return handler -} - -// configureLogging prepares the context with a logger using the -// configuration. -func configureLogging(ctx context.Context, config *configuration.Configuration) (context.Context, error) { - if config.Log.Level == "" && config.Log.Formatter == "" { - // If no config for logging is set, fallback to deprecated "Loglevel". - log.SetLevel(logLevel(config.Loglevel)) - ctx = context.WithLogger(ctx, context.GetLogger(ctx, "version")) - return ctx, nil - } - - log.SetLevel(logLevel(config.Log.Level)) - - formatter := config.Log.Formatter - if formatter == "" { - formatter = "text" // default formatter - } - - switch formatter { - case "json": - log.SetFormatter(&log.JSONFormatter{ - TimestampFormat: time.RFC3339Nano, - }) - case "text": - log.SetFormatter(&log.TextFormatter{ - TimestampFormat: time.RFC3339Nano, - }) - case "logstash": - log.SetFormatter(&logstash.LogstashFormatter{ - TimestampFormat: time.RFC3339Nano, - }) - default: - // just let the library use default on empty string. - if config.Log.Formatter != "" { - return ctx, fmt.Errorf("unsupported logging formatter: %q", config.Log.Formatter) - } - } - - if config.Log.Formatter != "" { - log.Debugf("using %q logging formatter", config.Log.Formatter) - } - - // log the application version with messages - ctx = context.WithLogger(ctx, context.GetLogger(ctx, "version")) - - if len(config.Log.Fields) > 0 { - // build up the static fields, if present. - var fields []interface{} - for k := range config.Log.Fields { - fields = append(fields, k) - } - - ctx = context.WithValues(ctx, config.Log.Fields) - ctx = context.WithLogger(ctx, context.GetLogger(ctx, fields...)) - } - - return ctx, nil -} - -func logLevel(level configuration.Loglevel) log.Level { - l, err := log.ParseLevel(string(level)) - if err != nil { - l = log.InfoLevel - log.Warnf("error parsing level %q: %v, using %q ", level, err, l) - } - - return l -} - -// debugServer starts the debug server with pprof, expvar among other -// endpoints. The addr should not be exposed externally. For most of these to -// work, tls cannot be enabled on the endpoint, so it is generally separate. -func debugServer(addr string) { - log.Infof("debug server listening %v", addr) - if err := http.ListenAndServe(addr, nil); err != nil { - log.Fatalf("error listening on debug interface: %v", err) - } -} - -// panicHandler add a HTTP handler to web app. The handler recover the happening -// panic. logrus.Panic transmits panic message to pre-config log hooks, which is -// defined in config.yml. -func panicHandler(handler http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer func() { - if err := recover(); err != nil { - log.Panic(fmt.Sprintf("%v", err)) - } - }() - handler.ServeHTTP(w, r) - }) -} - -// alive simply wraps the handler with a route that always returns an http 200 -// response when the path is matched. If the path is not matched, the request -// is passed to the provided handler. There is no guarantee of anything but -// that the server is up. Wrap with other handlers (such as health.Handler) -// for greater affect. -func alive(path string, handler http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == path { - w.Header().Set("Cache-Control", "no-cache") - w.WriteHeader(http.StatusOK) - return - } - - handler.ServeHTTP(w, r) - }) -} diff --git a/registry/doc.go b/registry/doc.go index 1c01e42e..a1ba7f3a 100644 --- a/registry/doc.go +++ b/registry/doc.go @@ -1,3 +1,2 @@ -// Package registry is a placeholder package for registry interface -// definitions and utilities. +// Package registry provides the main entrypoints for running a registry. package registry diff --git a/registry/handlers/api_test.go b/registry/handlers/api_test.go index 3473baf5..52a74a2b 100644 --- a/registry/handlers/api_test.go +++ b/registry/handlers/api_test.go @@ -1038,7 +1038,7 @@ func newTestEnv(t *testing.T, deleteEnabled bool) *testEnv { func newTestEnvWithConfig(t *testing.T, config *configuration.Configuration) *testEnv { ctx := context.Background() - app := NewApp(ctx, *config) + app := NewApp(ctx, config) server := httptest.NewServer(handlers.CombinedLoggingHandler(os.Stderr, app)) builder, err := v2.NewURLBuilderFromString(server.URL + config.HTTP.Prefix) diff --git a/registry/handlers/app.go b/registry/handlers/app.go index b1e46b02..8c67c20b 100644 --- a/registry/handlers/app.go +++ b/registry/handlers/app.go @@ -47,7 +47,7 @@ const defaultCheckInterval = 10 * time.Second type App struct { context.Context - Config configuration.Configuration + Config *configuration.Configuration router *mux.Router // main application router, configured with dispatchers driver storagedriver.StorageDriver // driver maintains the app global storage driver instance. @@ -69,7 +69,7 @@ type App struct { // NewApp takes a configuration and returns a configured app, ready to serve // requests. The app only implements ServeHTTP and can be wrapped in other // handlers accordingly. -func NewApp(ctx context.Context, configuration configuration.Configuration) *App { +func NewApp(ctx context.Context, configuration *configuration.Configuration) *App { app := &App{ Config: configuration, Context: ctx, @@ -117,10 +117,10 @@ func NewApp(ctx context.Context, configuration configuration.Configuration) *App panic(err) } - app.configureSecret(&configuration) - app.configureEvents(&configuration) - app.configureRedis(&configuration) - app.configureLogHook(&configuration) + app.configureSecret(configuration) + app.configureEvents(configuration) + app.configureRedis(configuration) + app.configureLogHook(configuration) options := []storage.RegistryOption{} diff --git a/registry/handlers/app_test.go b/registry/handlers/app_test.go index 0038a97d..9e2514d8 100644 --- a/registry/handlers/app_test.go +++ b/registry/handlers/app_test.go @@ -31,7 +31,7 @@ func TestAppDispatcher(t *testing.T) { t.Fatalf("error creating registry: %v", err) } app := &App{ - Config: configuration.Configuration{}, + Config: &configuration.Configuration{}, Context: ctx, router: v2.Router(), driver: driver, @@ -164,7 +164,7 @@ func TestNewApp(t *testing.T) { // Mostly, with this test, given a sane configuration, we are simply // ensuring that NewApp doesn't panic. We might want to tweak this // behavior. - app := NewApp(ctx, config) + app := NewApp(ctx, &config) server := httptest.NewServer(app) builder, err := v2.NewURLBuilderFromString(server.URL) diff --git a/registry/handlers/health_test.go b/registry/handlers/health_test.go index bb460b47..5fe65ede 100644 --- a/registry/handlers/health_test.go +++ b/registry/handlers/health_test.go @@ -23,7 +23,7 @@ func TestFileHealthCheck(t *testing.T) { } defer tmpfile.Close() - config := configuration.Configuration{ + config := &configuration.Configuration{ Storage: configuration.Storage{ "inmemory": configuration.Parameters{}, }, @@ -83,7 +83,7 @@ func TestTCPHealthCheck(t *testing.T) { } }() - config := configuration.Configuration{ + config := &configuration.Configuration{ Storage: configuration.Storage{ "inmemory": configuration.Parameters{}, }, @@ -142,7 +142,7 @@ func TestHTTPHealthCheck(t *testing.T) { } })) - config := configuration.Configuration{ + config := &configuration.Configuration{ Storage: configuration.Storage{ "inmemory": configuration.Parameters{}, }, diff --git a/registry/registry.go b/registry/registry.go new file mode 100644 index 00000000..68525040 --- /dev/null +++ b/registry/registry.go @@ -0,0 +1,294 @@ +package registry + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + "net" + "net/http" + "os" + "time" + + log "github.com/Sirupsen/logrus" + "github.com/Sirupsen/logrus/formatters/logstash" + "github.com/bugsnag/bugsnag-go" + "github.com/docker/distribution/configuration" + ctxu "github.com/docker/distribution/context" + "github.com/docker/distribution/health" + "github.com/docker/distribution/registry/handlers" + "github.com/docker/distribution/registry/listener" + "github.com/docker/distribution/uuid" + "github.com/docker/distribution/version" + gorhandlers "github.com/gorilla/handlers" + "github.com/yvasiyarov/gorelic" + "golang.org/x/net/context" +) + +// A Registry represents a complete instance of the registry. +type Registry struct { + config *configuration.Configuration + app *handlers.App + server *http.Server + ln net.Listener + debugLn net.Listener +} + +// NewRegistry creates a new registry from a context and configuration struct. +func NewRegistry(ctx context.Context, config *configuration.Configuration) (*Registry, error) { + // Note this + ctx = ctxu.WithValue(ctx, "version", version.Version) + + var err error + ctx, err = configureLogging(ctx, config) + if err != nil { + return nil, fmt.Errorf("error configuring logger: %v", err) + } + + // inject a logger into the uuid library. warns us if there is a problem + // with uuid generation under low entropy. + uuid.Loggerf = ctxu.GetLogger(ctx).Warnf + + app := handlers.NewApp(ctx, config) + // TODO(aaronl): The global scope of the health checks means NewRegistry + // can only be called once per process. + app.RegisterHealthChecks() + handler := configureReporting(app) + handler = alive("/", handler) + handler = health.Handler(handler) + handler = panicHandler(handler) + handler = gorhandlers.CombinedLoggingHandler(os.Stdout, handler) + + server := &http.Server{ + Handler: handler, + } + + ln, err := listener.NewListener(config.HTTP.Net, config.HTTP.Addr) + if err != nil { + return nil, err + } + + var debugLn net.Listener + if config.HTTP.Debug.Addr != "" { + debugLn, err = listener.NewListener("tcp", config.HTTP.Debug.Addr) + if err != nil { + return nil, fmt.Errorf("error listening on debug interface: %v", err) + } + log.Infof("debug server listening %v", config.HTTP.Debug.Addr) + } + + if config.HTTP.TLS.Certificate != "" { + tlsConf := &tls.Config{ + ClientAuth: tls.NoClientCert, + NextProtos: []string{"http/1.1"}, + Certificates: make([]tls.Certificate, 1), + MinVersion: tls.VersionTLS10, + PreferServerCipherSuites: true, + CipherSuites: []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_RSA_WITH_AES_256_CBC_SHA, + }, + } + + tlsConf.Certificates[0], err = tls.LoadX509KeyPair(config.HTTP.TLS.Certificate, config.HTTP.TLS.Key) + if err != nil { + return nil, err + } + + if len(config.HTTP.TLS.ClientCAs) != 0 { + pool := x509.NewCertPool() + + for _, ca := range config.HTTP.TLS.ClientCAs { + caPem, err := ioutil.ReadFile(ca) + if err != nil { + return nil, err + } + + if ok := pool.AppendCertsFromPEM(caPem); !ok { + return nil, fmt.Errorf("Could not add CA to pool") + } + } + + for _, subj := range pool.Subjects() { + ctxu.GetLogger(app).Debugf("CA Subject: %s", string(subj)) + } + + tlsConf.ClientAuth = tls.RequireAndVerifyClientCert + tlsConf.ClientCAs = pool + } + + ln = tls.NewListener(ln, tlsConf) + ctxu.GetLogger(app).Infof("listening on %v, tls", ln.Addr()) + } else { + ctxu.GetLogger(app).Infof("listening on %v", ln.Addr()) + } + + return &Registry{ + app: app, + config: config, + server: server, + ln: ln, + debugLn: debugLn, + }, nil +} + +// Serve runs the registry's HTTP server(s). +func (registry *Registry) Serve() error { + defer registry.ln.Close() + + errChan := make(chan error) + + if registry.debugLn != nil { + defer registry.debugLn.Close() + go func() { + errChan <- http.Serve(registry.debugLn, nil) + }() + } + + go func() { + errChan <- registry.server.Serve(registry.ln) + }() + + return <-errChan +} + +func configureReporting(app *handlers.App) http.Handler { + var handler http.Handler = app + + if app.Config.Reporting.Bugsnag.APIKey != "" { + bugsnagConfig := bugsnag.Configuration{ + APIKey: app.Config.Reporting.Bugsnag.APIKey, + // TODO(brianbland): provide the registry version here + // AppVersion: "2.0", + } + if app.Config.Reporting.Bugsnag.ReleaseStage != "" { + bugsnagConfig.ReleaseStage = app.Config.Reporting.Bugsnag.ReleaseStage + } + if app.Config.Reporting.Bugsnag.Endpoint != "" { + bugsnagConfig.Endpoint = app.Config.Reporting.Bugsnag.Endpoint + } + bugsnag.Configure(bugsnagConfig) + + handler = bugsnag.Handler(handler) + } + + if app.Config.Reporting.NewRelic.LicenseKey != "" { + agent := gorelic.NewAgent() + agent.NewrelicLicense = app.Config.Reporting.NewRelic.LicenseKey + if app.Config.Reporting.NewRelic.Name != "" { + agent.NewrelicName = app.Config.Reporting.NewRelic.Name + } + agent.CollectHTTPStat = true + agent.Verbose = app.Config.Reporting.NewRelic.Verbose + agent.Run() + + handler = agent.WrapHTTPHandler(handler) + } + + return handler +} + +// configureLogging prepares the context with a logger using the +// configuration. +func configureLogging(ctx ctxu.Context, config *configuration.Configuration) (context.Context, error) { + if config.Log.Level == "" && config.Log.Formatter == "" { + // If no config for logging is set, fallback to deprecated "Loglevel". + log.SetLevel(logLevel(config.Loglevel)) + ctx = ctxu.WithLogger(ctx, ctxu.GetLogger(ctx, "version")) + return ctx, nil + } + + log.SetLevel(logLevel(config.Log.Level)) + + formatter := config.Log.Formatter + if formatter == "" { + formatter = "text" // default formatter + } + + switch formatter { + case "json": + log.SetFormatter(&log.JSONFormatter{ + TimestampFormat: time.RFC3339Nano, + }) + case "text": + log.SetFormatter(&log.TextFormatter{ + TimestampFormat: time.RFC3339Nano, + }) + case "logstash": + log.SetFormatter(&logstash.LogstashFormatter{ + TimestampFormat: time.RFC3339Nano, + }) + default: + // just let the library use default on empty string. + if config.Log.Formatter != "" { + return ctx, fmt.Errorf("unsupported logging formatter: %q", config.Log.Formatter) + } + } + + if config.Log.Formatter != "" { + log.Debugf("using %q logging formatter", config.Log.Formatter) + } + + // log the application version with messages + ctx = ctxu.WithLogger(ctx, ctxu.GetLogger(ctx, "version")) + + if len(config.Log.Fields) > 0 { + // build up the static fields, if present. + var fields []interface{} + for k := range config.Log.Fields { + fields = append(fields, k) + } + + ctx = ctxu.WithValues(ctx, config.Log.Fields) + ctx = ctxu.WithLogger(ctx, ctxu.GetLogger(ctx, fields...)) + } + + return ctx, nil +} + +func logLevel(level configuration.Loglevel) log.Level { + l, err := log.ParseLevel(string(level)) + if err != nil { + l = log.InfoLevel + log.Warnf("error parsing level %q: %v, using %q ", level, err, l) + } + + return l +} + +// panicHandler add a HTTP handler to web app. The handler recover the happening +// panic. logrus.Panic transmits panic message to pre-config log hooks, which is +// defined in config.yml. +func panicHandler(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := recover(); err != nil { + log.Panic(fmt.Sprintf("%v", err)) + } + }() + handler.ServeHTTP(w, r) + }) +} + +// alive simply wraps the handler with a route that always returns an http 200 +// response when the path is matched. If the path is not matched, the request +// is passed to the provided handler. There is no guarantee of anything but +// that the server is up. Wrap with other handlers (such as health.Handler) +// for greater affect. +func alive(path string, handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == path { + w.Header().Set("Cache-Control", "no-cache") + w.WriteHeader(http.StatusOK) + return + } + + handler.ServeHTTP(w, r) + }) +}