From 3cad3c7b6acfbb87d241fad56fa3f41fd835d641 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Wed, 1 Apr 2015 16:27:24 -0700 Subject: [PATCH] Add redis pool to registry webapp Redis has been integrated with the web application for use with various services. The configuraiton exposes connection details, timeouts and pool parameters. Documentation has been updated accordingly. A few convenience methods have been added to the context package to get loggers with certain fields, exposing some missing functionality from logrus. Signed-off-by: Stephen J Day --- cmd/registry/config.yml | 9 ++++ configuration/configuration.go | 30 ++++++++++++ context/logger.go | 14 ++++++ doc/configuration.md | 56 +++++++++++++++++++++++ registry/handlers/app.go | 83 ++++++++++++++++++++++++++++++++++ 5 files changed, 192 insertions(+) diff --git a/cmd/registry/config.yml b/cmd/registry/config.yml index 57541e41..a8d469b6 100644 --- a/cmd/registry/config.yml +++ b/cmd/registry/config.yml @@ -12,6 +12,15 @@ http: secret: asecretforlocaldevelopment debug: addr: localhost:5001 +redis: + addr: localhost:6379 + pool: + maxidle: 16 + maxactive: 64 + idletimeout: 300s + dialtimeout: 10ms + readtimeout: 10ms + writetimeout: 10ms notifications: endpoints: - name: local-8082 diff --git a/configuration/configuration.go b/configuration/configuration.go index 5b76d029..c38cf7c6 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -91,6 +91,36 @@ type Configuration struct { // Notifications specifies configuration about various endpoint to which // registry events are dispatched. Notifications Notifications `yaml:"notifications,omitempty"` + + // Redis configures the redis pool available to the registry webapp. + Redis struct { + // Addr specifies the the redis instance available to the application. + Addr string `yaml:"addr,omitempty"` + + // Password string to use when making a connection. + Password string `yaml:"password,omitempty"` + + // DB specifies the database to connect to on the redis instance. + DB int `yaml:"db,omitempty"` + + DialTimeout time.Duration `yaml:"dialtimeout,omitempty"` // timeout for connect + ReadTimeout time.Duration `yaml:"readtimeout,omitempty"` // timeout for reads of data + WriteTimeout time.Duration `yaml:"writetimeout,omitempty"` // timeout for writes of data + + // Pool configures the behavior of the redis connection pool. + Pool struct { + // MaxIdle sets the maximum number of idle connections. + MaxIdle int `yaml:"maxidle,omitempty"` + + // MaxActive sets the maximum number of connections that should be + // opened before blocking a connection request. + MaxActive int `yaml:"maxactive,omitempty"` + + // IdleTimeout sets the amount time to wait before closing + // inactive connections. + IdleTimeout time.Duration `yaml:"idletimeout,omitempty"` + } `yaml:"pool,omitempty"` + } `yaml:"redis,omitempty"` } // v0_1Configuration is a Version 0.1 Configuration struct diff --git a/context/logger.go b/context/logger.go index bec8fade..1726e095 100644 --- a/context/logger.go +++ b/context/logger.go @@ -45,6 +45,20 @@ func WithLogger(ctx context.Context, logger Logger) context.Context { return context.WithValue(ctx, "logger", logger) } +// GetLoggerWithField returns a logger instance with the specified field key +// and value without affecting the context. Extra specified keys will be +// resolved from the context. +func GetLoggerWithField(ctx context.Context, key, value interface{}, keys ...interface{}) Logger { + return getLogrusLogger(ctx, keys...).WithField(fmt.Sprint(key), value) +} + +// GetLoggerWithFields returns a logger instance with the specified fields +// without affecting the context. Extra specified keys will be resolved from +// the context. +func GetLoggerWithFields(ctx context.Context, fields map[string]interface{}, keys ...interface{}) Logger { + return getLogrusLogger(ctx, keys...).WithFields(logrus.Fields(fields)) +} + // GetLogger returns the logger from the current context, if present. If one // or more keys are provided, they will be resolved on the context and // included in the logger. While context.Value takes an interface, any key diff --git a/doc/configuration.md b/doc/configuration.md index 5ac11711..6071efe6 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -82,6 +82,17 @@ notifications: timeout: 500 threshold: 5 backoff: 1000 +redis: + addr: localhost:6379 + password: asecret + db: 0 + dialtimeout: 10ms + readtimeout: 10ms + writetimeout: 10ms + pool: + maxidle: 16 + maxactive: 64 + idletimeout: 300s ``` N.B. In some instances a configuration option may be marked **optional** but contain child options marked as **required**. This indicates that a parent may be omitted with all its children, however, if the parent is included, the children marked **required** must be included. @@ -347,3 +358,48 @@ Endpoints is a list of named services (URLs) that can accept event notifications - threshold: **Required** - TODO: fill in description - backoff: **Required** - TODO: fill in description +## redis + +```yaml +redis: + addr: localhost:6379 + password: asecret + db: 0 + dialtimeout: 10ms + readtimeout: 10ms + writetimeout: 10ms + pool: + maxidle: 16 + maxactive: 64 + idletimeout: 300s +``` + +Declare parameters for constructing the redis connections. Registry instances +may use the redis instance for several applications. The current purpose is +caching information about immutable blobs. Most of the options below control +how the registry connects to redis. The behavior of the pool can be controlled +with the [pool](#pool) subsection. + +- addr: **Required** - Address (host and port) of redis instance. +- password: **Optional** - A password used to authenticate to the redis instance. +- db: **Optional** - Selects the db for each connection. +- dialtimeout: **Optional** - Timeout for connecting to a redis instance. +- readtimeout: **Optional** - Timeout for reading from redis connections. +- writetimeout: **Optional** - Timeout for writing to redis connections. + +### pool + +```yaml +pool: + maxidle: 16 + maxactive: 64 + idletimeout: 300s +``` + +Configure the behavior of the redis connection pool. + +- maxidle: **Optional** - sets the maximum number of idle connections. +- maxactive: **Optional** - sets the maximum number of connections that should + be opened before blocking a connection request. +- idletimeout: **Optional** - sets the amount time to wait before closing + inactive connections. diff --git a/registry/handlers/app.go b/registry/handlers/app.go index 1b5effbc..f837e861 100644 --- a/registry/handlers/app.go +++ b/registry/handlers/app.go @@ -1,10 +1,12 @@ package handlers import ( + "expvar" "fmt" "net" "net/http" "os" + "time" "code.google.com/p/go-uuid/uuid" "github.com/docker/distribution" @@ -19,6 +21,7 @@ import ( storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/factory" storagemiddleware "github.com/docker/distribution/registry/storage/driver/middleware" + "github.com/garyburd/redigo/redis" "github.com/gorilla/mux" "golang.org/x/net/context" ) @@ -44,6 +47,8 @@ type App struct { sink notifications.Sink source notifications.SourceRecord } + + redis *redis.Pool } // Value intercepts calls context.Context.Value, returning the current app id, @@ -95,6 +100,7 @@ func NewApp(ctx context.Context, configuration configuration.Configuration) *App } app.configureEvents(&configuration) + app.configureRedis(&configuration) app.registry = storage.NewRegistryWithDriver(app.driver) app.registry, err = applyRegistryMiddleware(app.registry, configuration.Middleware["registry"]) @@ -174,6 +180,83 @@ func (app *App) configureEvents(configuration *configuration.Configuration) { } } +func (app *App) configureRedis(configuration *configuration.Configuration) { + if configuration.Redis.Addr == "" { + ctxu.GetLogger(app).Infof("redis not configured") + return + } + + pool := &redis.Pool{ + Dial: func() (redis.Conn, error) { + // TODO(stevvooe): Yet another use case for contextual timing. + ctx := context.WithValue(app, "redis.connect.startedat", time.Now()) + + done := func(err error) { + logger := ctxu.GetLoggerWithField(ctx, "redis.connect.duration", + ctxu.Since(ctx, "redis.connect.startedat")) + if err != nil { + logger.Errorf("redis: error connecting: %v", err) + } else { + logger.Infof("redis: connect %v", configuration.Redis.Addr) + } + } + + conn, err := redis.DialTimeout("tcp", + configuration.Redis.Addr, + configuration.Redis.DialTimeout, + configuration.Redis.ReadTimeout, + configuration.Redis.WriteTimeout) + if err != nil { + ctxu.GetLogger(app).Errorf("error connecting to redis instance %s: %v", + configuration.Redis.Addr, err) + done(err) + return nil, err + } + + // authorize the connection + if configuration.Redis.Password != "" { + if _, err = conn.Do("AUTH", configuration.Redis.Password); err != nil { + defer conn.Close() + done(err) + return nil, err + } + } + + // select the database to use + if configuration.Redis.DB != 0 { + if _, err = conn.Do("SELECT", configuration.Redis.DB); err != nil { + defer conn.Close() + done(err) + return nil, err + } + } + + done(nil) + return conn, nil + }, + MaxIdle: configuration.Redis.Pool.MaxIdle, + MaxActive: configuration.Redis.Pool.MaxActive, + IdleTimeout: configuration.Redis.Pool.IdleTimeout, + TestOnBorrow: func(c redis.Conn, t time.Time) error { + // TODO(stevvooe): We can probably do something more interesting + // here with the health package. + _, err := c.Do("PING") + return err + }, + Wait: false, // if a connection is not avialable, proceed without cache. + } + + app.redis = pool + + expvar.Publish("redis", expvar.Func(func() interface{} { + return map[string]interface{}{ + "Config": configuration.Redis, + "Active": app.redis.ActiveCount(), + } + })) + +} + func (app *App) ServeHTTP(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() // ensure that request body is always closed.