plugins: experimental support for new plugin management

This patch introduces a new experimental engine-level plugin management
with a new API and command line. Plugins can be distributed via a Docker
registry, and their lifecycle is managed by the engine.
This makes plugins a first-class construct.

For more background, have a look at issue #20363.

Documentation is in a separate commit. If you want to understand how the
new plugin system works, you can start by reading the documentation.

Note: backwards compatibility with existing plugins is maintained,
albeit they won't benefit from the advantages of the new system.

Signed-off-by: Tibor Vass <tibor@docker.com>
Signed-off-by: Anusha Ragunathan <anusha@docker.com>

authorization/authz_unix_test.go

@@ -203,18 +203,17 @@ func TestResponseModifierOverride(t *testing.T) {
 
 // createTestPlugin creates a new sample authorization plugin
 func createTestPlugin(t *testing.T) *authorizationPlugin {
-	plugin := &plugins.Plugin{Name: "authz"}
 	pwd, err := os.Getwd()
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	plugin.Client, err = plugins.NewClient("unix:///"+path.Join(pwd, pluginAddress), tlsconfig.Options{InsecureSkipVerify: true})
+	client, err := plugins.NewClient("unix:///"+path.Join(pwd, pluginAddress), &tlsconfig.Options{InsecureSkipVerify: true})
 	if err != nil {
 		t.Fatalf("Failed to create client %v", err)
 	}
 
-	return &authorizationPlugin{name: "plugin", plugin: plugin}
+	return &authorizationPlugin{name: "plugin", plugin: client}
 }
 
 // AuthZPluginTestServer is a simple server that implements the authZ plugin interface

authorization/plugin.go

@@ -35,7 +35,7 @@ func NewPlugins(names []string) []Plugin {
 
 // authorizationPlugin is an internal adapter to docker plugin system
 type authorizationPlugin struct {
-	plugin *plugins.Plugin
+	plugin *plugins.Client
 	name   string
 	once   sync.Once
 }
@@ -54,7 +54,7 @@ func (a *authorizationPlugin) AuthZRequest(authReq *Request) (*Response, error)
 	}
 
 	authRes := &Response{}
-	if err := a.plugin.Client.Call(AuthZApiRequest, authReq, authRes); err != nil {
+	if err := a.plugin.Call(AuthZApiRequest, authReq, authRes); err != nil {
 		return nil, err
 	}
 
@@ -67,7 +67,7 @@ func (a *authorizationPlugin) AuthZResponse(authReq *Request) (*Response, error)
 	}
 
 	authRes := &Response{}
-	if err := a.plugin.Client.Call(AuthZApiResponse, authReq, authRes); err != nil {
+	if err := a.plugin.Call(AuthZApiResponse, authReq, authRes); err != nil {
 		return nil, err
 	}
 
@@ -80,7 +80,12 @@ func (a *authorizationPlugin) initPlugin() error {
 	var err error
 	a.once.Do(func() {
 		if a.plugin == nil {
-			a.plugin, err = plugins.Get(a.name, AuthZApiImplements)
+			plugin, e := plugins.Get(a.name, AuthZApiImplements)
+			if e != nil {
+				err = e
+				return
+			}
+			a.plugin = plugin.Client()
 		}
 	})
 	return err

plugins/client.go

@@ -20,14 +20,16 @@ const (
 )
 
 // NewClient creates a new plugin client (http).
-func NewClient(addr string, tlsConfig tlsconfig.Options) (*Client, error) {
+func NewClient(addr string, tlsConfig *tlsconfig.Options) (*Client, error) {
 	tr := &http.Transport{}
 
-	c, err := tlsconfig.Client(tlsConfig)
+	if tlsConfig != nil {
-	if err != nil {
+		c, err := tlsconfig.Client(*tlsConfig)
-		return nil, err
+		if err != nil {
+			return nil, err
+		}
+		tr.TLSClientConfig = c
 	}
-	tr.TLSClientConfig = c
 
 	u, err := url.Parse(addr)
 	if err != nil {

plugins/client_test.go

@@ -31,7 +31,7 @@ func teardownRemotePluginServer() {
 }
 
 func TestFailedConnection(t *testing.T) {
-	c, _ := NewClient("tcp://127.0.0.1:1", tlsconfig.Options{InsecureSkipVerify: true})
+	c, _ := NewClient("tcp://127.0.0.1:1", &tlsconfig.Options{InsecureSkipVerify: true})
 	_, err := c.callWithRetry("Service.Method", nil, false)
 	if err == nil {
 		t.Fatal("Unexpected successful connection")
@@ -55,7 +55,7 @@ func TestEchoInputOutput(t *testing.T) {
 		io.Copy(w, r.Body)
 	})
 
-	c, _ := NewClient(addr, tlsconfig.Options{InsecureSkipVerify: true})
+	c, _ := NewClient(addr, &tlsconfig.Options{InsecureSkipVerify: true})
 	var output Manifest
 	err := c.Call("Test.Echo", m, &output)
 	if err != nil {

plugins/discovery.go

@@ -64,7 +64,7 @@ func (l *localRegistry) Plugin(name string) (*Plugin, error) {
 
 	for _, p := range socketpaths {
 		if fi, err := os.Stat(p); err == nil && fi.Mode()&os.ModeSocket != 0 {
-			return newLocalPlugin(name, "unix://"+p), nil
+			return NewLocalPlugin(name, "unix://"+p), nil
 		}
 	}
 
@@ -101,7 +101,7 @@ func readPluginInfo(name, path string) (*Plugin, error) {
 		return nil, fmt.Errorf("Unknown protocol")
 	}
 
-	return newLocalPlugin(name, addr), nil
+	return NewLocalPlugin(name, addr), nil
 }
 
 func readPluginJSONInfo(name, path string) (*Plugin, error) {
@@ -115,7 +115,7 @@ func readPluginJSONInfo(name, path string) (*Plugin, error) {
 	if err := json.NewDecoder(f).Decode(&p); err != nil {
 		return nil, err
 	}
-	p.Name = name
+	p.name = name
 	if len(p.TLSConfig.CAFile) == 0 {
 		p.TLSConfig.InsecureSkipVerify = true
 	}

plugins/discovery_test.go

@@ -58,7 +58,7 @@ func TestFileSpecPlugin(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		if p.Name != c.name {
+		if p.name != c.name {
 			t.Fatalf("Expected plugin `%s`, got %s\n", c.name, p.Name)
 		}
 
@@ -97,7 +97,7 @@ func TestFileJSONSpecPlugin(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if plugin.Name != "example" {
+	if plugin.name != "example" {
 		t.Fatalf("Expected plugin `plugin-example`, got %s\n", plugin.Name)
 	}
 

plugins/discovery_unix_test.go

@@ -45,7 +45,7 @@ func TestLocalSocket(t *testing.T) {
 			t.Fatalf("Expected %v, was %v\n", p, pp)
 		}
 
-		if p.Name != "echo" {
+		if p.name != "echo" {
 			t.Fatalf("Expected plugin `echo`, got %s\n", p.Name)
 		}
 

plugins/plugins.go

@@ -55,13 +55,13 @@ type Manifest struct {
 // Plugin is the definition of a docker plugin.
 type Plugin struct {
 	// Name of the plugin
-	Name string `json:"-"`
+	name string
 	// Address of the plugin
 	Addr string
 	// TLS configuration of the plugin
-	TLSConfig tlsconfig.Options
+	TLSConfig *tlsconfig.Options
 	// Client attached to the plugin
-	Client *Client `json:"-"`
+	client *Client
 	// Manifest of the plugin (see above)
 	Manifest *Manifest `json:"-"`
 
@@ -73,11 +73,23 @@ type Plugin struct {
 	activateWait *sync.Cond
 }
 
-func newLocalPlugin(name, addr string) *Plugin {
+// Name returns the name of the plugin.
+func (p *Plugin) Name() string {
+	return p.name
+}
+
+// Client returns a ready-to-use plugin client that can be used to communicate with the plugin.
+func (p *Plugin) Client() *Client {
+	return p.client
+}
+
+// NewLocalPlugin creates a new local plugin.
+func NewLocalPlugin(name, addr string) *Plugin {
 	return &Plugin{
-		Name:         name,
+		name: name,
-		Addr:         addr,
+		Addr: addr,
-		TLSConfig:    tlsconfig.Options{InsecureSkipVerify: true},
+		// TODO: change to nil
+		TLSConfig:    &tlsconfig.Options{InsecureSkipVerify: true},
 		activateWait: sync.NewCond(&sync.Mutex{}),
 	}
 }
@@ -102,10 +114,10 @@ func (p *Plugin) activateWithLock() error {
 	if err != nil {
 		return err
 	}
-	p.Client = c
+	p.client = c
 
 	m := new(Manifest)
-	if err = p.Client.Call("Plugin.Activate", nil, m); err != nil {
+	if err = p.client.Call("Plugin.Activate", nil, m); err != nil {
 		return err
 	}
 
@@ -116,7 +128,7 @@ func (p *Plugin) activateWithLock() error {
 		if !handled {
 			continue
 		}
-		handler(p.Name, p.Client)
+		handler(p.name, p.client)
 	}
 	return nil
 }