Merge pull request #217 from andrewhsu/v

[17.06] bump version to 17.06.2-ce

CHANGELOG.md

@@ -5,7 +5,22 @@ information on the list of deprecated flags and APIs please have a look at
 https://docs.docker.com/engine/deprecated/ where target removal dates can also
 be found.
 
-## 17.06.1-ce (2017-07-XX)
+## 17.06.2-ce (2017-09-05)
+
+### Client
+
+- Enable TCP keepalive in the client to prevent loss of connection [docker/cli#415](https://github.com/docker/cli/pull/415)
+
+### Runtime
+
+- Devmapper: ensure UdevWait is called after calls to setCookie [moby/moby#33732](https://github.com/moby/moby/pull/33732)
+- Aufs: ensure diff layers are correctly removed to prevent leftover files from using up storage [moby/moby#34587](https://github.com/moby/moby/pull/34587)
+
+### Swarm mode
+
+- Ignore PullOptions for running tasks [docker/swarmkit#2351](https://github.com/docker/swarmkit/pull/2351)
+
+## 17.06.1-ce (2017-08-17)
 
 ### Builder
 
@@ -31,6 +46,7 @@ be found.
 ### Networking
 
 * Fixed issue with driver options not received by network drivers [#127](https://github.com/docker/docker-ce/pull/127)
+* Fixed issue with overlay network IP address reuse [#197](https://github.com/docker/docker-ce/pull/197)
 
 ### Plugins
 

VERSION

@@ -1 +1 @@
-17.06.1-ce-rc3
+17.06.2-ce

components/cli/VERSION

@@ -1 +1 @@
-17.06.1-ce-rc3
+17.06.2-ce

components/cli/cli/command/cli.go

@@ -3,9 +3,11 @@ package command
 import (
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"os"
 	"runtime"
+	"time"
 
 	"github.com/docker/cli/cli"
 	cliconfig "github.com/docker/cli/cli/config"
@@ -285,6 +287,10 @@ func newHTTPClient(host string, tlsOptions *tlsconfig.Options) (*http.Client, er
 	}
 	tr := &http.Transport{
 		TLSClientConfig: config,
+		DialContext: (&net.Dialer{
+			KeepAlive: 30 * time.Second,
+			Timeout:   30 * time.Second,
+		}).DialContext,
 	}
 	proto, addr, _, err := client.ParseHost(host)
 	if err != nil {

components/cli/cli/command/node/ps_test.go

@@ -103,11 +103,11 @@ func TestNodePs(t *testing.T) {
 			},
 			taskListFunc: func(options types.TaskListOptions) ([]swarm.Task, error) {
 				return []swarm.Task{
-					*Task(TaskID("taskID1"), ServiceID("failure"),
+					*Task(TaskID("taskID1"), TaskServiceID("failure"),
 						WithStatus(Timestamp(time.Now().Add(-2*time.Hour)), StatusErr("a task error"))),
-					*Task(TaskID("taskID2"), ServiceID("failure"),
+					*Task(TaskID("taskID2"), TaskServiceID("failure"),
 						WithStatus(Timestamp(time.Now().Add(-3*time.Hour)), StatusErr("a task error"))),
-					*Task(TaskID("taskID3"), ServiceID("failure"),
+					*Task(TaskID("taskID3"), TaskServiceID("failure"),
 						WithStatus(Timestamp(time.Now().Add(-4*time.Hour)), StatusErr("a task error"))),
 				}, nil
 			},

components/cli/cli/command/stack/client_test.go

@@ -15,6 +15,8 @@ import (
 type fakeClient struct {
 	client.Client
 
+	version string
+
 	services []string
 	networks []string
 	secrets  []string
@@ -25,14 +27,17 @@ type fakeClient struct {
 	removedSecrets  []string
 	removedConfigs  []string
 
-	serviceListFunc   func(options types.ServiceListOptions) ([]swarm.Service, error)
-	networkListFunc   func(options types.NetworkListOptions) ([]types.NetworkResource, error)
-	secretListFunc    func(options types.SecretListOptions) ([]swarm.Secret, error)
-	configListFunc    func(options types.ConfigListOptions) ([]swarm.Config, error)
-	serviceRemoveFunc func(serviceID string) error
-	networkRemoveFunc func(networkID string) error
-	secretRemoveFunc  func(secretID string) error
-	configRemoveFunc  func(configID string) error
+	serviceListFunc    func(options types.ServiceListOptions) ([]swarm.Service, error)
+	networkListFunc    func(options types.NetworkListOptions) ([]types.NetworkResource, error)
+	secretListFunc     func(options types.SecretListOptions) ([]swarm.Secret, error)
+	configListFunc     func(options types.ConfigListOptions) ([]swarm.Config, error)
+	nodeListFunc       func(options types.NodeListOptions) ([]swarm.Node, error)
+	taskListFunc       func(options types.TaskListOptions) ([]swarm.Task, error)
+	nodeInspectWithRaw func(ref string) (swarm.Node, []byte, error)
+	serviceRemoveFunc  func(serviceID string) error
+	networkRemoveFunc  func(networkID string) error
+	secretRemoveFunc   func(secretID string) error
+	configRemoveFunc   func(configID string) error
 }
 
 func (cli *fakeClient) ServerVersion(ctx context.Context) (types.Version, error) {
@@ -42,6 +47,10 @@ func (cli *fakeClient) ServerVersion(ctx context.Context) (types.Version, error)
 	}, nil
 }
 
+func (cli *fakeClient) ClientVersion() string {
+	return cli.version
+}
+
 func (cli *fakeClient) ServiceList(ctx context.Context, options types.ServiceListOptions) ([]swarm.Service, error) {
 	if cli.serviceListFunc != nil {
 		return cli.serviceListFunc(options)
@@ -102,6 +111,27 @@ func (cli *fakeClient) ConfigList(ctx context.Context, options types.ConfigListO
 	return configsList, nil
 }
 
+func (cli *fakeClient) TaskList(ctx context.Context, options types.TaskListOptions) ([]swarm.Task, error) {
+	if cli.taskListFunc != nil {
+		return cli.taskListFunc(options)
+	}
+	return []swarm.Task{}, nil
+}
+
+func (cli *fakeClient) NodeList(ctx context.Context, options types.NodeListOptions) ([]swarm.Node, error) {
+	if cli.nodeListFunc != nil {
+		return cli.nodeListFunc(options)
+	}
+	return []swarm.Node{}, nil
+}
+
+func (cli *fakeClient) NodeInspectWithRaw(ctx context.Context, ref string) (swarm.Node, []byte, error) {
+	if cli.nodeInspectWithRaw != nil {
+		return cli.nodeInspectWithRaw(ref)
+	}
+	return swarm.Node{}, nil, nil
+}
+
 func (cli *fakeClient) ServiceRemove(ctx context.Context, serviceID string) error {
 	if cli.serviceRemoveFunc != nil {
 		return cli.serviceRemoveFunc(serviceID)

components/cli/cli/command/stack/list.go

@@ -18,7 +18,7 @@ type listOptions struct {
 	format string
 }
 
-func newListCommand(dockerCli *command.DockerCli) *cobra.Command {
+func newListCommand(dockerCli command.Cli) *cobra.Command {
 	opts := listOptions{}
 
 	cmd := &cobra.Command{
@@ -36,7 +36,7 @@ func newListCommand(dockerCli *command.DockerCli) *cobra.Command {
 	return cmd
 }
 
-func runList(dockerCli *command.DockerCli, opts listOptions) error {
+func runList(dockerCli command.Cli, opts listOptions) error {
 	client := dockerCli.Client()
 	ctx := context.Background()
 

components/cli/cli/command/stack/list_test.go

@@ -0,0 +1,122 @@
+package stack
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	"github.com/docker/cli/cli/internal/test"
+	// Import builders to get the builder function as package function
+	. "github.com/docker/cli/cli/internal/test/builders"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/pkg/testutil"
+	"github.com/docker/docker/pkg/testutil/golden"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestListErrors(t *testing.T) {
+	testCases := []struct {
+		args            []string
+		flags           map[string]string
+		serviceListFunc func(options types.ServiceListOptions) ([]swarm.Service, error)
+		expectedError   string
+	}{
+		{
+			args:          []string{"foo"},
+			expectedError: "accepts no argument",
+		},
+		{
+			flags: map[string]string{
+				"format": "{{invalid format}}",
+			},
+			expectedError: "Template parsing error",
+		},
+		{
+			serviceListFunc: func(options types.ServiceListOptions) ([]swarm.Service, error) {
+				return []swarm.Service{}, errors.Errorf("error getting services")
+			},
+			expectedError: "error getting services",
+		},
+		{
+			serviceListFunc: func(options types.ServiceListOptions) ([]swarm.Service, error) {
+				return []swarm.Service{*Service()}, nil
+			},
+			expectedError: "cannot get label",
+		},
+	}
+
+	for _, tc := range testCases {
+		cmd := newListCommand(test.NewFakeCli(&fakeClient{
+			serviceListFunc: tc.serviceListFunc,
+		}, &bytes.Buffer{}))
+		cmd.SetArgs(tc.args)
+		cmd.SetOutput(ioutil.Discard)
+		for key, value := range tc.flags {
+			cmd.Flags().Set(key, value)
+		}
+		testutil.ErrorContains(t, cmd.Execute(), tc.expectedError)
+	}
+}
+
+func TestListWithFormat(t *testing.T) {
+	buf := new(bytes.Buffer)
+	cmd := newListCommand(test.NewFakeCli(&fakeClient{
+		serviceListFunc: func(options types.ServiceListOptions) ([]swarm.Service, error) {
+			return []swarm.Service{
+				*Service(
+					ServiceLabels(map[string]string{
+						"com.docker.stack.namespace": "service-name-foo",
+					}),
+				)}, nil
+		},
+	}, buf))
+	cmd.Flags().Set("format", "{{ .Name }}")
+	assert.NoError(t, cmd.Execute())
+	actual := buf.String()
+	expected := golden.Get(t, []byte(actual), "stack-list-with-format.golden")
+	testutil.EqualNormalizedString(t, testutil.RemoveSpace, actual, string(expected))
+}
+
+func TestListWithoutFormat(t *testing.T) {
+	buf := new(bytes.Buffer)
+	cmd := newListCommand(test.NewFakeCli(&fakeClient{
+		serviceListFunc: func(options types.ServiceListOptions) ([]swarm.Service, error) {
+			return []swarm.Service{
+				*Service(
+					ServiceLabels(map[string]string{
+						"com.docker.stack.namespace": "service-name-foo",
+					}),
+				)}, nil
+		},
+	}, buf))
+	assert.NoError(t, cmd.Execute())
+	actual := buf.String()
+	expected := golden.Get(t, []byte(actual), "stack-list-without-format.golden")
+	testutil.EqualNormalizedString(t, testutil.RemoveSpace, actual, string(expected))
+}
+
+func TestListOrder(t *testing.T) {
+	buf := new(bytes.Buffer)
+	cmd := newListCommand(test.NewFakeCli(&fakeClient{
+		serviceListFunc: func(options types.ServiceListOptions) ([]swarm.Service, error) {
+			return []swarm.Service{
+				*Service(
+					ServiceLabels(map[string]string{
+						"com.docker.stack.namespace": "service-name-foo",
+					}),
+				),
+				*Service(
+					ServiceLabels(map[string]string{
+						"com.docker.stack.namespace": "service-name-bar",
+					}),
+				),
+			}, nil
+		},
+	}, buf))
+	assert.NoError(t, cmd.Execute())
+	actual := buf.String()
+	expected := golden.Get(t, []byte(actual), "stack-list-sort.golden")
+	testutil.EqualNormalizedString(t, testutil.RemoveSpace, actual, string(expected))
+}

components/cli/cli/command/stack/opts_test.go

@@ -0,0 +1,49 @@
+package stack
+
+import (
+	"bytes"
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadBundlefileErrors(t *testing.T) {
+	testCases := []struct {
+		namespace     string
+		path          string
+		expectedError error
+	}{
+		{
+			namespace:     "namespace_foo",
+			expectedError: fmt.Errorf("Bundle %s.dab not found", "namespace_foo"),
+		},
+		{
+			namespace:     "namespace_foo",
+			path:          "invalid_path",
+			expectedError: fmt.Errorf("Bundle %s not found", "invalid_path"),
+		},
+		{
+			namespace:     "namespace_foo",
+			path:          filepath.Join("testdata", "bundlefile_with_invalid_syntax"),
+			expectedError: fmt.Errorf("Error reading"),
+		},
+	}
+
+	for _, tc := range testCases {
+		_, err := loadBundlefile(&bytes.Buffer{}, tc.namespace, tc.path)
+		assert.Error(t, err, tc.expectedError)
+	}
+}
+
+func TestLoadBundlefile(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	namespace := ""
+	path := filepath.Join("testdata", "bundlefile_with_two_services.dab")
+	bundleFile, err := loadBundlefile(buf, namespace, path)
+
+	assert.NoError(t, err)
+	assert.Equal(t, len(bundleFile.Services), 2)
+}

components/cli/cli/command/stack/ps.go

@@ -58,7 +58,7 @@ func runPS(dockerCli command.Cli, options psOptions) error {
 	}
 
 	if len(tasks) == 0 {
-		fmt.Fprintf(dockerCli.Out(), "Nothing found in stack: %s\n", namespace)
+		fmt.Fprintf(dockerCli.Err(), "Nothing found in stack: %s\n", namespace)
 		return nil
 	}
 

components/cli/cli/command/stack/ps_test.go

@@ -0,0 +1,190 @@
+package stack
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+	"time"
+
+	"github.com/docker/cli/cli/config/configfile"
+	"github.com/docker/cli/cli/internal/test"
+	// Import builders to get the builder function as package function
+	. "github.com/docker/cli/cli/internal/test/builders"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/pkg/testutil"
+	"github.com/docker/docker/pkg/testutil/golden"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStackPsErrors(t *testing.T) {
+	testCases := []struct {
+		args          []string
+		taskListFunc  func(options types.TaskListOptions) ([]swarm.Task, error)
+		expectedError string
+	}{
+
+		{
+			args:          []string{},
+			expectedError: "requires exactly 1 argument",
+		},
+		{
+			args:          []string{"foo", "bar"},
+			expectedError: "requires exactly 1 argument",
+		},
+		{
+			args: []string{"foo"},
+			taskListFunc: func(options types.TaskListOptions) ([]swarm.Task, error) {
+				return nil, errors.Errorf("error getting tasks")
+			},
+			expectedError: "error getting tasks",
+		},
+	}
+
+	for _, tc := range testCases {
+		cmd := newPsCommand(test.NewFakeCli(&fakeClient{
+			taskListFunc: tc.taskListFunc,
+		}, &bytes.Buffer{}))
+		cmd.SetArgs(tc.args)
+		cmd.SetOutput(ioutil.Discard)
+		testutil.ErrorContains(t, cmd.Execute(), tc.expectedError)
+	}
+}
+
+func TestStackPsEmptyStack(t *testing.T) {
+	out := new(bytes.Buffer)
+	stderr := new(bytes.Buffer)
+	fakeCli := test.NewFakeCli(&fakeClient{
+		taskListFunc: func(options types.TaskListOptions) ([]swarm.Task, error) {
+			return []swarm.Task{}, nil
+		},
+	}, out)
+	fakeCli.SetErr(stderr)
+	cmd := newPsCommand(fakeCli)
+	cmd.SetArgs([]string{"foo"})
+
+	assert.NoError(t, cmd.Execute())
+	assert.Equal(t, "", out.String())
+	assert.Equal(t, "Nothing found in stack: foo\n", stderr.String())
+}
+
+func TestStackPsWithQuietOption(t *testing.T) {
+	buf := new(bytes.Buffer)
+	cli := test.NewFakeCli(&fakeClient{
+		taskListFunc: func(options types.TaskListOptions) ([]swarm.Task, error) {
+			return []swarm.Task{*Task(TaskID("id-foo"))}, nil
+		},
+	}, buf)
+	cli.SetConfigfile(&configfile.ConfigFile{})
+	cmd := newPsCommand(cli)
+	cmd.SetArgs([]string{"foo"})
+	cmd.Flags().Set("quiet", "true")
+	assert.NoError(t, cmd.Execute())
+	actual := buf.String()
+	expected := golden.Get(t, []byte(actual), "stack-ps-with-quiet-option.golden")
+	testutil.EqualNormalizedString(t, testutil.RemoveSpace, actual, string(expected))
+
+}
+
+func TestStackPsWithNoTruncOption(t *testing.T) {
+	buf := new(bytes.Buffer)
+	cli := test.NewFakeCli(&fakeClient{
+		taskListFunc: func(options types.TaskListOptions) ([]swarm.Task, error) {
+			return []swarm.Task{*Task(TaskID("xn4cypcov06f2w8gsbaf2lst3"))}, nil
+		},
+	}, buf)
+	cli.SetConfigfile(&configfile.ConfigFile{})
+	cmd := newPsCommand(cli)
+	cmd.SetArgs([]string{"foo"})
+	cmd.Flags().Set("no-trunc", "true")
+	cmd.Flags().Set("format", "{{ .ID }}")
+	assert.NoError(t, cmd.Execute())
+	actual := buf.String()
+	expected := golden.Get(t, []byte(actual), "stack-ps-with-no-trunc-option.golden")
+	testutil.EqualNormalizedString(t, testutil.RemoveSpace, actual, string(expected))
+}
+
+func TestStackPsWithNoResolveOption(t *testing.T) {
+	buf := new(bytes.Buffer)
+	cli := test.NewFakeCli(&fakeClient{
+		taskListFunc: func(options types.TaskListOptions) ([]swarm.Task, error) {
+			return []swarm.Task{*Task(
+				TaskNodeID("id-node-foo"),
+			)}, nil
+		},
+		nodeInspectWithRaw: func(ref string) (swarm.Node, []byte, error) {
+			return *Node(NodeName("node-name-bar")), nil, nil
+		},
+	}, buf)
+	cli.SetConfigfile(&configfile.ConfigFile{})
+	cmd := newPsCommand(cli)
+	cmd.SetArgs([]string{"foo"})
+	cmd.Flags().Set("no-resolve", "true")
+	cmd.Flags().Set("format", "{{ .Node }}")
+	assert.NoError(t, cmd.Execute())
+	actual := buf.String()
+	expected := golden.Get(t, []byte(actual), "stack-ps-with-no-resolve-option.golden")
+	testutil.EqualNormalizedString(t, testutil.RemoveSpace, actual, string(expected))
+}
+
+func TestStackPsWithFormat(t *testing.T) {
+	buf := new(bytes.Buffer)
+	cli := test.NewFakeCli(&fakeClient{
+		taskListFunc: func(options types.TaskListOptions) ([]swarm.Task, error) {
+			return []swarm.Task{*Task(TaskServiceID("service-id-foo"))}, nil
+		},
+	}, buf)
+	cli.SetConfigfile(&configfile.ConfigFile{})
+	cmd := newPsCommand(cli)
+	cmd.SetArgs([]string{"foo"})
+	cmd.Flags().Set("format", "{{ .Name }}")
+	assert.NoError(t, cmd.Execute())
+	actual := buf.String()
+	expected := golden.Get(t, []byte(actual), "stack-ps-with-format.golden")
+	testutil.EqualNormalizedString(t, testutil.RemoveSpace, actual, string(expected))
+}
+
+func TestStackPsWithConfigFormat(t *testing.T) {
+	buf := new(bytes.Buffer)
+	cli := test.NewFakeCli(&fakeClient{
+		taskListFunc: func(options types.TaskListOptions) ([]swarm.Task, error) {
+			return []swarm.Task{*Task(TaskServiceID("service-id-foo"))}, nil
+		},
+	}, buf)
+	cli.SetConfigfile(&configfile.ConfigFile{
+		TasksFormat: "{{ .Name }}",
+	})
+	cmd := newPsCommand(cli)
+	cmd.SetArgs([]string{"foo"})
+	assert.NoError(t, cmd.Execute())
+	actual := buf.String()
+	expected := golden.Get(t, []byte(actual), "stack-ps-with-config-format.golden")
+	testutil.EqualNormalizedString(t, testutil.RemoveSpace, actual, string(expected))
+}
+
+func TestStackPsWithoutFormat(t *testing.T) {
+	buf := new(bytes.Buffer)
+	cli := test.NewFakeCli(&fakeClient{
+		taskListFunc: func(options types.TaskListOptions) ([]swarm.Task, error) {
+			return []swarm.Task{*Task(
+				TaskID("id-foo"),
+				TaskServiceID("service-id-foo"),
+				TaskNodeID("id-node"),
+				WithTaskSpec(TaskImage("myimage:mytag")),
+				TaskDesiredState(swarm.TaskStateReady),
+				WithStatus(TaskState(swarm.TaskStateFailed), Timestamp(time.Now().Add(-2*time.Hour))),
+			)}, nil
+		},
+		nodeInspectWithRaw: func(ref string) (swarm.Node, []byte, error) {
+			return *Node(NodeName("node-name-bar")), nil, nil
+		},
+	}, buf)
+	cli.SetConfigfile(&configfile.ConfigFile{})
+	cmd := newPsCommand(cli)
+	cmd.SetArgs([]string{"foo"})
+	assert.NoError(t, cmd.Execute())
+	actual := buf.String()
+	expected := golden.Get(t, []byte(actual), "stack-ps-without-format.golden")
+	testutil.EqualNormalizedString(t, testutil.RemoveSpace, actual, string(expected))
+}

components/cli/cli/command/stack/remove.go

@@ -51,20 +51,16 @@ func runRemove(dockerCli command.Cli, opts removeOptions) error {
 			return err
 		}
 
-		secrets, err := getStackSecrets(ctx, client, namespace)
-		if err != nil {
-			return err
+		var secrets []swarm.Secret
+		if versions.GreaterThanOrEqualTo(client.ClientVersion(), "1.25") {
+			secrets, err = getStackSecrets(ctx, client, namespace)
+			if err != nil {
+				return err
+			}
 		}
 
 		var configs []swarm.Config
-
-		version, err := client.ServerVersion(ctx)
-		if err != nil {
-			return err
-		}
-		if versions.LessThan(version.APIVersion, "1.30") {
-			fmt.Fprintf(dockerCli.Err(), `WARNING: ignoring "configs" (requires API version 1.30, but the Docker daemon API version is %s)`, version.APIVersion)
-		} else {
+		if versions.GreaterThanOrEqualTo(client.ClientVersion(), "1.30") {
 			configs, err = getStackConfigs(ctx, client, namespace)
 			if err != nil {
 				return err
@@ -72,7 +68,7 @@ func runRemove(dockerCli command.Cli, opts removeOptions) error {
 		}
 
 		if len(services)+len(networks)+len(secrets)+len(configs) == 0 {
-			fmt.Fprintf(dockerCli.Out(), "Nothing found in stack: %s\n", namespace)
+			fmt.Fprintf(dockerCli.Err(), "Nothing found in stack: %s\n", namespace)
 			continue
 		}
 

components/cli/cli/command/stack/remove_test.go

@@ -39,7 +39,9 @@ func TestRemoveStack(t *testing.T) {
 	}
 	allConfigIDs := buildObjectIDs(allConfigs)
 
+	// Using API 1.24; removes services, networks, but doesn't remove configs and secrets
 	cli := &fakeClient{
+		version:  "1.24",
 		services: allServices,
 		networks: allNetworks,
 		secrets:  allSecrets,
@@ -48,6 +50,40 @@ func TestRemoveStack(t *testing.T) {
 	cmd := newRemoveCommand(test.NewFakeCli(cli, &bytes.Buffer{}))
 	cmd.SetArgs([]string{"foo", "bar"})
 
+	assert.NoError(t, cmd.Execute())
+	assert.Equal(t, allServiceIDs, cli.removedServices)
+	assert.Equal(t, allNetworkIDs, cli.removedNetworks)
+	assert.Nil(t, cli.removedSecrets)
+	assert.Nil(t, cli.removedConfigs)
+
+	// Using API 1.25; removes services, networks, but doesn't remove configs
+	cli = &fakeClient{
+		version:  "1.25",
+		services: allServices,
+		networks: allNetworks,
+		secrets:  allSecrets,
+		configs:  allConfigs,
+	}
+	cmd = newRemoveCommand(test.NewFakeCli(cli, &bytes.Buffer{}))
+	cmd.SetArgs([]string{"foo", "bar"})
+
+	assert.NoError(t, cmd.Execute())
+	assert.Equal(t, allServiceIDs, cli.removedServices)
+	assert.Equal(t, allNetworkIDs, cli.removedNetworks)
+	assert.Equal(t, allSecretIDs, cli.removedSecrets)
+	assert.Nil(t, cli.removedConfigs)
+
+	// Using API 1.30; removes services, networks, configs, and secrets
+	cli = &fakeClient{
+		version:  "1.30",
+		services: allServices,
+		networks: allNetworks,
+		secrets:  allSecrets,
+		configs:  allConfigs,
+	}
+	cmd = newRemoveCommand(test.NewFakeCli(cli, &bytes.Buffer{}))
+	cmd.SetArgs([]string{"foo", "bar"})
+
 	assert.NoError(t, cmd.Execute())
 	assert.Equal(t, allServiceIDs, cli.removedServices)
 	assert.Equal(t, allNetworkIDs, cli.removedNetworks)
@@ -55,8 +91,9 @@ func TestRemoveStack(t *testing.T) {
 	assert.Equal(t, allConfigIDs, cli.removedConfigs)
 }
 
-func TestSkipEmptyStack(t *testing.T) {
-	buf := new(bytes.Buffer)
+func TestRemoveStackSkipEmpty(t *testing.T) {
+	out := new(bytes.Buffer)
+	stderr := new(bytes.Buffer)
 	allServices := []string{objectName("bar", "service1"), objectName("bar", "service2")}
 	allServiceIDs := buildObjectIDs(allServices)
 
@@ -69,24 +106,28 @@ func TestSkipEmptyStack(t *testing.T) {
 	allConfigs := []string{objectName("bar", "config1")}
 	allConfigIDs := buildObjectIDs(allConfigs)
 
-	cli := &fakeClient{
+	fakeClient := &fakeClient{
+		version:  "1.30",
 		services: allServices,
 		networks: allNetworks,
 		secrets:  allSecrets,
 		configs:  allConfigs,
 	}
-	cmd := newRemoveCommand(test.NewFakeCli(cli, buf))
+	fakeCli := test.NewFakeCli(fakeClient, out)
+	fakeCli.SetErr(stderr)
+	cmd := newRemoveCommand(fakeCli)
 	cmd.SetArgs([]string{"foo", "bar"})
 
 	assert.NoError(t, cmd.Execute())
-	assert.Contains(t, buf.String(), "Nothing found in stack: foo")
-	assert.Equal(t, allServiceIDs, cli.removedServices)
-	assert.Equal(t, allNetworkIDs, cli.removedNetworks)
-	assert.Equal(t, allSecretIDs, cli.removedSecrets)
-	assert.Equal(t, allConfigIDs, cli.removedConfigs)
+	assert.Equal(t, "", out.String())
+	assert.Contains(t, stderr.String(), "Nothing found in stack: foo\n")
+	assert.Equal(t, allServiceIDs, fakeClient.removedServices)
+	assert.Equal(t, allNetworkIDs, fakeClient.removedNetworks)
+	assert.Equal(t, allSecretIDs, fakeClient.removedSecrets)
+	assert.Equal(t, allConfigIDs, fakeClient.removedConfigs)
 }
 
-func TestContinueAfterError(t *testing.T) {
+func TestRemoveContinueAfterError(t *testing.T) {
 	allServices := []string{objectName("foo", "service1"), objectName("bar", "service1")}
 	allServiceIDs := buildObjectIDs(allServices)
 
@@ -101,6 +142,7 @@ func TestContinueAfterError(t *testing.T) {
 
 	removedServices := []string{}
 	cli := &fakeClient{
+		version:  "1.30",
 		services: allServices,
 		networks: allNetworks,
 		secrets:  allSecrets,

components/cli/cli/command/stack/services.go

@@ -21,7 +21,7 @@ type servicesOptions struct {
 	namespace string
 }
 
-func newServicesCommand(dockerCli *command.DockerCli) *cobra.Command {
+func newServicesCommand(dockerCli command.Cli) *cobra.Command {
 	options := servicesOptions{filter: opts.NewFilterOpt()}
 
 	cmd := &cobra.Command{
@@ -41,7 +41,7 @@ func newServicesCommand(dockerCli *command.DockerCli) *cobra.Command {
 	return cmd
 }
 
-func runServices(dockerCli *command.DockerCli, options servicesOptions) error {
+func runServices(dockerCli command.Cli, options servicesOptions) error {
 	ctx := context.Background()
 	client := dockerCli.Client()
 
@@ -51,11 +51,9 @@ func runServices(dockerCli *command.DockerCli, options servicesOptions) error {
 		return err
 	}
 
-	out := dockerCli.Out()
-
 	// if no services in this stack, print message and exit 0
 	if len(services) == 0 {
-		fmt.Fprintf(out, "Nothing found in stack: %s\n", options.namespace)
+		fmt.Fprintf(dockerCli.Err(), "Nothing found in stack: %s\n", options.namespace)
 		return nil
 	}
 

components/cli/cli/command/stack/services_test.go

@@ -0,0 +1,182 @@
+package stack
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	"github.com/docker/cli/cli/config/configfile"
+	"github.com/docker/cli/cli/internal/test"
+	// Import builders to get the builder function as package function
+	. "github.com/docker/cli/cli/internal/test/builders"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/pkg/testutil"
+	"github.com/docker/docker/pkg/testutil/golden"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStackServicesErrors(t *testing.T) {
+	testCases := []struct {
+		args            []string
+		flags           map[string]string
+		serviceListFunc func(options types.ServiceListOptions) ([]swarm.Service, error)
+		nodeListFunc    func(options types.NodeListOptions) ([]swarm.Node, error)
+		taskListFunc    func(options types.TaskListOptions) ([]swarm.Task, error)
+		expectedError   string
+	}{
+		{
+			args: []string{"foo"},
+			serviceListFunc: func(options types.ServiceListOptions) ([]swarm.Service, error) {
+				return nil, errors.Errorf("error getting services")
+			},
+			expectedError: "error getting services",
+		},
+		{
+			args: []string{"foo"},
+			serviceListFunc: func(options types.ServiceListOptions) ([]swarm.Service, error) {
+				return []swarm.Service{*Service()}, nil
+			},
+			nodeListFunc: func(options types.NodeListOptions) ([]swarm.Node, error) {
+				return nil, errors.Errorf("error getting nodes")
+			},
+			expectedError: "error getting nodes",
+		},
+		{
+			args: []string{"foo"},
+			serviceListFunc: func(options types.ServiceListOptions) ([]swarm.Service, error) {
+				return []swarm.Service{*Service()}, nil
+			},
+			taskListFunc: func(options types.TaskListOptions) ([]swarm.Task, error) {
+				return nil, errors.Errorf("error getting tasks")
+			},
+			expectedError: "error getting tasks",
+		},
+		{
+			args: []string{"foo"},
+			flags: map[string]string{
+				"format": "{{invalid format}}",
+			},
+			serviceListFunc: func(options types.ServiceListOptions) ([]swarm.Service, error) {
+				return []swarm.Service{*Service()}, nil
+			},
+			expectedError: "Template parsing error",
+		},
+	}
+
+	for _, tc := range testCases {
+		cli := test.NewFakeCli(&fakeClient{
+			serviceListFunc: tc.serviceListFunc,
+			nodeListFunc:    tc.nodeListFunc,
+			taskListFunc:    tc.taskListFunc,
+		}, &bytes.Buffer{})
+		cli.SetConfigfile(&configfile.ConfigFile{})
+		cmd := newServicesCommand(cli)
+		cmd.SetArgs(tc.args)
+		for key, value := range tc.flags {
+			cmd.Flags().Set(key, value)
+		}
+		cmd.SetOutput(ioutil.Discard)
+		testutil.ErrorContains(t, cmd.Execute(), tc.expectedError)
+	}
+}
+
+func TestStackServicesEmptyServiceList(t *testing.T) {
+	out := new(bytes.Buffer)
+	stderr := new(bytes.Buffer)
+	fakeCli := test.NewFakeCli(&fakeClient{
+		serviceListFunc: func(options types.ServiceListOptions) ([]swarm.Service, error) {
+			return []swarm.Service{}, nil
+		},
+	}, out)
+	fakeCli.SetErr(stderr)
+	cmd := newServicesCommand(fakeCli)
+	cmd.SetArgs([]string{"foo"})
+	assert.NoError(t, cmd.Execute())
+	assert.Equal(t, "", out.String())
+	assert.Equal(t, "Nothing found in stack: foo\n", stderr.String())
+}
+
+func TestStackServicesWithQuietOption(t *testing.T) {
+	buf := new(bytes.Buffer)
+	cli := test.NewFakeCli(&fakeClient{
+		serviceListFunc: func(options types.ServiceListOptions) ([]swarm.Service, error) {
+			return []swarm.Service{*Service(ServiceID("id-foo"))}, nil
+		},
+	}, buf)
+	cli.SetConfigfile(&configfile.ConfigFile{})
+	cmd := newServicesCommand(cli)
+	cmd.Flags().Set("quiet", "true")
+	cmd.SetArgs([]string{"foo"})
+	assert.NoError(t, cmd.Execute())
+	actual := buf.String()
+	expected := golden.Get(t, []byte(actual), "stack-services-with-quiet-option.golden")
+	testutil.EqualNormalizedString(t, testutil.RemoveSpace, actual, string(expected))
+}
+
+func TestStackServicesWithFormat(t *testing.T) {
+	buf := new(bytes.Buffer)
+	cli := test.NewFakeCli(&fakeClient{
+		serviceListFunc: func(options types.ServiceListOptions) ([]swarm.Service, error) {
+			return []swarm.Service{
+				*Service(ServiceName("service-name-foo")),
+			}, nil
+		},
+	}, buf)
+	cli.SetConfigfile(&configfile.ConfigFile{})
+	cmd := newServicesCommand(cli)
+	cmd.SetArgs([]string{"foo"})
+	cmd.Flags().Set("format", "{{ .Name }}")
+	assert.NoError(t, cmd.Execute())
+	actual := buf.String()
+	expected := golden.Get(t, []byte(actual), "stack-services-with-format.golden")
+	testutil.EqualNormalizedString(t, testutil.RemoveSpace, actual, string(expected))
+}
+
+func TestStackServicesWithConfigFormat(t *testing.T) {
+	buf := new(bytes.Buffer)
+	cli := test.NewFakeCli(&fakeClient{
+		serviceListFunc: func(options types.ServiceListOptions) ([]swarm.Service, error) {
+			return []swarm.Service{
+				*Service(ServiceName("service-name-foo")),
+			}, nil
+		},
+	}, buf)
+	cli.SetConfigfile(&configfile.ConfigFile{
+		ServicesFormat: "{{ .Name }}",
+	})
+	cmd := newServicesCommand(cli)
+	cmd.SetArgs([]string{"foo"})
+	assert.NoError(t, cmd.Execute())
+	actual := buf.String()
+	expected := golden.Get(t, []byte(actual), "stack-services-with-config-format.golden")
+	testutil.EqualNormalizedString(t, testutil.RemoveSpace, actual, string(expected))
+}
+
+func TestStackServicesWithoutFormat(t *testing.T) {
+	buf := new(bytes.Buffer)
+	cli := test.NewFakeCli(&fakeClient{
+		serviceListFunc: func(options types.ServiceListOptions) ([]swarm.Service, error) {
+			return []swarm.Service{*Service(
+				ServiceName("name-foo"),
+				ServiceID("id-foo"),
+				ReplicatedService(2),
+				ServiceImage("busybox:latest"),
+				ServicePort(swarm.PortConfig{
+					PublishMode:   swarm.PortConfigPublishModeIngress,
+					PublishedPort: 0,
+					TargetPort:    3232,
+					Protocol:      swarm.PortConfigProtocolTCP,
+				}),
+			)}, nil
+		},
+	}, buf)
+	cli.SetConfigfile(&configfile.ConfigFile{})
+	cmd := newServicesCommand(cli)
+	cmd.SetArgs([]string{"foo"})
+	assert.NoError(t, cmd.Execute())
+	actual := buf.String()
+	expected := golden.Get(t, []byte(actual), "stack-services-without-format.golden")
+	testutil.EqualNormalizedString(t, testutil.RemoveSpace, actual, string(expected))
+}

components/cli/cli/command/stack/testdata/bundlefile_with_two_services.dab

@@ -0,0 +1,29 @@
+{
+  "Services": {
+    "visualizer": {
+      "Image": "busybox@sha256:32f093055929dbc23dec4d03e09dfe971f5973a9ca5cf059cbfb644c206aa83f",
+      "Networks": [
+        "webnet"
+      ],
+      "Ports": [
+        {
+          "Port": 8080,
+          "Protocol": "tcp"
+        }
+      ]
+    },
+    "web": {
+      "Image": "busybox@sha256:32f093055929dbc23dec4d03e09dfe971f5973a9ca5cf059cbfb644c206aa83f",
+      "Networks": [
+        "webnet"
+      ],
+      "Ports": [
+        {
+          "Port": 80,
+          "Protocol": "tcp"
+        }
+      ]
+    }
+  },
+  "Version": "0.1"
+}

components/cli/cli/command/stack/testdata/stack-list-sort.golden

@@ -0,0 +1,3 @@
+NAME                SERVICES
+service-name-bar    1
+service-name-foo    1

components/cli/cli/command/stack/testdata/stack-list-with-format.golden

@@ -0,0 +1 @@
+service-name-foo

components/cli/cli/command/stack/testdata/stack-list-without-format.golden

@@ -0,0 +1,2 @@
+NAME                SERVICES
+service-name-foo    1

components/cli/cli/command/stack/testdata/stack-ps-with-config-format.golden

@@ -0,0 +1 @@
+service-id-foo.1

components/cli/cli/command/stack/testdata/stack-ps-with-format.golden

@@ -0,0 +1 @@
+service-id-foo.1

components/cli/cli/command/stack/testdata/stack-ps-with-no-resolve-option.golden

@@ -0,0 +1 @@
+id-node-foo

components/cli/cli/command/stack/testdata/stack-ps-with-no-trunc-option.golden

@@ -0,0 +1 @@
+xn4cypcov06f2w8gsbaf2lst3

components/cli/cli/command/stack/testdata/stack-ps-with-quiet-option.golden

@@ -0,0 +1 @@
+id-foo

components/cli/cli/command/stack/testdata/stack-ps-without-format.golden

@@ -0,0 +1,2 @@
+ID                  NAME                IMAGE                     NODE                DESIRED STATE       CURRENT STATE                ERROR                              PORTS
+id-foo service-id-foo.1            myimage:mytag                  node-name-bar     Ready               Failed 2 hours ago

components/cli/cli/command/stack/testdata/stack-services-with-config-format.golden

@@ -0,0 +1 @@
+service-name-foo

components/cli/cli/command/stack/testdata/stack-services-with-format.golden

@@ -0,0 +1 @@
+service-name-foo

components/cli/cli/command/stack/testdata/stack-services-with-quiet-option.golden

@@ -0,0 +1 @@
+id-foo

components/cli/cli/command/stack/testdata/stack-services-without-format.golden

@@ -0,0 +1,2 @@
+ID                  NAME                MODE                REPLICAS            IMAGE               PORTS
+id-foo              name-foo            replicated          0/2                 busybox:latest      *:0->3232/tcp

components/cli/cli/internal/test/builders/service.go

@@ -14,6 +14,7 @@ func Service(builders ...func(*swarm.Service)) *swarm.Service {
 			Annotations: swarm.Annotations{
 				Name: "defaultServiceName",
 			},
+			EndpointSpec: &swarm.EndpointSpec{},
 		},
 	}
 
@@ -24,9 +25,44 @@ func Service(builders ...func(*swarm.Service)) *swarm.Service {
 	return service
 }
 
+// ServiceID sets the service ID
+func ServiceID(ID string) func(*swarm.Service) {
+	return func(service *swarm.Service) {
+		service.ID = ID
+	}
+}
+
 // ServiceName sets the service name
 func ServiceName(name string) func(*swarm.Service) {
 	return func(service *swarm.Service) {
 		service.Spec.Annotations.Name = name
 	}
 }
+
+// ServiceLabels sets the service's labels
+func ServiceLabels(labels map[string]string) func(*swarm.Service) {
+	return func(service *swarm.Service) {
+		service.Spec.Annotations.Labels = labels
+	}
+}
+
+// ReplicatedService sets the number of replicas for the service
+func ReplicatedService(replicas uint64) func(*swarm.Service) {
+	return func(service *swarm.Service) {
+		service.Spec.Mode = swarm.ServiceMode{Replicated: &swarm.ReplicatedService{Replicas: &replicas}}
+	}
+}
+
+// ServiceImage sets the service's image
+func ServiceImage(image string) func(*swarm.Service) {
+	return func(service *swarm.Service) {
+		service.Spec.TaskTemplate = swarm.TaskSpec{ContainerSpec: swarm.ContainerSpec{Image: image}}
+	}
+}
+
+// ServicePort sets the service's port
+func ServicePort(port swarm.PortConfig) func(*swarm.Service) {
+	return func(service *swarm.Service) {
+		service.Spec.EndpointSpec.Ports = append(service.Spec.EndpointSpec.Ports, port)
+	}
+}

components/cli/cli/internal/test/builders/task.go

@@ -42,13 +42,34 @@ func TaskID(id string) func(*swarm.Task) {
 	}
 }
 
-// ServiceID sets the task service's ID
-func ServiceID(id string) func(*swarm.Task) {
+// TaskName sets the task name
+func TaskName(name string) func(*swarm.Task) {
+	return func(task *swarm.Task) {
+		task.Annotations.Name = name
+	}
+}
+
+// TaskServiceID sets the task service's ID
+func TaskServiceID(id string) func(*swarm.Task) {
 	return func(task *swarm.Task) {
 		task.ServiceID = id
 	}
 }
 
+// TaskNodeID sets the task's node id
+func TaskNodeID(id string) func(*swarm.Task) {
+	return func(task *swarm.Task) {
+		task.NodeID = id
+	}
+}
+
+// TaskDesiredState sets the task's desired state
+func TaskDesiredState(state swarm.TaskState) func(*swarm.Task) {
+	return func(task *swarm.Task) {
+		task.DesiredState = state
+	}
+}
+
 // WithStatus sets the task status
 func WithStatus(statusBuilders ...func(*swarm.TaskStatus)) func(*swarm.Task) {
 	return func(task *swarm.Task) {
@@ -86,6 +107,13 @@ func StatusErr(err string) func(*swarm.TaskStatus) {
 	}
 }
 
+// TaskState sets the task's current state
+func TaskState(state swarm.TaskState) func(*swarm.TaskStatus) {
+	return func(taskStatus *swarm.TaskStatus) {
+		taskStatus.State = state
+	}
+}
+
 // PortStatus sets the tasks port config status
 // FIXME(vdemeester) should be a sub builder 👼
 func PortStatus(portConfigs []swarm.PortConfig) func(*swarm.TaskStatus) {
@@ -94,6 +122,13 @@ func PortStatus(portConfigs []swarm.PortConfig) func(*swarm.TaskStatus) {
 	}
 }
 
+// WithTaskSpec sets the task spec
+func WithTaskSpec(specBuilders ...func(*swarm.TaskSpec)) func(*swarm.Task) {
+	return func(task *swarm.Task) {
+		task.Spec = *TaskSpec(specBuilders...)
+	}
+}
+
 // TaskSpec creates a task spec with default values .
 // Any number of taskSpec function builder can be pass to augment it.
 func TaskSpec(specBuilders ...func(*swarm.TaskSpec)) *swarm.TaskSpec {
@@ -109,3 +144,10 @@ func TaskSpec(specBuilders ...func(*swarm.TaskSpec)) *swarm.TaskSpec {
 
 	return taskSpec
 }
+
+// TaskImage sets the task's image
+func TaskImage(image string) func(*swarm.TaskSpec) {
+	return func(taskSpec *swarm.TaskSpec) {
+		taskSpec.ContainerSpec.Image = image
+	}
+}

components/cli/docs/extend/legacy_plugins.md

@@ -47,49 +47,50 @@ The sections below provide an inexhaustive overview of available plugins.
 
 ### Network plugins
 
-Plugin                                                                              | Description
------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-[Contiv Networking](https://github.com/contiv/netplugin)                            | An open source network plugin to provide infrastructure and security policies for a multi-tenant micro services deployment, while providing an integration to physical network for non-container workload. Contiv Networking implements the remote driver and IPAM APIs available in Docker 1.9 onwards.
-[Kuryr Network Plugin](https://github.com/openstack/kuryr)                          | A network plugin is developed as part of the OpenStack Kuryr project and implements the Docker networking (libnetwork) remote driver API by utilizing Neutron, the OpenStack networking service. It includes an IPAM driver as well.
-[Weave Network Plugin](https://www.weave.works/docs/net/latest/introducing-weave/)    | A network plugin that creates a virtual network that connects your Docker containers - across multiple hosts or clouds and enables automatic discovery of applications. Weave networks are resilient, partition tolerant, secure and work in partially connected networks, and other adverse environments - all configured with delightful simplicity.
+| Plugin                                                                             | Description                                                                                                                                                                                                                                                                                                                                            |
+|:-----------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| [Contiv Networking](https://github.com/contiv/netplugin)                           | An open source network plugin to provide infrastructure and security policies for a multi-tenant micro services deployment, while providing an integration to physical network for non-container workload. Contiv Networking implements the remote driver and IPAM APIs available in Docker 1.9 onwards.                                               |
+| [Kuryr Network Plugin](https://github.com/openstack/kuryr)                         | A network plugin is developed as part of the OpenStack Kuryr project and implements the Docker networking (libnetwork) remote driver API by utilizing Neutron, the OpenStack networking service. It includes an IPAM driver as well.                                                                                                                   |
+| [Weave Network Plugin](https://www.weave.works/docs/net/latest/introducing-weave/) | A network plugin that creates a virtual network that connects your Docker containers - across multiple hosts or clouds and enables automatic discovery of applications. Weave networks are resilient, partition tolerant, secure and work in partially connected networks, and other adverse environments - all configured with delightful simplicity. |
 
 ### Volume plugins
 
-Plugin                                                                              | Description
------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-[Azure File Storage plugin](https://github.com/Azure/azurefile-dockervolumedriver)  | Lets you mount Microsoft [Azure File Storage](https://azure.microsoft.com/blog/azure-file-storage-now-generally-available/) shares to Docker containers as volumes using the SMB 3.0 protocol. [Learn more](https://azure.microsoft.com/blog/persistent-docker-volumes-with-azure-file-storage/).
-[BeeGFS Volume Plugin](https://github.com/RedCoolBeans/docker-volume-beegfs)        | An open source volume plugin to create persistent volumes in a BeeGFS parallel file system.
-[Blockbridge plugin](https://github.com/blockbridge/blockbridge-docker-volume)      | A volume plugin that provides access to an extensible set of container-based persistent storage options. It supports single and multi-host Docker environments with features that include tenant isolation, automated provisioning, encryption, secure deletion, snapshots and QoS.
-[Contiv Volume Plugin](https://github.com/contiv/volplugin)                         | An open source volume plugin that provides multi-tenant, persistent, distributed storage with intent based consumption. It has support for Ceph and NFS.
-[Convoy plugin](https://github.com/rancher/convoy)                                  | A volume plugin for a variety of storage back-ends including device mapper and NFS. It's a simple standalone executable written in Go and provides the framework to support vendor-specific extensions such as snapshots, backups and restore.
-[DigitalOcean Block Storage plugin](https://github.com/omallo/docker-volume-plugin-dostorage) | Integrates DigitalOcean's [block storage solution](https://www.digitalocean.com/products/storage/) into the Docker ecosystem by automatically attaching a given block storage volume to a DigitalOcean droplet and making the contents of the volume available to Docker containers running on that droplet.
-[DRBD plugin](https://www.drbd.org/en/supported-projects/docker)                    | A volume plugin that provides highly available storage replicated by [DRBD](https://www.drbd.org). Data written to the docker volume is replicated in a cluster of DRBD nodes.
-[Flocker plugin](https://clusterhq.com/docker-plugin/)                              | A volume plugin that provides multi-host portable volumes for Docker, enabling you to run databases and other stateful containers and move them around across a cluster of machines.
-[Fuxi Volume Plugin](https://github.com/openstack/fuxi)                             | A volume plugin that is developed as part of the OpenStack Kuryr project and implements the Docker volume plugin API by utilizing Cinder, the OpenStack block storage service.
-[gce-docker plugin](https://github.com/mcuadros/gce-docker)                         | A volume plugin able to attach, format and mount Google Compute [persistent-disks](https://cloud.google.com/compute/docs/disks/persistent-disks).
-[GlusterFS plugin](https://github.com/calavera/docker-volume-glusterfs)             | A volume plugin that provides multi-host volumes management for Docker using GlusterFS.
-[Horcrux Volume Plugin](https://github.com/muthu-r/horcrux)                         | A volume plugin that allows on-demand, version controlled access to your data. Horcrux is an open-source plugin, written in Go, and supports SCP, [Minio](https://www.minio.io) and Amazon S3.
-[HPE 3Par Volume Plugin](https://github.com/hpe-storage/python-hpedockerplugin/)    | A volume plugin that supports HPE 3Par and StoreVirtual iSCSI storage arrays.
-[IPFS Volume Plugin](http://github.com/vdemeester/docker-volume-ipfs)               | An open source volume plugin that allows using an [ipfs](https://ipfs.io/) filesystem as a volume.
-[Keywhiz plugin](https://github.com/calavera/docker-volume-keywhiz)                 | A plugin that provides credentials and secret management using Keywhiz as a central repository.
-[Local Persist Plugin](https://github.com/CWSpear/local-persist)                    | A volume plugin that extends the default `local` driver's functionality by allowing you specify a mountpoint anywhere on the host, which enables the files to *always persist*, even if the volume is removed via `docker volume rm`.
-[NetApp Plugin](https://github.com/NetApp/netappdvp) (nDVP)                         | A volume plugin that provides direct integration with the Docker ecosystem for the NetApp storage portfolio. The nDVP package supports the provisioning and management of storage resources from the storage platform to Docker hosts, with a robust framework for adding additional platforms in the future.
-[Netshare plugin](https://github.com/ContainX/docker-volume-netshare)                 | A volume plugin that provides volume management for NFS 3/4, AWS EFS and CIFS file systems.
-[Nimble Storage Volume Plugin](https://connect.nimblestorage.com/community/app-integration/docker)| A volume plug-in that integrates with Nimble Storage Unified Flash Fabric arrays. The plug-in abstracts array volume capabilities to the Docker administrator to allow self-provisioning of secure multi-tenant volumes and clones.
-[OpenStorage Plugin](https://github.com/libopenstorage/openstorage)                 | A cluster-aware volume plugin that provides volume management for file and block storage solutions.  It implements a vendor neutral specification for implementing extensions such as CoS, encryption, and snapshots. It has example drivers based on FUSE, NFS, NBD and EBS to name a few.
-[Portworx Volume Plugin](https://github.com/portworx/px-dev)                        | A volume plugin that turns any server into a scale-out converged compute/storage node, providing container granular storage and highly available volumes across any node, using a shared-nothing storage backend that works with any docker scheduler.
-[Quobyte Volume Plugin](https://github.com/quobyte/docker-volume)                   | A volume plugin that connects Docker to [Quobyte](http://www.quobyte.com/containers)'s data center file system, a general-purpose scalable and fault-tolerant storage platform.
-[REX-Ray plugin](https://github.com/emccode/rexray)                                 | A volume plugin which is written in Go and provides advanced storage functionality for many platforms including VirtualBox, EC2, Google Compute Engine, OpenStack, and EMC.
-[Virtuozzo Storage and Ploop plugin](https://github.com/virtuozzo/docker-volume-ploop) | A volume plugin with support for Virtuozzo Storage distributed cloud file system as well as ploop devices.
-[VMware vSphere Storage Plugin](https://github.com/vmware/docker-volume-vsphere)    | Docker Volume Driver for vSphere enables customers to address persistent storage requirements for Docker containers in vSphere environments.
+| Plugin                                                                                             | Description                                                                                                                                                                                                                                                                                                   |
+|:---------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| [Azure File Storage plugin](https://github.com/Azure/azurefile-dockervolumedriver)                 | Lets you mount Microsoft [Azure File Storage](https://azure.microsoft.com/blog/azure-file-storage-now-generally-available/) shares to Docker containers as volumes using the SMB 3.0 protocol. [Learn more](https://azure.microsoft.com/blog/persistent-docker-volumes-with-azure-file-storage/).             |
+| [BeeGFS Volume Plugin](https://github.com/RedCoolBeans/docker-volume-beegfs)                       | An open source volume plugin to create persistent volumes in a BeeGFS parallel file system.                                                                                                                                                                                                                   |
+| [Blockbridge plugin](https://github.com/blockbridge/blockbridge-docker-volume)                     | A volume plugin that provides access to an extensible set of container-based persistent storage options. It supports single and multi-host Docker environments with features that include tenant isolation, automated provisioning, encryption, secure deletion, snapshots and QoS.                           |
+| [Contiv Volume Plugin](https://github.com/contiv/volplugin)                                        | An open source volume plugin that provides multi-tenant, persistent, distributed storage with intent based consumption. It has support for Ceph and NFS.                                                                                                                                                      |
+| [Convoy plugin](https://github.com/rancher/convoy)                                                 | A volume plugin for a variety of storage back-ends including device mapper and NFS. It's a simple standalone executable written in Go and provides the framework to support vendor-specific extensions such as snapshots, backups and restore.                                                                |
+| [DigitalOcean Block Storage plugin](https://github.com/omallo/docker-volume-plugin-dostorage)      | Integrates DigitalOcean's [block storage solution](https://www.digitalocean.com/products/storage/) into the Docker ecosystem by automatically attaching a given block storage volume to a DigitalOcean droplet and making the contents of the volume available to Docker containers running on that droplet.  |
+| [DRBD plugin](https://www.drbd.org/en/supported-projects/docker)                                   | A volume plugin that provides highly available storage replicated by [DRBD](https://www.drbd.org). Data written to the docker volume is replicated in a cluster of DRBD nodes.                                                                                                                                |
+| [Flocker plugin](https://clusterhq.com/docker-plugin/)                                             | A volume plugin that provides multi-host portable volumes for Docker, enabling you to run databases and other stateful containers and move them around across a cluster of machines.                                                                                                                          |
+| [Fuxi Volume Plugin](https://github.com/openstack/fuxi)                                            | A volume plugin that is developed as part of the OpenStack Kuryr project and implements the Docker volume plugin API by utilizing Cinder, the OpenStack block storage service.                                                                                                                                |
+| [gce-docker plugin](https://github.com/mcuadros/gce-docker)                                        | A volume plugin able to attach, format and mount Google Compute [persistent-disks](https://cloud.google.com/compute/docs/disks/persistent-disks).                                                                                                                                                             |
+| [GlusterFS plugin](https://github.com/calavera/docker-volume-glusterfs)                            | A volume plugin that provides multi-host volumes management for Docker using GlusterFS.                                                                                                                                                                                                                       |
+| [Horcrux Volume Plugin](https://github.com/muthu-r/horcrux)                                        | A volume plugin that allows on-demand, version controlled access to your data. Horcrux is an open-source plugin, written in Go, and supports SCP, [Minio](https://www.minio.io) and Amazon S3.                                                                                                                |
+| [HPE 3Par Volume Plugin](https://github.com/hpe-storage/python-hpedockerplugin/)                   | A volume plugin that supports HPE 3Par and StoreVirtual iSCSI storage arrays.                                                                                                                                                                                                                                 |
+| [Infinit volume plugin](https://infinit.sh/documentation/docker/volume-plugin)                     | A volume plugin that makes it easy to mount and manage Infinit volumes using Docker.                                                                                                                                                                                                                          |
+| [IPFS Volume Plugin](http://github.com/vdemeester/docker-volume-ipfs)                              | An open source volume plugin that allows using an [ipfs](https://ipfs.io/) filesystem as a volume.                                                                                                                                                                                                            |
+| [Keywhiz plugin](https://github.com/calavera/docker-volume-keywhiz)                                | A plugin that provides credentials and secret management using Keywhiz as a central repository.                                                                                                                                                                                                               |
+| [Local Persist Plugin](https://github.com/CWSpear/local-persist)                                   | A volume plugin that extends the default `local` driver's functionality by allowing you specify a mountpoint anywhere on the host, which enables the files to *always persist*, even if the volume is removed via `docker volume rm`.                                                                         |
+| [NetApp Plugin](https://github.com/NetApp/netappdvp) (nDVP)                                        | A volume plugin that provides direct integration with the Docker ecosystem for the NetApp storage portfolio. The nDVP package supports the provisioning and management of storage resources from the storage platform to Docker hosts, with a robust framework for adding additional platforms in the future. |
+| [Netshare plugin](https://github.com/ContainX/docker-volume-netshare)                              | A volume plugin that provides volume management for NFS 3/4, AWS EFS and CIFS file systems.                                                                                                                                                                                                                   |
+| [Nimble Storage Volume Plugin](https://connect.nimblestorage.com/community/app-integration/docker) | A volume plug-in that integrates with Nimble Storage Unified Flash Fabric arrays. The plug-in abstracts array volume capabilities to the Docker administrator to allow self-provisioning of secure multi-tenant volumes and clones.                                                                           |
+| [OpenStorage Plugin](https://github.com/libopenstorage/openstorage)                                | A cluster-aware volume plugin that provides volume management for file and block storage solutions.  It implements a vendor neutral specification for implementing extensions such as CoS, encryption, and snapshots. It has example drivers based on FUSE, NFS, NBD and EBS to name a few.                   |
+| [Portworx Volume Plugin](https://github.com/portworx/px-dev)                                       | A volume plugin that turns any server into a scale-out converged compute/storage node, providing container granular storage and highly available volumes across any node, using a shared-nothing storage backend that works with any docker scheduler.                                                        |
+| [Quobyte Volume Plugin](https://github.com/quobyte/docker-volume)                                  | A volume plugin that connects Docker to [Quobyte](http://www.quobyte.com/containers)'s data center file system, a general-purpose scalable and fault-tolerant storage platform.                                                                                                                               |
+| [REX-Ray plugin](https://github.com/emccode/rexray)                                                | A volume plugin which is written in Go and provides advanced storage functionality for many platforms including VirtualBox, EC2, Google Compute Engine, OpenStack, and EMC.                                                                                                                                   |
+| [Virtuozzo Storage and Ploop plugin](https://github.com/virtuozzo/docker-volume-ploop)             | A volume plugin with support for Virtuozzo Storage distributed cloud file system as well as ploop devices.                                                                                                                                                                                                    |
+| [VMware vSphere Storage Plugin](https://github.com/vmware/docker-volume-vsphere)                   | Docker Volume Driver for vSphere enables customers to address persistent storage requirements for Docker containers in vSphere environments.                                                                                                                                                                  |
 
 ### Authorization plugins
 
- Plugin                                                       | Description
-------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-[Casbin AuthZ Plugin](https://github.com/casbin/casbin-authz-plugin) | An authorization plugin based on [Casbin](https://github.com/casbin/casbin), which supports access control models like ACL, RBAC, ABAC. The access control model can be customized. The policy can be persisted into file or DB.
-[HBM plugin](https://github.com/kassisol/hbm)                 | An authorization plugin that prevents from executing commands with certains parameters.
-[Twistlock AuthZ Broker](https://github.com/twistlock/authz)  | A basic extendable authorization plugin that runs directly on the host or inside a container. This plugin allows you to define user policies that it evaluates during authorization. Basic authorization is provided if Docker daemon is started with the --tlsverify flag (username is extracted from the certificate common name).
+| Plugin                                                               | Description                                                                                                                                                                                                                                                                                                                          |
+|:---------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| [Casbin AuthZ Plugin](https://github.com/casbin/casbin-authz-plugin) | An authorization plugin based on [Casbin](https://github.com/casbin/casbin), which supports access control models like ACL, RBAC, ABAC. The access control model can be customized. The policy can be persisted into file or DB.                                                                                                     |
+| [HBM plugin](https://github.com/kassisol/hbm)                        | An authorization plugin that prevents from executing commands with certains parameters.                                                                                                                                                                                                                                              |
+| [Twistlock AuthZ Broker](https://github.com/twistlock/authz)         | A basic extendable authorization plugin that runs directly on the host or inside a container. This plugin allows you to define user policies that it evaluates during authorization. Basic authorization is provided if Docker daemon is started with the --tlsverify flag (username is extracted from the certificate common name). |
 
 ## Troubleshooting a plugin
 

components/cli/docs/reference/commandline/cp.md

@@ -105,11 +105,11 @@ running `tar` in `docker exec`. Both of the following examples do the same thing
 in different ways (consider `SRC_PATH` and `DEST_PATH` are directories):
 
 ```bash
-$ docker exec foo tar Ccf $(dirname SRC_PATH) - $(basename SRC_PATH) | tar Cxf DEST_PATH -
+$ docker exec CONTAINER tar Ccf $(dirname SRC_PATH) - $(basename SRC_PATH) | tar Cxf DEST_PATH -
 ```
 
 ```bash
-$ tar Ccf $(dirname SRC_PATH) - $(basename SRC_PATH) | docker exec -i foo tar Cxf DEST_PATH -
+$ tar Ccf $(dirname SRC_PATH) - $(basename SRC_PATH) | docker exec -i CONTAINER tar Cxf DEST_PATH -
 ```
 
 Using `-` as the `SRC_PATH` streams the contents of `STDIN` as a tar archive.

components/cli/docs/reference/commandline/ps.md

@@ -84,18 +84,19 @@ The currently supported filters are:
 
 | Filter                | Description                                                                                                                          |
 |:----------------------|:-------------------------------------------------------------------------------------------------------------------------------------|
-| `id`                  | container's ID                                                                                                                       |
-| `name`                | container's name                                                                                                                     |
-| `label`               | An arbitrary string representing either a key or a key-value pair                                                                    |
+| `id`                  | Container's ID                                                                                                                       |
+| `name`                | Container's name                                                                                                                     |
+| `label`               | An arbitrary string representing either a key or a key-value pair. Expressed as `<key>` or `<key>=<value>`                           |
 | `exited`              | An integer representing the container's exit code. Only useful with `--all`.                                                         |
-| `status`              | One of `created|restarting|running|removing|paused|exited|dead`                                                                      |
+| `status`              | One of `created`, `restarting`, `running`, `removing`, `paused`, `exited`, or `dead`                                                 |
 | `ancestor`            | Filters containers which share a given image as an ancestor. Expressed as `<image-name>[:<tag>]`,  `<image id>`, or `<image@digest>` |
 | `before` or `since`   | Filters containers created before or after a given container ID or name                                                              |
 | `volume`              | Filters running containers which have mounted a given volume or bind mount.                                                          |
 | `network`             | Filters running containers connected to a given network.                                                                             |
-| `publish` or `expose` | Filters containers which publish or expose a given port.                                                                             |
-| `health`              | One of `starting|healthy|unhealthy|none`. Filters containers based on their healthcheck status.                                      |
-| `isolation`           | Windows daemon only. One of `default|process|hyperv`.                                                                                |
+| `publish` or `expose` | Filters containers which publish or expose a given port. Expressed as `<port>[/<proto>]` or `<startport-endport>/[<proto>]`          |
+| `health`              | Filters containers based on their healthcheck status. One of `starting`, `healthy`, `unhealthy` or `none`.                           |
+| `isolation`           | Windows daemon only. One of `default`, `process`, or `hyperv`.                                                                       |
+| `is-task`             | Filters containers that are a "task" for a service. Boolean option (`true` or `false`)                                               |
 
 
 #### label
@@ -208,11 +209,11 @@ CONTAINER ID        IMAGE               COMMAND             CREATED
 The `ancestor` filter matches containers based on its image or a descendant of
 it. The filter supports the following image representation:
 
-- image
-- image:tag
-- image:tag@digest
-- short-id
-- full-id
+- `image`
+- `image:tag`
+- `image:tag@digest`
+- `short-id`
+- `full-id`
 
 If you don't specify a `tag`, the `latest` tag is used. For example, to filter
 for containers that use the latest `ubuntu` image:

components/cli/docs/reference/commandline/service_create.md

@@ -21,9 +21,10 @@ Usage:  docker service create [OPTIONS] IMAGE [COMMAND] [ARG...]
 Create a new service
 
 Options:
+      --config config                      Specify configurations to expose to the service
       --constraint list                    Placement constraints
       --container-label list               Container labels
-      --credential-spec                    Credential spec for managed service account (Windows only)
+      --credential-spec credential-spec    Credential spec for managed service account (Windows only)
   -d, --detach                             Exit immediately instead of waiting for the service to converge (default true)
       --dns list                           Set custom DNS servers
       --dns-option list                    Set DNS options
@@ -49,8 +50,9 @@ Options:
       --mode string                        Service mode (replicated or global) (default "replicated")
       --mount mount                        Attach a filesystem mount to the service
       --name string                        Service name
-      --network list                       Network attachments
+      --network network                    Network attachments
       --no-healthcheck                     Disable any container-specified HEALTHCHECK
+      --no-resolve-image                   Do not query the registry to resolve image digest and supported platforms
       --placement-pref pref                Add a placement preference
   -p, --publish port                       Publish a port as a node port
   -q, --quiet                              Suppress progress output

components/cli/docs/reference/commandline/service_update.md

@@ -22,11 +22,13 @@ Update a service
 
 Options:
       --args command                       Service command args
+      --config-add config                  Add or update a config file on a service
+      --config-rm list                     Remove a configuration file
       --constraint-add list                Add or update a placement constraint
       --constraint-rm list                 Remove a constraint
       --container-label-add list           Add or update a container label
       --container-label-rm list            Remove a container label by its key
-      --credential-spec                    Credential spec for managed service account (Windows only)
+      --credential-spec credential-spec    Credential spec for managed service account (Windows only)
   -d, --detach                             Exit immediately instead of waiting for the service to converge (default true)
       --dns-add list                       Add or update a custom DNS server
       --dns-option-add list                Add or update a DNS option
@@ -59,9 +61,10 @@ Options:
       --log-opt list                       Logging driver options
       --mount-add mount                    Add or update a mount on a service
       --mount-rm list                      Remove a mount by its target path
-      --network-add list                   Add a network
+      --network-add network                Add a network
       --network-rm list                    Remove a network
       --no-healthcheck                     Disable any container-specified HEALTHCHECK
+      --no-resolve-image                   Do not query the registry to resolve image digest and supported platforms
       --placement-pref-add pref            Add a placement preference
       --placement-pref-rm pref             Remove a placement preference
       --publish-add port                   Add or update a published port
@@ -80,7 +83,7 @@ Options:
       --rollback-failure-action string     Action on rollback failure ("pause"|"continue")
       --rollback-max-failure-ratio float   Failure rate to tolerate during a rollback
       --rollback-monitor duration          Duration after each task rollback to monitor for failure (ns|us|ms|s|m|h)
-      --rollback-order string              Rollback order ("start-first"|"stop-first") (default "stop-first")
+      --rollback-order string              Rollback order ("start-first"|"stop-first")
       --rollback-parallelism uint          Maximum number of tasks rolled back simultaneously (0 to roll back all at once)
       --secret-add secret                  Add or update a secret on a service
       --secret-rm list                     Remove a secret

components/cli/man/src/container/ls.md

@@ -4,39 +4,46 @@ the running containers.
 ## Filters
 
 Filter output based on these conditions:
-   - exited=<int> an exit code of <int>
-   - label=<key> or label=<key>=<value>
-   - status=(created|restarting|running|paused|exited|dead)
-   - name=<string> a container's name
-   - id=<ID> a container's ID
-   - is-task=(true|false) - containers that are a task (part of a service managed by swarm)
+   - ancestor=(<image-name>[:tag]|<image-id>|<image@digest>)
+     containers created from an image or a descendant.
    - before=(<container-name>|<container-id>)
+   - expose=(<port>[/<proto>]|<startport-endport>/[<proto>])
+   - exited=<int> an exit code of <int>
+   - health=(starting|healthy|unhealthy|none)
+   - id=<ID> a container's ID
+   - isolation=(`default`|`process`|`hyperv`) (Windows daemon only)
+   - is-task=(true|false)
+   - label=<key> or label=<key>=<value>
+   - name=<string> a container's name
+   - network=(<network-id>|<network-name>)
+   - publish=(<port>[/<proto>]|<startport-endport>/[<proto>])
    - since=(<container-name>|<container-id>)
-   - ancestor=(<image-name>[:tag]|<image-id>|<image@digest>) - containers created from an image or a descendant.
-   - volume=(<volume-name>|<mount-point-destination>)
-   - network=(<network-name>|<network-id>) - containers connected to the provided network
-   - health=(starting|healthy|unhealthy|none) - filters containers based on healthcheck status
-   - publish=(<port>[/<proto>]|<startport-endport>/[<proto>]) - filters containers based on published ports
-   - expose=(<port>[/<proto>]|<startport-endport>/[<proto>]) - filters containers based on exposed ports
+   - status=(created|restarting|removing|running|paused|exited)
+   - volume=(<volume name>|<mount point destination>)
 
 ## Format
 
-   Pretty-print containers using a Go template.
-   Valid placeholders:
-      .ID - Container ID
-      .Image - Image ID
-      .Command - Quoted command
-      .CreatedAt - Time when the container was created.
-      .RunningFor - Elapsed time since the container was started.
-      .Ports - Exposed ports.
-      .Status - Container status.
-      .Size - Container disk size.
-      .Names - Container names.
-      .Labels - All labels assigned to the container.
-      .Label - Value of a specific label for this container. For example `{{.Label "com.docker.swarm.cpu"}}`
-      .Mounts - Names of the volumes mounted in this container.
+The formatting option (**--format**) pretty-prints container output
+using a Go template.
+
+Valid placeholders for the Go template are listed below:
+   - .ID           - Container ID.
+   - .Image        - Image ID.
+   - .Command      - Quoted command.
+   - .CreatedAt    - Time when the container was created.
+   - .RunningFor   - Elapsed time since the container was started.
+   - .Ports        - Exposed ports.
+   - .Status       - Container status.
+   - .Size         - Container disk size.
+   - .Names        - Container names.
+   - .Labels       - All labels assigned to the container.
+   - .Label        - Value of a specific label for this container.
+                     For example **'{{.Label "com.docker.swarm.cpu"}}'**.
+   - .Mounts       - Names of the volumes mounted in this container.
+   - .Networks     - Names of the networks attached to this container.
 
 # EXAMPLES
+
 ## Display all containers, including non-running
 
     $ docker container ls -a

components/engine/VERSION

@@ -1 +1 @@
-17.06.1-ce-rc3
+17.06.2-ce

components/engine/api/server/router/network/network_routes.go

@@ -398,7 +398,9 @@ func buildIpamResources(r *types.NetworkResource, nwInfo libnetwork.NetworkInfo)
 		for _, ip4Info := range ipv4Info {
 			iData := network.IPAMConfig{}
 			iData.Subnet = ip4Info.IPAMData.Pool.String()
-			iData.Gateway = ip4Info.IPAMData.Gateway.IP.String()
+			if ip4Info.IPAMData.Gateway != nil {
+				iData.Gateway = ip4Info.IPAMData.Gateway.IP.String()
+			}
 			r.IPAM.Config = append(r.IPAM.Config, iData)
 		}
 	}

components/engine/api/swagger.yaml

@@ -1346,26 +1346,33 @@ definitions:
       Name:
         type: "string"
         x-nullable: false
+        example: "some-mount"
       Description:
         type: "string"
         x-nullable: false
+        example: "This is a mount that's used by the plugin."
       Settable:
         type: "array"
         items:
           type: "string"
       Source:
         type: "string"
+        example: "/var/lib/docker/plugins/"
       Destination:
         type: "string"
         x-nullable: false
+        example: "/mnt/state"
       Type:
         type: "string"
         x-nullable: false
+        example: "bind"
       Options:
         type: "array"
         items:
           type: "string"
-
+        example:
+          - "rbind"
+          - "rw"
   PluginDevice:
     type: "object"
     required: [Name, Description, Settable, Path]
@@ -1383,6 +1390,7 @@ definitions:
           type: "string"
       Path:
         type: "string"
+        example: "/dev/fuse"
 
   PluginEnv:
     type: "object"
@@ -1424,13 +1432,16 @@ definitions:
     properties:
       Id:
         type: "string"
+        example: "5724e2c8652da337ab2eedd19fc6fc0ec908e4bd907c7421bf6a8dfc70c4c078"
       Name:
         type: "string"
         x-nullable: false
+        example: "tiborvass/sample-volume-plugin"
       Enabled:
-        description: "True when the plugin is running. False when the plugin is not running, only installed."
+        description: "True if the plugin is running. False if the plugin is not running, only installed."
         type: "boolean"
         x-nullable: false
+        example: true
       Settings:
         description: "Settings that can be modified by users."
         type: "object"
@@ -1445,6 +1456,8 @@ definitions:
             type: "array"
             items:
               type: "string"
+            example:
+              - "DEBUG=0"
           Args:
             type: "array"
             items:
@@ -1457,6 +1470,7 @@ definitions:
         description: "plugin remote reference used to push/pull the plugin"
         type: "string"
         x-nullable: false
+        example: "localhost:5000/tiborvass/sample-volume-plugin:latest"
       Config:
         description: "The config of a plugin."
         type: "object"
@@ -1480,12 +1494,15 @@ definitions:
             description: "Docker Version used to create the plugin"
             type: "string"
             x-nullable: false
+            example: "17.06.0-ce"
           Description:
             type: "string"
             x-nullable: false
+            example: "A sample volume plugin for Docker"
           Documentation:
             type: "string"
             x-nullable: false
+            example: "https://docs.docker.com/engine/extend/plugins/"
           Interface:
             description: "The interface between Docker and the plugin"
             x-nullable: false
@@ -1496,16 +1513,23 @@ definitions:
                 type: "array"
                 items:
                   $ref: "#/definitions/PluginInterfaceType"
+                example:
+                  - "docker.volumedriver/1.0"
               Socket:
                 type: "string"
                 x-nullable: false
+                example: "plugins.sock"
           Entrypoint:
             type: "array"
             items:
               type: "string"
+            example:
+              - "/usr/bin/sample-volume-plugin"
+              - "/data"
           WorkDir:
             type: "string"
             x-nullable: false
+            example: "/bin/"
           User:
             type: "object"
             x-nullable: false
@@ -1513,9 +1537,11 @@ definitions:
               UID:
                 type: "integer"
                 format: "uint32"
+                example: 1000
               GID:
                 type: "integer"
                 format: "uint32"
+                example: 1000
           Network:
             type: "object"
             x-nullable: false
@@ -1524,6 +1550,7 @@ definitions:
               Type:
                 x-nullable: false
                 type: "string"
+                example: "host"
           Linux:
             type: "object"
             x-nullable: false
@@ -1533,9 +1560,13 @@ definitions:
                 type: "array"
                 items:
                   type: "string"
+                example:
+                  - "CAP_SYS_ADMIN"
+                  - "CAP_SYSLOG"
               AllowAllDevices:
                 type: "boolean"
                 x-nullable: false
+                example: false
               Devices:
                 type: "array"
                 items:
@@ -1543,12 +1574,15 @@ definitions:
           PropagatedMount:
             type: "string"
             x-nullable: false
+            example: "/mnt/volumes"
           IpcHost:
             type: "boolean"
             x-nullable: false
+            example: false
           PidHost:
             type: "boolean"
             x-nullable: false
+            example: false
           Mounts:
             type: "array"
             items:
@@ -1557,6 +1591,11 @@ definitions:
             type: "array"
             items:
               $ref: "#/definitions/PluginEnv"
+            example:
+              - Name: "DEBUG"
+                Description: "If set, prints debug messages"
+                Settable: null
+                Value: "0"
           Args:
             type: "object"
             x-nullable: false
@@ -1565,9 +1604,11 @@ definitions:
               Name:
                 x-nullable: false
                 type: "string"
+                example: "args"
               Description:
                 x-nullable: false
                 type: "string"
+                example: "command line arguments"
               Settable:
                 type: "array"
                 items:
@@ -1581,50 +1622,14 @@ definitions:
             properties:
               type:
                 type: "string"
+                example: "layers"
               diff_ids:
                 type: "array"
                 items:
                   type: "string"
-    example:
-      Id: "5724e2c8652da337ab2eedd19fc6fc0ec908e4bd907c7421bf6a8dfc70c4c078"
-      Name: "tiborvass/sample-volume-plugin"
-      Tag: "latest"
-      Active: true
-      Settings:
-        Env:
-          - "DEBUG=0"
-        Args: null
-        Devices: null
-      Config:
-        Description: "A sample volume plugin for Docker"
-        Documentation: "https://docs.docker.com/engine/extend/plugins/"
-        Interface:
-          Types:
-            - "docker.volumedriver/1.0"
-          Socket: "plugins.sock"
-        Entrypoint:
-          - "/usr/bin/sample-volume-plugin"
-          - "/data"
-        WorkDir: ""
-        User: {}
-        Network:
-          Type: ""
-        Linux:
-          Capabilities: null
-          AllowAllDevices: false
-          Devices: null
-        Mounts: null
-        PropagatedMount: "/data"
-        Env:
-          - Name: "DEBUG"
-            Description: "If set, prints debug messages"
-            Settable: null
-            Value: "0"
-        Args:
-          Name: "args"
-          Description: "command line arguments"
-          Settable: null
-          Value: []
+                example:
+                  - "sha256:675532206fbf3030b8458f88d6e26d4eb1577688a25efec97154c94e8b6b4887"
+                  - "sha256:e216a057b1cb1efc11f8a268f37ef62083e70b1b38323ba252e25ac88904a7e8"
 
   ObjectVersion:
     description: |
@@ -6823,46 +6828,6 @@ paths:
             type: "array"
             items:
               $ref: "#/definitions/Plugin"
-            example:
-              - Id: "5724e2c8652da337ab2eedd19fc6fc0ec908e4bd907c7421bf6a8dfc70c4c078"
-                Name: "tiborvass/sample-volume-plugin"
-                Tag: "latest"
-                Active: true
-                Settings:
-                  Env:
-                    - "DEBUG=0"
-                  Args: null
-                  Devices: null
-                Config:
-                  Description: "A sample volume plugin for Docker"
-                  Documentation: "https://docs.docker.com/engine/extend/plugins/"
-                  Interface:
-                    Types:
-                      - "docker.volumedriver/1.0"
-                    Socket: "plugins.sock"
-                  Entrypoint:
-                    - "/usr/bin/sample-volume-plugin"
-                    - "/data"
-                  WorkDir: ""
-                  User: {}
-                  Network:
-                    Type: ""
-                  Linux:
-                    Capabilities: null
-                    AllowAllDevices: false
-                    Devices: null
-                  Mounts: null
-                  PropagatedMount: "/data"
-                  Env:
-                    - Name: "DEBUG"
-                      Description: "If set, prints debug messages"
-                      Settable: null
-                      Value: "0"
-                  Args:
-                    Name: "args"
-                    Description: "command line arguments"
-                    Settable: null
-                    Value: []
         500:
           description: "Server error"
           schema:

components/engine/api/types/plugin.go

@@ -11,7 +11,7 @@ type Plugin struct {
 	// Required: true
 	Config PluginConfig `json:"Config"`
 
-	// True when the plugin is running. False when the plugin is not running, only installed.
+	// True if the plugin is running. False if the plugin is not running, only installed.
 	// Required: true
 	Enabled bool `json:"Enabled"`
 

components/engine/daemon/graphdriver/aufs/aufs.go

@@ -143,6 +143,23 @@ func Init(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		}
 	}
 
+	for _, path := range []string{"mnt", "diff"} {
+		p := filepath.Join(root, path)
+		dirs, err := ioutil.ReadDir(p)
+		if err != nil {
+			logrus.WithError(err).WithField("dir", p).Error("error reading dir entries")
+			continue
+		}
+		for _, dir := range dirs {
+			if strings.HasSuffix(dir.Name(), "-removing") {
+				logrus.WithField("dir", dir.Name()).Debug("Cleaning up stale layer dir")
+				if err := system.EnsureRemoveAll(filepath.Join(p, dir.Name())); err != nil {
+					logrus.WithField("dir", dir.Name()).WithError(err).Error("Error removing stale layer dir")
+				}
+			}
+		}
+	}
+
 	a.naiveDiff = graphdriver.NewNaiveDiffDriver(a, uidMaps, gidMaps)
 	return a, nil
 }
@@ -318,26 +335,6 @@ func (a *Driver) Remove(id string) error {
 		retries++
 		logger.Warnf("unmount failed due to EBUSY: retry count: %d", retries)
 		time.Sleep(100 * time.Millisecond)
-		continue
-	}
-
-	// Atomically remove each directory in turn by first moving it out of the
-	// way (so that docker doesn't find it anymore) before doing removal of
-	// the whole tree.
-	tmpMntPath := path.Join(a.mntPath(), fmt.Sprintf("%s-removing", id))
-	if err := os.Rename(mountpoint, tmpMntPath); err != nil && !os.IsNotExist(err) {
-		if err == syscall.EBUSY {
-			logger.WithField("dir", mountpoint).WithError(err).Warn("os.Rename err due to EBUSY")
-		}
-		return errors.Wrapf(err, "error preparing atomic delete of aufs mountpoint for id: %s", id)
-	}
-	if err := system.EnsureRemoveAll(tmpMntPath); err != nil {
-		return errors.Wrapf(err, "error removing aufs layer %s", id)
-	}
-
-	tmpDiffpath := path.Join(a.diffPath(), fmt.Sprintf("%s-removing", id))
-	if err := os.Rename(a.getDiffPath(id), tmpDiffpath); err != nil && !os.IsNotExist(err) {
-		return errors.Wrapf(err, "error preparing atomic delete of aufs diff dir for id: %s", id)
 	}
 
 	// Remove the layers file for the id
@@ -345,12 +342,44 @@ func (a *Driver) Remove(id string) error {
 		return errors.Wrapf(err, "error removing layers dir for %s", id)
 	}
 
+	if err := atomicRemove(a.getDiffPath(id)); err != nil {
+		return errors.Wrapf(err, "could not remove diff path for id %s", id)
+	}
+
+	// Atomically remove each directory in turn by first moving it out of the
+	// way (so that docker doesn't find it anymore) before doing removal of
+	// the whole tree.
+	if err := atomicRemove(mountpoint); err != nil {
+		if errors.Cause(err) == syscall.EBUSY {
+			logger.WithField("dir", mountpoint).WithError(err).Warn("error performing atomic remove due to EBUSY")
+		}
+		return errors.Wrapf(err, "could not remove mountpoint for id %s", id)
+	}
+
 	a.pathCacheLock.Lock()
 	delete(a.pathCache, id)
 	a.pathCacheLock.Unlock()
 	return nil
 }
 
+func atomicRemove(source string) error {
+	target := source + "-removing"
+
+	err := os.Rename(source, target)
+	switch {
+	case err == nil, os.IsNotExist(err):
+	case os.IsExist(err):
+		// Got error saying the target dir already exists, maybe the source doesn't exist due to a previous (failed) remove
+		if _, e := os.Stat(source); !os.IsNotExist(e) {
+			return errors.Wrapf(err, "target rename dir '%s' exists but should not, this needs to be manually cleaned up")
+		}
+	default:
+		return errors.Wrapf(err, "error preparing atomic delete")
+	}
+
+	return system.EnsureRemoveAll(target)
+}
+
 // Get returns the rootfs path for the id.
 // This will mount the dir at its given path
 func (a *Driver) Get(id, mountLabel string) (string, error) {

components/engine/daemon/graphdriver/aufs/aufs_test.go

@@ -12,6 +12,8 @@ import (
 	"sync"
 	"testing"
 
+	"path/filepath"
+
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/reexec"
@@ -147,7 +149,10 @@ func TestRemoveImage(t *testing.T) {
 
 	for _, p := range paths {
 		if _, err := os.Stat(path.Join(tmp, p, "1")); err == nil {
-			t.Fatalf("Error should not be nil because dirs with id 1 should be delted: %s", p)
+			t.Fatalf("Error should not be nil because dirs with id 1 should be deleted: %s", p)
+		}
+		if _, err := os.Stat(path.Join(tmp, p, "1-removing")); err == nil {
+			t.Fatalf("Error should not be nil because dirs with id 1-removing should be deleted: %s", p)
 		}
 	}
 }
@@ -800,3 +805,23 @@ func BenchmarkConcurrentAccess(b *testing.B) {
 		}
 	}
 }
+
+func TestInitStaleCleanup(t *testing.T) {
+	if err := os.MkdirAll(tmp, 0755); err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+
+	for _, d := range []string{"diff", "mnt"} {
+		if err := os.MkdirAll(filepath.Join(tmp, d, "123-removing"), 0755); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	testInit(tmp, t)
+	for _, d := range []string{"diff", "mnt"} {
+		if _, err := os.Stat(filepath.Join(tmp, d, "123-removing")); err == nil {
+			t.Fatal("cleanup failed")
+		}
+	}
+}

components/engine/integration-cli/docker_cli_network_unix_test.go

@@ -690,6 +690,21 @@ func (s *DockerNetworkSuite) TestDockerNetworkIPAMOptions(c *check.C) {
 	c.Assert(opts["opt2"], checker.Equals, "drv2")
 }
 
+func (s *DockerNetworkSuite) TestDockerNetworkNullIPAMDriver(c *check.C) {
+	// Create a network with null ipam driver
+	_, _, err := dockerCmdWithError("network", "create", "-d", dummyNetworkDriver, "--ipam-driver", "null", "test000")
+	c.Assert(err, check.IsNil)
+	assertNwIsAvailable(c, "test000")
+
+	// Verify the inspect data contains the default subnet provided by the null
+	// ipam driver and no gateway, as the null ipam driver does not provide one
+	nr := getNetworkResource(c, "test000")
+	c.Assert(nr.IPAM.Driver, checker.Equals, "null")
+	c.Assert(len(nr.IPAM.Config), checker.Equals, 1)
+	c.Assert(nr.IPAM.Config[0].Subnet, checker.Equals, "0.0.0.0/0")
+	c.Assert(nr.IPAM.Config[0].Gateway, checker.Equals, "")
+}
+
 func (s *DockerNetworkSuite) TestDockerNetworkInspectDefault(c *check.C) {
 	nr := getNetworkResource(c, "none")
 	c.Assert(nr.Driver, checker.Equals, "null")

components/engine/libcontainerd/remote_unix.go

@@ -56,7 +56,7 @@ type remote struct {
 	stateDir             string
 	rpcAddr              string
 	startDaemon          bool
-	closeManually        bool
+	closedManually       bool
 	debugLog             bool
 	rpcConn              *grpc.ClientConn
 	clients              []*client
@@ -195,7 +195,7 @@ func (r *remote) handleConnectionChange() {
 		logrus.Debugf("libcontainerd: containerd health check returned error: %v", err)
 
 		if r.daemonPid != -1 {
-			if r.closeManually {
+			if r.closedManually {
 				// Well, we asked for it to stop, just return
 				return
 			}
@@ -221,7 +221,7 @@ func (r *remote) Cleanup() {
 	if r.daemonPid == -1 {
 		return
 	}
-	r.closeManually = true
+	r.closedManually = true
 	r.rpcConn.Close()
 	// Ask the daemon to quit
 	syscall.Kill(r.daemonPid, syscall.SIGTERM)
@@ -321,10 +321,23 @@ func (r *remote) startEventsMonitor() error {
 	er := &containerd.EventsRequest{
 		Timestamp: tsp,
 	}
-	events, err := r.apiClient.Events(context.Background(), er, grpc.FailFast(false))
-	if err != nil {
-		return err
+
+	var events containerd.API_EventsClient
+	for {
+		events, err = r.apiClient.Events(context.Background(), er, grpc.FailFast(false))
+		if err == nil {
+			break
+		}
+		logrus.Warnf("libcontainerd: failed to get events from containerd: %q", err)
+
+		if r.closedManually {
+			// ignore error if grpc remote connection is closed manually
+			return nil
+		}
+
+		<-time.After(100 * time.Millisecond)
 	}
+
 	go r.handleEventStream(events)
 	return nil
 }
@@ -334,7 +347,7 @@ func (r *remote) handleEventStream(events containerd.API_EventsClient) {
 		e, err := events.Recv()
 		if err != nil {
 			if grpc.ErrorDesc(err) == transport.ErrConnClosing.Desc &&
-				r.closeManually {
+				r.closedManually {
 				// ignore error if grpc remote connection is closed manually
 				return
 			}

components/engine/pkg/devicemapper/devmapper.go

@@ -333,6 +333,7 @@ func RemoveDevice(name string) error {
 	if err := task.setCookie(cookie, 0); err != nil {
 		return fmt.Errorf("devicemapper: Can not set cookie: %s", err)
 	}
+	defer UdevWait(cookie)
 
 	dmSawBusy = false // reset before the task is run
 	if err = task.run(); err != nil {
@@ -342,7 +343,7 @@ func RemoveDevice(name string) error {
 		return fmt.Errorf("devicemapper: Error running RemoveDevice %s", err)
 	}
 
-	return UdevWait(cookie)
+	return nil
 }
 
 // RemoveDeviceDeferred is a useful helper for cleaning up a device, but deferred.
@@ -368,10 +369,6 @@ func RemoveDeviceDeferred(name string) error {
 		return fmt.Errorf("devicemapper: Can not set cookie: %s", err)
 	}
 
-	if err = task.run(); err != nil {
-		return fmt.Errorf("devicemapper: Error running RemoveDeviceDeferred %s", err)
-	}
-
 	// libdevmapper and udev relies on System V semaphore for synchronization,
 	// semaphores created in `task.setCookie` will be cleaned up in `UdevWait`.
 	// So these two function call must come in pairs, otherwise semaphores will
@@ -381,8 +378,13 @@ func RemoveDeviceDeferred(name string) error {
 	// this call will not wait for the deferred removal's final executing, since no
 	// udev event will be generated, and the semaphore's value will not be incremented
 	// by udev, what UdevWait is just cleaning up the semaphore.
+	defer UdevWait(cookie)
 
-	return UdevWait(cookie)
+	if err = task.run(); err != nil {
+		return fmt.Errorf("devicemapper: Error running RemoveDeviceDeferred %s", err)
+	}
+
+	return nil
 }
 
 // CancelDeferredRemove cancels a deferred remove for a device.
@@ -476,12 +478,13 @@ func CreatePool(poolName string, dataFile, metadataFile *os.File, poolBlockSize
 	if err := task.setCookie(cookie, flags); err != nil {
 		return fmt.Errorf("devicemapper: Can't set cookie %s", err)
 	}
+	defer UdevWait(cookie)
 
 	if err := task.run(); err != nil {
 		return fmt.Errorf("devicemapper: Error running deviceCreate (CreatePool) %s", err)
 	}
 
-	return UdevWait(cookie)
+	return nil
 }
 
 // ReloadPool is the programmatic example of "dmsetup reload".
@@ -661,12 +664,13 @@ func ResumeDevice(name string) error {
 	if err := task.setCookie(cookie, 0); err != nil {
 		return fmt.Errorf("devicemapper: Can't set cookie %s", err)
 	}
+	defer UdevWait(cookie)
 
 	if err := task.run(); err != nil {
 		return fmt.Errorf("devicemapper: Error running deviceResume %s", err)
 	}
 
-	return UdevWait(cookie)
+	return nil
 }
 
 // CreateDevice creates a device with the specified poolName with the specified device id.
@@ -759,11 +763,13 @@ func activateDevice(poolName string, name string, deviceID int, size uint64, ext
 		return fmt.Errorf("devicemapper: Can't set cookie %s", err)
 	}
 
+	defer UdevWait(cookie)
+
 	if err := task.run(); err != nil {
 		return fmt.Errorf("devicemapper: Error running deviceCreate (ActivateDevice) %s", err)
 	}
 
-	return UdevWait(cookie)
+	return nil
 }
 
 // CreateSnapDeviceRaw creates a snapshot device. Caller needs to suspend and resume the origin device if it is active.

components/engine/plugin/manager.go

@@ -102,6 +102,11 @@ func NewManager(config ManagerConfig) (*Manager, error) {
 	if err := os.MkdirAll(manager.tmpDir(), 0700); err != nil {
 		return nil, errors.Wrapf(err, "failed to mkdir %v", manager.tmpDir())
 	}
+
+	if err := setupRoot(manager.config.Root); err != nil {
+		return nil, err
+	}
+
 	var err error
 	manager.containerdClient, err = config.Executor.Client(manager) // todo: move to another struct
 	if err != nil {

components/engine/plugin/manager_linux.go

@@ -161,6 +161,13 @@ func shutdownPlugin(p *v2.Plugin, c *controller, containerdClient libcontainerd.
 	}
 }
 
+func setupRoot(root string) error {
+	if err := mount.MakePrivate(root); err != nil {
+		return errors.Wrap(err, "error setting plugin manager root to private")
+	}
+	return nil
+}
+
 func (pm *Manager) disable(p *v2.Plugin, c *controller) error {
 	if !p.IsEnabled() {
 		return fmt.Errorf("plugin %s is already disabled", p.Name())
@@ -189,6 +196,7 @@ func (pm *Manager) Shutdown() {
 			shutdownPlugin(p, c, pm.containerdClient)
 		}
 	}
+	mount.Unmount(pm.config.Root)
 }
 
 func (pm *Manager) upgradePlugin(p *v2.Plugin, configDigest digest.Digest, blobsums []digest.Digest, tmpRootFSDir string, privileges *types.PluginPrivileges) (err error) {

components/engine/plugin/manager_solaris.go

@@ -26,3 +26,5 @@ func (pm *Manager) restore(p *v2.Plugin) error {
 // Shutdown plugins
 func (pm *Manager) Shutdown() {
 }
+
+func setupRoot(root string) error { return nil }

components/engine/plugin/manager_windows.go

@@ -28,3 +28,5 @@ func (pm *Manager) restore(p *v2.Plugin) error {
 // Shutdown plugins
 func (pm *Manager) Shutdown() {
 }
+
+func setupRoot(root string) error { return nil }

components/engine/vendor.conf

@@ -27,7 +27,7 @@ github.com/imdario/mergo 0.2.1
 golang.org/x/sync de49d9dcd27d4f764488181bea099dfe6179bcf0
 
 #get libnetwork packages
-github.com/docker/libnetwork 0605cbcf31e36c2606461151f66b4e30695637a6
+github.com/docker/libnetwork 93976265048377e7644acfbfae2178a17fb04087
 github.com/docker/go-events 18b43f1bc85d9cdd42c05a6cd2d444c7a200a894
 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec
@@ -108,7 +108,7 @@ github.com/containerd/containerd 6e23458c129b551d5c9871e5174f6b1b7f6d1170 https:
 github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4
 
 # cluster
-github.com/docker/swarmkit a0a7f6f663c35c92ddcd73e2c1b97b0f4ed8caf3
+github.com/docker/swarmkit bf3d9a21fa618289839963138923b1534103486a
 github.com/gogo/protobuf v0.4
 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a
 github.com/google/certificate-transparency d90e65c3a07988180c5b1ece71791c0b6506826e

components/engine/vendor/github.com/docker/libnetwork/agent.go

@@ -738,11 +738,12 @@ func (n *network) addDriverWatches() {
 			return
 		}
 
-		agent.networkDB.WalkTable(table.name, func(nid, key string, value []byte) bool {
-			if nid == n.ID() {
+		agent.networkDB.WalkTable(table.name, func(nid, key string, value []byte, deleted bool) bool {
+			// skip the entries that are mark for deletion, this is safe because this function is
+			// called at initialization time so there is no state to delete
+			if nid == n.ID() && !deleted {
 				d.EventNotify(driverapi.Create, nid, table.name, key, value)
 			}
-
 			return false
 		})
 	}

components/engine/vendor/github.com/docker/libnetwork/common/caller.go

@@ -0,0 +1,29 @@
+package common
+
+import (
+	"runtime"
+	"strings"
+)
+
+func callerInfo(i int) string {
+	ptr, _, _, ok := runtime.Caller(i)
+	fName := "unknown"
+	if ok {
+		f := runtime.FuncForPC(ptr)
+		if f != nil {
+			// f.Name() is like: github.com/docker/libnetwork/common.MethodName
+			tmp := strings.Split(f.Name(), ".")
+			if len(tmp) > 0 {
+				fName = tmp[len(tmp)-1]
+			}
+		}
+	}
+
+	return fName
+}
+
+// CallerName returns the name of the function at the specified level
+// level == 0 means current method name
+func CallerName(level int) string {
+	return callerInfo(2 + level)
+}

components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/joinleave.go

@@ -120,8 +120,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 		}
 	}
 
-	d.peerDbAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac,
-		net.ParseIP(d.advertiseAddress), true)
+	d.peerAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), true, false, false, true)
 
 	if err := d.checkEncryption(nid, nil, n.vxlanID(s), true, true); err != nil {
 		logrus.Warn(err)
@@ -205,7 +204,7 @@ func (d *driver) EventNotify(etype driverapi.EventType, nid, tableName, key stri
 		return
 	}
 
-	d.peerAdd(nid, eid, addr.IP, addr.Mask, mac, vtep, true, false, false)
+	d.peerAdd(nid, eid, addr.IP, addr.Mask, mac, vtep, true, false, false, false)
 }
 
 // Leave method is invoked when a Sandbox detaches from an endpoint.

components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go

@@ -8,10 +8,12 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
 	"syscall"
+	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/pkg/reexec"
@@ -80,6 +82,10 @@ func setDefaultVlan() {
 		logrus.Error("insufficient number of arguments")
 		os.Exit(1)
 	}
+
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
 	nsPath := os.Args[1]
 	ns, err := netns.GetFromPath(nsPath)
 	if err != nil {
@@ -682,10 +688,12 @@ func (n *network) initSandbox(restore bool) error {
 		return fmt.Errorf("could not get network sandbox (oper %t): %v", restore, err)
 	}
 
+	// this is needed to let the peerAdd configure the sandbox
 	n.setSandbox(sbox)
 
 	if !restore {
-		n.driver.peerDbUpdateSandbox(n.id)
+		// Initialize the sandbox with all the peers previously received from networkdb
+		n.driver.initSandboxPeerDB(n.id)
 	}
 
 	var nlSock *nl.NetlinkSocket
@@ -705,6 +713,7 @@ func (n *network) initSandbox(restore bool) error {
 }
 
 func (n *network) watchMiss(nlSock *nl.NetlinkSocket) {
+	t := time.Now()
 	for {
 		msgs, err := nlSock.Receive()
 		if err != nil {
@@ -757,23 +766,52 @@ func (n *network) watchMiss(nlSock *nl.NetlinkSocket) {
 				continue
 			}
 
-			if !n.driver.isSerfAlive() {
-				continue
-			}
-
-			mac, IPmask, vtep, err := n.driver.resolvePeer(n.id, ip)
-			if err != nil {
-				logrus.Errorf("could not resolve peer %q: %v", ip, err)
-				continue
-			}
-
-			if err := n.driver.peerAdd(n.id, "dummy", ip, IPmask, mac, vtep, true, l2Miss, l3Miss); err != nil {
-				logrus.Errorf("could not add neighbor entry for missed peer %q: %v", ip, err)
+			if n.driver.isSerfAlive() {
+				mac, IPmask, vtep, err := n.driver.resolvePeer(n.id, ip)
+				if err != nil {
+					logrus.Errorf("could not resolve peer %q: %v", ip, err)
+					continue
+				}
+				n.driver.peerAdd(n.id, "dummy", ip, IPmask, mac, vtep, true, l2Miss, l3Miss, false)
+			} else {
+				// If the gc_thresh values are lower kernel might knock off the neighor entries.
+				// When we get a L3 miss check if its a valid peer and reprogram the neighbor
+				// entry again. Rate limit it to once attempt every 500ms, just in case a faulty
+				// container sends a flood of packets to invalid peers
+				if !l3Miss {
+					continue
+				}
+				if time.Since(t) > 500*time.Millisecond {
+					t = time.Now()
+					n.programNeighbor(ip)
+				}
 			}
 		}
 	}
 }
 
+func (n *network) programNeighbor(ip net.IP) {
+	peerMac, _, _, err := n.driver.peerDbSearch(n.id, ip)
+	if err != nil {
+		logrus.Errorf("Reprogramming on L3 miss failed for %s, no peer entry", ip)
+		return
+	}
+	s := n.getSubnetforIPAddr(ip)
+	if s == nil {
+		logrus.Errorf("Reprogramming on L3 miss failed for %s, not a valid subnet", ip)
+		return
+	}
+	sbox := n.sandbox()
+	if sbox == nil {
+		logrus.Errorf("Reprogramming on L3 miss failed for %s, overlay sandbox missing", ip)
+		return
+	}
+	if err := sbox.AddNeighbor(ip, peerMac, true, sbox.NeighborOptions().LinkName(s.vxlanName)); err != nil {
+		logrus.Errorf("Reprogramming on L3 miss failed for %s: %v", ip, err)
+		return
+	}
+}
+
 func (d *driver) addNetwork(n *network) {
 	d.Lock()
 	d.networks[n.id] = n
@@ -1057,6 +1095,15 @@ func (n *network) contains(ip net.IP) bool {
 	return false
 }
 
+func (n *network) getSubnetforIPAddr(ip net.IP) *subnet {
+	for _, s := range n.subnets {
+		if s.subnetIP.Contains(ip) {
+			return s
+		}
+	}
+	return nil
+}
+
 // getSubnetforIP returns the subnet to which the given IP belongs
 func (n *network) getSubnetforIP(ip *net.IPNet) *subnet {
 	for _, s := range n.subnets {

components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_serf.go

@@ -120,15 +120,10 @@ func (d *driver) processEvent(u serf.UserEvent) {
 
 	switch action {
 	case "join":
-		if err := d.peerAdd(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac,
-			net.ParseIP(vtepStr), true, false, false); err != nil {
-			logrus.Errorf("Peer add failed in the driver: %v\n", err)
-		}
+		d.peerAdd(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac, net.ParseIP(vtepStr),
+			true, false, false, false)
 	case "leave":
-		if err := d.peerDelete(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac,
-			net.ParseIP(vtepStr), true); err != nil {
-			logrus.Errorf("Peer delete failed in the driver: %v\n", err)
-		}
+		d.peerDelete(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac, net.ParseIP(vtepStr), true)
 	}
 }
 

components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/overlay.go

@@ -3,6 +3,7 @@ package overlay
 //go:generate protoc -I.:../../Godeps/_workspace/src/github.com/gogo/protobuf  --gogo_out=import_path=github.com/docker/libnetwork/drivers/overlay,Mgogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto:. overlay.proto
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"sync"
@@ -50,6 +51,8 @@ type driver struct {
 	joinOnce         sync.Once
 	localJoinOnce    sync.Once
 	keys             []*key
+	peerOpCh         chan *peerOperation
+	peerOpCancel     context.CancelFunc
 	sync.Mutex
 }
 
@@ -64,10 +67,16 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 		peerDb: peerNetworkMap{
 			mp: map[string]*peerMap{},
 		},
-		secMap: &encrMap{nodes: map[string][]*spi{}},
-		config: config,
+		secMap:   &encrMap{nodes: map[string][]*spi{}},
+		config:   config,
+		peerOpCh: make(chan *peerOperation),
 	}
 
+	// Launch the go routine for processing peer operations
+	ctx, cancel := context.WithCancel(context.Background())
+	d.peerOpCancel = cancel
+	go d.peerOpRoutine(ctx, d.peerOpCh)
+
 	if data, ok := config[netlabel.GlobalKVClient]; ok {
 		var err error
 		dsc, ok := data.(discoverapi.DatastoreConfigData)
@@ -161,7 +170,7 @@ func (d *driver) restoreEndpoints() error {
 		}
 
 		n.incEndpointCount()
-		d.peerDbAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), true)
+		d.peerAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), true, false, false, true)
 	}
 	return nil
 }
@@ -170,6 +179,11 @@ func (d *driver) restoreEndpoints() error {
 func Fini(drv driverapi.Driver) {
 	d := drv.(*driver)
 
+	// Notify the peer go routine to return
+	if d.peerOpCancel != nil {
+		d.peerOpCancel()
+	}
+
 	if d.exitCh != nil {
 		waitCh := make(chan struct{})
 

components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/peerdb.go

@@ -1,12 +1,14 @@
 package overlay
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"sync"
 	"syscall"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/docker/libnetwork/common"
 )
 
 const ovPeerTable = "overlay_peer_table"
@@ -59,8 +61,6 @@ func (pKey *peerKey) Scan(state fmt.ScanState, verb rune) error {
 	return nil
 }
 
-var peerDbWg sync.WaitGroup
-
 func (d *driver) peerDbWalk(f func(string, *peerKey, *peerEntry) bool) error {
 	d.peerDb.Lock()
 	nids := []string{}
@@ -141,8 +141,6 @@ func (d *driver) peerDbSearch(nid string, peerIP net.IP) (net.HardwareAddr, net.
 func (d *driver) peerDbAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
 	peerMac net.HardwareAddr, vtep net.IP, isLocal bool) {
 
-	peerDbWg.Wait()
-
 	d.peerDb.Lock()
 	pMap, ok := d.peerDb.mp[nid]
 	if !ok {
@@ -173,7 +171,6 @@ func (d *driver) peerDbAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask
 
 func (d *driver) peerDbDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
 	peerMac net.HardwareAddr, vtep net.IP) peerEntry {
-	peerDbWg.Wait()
 
 	d.peerDb.Lock()
 	pMap, ok := d.peerDb.mp[nid]
@@ -206,61 +203,109 @@ func (d *driver) peerDbDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPM
 	return pEntry
 }
 
-func (d *driver) peerDbUpdateSandbox(nid string) {
-	// The following logic is useful only in non swarm mode
-	// In swarm mode instead the programmig will come directly from networkDB
-	if !d.isSerfAlive() {
-		return
-	}
+// The overlay uses a lazy initialization approach, this means that when a network is created
+// and the driver registered the overlay does not allocate resources till the moment that a
+// sandbox is actually created.
+// At the moment of this call, that happens when a sandbox is initialized, is possible that
+// networkDB has already delivered some events of peers already available on remote nodes,
+// these peers are saved into the peerDB and this function is used to properly configure
+// the network sandbox with all those peers that got previously notified.
+// Note also that this method sends a single message on the channel and the go routine on the
+// other side, will atomically loop on the whole table of peers and will program their state
+// in one single atomic operation. This is fundamental to guarantee consistency, and avoid that
+// new peerAdd or peerDelete gets reordered during the sandbox init.
+func (d *driver) initSandboxPeerDB(nid string) {
+	d.peerInit(nid)
+}
 
-	d.peerDb.Lock()
-	pMap, ok := d.peerDb.mp[nid]
-	if !ok {
-		d.peerDb.Unlock()
-		return
-	}
-	d.peerDb.Unlock()
+type peerOperationType int32
 
-	peerDbWg.Add(1)
+const (
+	peerOperationINIT peerOperationType = iota
+	peerOperationADD
+	peerOperationDELETE
+)
 
-	var peerOps []func()
-	pMap.Lock()
-	for pKeyStr, pEntry := range pMap.mp {
-		var pKey peerKey
-		if _, err := fmt.Sscan(pKeyStr, &pKey); err != nil {
-			fmt.Printf("peer key scan failed: %v", err)
-		}
+type peerOperation struct {
+	opType     peerOperationType
+	networkID  string
+	endpointID string
+	peerIP     net.IP
+	peerIPMask net.IPMask
+	peerMac    net.HardwareAddr
+	vtepIP     net.IP
+	updateDB   bool
+	l2Miss     bool
+	l3Miss     bool
+	localPeer  bool
+	callerName string
+}
 
-		if pEntry.isLocal {
-			continue
-		}
-
-		// Go captures variables by reference. The pEntry could be
-		// pointing to the same memory location for every iteration. Make
-		// a copy of pEntry before capturing it in the following closure.
-		entry := pEntry
-		op := func() {
-			if err := d.peerAdd(nid, entry.eid, pKey.peerIP, entry.peerIPMask,
-				pKey.peerMac, entry.vtep,
-				false, false, false); err != nil {
-				fmt.Printf("peerdbupdate in sandbox failed for ip %s and mac %s: %v",
-					pKey.peerIP, pKey.peerMac, err)
+func (d *driver) peerOpRoutine(ctx context.Context, ch chan *peerOperation) {
+	var err error
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case op := <-ch:
+			switch op.opType {
+			case peerOperationINIT:
+				err = d.peerInitOp(op.networkID)
+			case peerOperationADD:
+				err = d.peerAddOp(op.networkID, op.endpointID, op.peerIP, op.peerIPMask, op.peerMac, op.vtepIP, op.updateDB, op.l2Miss, op.l3Miss, op.localPeer)
+			case peerOperationDELETE:
+				err = d.peerDeleteOp(op.networkID, op.endpointID, op.peerIP, op.peerIPMask, op.peerMac, op.vtepIP, op.updateDB)
+			}
+			if err != nil {
+				logrus.Warnf("Peer operation failed:%s op:%v", err, op)
 			}
 		}
-
-		peerOps = append(peerOps, op)
 	}
-	pMap.Unlock()
+}
 
-	for _, op := range peerOps {
-		op()
+func (d *driver) peerInit(nid string) {
+	callerName := common.CallerName(1)
+	d.peerOpCh <- &peerOperation{
+		opType:     peerOperationINIT,
+		networkID:  nid,
+		callerName: callerName,
 	}
+}
 
-	peerDbWg.Done()
+func (d *driver) peerInitOp(nid string) error {
+	return d.peerDbNetworkWalk(nid, func(pKey *peerKey, pEntry *peerEntry) bool {
+		// Local entries do not need to be added
+		if pEntry.isLocal {
+			return false
+		}
+
+		d.peerAddOp(nid, pEntry.eid, pKey.peerIP, pEntry.peerIPMask, pKey.peerMac, pEntry.vtep, false, false, false, false)
+		// return false to loop on all entries
+		return false
+	})
 }
 
 func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
-	peerMac net.HardwareAddr, vtep net.IP, updateDb, l2Miss, l3Miss bool) error {
+	peerMac net.HardwareAddr, vtep net.IP, updateDb, l2Miss, l3Miss, localPeer bool) {
+	callerName := common.CallerName(1)
+	d.peerOpCh <- &peerOperation{
+		opType:     peerOperationADD,
+		networkID:  nid,
+		endpointID: eid,
+		peerIP:     peerIP,
+		peerIPMask: peerIPMask,
+		peerMac:    peerMac,
+		vtepIP:     vtep,
+		updateDB:   updateDb,
+		l2Miss:     l2Miss,
+		l3Miss:     l3Miss,
+		localPeer:  localPeer,
+		callerName: callerName,
+	}
+}
+
+func (d *driver) peerAddOp(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
+	peerMac net.HardwareAddr, vtep net.IP, updateDb, l2Miss, l3Miss, updateOnlyDB bool) error {
 
 	if err := validateID(nid, eid); err != nil {
 		return err
@@ -268,6 +313,9 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
 
 	if updateDb {
 		d.peerDbAdd(nid, eid, peerIP, peerIPMask, peerMac, vtep, false)
+		if updateOnlyDB {
+			return nil
+		}
 	}
 
 	n := d.network(nid)
@@ -277,6 +325,9 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
 
 	sbox := n.sandbox()
 	if sbox == nil {
+		// We are hitting this case for all the events that are arriving before that the sandbox
+		// is being created. The peer got already added into the database and the sanbox init will
+		// call the peerDbUpdateSandbox that will configure all these peers from the database
 		return nil
 	}
 
@@ -317,6 +368,22 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
 }
 
 func (d *driver) peerDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
+	peerMac net.HardwareAddr, vtep net.IP, updateDb bool) {
+	callerName := common.CallerName(1)
+	d.peerOpCh <- &peerOperation{
+		opType:     peerOperationDELETE,
+		networkID:  nid,
+		endpointID: eid,
+		peerIP:     peerIP,
+		peerIPMask: peerIPMask,
+		peerMac:    peerMac,
+		vtepIP:     vtep,
+		updateDB:   updateDb,
+		callerName: callerName,
+	}
+}
+
+func (d *driver) peerDeleteOp(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,
 	peerMac net.HardwareAddr, vtep net.IP, updateDb bool) error {
 
 	if err := validateID(nid, eid); err != nil {

components/engine/vendor/github.com/docker/libnetwork/endpoint_info.go

@@ -415,7 +415,7 @@ func (epj *endpointJoinInfo) UnmarshalJSON(b []byte) error {
 		return err
 	}
 	if v, ok := epMap["gw"]; ok {
-		epj.gw6 = net.ParseIP(v.(string))
+		epj.gw = net.ParseIP(v.(string))
 	}
 	if v, ok := epMap["gw6"]; ok {
 		epj.gw6 = net.ParseIP(v.(string))
@@ -444,6 +444,6 @@ func (epj *endpointJoinInfo) CopyTo(dstEpj *endpointJoinInfo) error {
 	dstEpj.driverTableEntries = make([]*tableEntry, len(epj.driverTableEntries))
 	copy(dstEpj.driverTableEntries, epj.driverTableEntries)
 	dstEpj.gw = types.GetIPCopy(epj.gw)
-	dstEpj.gw = types.GetIPCopy(epj.gw6)
+	dstEpj.gw6 = types.GetIPCopy(epj.gw6)
 	return nil
 }

components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.go

@@ -467,7 +467,7 @@ func (nDB *NetworkDB) deleteNodeTableEntries(node string) {
 // WalkTable walks a single table in NetworkDB and invokes the passed
 // function for each entry in the table passing the network, key,
 // value. The walk stops if the passed function returns a true.
-func (nDB *NetworkDB) WalkTable(tname string, fn func(string, string, []byte) bool) error {
+func (nDB *NetworkDB) WalkTable(tname string, fn func(string, string, []byte, bool) bool) error {
 	nDB.RLock()
 	values := make(map[string]interface{})
 	nDB.indexes[byTable].WalkPrefix(fmt.Sprintf("/%s", tname), func(path string, v interface{}) bool {
@@ -480,7 +480,7 @@ func (nDB *NetworkDB) WalkTable(tname string, fn func(string, string, []byte) bo
 		params := strings.Split(k[1:], "/")
 		nid := params[1]
 		key := params[2]
-		if fn(nid, key, v.(*entry).value) {
+		if fn(nid, key, v.(*entry).value, v.(*entry).deleting) {
 			return nil
 		}
 	}

components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/task.go

@@ -67,7 +67,29 @@ func IsTaskDirty(s *api.Service, t *api.Task) bool {
 		return false
 	}
 
-	return !reflect.DeepEqual(s.Spec.Task, t.Spec) ||
+	// Make a deep copy of the service and task spec for the comparison.
+	serviceTaskSpec := *s.Spec.Task.Copy()
+
+	// For non-failed tasks with a container spec runtime that have already
+	// pulled the required image (i.e., current state is between READY and
+	// RUNNING inclusively), ignore the value of the `PullOptions` field by
+	// setting the copied service to have the same PullOptions value as the
+	// task. A difference in only the `PullOptions` field should not cause
+	// a running (or ready to run) task to be considered 'dirty' when we
+	// handle updates.
+	// See https://github.com/docker/swarmkit/issues/971
+	currentState := t.Status.State
+	// Ignore PullOpts if the task is desired to be in a "runnable" state
+	// and its last known current state is between READY and RUNNING in
+	// which case we know that the task either successfully pulled its
+	// container image or didn't need to.
+	ignorePullOpts := t.DesiredState <= api.TaskStateRunning && currentState >= api.TaskStateReady && currentState <= api.TaskStateRunning
+	if ignorePullOpts && serviceTaskSpec.GetContainer() != nil && t.Spec.GetContainer() != nil {
+		// Modify the service's container spec.
+		serviceTaskSpec.GetContainer().PullOptions = t.Spec.GetContainer().PullOptions
+	}
+
+	return !reflect.DeepEqual(serviceTaskSpec, t.Spec) ||
 		(t.Endpoint != nil && !reflect.DeepEqual(s.Spec.Endpoint, t.Endpoint.Spec))
 }