From 9caba3315b4e3aea0426bad6d73724f763bce349 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 8 Jun 2018 20:34:29 -0700 Subject: [PATCH 1/3] bump libnetwork to 19279f0492417475b6bfbd0aa529f73e8f178fb5 includes; - docker/libnetwork#2178 Fix possible race on ingress programming - docker/libnetwork#2180 Fix spurious deadlock in overlay driver Signed-off-by: Sebastiaan van Stijn Upstream-commit: 6630f214fae3bc7c8b802a0b58e1bacd9adc6148 Component: engine --- .../engine/hack/dockerfile/install/proxy.installer | 2 +- components/engine/vendor.conf | 2 +- .../docker/libnetwork/drivers/overlay/ov_network.go | 12 ++++++++++-- .../github.com/docker/libnetwork/service_linux.go | 9 ++++----- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/components/engine/hack/dockerfile/install/proxy.installer b/components/engine/hack/dockerfile/install/proxy.installer index c10ce330ef..00c2f1dd05 100755 --- a/components/engine/hack/dockerfile/install/proxy.installer +++ b/components/engine/hack/dockerfile/install/proxy.installer @@ -3,7 +3,7 @@ # LIBNETWORK_COMMIT is used to build the docker-userland-proxy binary. When # updating the binary version, consider updating github.com/docker/libnetwork # in vendor.conf accordingly -LIBNETWORK_COMMIT=3931ba4d815e385ab97093c64477b82f14dadefb +LIBNETWORK_COMMIT=19279f0492417475b6bfbd0aa529f73e8f178fb5 install_proxy() { case "$1" in diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index 858182b841..719970d273 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -35,7 +35,7 @@ github.com/opentracing/opentracing-go 1361b9cd60be79c4c3a7fa9841b3c132e40066a7 #get libnetwork packages # When updating, also update LIBNETWORK_COMMIT in hack/dockerfile/install/proxy accordingly -github.com/docker/libnetwork 3931ba4d815e385ab97093c64477b82f14dadefb +github.com/docker/libnetwork 19279f0492417475b6bfbd0aa529f73e8f178fb5 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go index 9c83127d4f..8b6839656b 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go @@ -244,7 +244,15 @@ func (d *driver) DeleteNetwork(nid string) error { } d.Lock() - defer d.Unlock() + // Only perform a peer flush operation (if required) AFTER unlocking + // the driver lock to avoid deadlocking w/ the peerDB. + var doPeerFlush bool + defer func() { + d.Unlock() + if doPeerFlush { + d.peerFlush(nid) + } + }() // This is similar to d.network(), but we need to keep holding the lock // until we are done removing this network. @@ -270,7 +278,7 @@ func (d *driver) DeleteNetwork(nid string) error { } } // flush the peerDB entries - d.peerFlush(nid) + doPeerFlush = true delete(d.networks, nid) vnis, err := n.releaseVxlanID() diff --git a/components/engine/vendor/github.com/docker/libnetwork/service_linux.go b/components/engine/vendor/github.com/docker/libnetwork/service_linux.go index ef0590df78..a95ece8a08 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/service_linux.go +++ b/components/engine/vendor/github.com/docker/libnetwork/service_linux.go @@ -279,7 +279,7 @@ const ingressChain = "DOCKER-INGRESS" var ( ingressOnce sync.Once - ingressProxyMu sync.Mutex + ingressMu sync.Mutex // lock for operations on ingress ingressProxyTbl = make(map[string]io.Closer) portConfigMu sync.Mutex portConfigTbl = make(map[PortConfig]int) @@ -328,6 +328,9 @@ func programIngress(gwIP net.IP, ingressPorts []*PortConfig, isDelete bool) erro addDelOpt = "-D" } + ingressMu.Lock() + defer ingressMu.Unlock() + chainExists := iptables.ExistChain(ingressChain, iptables.Nat) filterChainExists := iptables.ExistChain(ingressChain, iptables.Filter) @@ -497,13 +500,11 @@ func plumbProxy(iPort *PortConfig, isDelete bool) error { portSpec := fmt.Sprintf("%d/%s", iPort.PublishedPort, strings.ToLower(PortConfig_Protocol_name[int32(iPort.Protocol)])) if isDelete { - ingressProxyMu.Lock() if listener, ok := ingressProxyTbl[portSpec]; ok { if listener != nil { listener.Close() } } - ingressProxyMu.Unlock() return nil } @@ -523,9 +524,7 @@ func plumbProxy(iPort *PortConfig, isDelete bool) error { return err } - ingressProxyMu.Lock() ingressProxyTbl[portSpec] = l - ingressProxyMu.Unlock() return nil } From ec203c5bb5fd38db43a50366f3df826cfb4e890d Mon Sep 17 00:00:00 2001 From: Arash Deshmeh Date: Sat, 9 Jun 2018 06:39:22 -0400 Subject: [PATCH 2/3] refactor delete network integration tests to use network package Signed-off-by: Arash Deshmeh Upstream-commit: 991d5121592eeb1e4442516f957ed484d8b687c7 Component: engine --- .../engine/integration/network/delete_test.go | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/components/engine/integration/network/delete_test.go b/components/engine/integration/network/delete_test.go index f937da5274..83d00bbdf1 100644 --- a/components/engine/integration/network/delete_test.go +++ b/components/engine/integration/network/delete_test.go @@ -6,15 +6,16 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/versions" + "github.com/docker/docker/integration/internal/network" "github.com/docker/docker/internal/test/request" "github.com/gotestyourself/gotestyourself/assert" is "github.com/gotestyourself/gotestyourself/assert/cmp" "github.com/gotestyourself/gotestyourself/skip" ) -func containsNetwork(nws []types.NetworkResource, nw types.NetworkCreateResponse) bool { +func containsNetwork(nws []types.NetworkResource, networkID string) bool { for _, n := range nws { - if n.ID == nw.ID { + if n.ID == networkID { return true } } @@ -26,16 +27,13 @@ func containsNetwork(nws []types.NetworkResource, nw types.NetworkCreateResponse // first network's ID as name. // // After successful creation, properties of all three networks is returned -func createAmbiguousNetworks(t *testing.T) (types.NetworkCreateResponse, types.NetworkCreateResponse, types.NetworkCreateResponse) { +func createAmbiguousNetworks(t *testing.T) (string, string, string) { client := request.NewAPIClient(t) ctx := context.Background() - testNet, err := client.NetworkCreate(ctx, "testNet", types.NetworkCreate{}) - assert.NilError(t, err) - idPrefixNet, err := client.NetworkCreate(ctx, testNet.ID[:12], types.NetworkCreate{}) - assert.NilError(t, err) - fullIDNet, err := client.NetworkCreate(ctx, testNet.ID, types.NetworkCreate{}) - assert.NilError(t, err) + testNet := network.CreateNoError(t, ctx, client, "testNet") + idPrefixNet := network.CreateNoError(t, ctx, client, testNet[:12]) + fullIDNet := network.CreateNoError(t, ctx, client, testNet) nws, err := client.NetworkList(ctx, types.NetworkListOptions{}) assert.NilError(t, err) @@ -58,12 +56,12 @@ func TestDockerNetworkDeletePreferID(t *testing.T) { // Delete the network using a prefix of the first network's ID as name. // This should the network name with the id-prefix, not the original network. - err := client.NetworkRemove(ctx, testNet.ID[:12]) + err := client.NetworkRemove(ctx, testNet[:12]) assert.NilError(t, err) // Delete the network using networkID. This should remove the original // network, not the network with the name equal to the networkID - err = client.NetworkRemove(ctx, testNet.ID) + err = client.NetworkRemove(ctx, testNet) assert.NilError(t, err) // networks "testNet" and "idPrefixNet" should be removed, but "fullIDNet" should still exist From 693307ee7dd075b9eca4250fc21f09dd58a60098 Mon Sep 17 00:00:00 2001 From: unclejack Date: Sun, 10 Jun 2018 17:41:22 +0300 Subject: [PATCH 3/3] Dockerfile*: bump Go to 1.10.3 Signed-off-by: Cristian Staretu Upstream-commit: 599a3b888cc96ba7bfac7f81a29d4e0688451805 Component: engine --- components/engine/Dockerfile | 4 ++-- components/engine/Dockerfile.e2e | 2 +- components/engine/Dockerfile.simple | 2 +- components/engine/Dockerfile.windows | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 4db75a6722..38ca482a5a 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -24,10 +24,10 @@ # the case. Therefore, you don't have to disable it anymore. # -FROM golang:1.10.2 AS base +FROM golang:1.10.3 AS base # FIXME(vdemeester) this is kept for other script depending on it to not fail right away # Remove this once the other scripts uses something else to detect the version -ENV GO_VERSION 1.10.2 +ENV GO_VERSION 1.10.3 # allow replacing httpredir or deb mirror ARG APT_MIRROR=deb.debian.org RUN sed -ri "s/(httpredir|deb).debian.org/$APT_MIRROR/g" /etc/apt/sources.list diff --git a/components/engine/Dockerfile.e2e b/components/engine/Dockerfile.e2e index 77c9205d45..663a58af33 100644 --- a/components/engine/Dockerfile.e2e +++ b/components/engine/Dockerfile.e2e @@ -1,5 +1,5 @@ ## Step 1: Build tests -FROM golang:1.10.2-alpine3.7 as builder +FROM golang:1.10.3-alpine3.7 as builder RUN apk add --update \ bash \ diff --git a/components/engine/Dockerfile.simple b/components/engine/Dockerfile.simple index 67a78003e1..b0338ac0c8 100644 --- a/components/engine/Dockerfile.simple +++ b/components/engine/Dockerfile.simple @@ -42,7 +42,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # will need updating, to avoid errors. Ping #docker-maintainers on IRC # with a heads-up. # IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored -ENV GO_VERSION 1.10.2 +ENV GO_VERSION 1.10.3 RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz" \ | tar -xzC /usr/local ENV PATH /go/bin:/usr/local/go/bin:$PATH diff --git a/components/engine/Dockerfile.windows b/components/engine/Dockerfile.windows index 0cc7aa4f5b..1b2c1f3c8a 100644 --- a/components/engine/Dockerfile.windows +++ b/components/engine/Dockerfile.windows @@ -161,7 +161,7 @@ SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPref # Environment variable notes: # - GO_VERSION must be consistent with 'Dockerfile' used by Linux. # - FROM_DOCKERFILE is used for detection of building within a container. -ENV GO_VERSION=1.10.2 ` +ENV GO_VERSION=1.10.3 ` GIT_VERSION=2.11.1 ` GOPATH=C:\go ` FROM_DOCKERFILE=1