Merge pull request #2373 from tiborvass/19.03-bump-grpc-1.23.1

[19.03 backport] Bump grpc to v1.23.1
vendor: bump google.golang.org/grpc v1.23.1
2026-08-28 02:24:15 -05:00 · 2020-03-03 22:59:52 +01:00 · 2020-03-03 21:39:18 +00:00 · 2020-03-03 21:39:13 +00:00 · 2020-02-26 17:18:59 +01:00 · 2020-02-26 16:35:08 +01:00
106 changed files with 5727 additions and 3153 deletions
@@ -4,7 +4,7 @@ clone_folder: c:\gopath\src\github.com\docker\cli

 environment:
  GOPATH: c:\gopath
-  GOVERSION: 1.12.15
+  GOVERSION: 1.12.17
  DEPVERSION: v0.4.1

 install:
@@ -44,8 +44,8 @@ func NewPruneCommand(dockerCli command.Cli) *cobra.Command {

 	flags := cmd.Flags()
 	flags.BoolVarP(&options.force, "force", "f", false, "Do not prompt for confirmation")
-	flags.BoolVarP(&options.all, "all", "a", false, "Remove all unused images, not just dangling ones")
-	flags.Var(&options.filter, "filter", "Provide filter values (e.g. 'unused-for=24h')")
+	flags.BoolVarP(&options.all, "all", "a", false, "Remove all unused build cache, not just dangling ones")
+	flags.Var(&options.filter, "filter", "Provide filter values (e.g. 'until=24h')")
 	flags.Var(&options.keepStorage, "keep-storage", "Amount of disk space to keep for cache")

 	return cmd
@@ -35,7 +35,7 @@ func NewRmCommand(dockerCli command.Cli) *cobra.Command {
 	}

 	flags := cmd.Flags()
-	flags.BoolVarP(&opts.rmVolumes, "volumes", "v", false, "Remove the volumes associated with the container")
+	flags.BoolVarP(&opts.rmVolumes, "volumes", "v", false, "Remove anonymous volumes associated with the container")
 	flags.BoolVarP(&opts.rmLink, "link", "l", false, "Remove the specified link")
 	flags.BoolVarP(&opts.force, "force", "f", false, "Force the removal of a running container (uses SIGKILL)")
 	return cmd
@@ -439,7 +439,7 @@ complete -c docker -f -n '__fish_docker_no_subcommand' -a rm -d 'Remove one or m
 complete -c docker -A -f -n '__fish_seen_subcommand_from rm' -s f -l force -d 'Force the removal of a running container (uses SIGKILL)'
 complete -c docker -A -f -n '__fish_seen_subcommand_from rm' -l help -d 'Print usage'
 complete -c docker -A -f -n '__fish_seen_subcommand_from rm' -s l -l link -d 'Remove the specified link and not the underlying container'
-complete -c docker -A -f -n '__fish_seen_subcommand_from rm' -s v -l volumes -d 'Remove the volumes associated with the container'
+complete -c docker -A -f -n '__fish_seen_subcommand_from rm' -s v -l volumes -d 'Remove anonymous volumes associated with the container'
 complete -c docker -A -f -n '__fish_seen_subcommand_from rm' -a '(__fish_print_docker_containers stopped)' -d "Container"
 complete -c docker -A -f -n '__fish_seen_subcommand_from rm' -s f -l force -a '(__fish_print_docker_containers all)' -d "Container"

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.12.15
+ARG GO_VERSION=1.12.17

 FROM    golang:${GO_VERSION}-alpine

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.12.15
+ARG GO_VERSION=1.12.17

 FROM	dockercore/golang-cross:${GO_VERSION}
 ENV     DISABLE_WARN_OUTSIDE_CONTAINER=1
@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.12.15
+ARG GO_VERSION=1.12.17

 FROM    golang:${GO_VERSION}-alpine

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.12.15
+ARG GO_VERSION=1.12.17

 # Use Debian based image as docker-compose requires glibc.
 FROM golang:${GO_VERSION}
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.1.3-experimental

-ARG GO_VERSION=1.12.15
+ARG GO_VERSION=1.12.17
 ARG GOLANGCI_LINTER_SHA="v1.21.0"

 FROM    golang:${GO_VERSION}-alpine AS build
@@ -15,15 +15,68 @@ keywords: "docker, documentation, about, technology, deprecate"

 # Deprecated Engine Features

-The following list of features are deprecated in Engine.
-To learn more about Docker Engine's deprecation policy,
-see [Feature Deprecation Policy](https://docs.docker.com/engine/#feature-deprecation-policy).
+The following list of features are deprecated in Engine. To learn more about Docker
+Engine's deprecation policy, see [Feature Deprecation Policy](https://docs.docker.com/engine/#feature-deprecation-policy).
+
+The table below provides an overview of the current status of deprecated features:
+
+- **Deprecated**: the feature is marked "deprecated" and should no longer be used.
+  The feature may be removed, disabled, or change behavior in a future release.
+  The _"Deprecated"_ column contains the release in which the feature was marked 
+  deprecated, whereas the _"Remove"_ column contains a tentative release in which
+  the feature is to be removed. If no release is included in the _"Remove"_ column,
+  the release is yet to be decided on.
+- **Removed**: the feature was removed, disabled, or hidden. Refer to the linked
+  section for details. Some features are "soft" deprecated, which means that they
+  remain functional for backward compatibility, and to allow users to migrate to
+  alternatives. In such cases, a warning may be printed, and users should not rely
+  on this feature.
+
+Status     | Feature                                                                                                                            | Deprecated | Remove
+-----------|------------------------------------------------------------------------------------------------------------------------------------|------------|------------
+Deprecated | [Pushing and pulling with image manifest v2 schema 1](#pushing-and-pulling-with-image-manifest-v2-schema-1)                        | v19.03.0   | v20.03.0
+Deprecated | [`docker engine` subcommands](#docker-engine-subcommands)                                                                          | v19.03.0   | v20.03.0
+Deprecated | [Top-level `docker deploy` subcommand (experimental)](#top-level-docker-deploy-subcommand-experimental)                            | v19.03.0   | v20.03.0
+Deprecated | [`docker stack deploy` using "dab" files (experimental)](#docker-stack-deploy-using-dab-files-experimental)                        | v19.03.0   | v20.03.0
+Deprecated | [AuFS storage driver](#aufs-storage-driver)                                                                                        | v19.03.0   | -
+Deprecated | [Legacy "overlay" storage driver](#legacy-overlay-storage-driver)                                                                  | v18.09.0   | -
+Deprecated | [Device mapper storage driver](#device-mapper-storage-driver)                                                                      | v18.09.0   | -
+Deprecated | [Reserved namespaces in engine labels](#reserved-namespaces-in-engine-labels)                                                      | v18.06.0   | v20.03.0
+Removed    | [`--disable-legacy-registry` override daemon option](#--disable-legacy-registry-override-daemon-option)                            | v17.12.0   | v19.03.0
+Removed    | [Interacting with V1 registries](#interacting-with-v1-registries)                                                                  | v17.06.0   | v17.12.0
+Removed    | [Asynchronous `service create` and `service update` as default](#asynchronous-service-create-and-service-update-as-default)        | v17.05.0   | v17.10.0
+Removed    | [`-g` and `--graph` flags on `dockerd`](#-g-and---graph-flags-on-dockerd)                                                          | v17.05.0   | -
+Deprecated | [Top-level network properties in NetworkSettings](#top-level-network-properties-in-networksettings)                                | v1.13.0    | v17.12.0
+Deprecated | [`filter` param for `/images/json` endpoint](#filter-param-for-imagesjson-endpoint)                                                | v1.13.0    | v17.12.0
+Removed    | [`repository:shortid` image references](#repositoryshortid-image-references)                                                       | v1.13.0    | v17.12.0
+Removed    | [`docker daemon` subcommand](#docker-daemon-subcommand)                                                                            | v1.13.0    | v17.12.0
+Removed    | [Duplicate keys with conflicting values in engine labels](#duplicate-keys-with-conflicting-values-in-engine-labels)                | v1.13.0    | v17.12.0
+Deprecated | [`MAINTAINER` in Dockerfile](#maintainer-in-dockerfile)                                                                            | v1.13.0    | -
+Deprecated | [API calls without a version](#api-calls-without-a-version)                                                                        | v1.13.0    | v17.12.0
+Removed    | [Backing filesystem without `d_type` support for overlay/overlay2](#backing-filesystem-without-d_type-support-for-overlayoverlay2) | v1.13.0    | v17.12.0
+Deprecated | [`--automated` and `--stars` flags on `docker search`](#--automated-and---stars-flags-on-docker-search)                            | v1.12.0    | v17.09.0
+Deprecated | [`-h` shorthand for `--help`](#-h-shorthand-for---help)                                                                            | v1.12.0    | v17.09.0
+Removed    | [`-e` and `--email` flags on `docker login`](#-e-and---email-flags-on-docker-login)                                                | v1.11.0    | v17.06.0
+Deprecated | [Separator (`:`) of `--security-opt` flag on `docker run`](#separator--of---security-opt-flag-on-docker-run)                       | v1.11.0    | v17.06.0
+Deprecated | [Ambiguous event fields in API](#ambiguous-event-fields-in-api)                                                                    | v1.10.0    | -
+Removed    | [`-f` flag on `docker tag`](#-f-flag-on-docker-tag)                                                                                | v1.10.0    | v1.12.0
+Removed    | [HostConfig at API container start](#hostconfig-at-api-container-start)                                                            | v1.10.0    | v1.12.0
+Removed    | [`--before` and `--since` flags on `docker ps`](#--before-and---since-flags-on-docker-ps)                                          | v1.10.0    | v1.12.0
+Removed    | [Driver-specific log tags](#driver-specific-log-tags)                                                                              | v1.9.0     | v1.12.0
+Removed    | [Docker Content Trust `ENV` passphrase variables name change](#docker-content-trust-env-passphrase-variables-name-change)          | v1.9.0     | v1.12.0
+Removed    | [`/containers/(id or name)/copy` endpoint](#containersid-or-namecopy-endpoint)                                                     | v1.8.0     | v1.12.0
+Removed    | [LXC built-in exec driver](#lxc-built-in-exec-driver)                                                                              | v1.8.0     | v1.10.0
+Removed    | [Old Command Line Options](#old-command-line-options)                                                                              | v1.8.0     | v1.10.0
+Removed    | [`--api-enable-cors` flag on `dockerd`](#--api-enable-cors-flag-on-dockerd)                                                        | v1.6.0     | v17.09.0
+Removed    | [`--run` flag on `docker commit`](#--run-flag-on-docker-commit)                                                                    | v0.10.0    | v1.13.0
+Removed    | [Three arguments form in `docker import`](#three-arguments-form-in-docker-import)                                                  | v0.6.7     | v1.12.0
+

 ### Pushing and pulling with image manifest v2 schema 1

 **Deprecated in Release: v19.03.0**

-**Target For Removal In Release: v19.09.0**
+**Target For Removal In Release: v20.03.0**

 The image manifest
 [v2 schema 1](https://github.com/docker/distribution/blob/fda42e5ef908bdba722d435ff1f330d40dfcd56c/docs/spec/manifest-v2-1.md)
@@ -32,36 +85,46 @@ format is deprecated in favor of the

 If the registry you are using still supports v2 schema 1, urge their administrators to move to v2 schema 2.

-### Legacy "overlay" storage driver

-**Deprecated in Release: v18.09.0**
+### `docker engine` subcommands

-The `overlay` storage driver is deprecated in favor of the `overlay2` storage
-driver, which has all the benefits of `overlay`, without its limitations (excessive
-inode consumption). The legacy `overlay` storage driver will be removed in a future
-release. Users of the `overlay` storage driver should migrate to the `overlay2`
-storage driver.
+**Deprecated in Release: v19.03.0**

-The legacy `overlay` storage driver allowed using overlayFS-backed filesystems
-on pre 4.x kernels. Now that all supported distributions are able to run `overlay2`
-(as they are either on kernel 4.x, or have support for multiple lowerdirs
-backported), there is no reason to keep maintaining the `overlay` storage driver.
+**Target For Removal In Release: v20.03.0**

-### device mapper storage driver
+The `docker engine activate`, `docker engine check`, and `docker engine update`
+provided an alternative installation method to upgrade Docker Community engines
+to Docker Enterprise, using an image-based distribution of the Docker Engine.

-**Deprecated in Release: v18.09.0**
+This feature was only available on Linux, and only when executed on a local node.
+Given the limitations of this feature, and the feature not getting widely adopted,
+the `docker engine` subcommands will be removed, in favor of installation through
+standard package  managers.

-The `devicemapper` storage driver is deprecated in favor of `overlay2`, and will
-be removed in a future release. Users of the `devicemapper` storage driver are
-recommended to migrate to a different storage driver, such as `overlay2`, which
-is now the default storage driver.

-The `devicemapper` storage driver facilitates running Docker on older (3.x) kernels
-that have no support for other storage drivers (such as overlay2, or AUFS).
+### Top-level `docker deploy` subcommand (experimental)
+
+**Deprecated in Release: v19.03.0**
+
+**Target For Removal In Release: v20.03.0**
+
+The top-level `docker deploy` command (using the "Docker Application Bundle"
+(.dab) file format was introduced as an experimental feature in Docker 1.13 /
+17.03, but superseded by support for Docker Compose files using the `docker stack deploy`
+subcommand.
+
+
+### `docker stack deploy` using "dab" files (experimental)
+
+**Deprecated in Release: v19.03.0**
+
+**Target For Removal In Release: v20.03.0**
+
+With no development being done on this feature, and no active use of the file
+format, support for the DAB file format and the top-level docker deploy command
+(hidden by default in 19.03), will be removed, in favour of `docker stack deploy`
+using compose files.

-Now that support for `overlay2` is added to all supported distros (as they are
-either on kernel 4.x, or have support for multiple lowerdirs backported), there
-is no reason to continue maintenance of the `devicemapper` storage driver.

 ### AuFS storage driver

@@ -81,6 +144,39 @@ is available to all supported distros (as they are either on kernel 4.x, or have
 support for multiple lowerdirs backported), there is no reason to continue
 maintenance of the `aufs` storage driver.

+
+### Legacy "overlay" storage driver
+
+**Deprecated in Release: v18.09.0**
+
+The `overlay` storage driver is deprecated in favor of the `overlay2` storage
+driver, which has all the benefits of `overlay`, without its limitations (excessive
+inode consumption). The legacy `overlay` storage driver will be removed in a future
+release. Users of the `overlay` storage driver should migrate to the `overlay2`
+storage driver.
+
+The legacy `overlay` storage driver allowed using overlayFS-backed filesystems
+on pre 4.x kernels. Now that all supported distributions are able to run `overlay2`
+(as they are either on kernel 4.x, or have support for multiple lowerdirs
+backported), there is no reason to keep maintaining the `overlay` storage driver.
+
+### Device mapper storage driver
+
+**Deprecated in Release: v18.09.0**
+
+The `devicemapper` storage driver is deprecated in favor of `overlay2`, and will
+be removed in a future release. Users of the `devicemapper` storage driver are
+recommended to migrate to a different storage driver, such as `overlay2`, which
+is now the default storage driver.
+
+The `devicemapper` storage driver facilitates running Docker on older (3.x) kernels
+that have no support for other storage drivers (such as overlay2, or AUFS).
+
+Now that support for `overlay2` is added to all supported distros (as they are
+either on kernel 4.x, or have support for multiple lowerdirs backported), there
+is no reason to continue maintenance of the `devicemapper` storage driver.
+
+
 ### Reserved namespaces in engine labels

 **Deprecated in Release: v18.06.0**
@@ -89,9 +185,41 @@ The namespaces `com.docker.*`, `io.docker.*`, and `org.dockerproject.*` in engin
 were always documented to be reserved, but there was never any enforcement.

 Usage of these namespaces will now cause a warning in the engine logs to discourage their
-use, and will error instead in 18.12 and above.
+use, and will error instead in v20.03.0 and above.

-### Asynchronous `service create` and `service update`
+
+### `--disable-legacy-registry` override daemon option
+
+**Disabled In Release: v17.12**
+
+**Removed In Release: v19.03**
+
+The `--disable-legacy-registry` flag was disabled in Docker 17.12 and will print
+an error when used. For this error to be printed, the flag itself is still present,
+but hidden. The flag has been removed in Docker 19.03.
+
+
+### Interacting with V1 registries
+
+**Disabled By Default In Release: v17.06**
+
+**Removed In Release: v17.12**
+
+Version 1.8.3 added a flag (`--disable-legacy-registry=false`) which prevents the
+docker daemon from `pull`, `push`, and `login` operations against v1
+registries.  Though enabled by default, this signals the intent to deprecate
+the v1 protocol.
+
+Support for the v1 protocol to the public registry was removed in 1.13. Any
+mirror configurations using v1 should be updated to use a
+[v2 registry mirror](https://docs.docker.com/registry/recipes/mirror/).
+
+Starting with Docker 17.12, support for V1 registries has been removed, and the
+`--disable-legacy-registry` flag can no longer be used, and `dockerd` will fail to
+start when set.
+
+
+### Asynchronous `service create` and `service update` as default

 **Deprecated In Release: v17.05.0**

@@ -199,12 +327,16 @@ to 17.12, a warning will be printed.
 Please also refer to [#27358](https://github.com/docker/docker/issues/27358) for
 further information.

-### Three arguments form in `docker import`
-**Deprecated In Release: [v0.6.7](https://github.com/docker/docker/releases/tag/v0.6.7)**

-**Removed In Release: [v1.12.0](https://github.com/docker/docker/releases/tag/v1.12.0)**
+### `--automated` and `--stars` flags on `docker search`
+
+**Deprecated in Release: [v1.12.0](https://github.com/docker/docker/releases/tag/v1.12.0)**
+
+**Target For Removal In Release: v17.09**
+
+The `docker search --automated` and `docker search --stars` options are deprecated.
+Use `docker search --filter=is-automated=...` and `docker search --filter=stars=...` instead.

-The `docker import` command format `file|URL|- [REPOSITORY [TAG]]` is deprecated since November 2013. It's no more supported.

 ### `-h` shorthand for `--help`

@@ -231,14 +363,6 @@ The docker login command is removing the ability to automatically register for a

 The flag `--security-opt` doesn't use the colon separator (`:`) anymore to divide keys and values, it uses the equal symbol (`=`) for consistency with other similar flags, like `--storage-opt`.

-### `/containers/(id or name)/copy` endpoint
-
-**Deprecated In Release: [v1.8.0](https://github.com/docker/docker/releases/tag/v1.8.0)**
-
-**Removed In Release: [v1.12.0](https://github.com/docker/docker/releases/tag/v1.12.0)**
-
-The endpoint `/containers/(id or name)/copy` is deprecated in favor of `/containers/(id or name)/archive`.
-
 ### Ambiguous event fields in API
 **Deprecated In Release: [v1.10.0](https://github.com/docker/docker/releases/tag/v1.10.0)**

@@ -269,16 +393,8 @@ defining it at container creation (`POST /containers/create`).
 The `docker ps --before` and `docker ps --since` options are deprecated.
 Use `docker ps --filter=before=...` and `docker ps --filter=since=...` instead.

-### `--automated` and `--stars` flags on `docker search`

-**Deprecated in Release: [v1.12.0](https://github.com/docker/docker/releases/tag/v1.12.0)**
-
-**Target For Removal In Release: v17.09**
-
-The `docker search --automated` and `docker search --stars` options are deprecated.
-Use `docker search --filter=is-automated=...` and `docker search --filter=stars=...` instead.
-
-### Driver Specific Log Tags
+### Driver-specific log tags
 **Deprecated In Release: [v1.9.0](https://github.com/docker/docker/releases/tag/v1.9.0)**

 **Removed In Release: [v1.12.0](https://github.com/docker/docker/releases/tag/v1.12.0)**
@@ -293,6 +409,27 @@ docker --log-driver=syslog --log-opt tag="{{.ImageName}}/{{.Name}}/{{.ID}}"
 {% endraw %}
 ```

+
+### Docker Content Trust ENV passphrase variables name change
+**Deprecated In Release: [v1.9.0](https://github.com/docker/docker/releases/tag/v1.9.0)**
+
+**Removed In Release: [v1.12.0](https://github.com/docker/docker/releases/tag/v1.12.0)**
+
+Since 1.9, Docker Content Trust Offline key has been renamed to Root key and the Tagging key has been renamed to Repository key. Due to this renaming, we're also changing the corresponding environment variables
+
+- DOCKER_CONTENT_TRUST_OFFLINE_PASSPHRASE is now named DOCKER_CONTENT_TRUST_ROOT_PASSPHRASE
+- DOCKER_CONTENT_TRUST_TAGGING_PASSPHRASE is now named DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE
+
+
+### `/containers/(id or name)/copy` endpoint
+
+**Deprecated In Release: [v1.8.0](https://github.com/docker/docker/releases/tag/v1.8.0)**
+
+**Removed In Release: [v1.12.0](https://github.com/docker/docker/releases/tag/v1.12.0)**
+
+The endpoint `/containers/(id or name)/copy` is deprecated in favor of `/containers/(id or name)/archive`.
+
+
 ### LXC built-in exec driver
 **Deprecated In Release: [v1.8.0](https://github.com/docker/docker/releases/tag/v1.8.0)**

@@ -300,6 +437,7 @@ docker --log-driver=syslog --log-opt tag="{{.ImageName}}/{{.Name}}/{{.ID}}"

 The built-in LXC execution driver, the lxc-conf flag, and API fields have been removed.

+
 ### Old Command Line Options
 **Deprecated In Release: [v1.8.0](https://github.com/docker/docker/releases/tag/v1.8.0)**

@@ -358,6 +496,16 @@ The single-dash (`-help`) was removed, in favor of the double-dash `--help`
    docker -help
    docker [COMMAND] -help

+
+### `--api-enable-cors` flag on dockerd
+
+**Deprecated In Release: [v1.6.0](https://github.com/docker/docker/releases/tag/v1.6.0)**
+
+**Removed In Release: [v17.09](https://github.com/docker/docker-ce/releases/tag/v17.09.0-ce)**
+
+The flag `--api-enable-cors` is deprecated since v1.6.0. Use the flag
+`--api-cors-header` instead.
+
 ### `--run` flag on docker commit

 **Deprecated In Release: [v0.10.0](https://github.com/docker/docker/releases/tag/v0.10.0)**
@@ -368,51 +516,10 @@ The flag `--run` of the docker commit (and its short version `-run`) were deprec
 of the `--changes` flag that allows to pass `Dockerfile` commands.


-### Interacting with V1 registries
-
-**Disabled By Default In Release: v17.06**
-
-**Removed In Release: v17.12**
-
-Version 1.8.3 added a flag (`--disable-legacy-registry=false`) which prevents the
-docker daemon from `pull`, `push`, and `login` operations against v1
-registries.  Though enabled by default, this signals the intent to deprecate
-the v1 protocol.
-
-Support for the v1 protocol to the public registry was removed in 1.13. Any
-mirror configurations using v1 should be updated to use a
-[v2 registry mirror](https://docs.docker.com/registry/recipes/mirror/).
-
-Starting with Docker 17.12, support for V1 registries has been removed, and the
-`--disable-legacy-registry` flag can no longer be used, and `dockerd` will fail to
-start when set.
-
-### `--disable-legacy-registry` override daemon option
-
-**Disabled In Release: v17.12**
-
-**Target For Removal In Release: v18.03**
-
-The `--disable-legacy-registry` flag was disabled in Docker 17.12 and will print
-an error when used. For this error to be printed, the flag itself is still present,
-but hidden. The flag will be removed in Docker 18.03.
-
-
-### Docker Content Trust ENV passphrase variables name change
-**Deprecated In Release: [v1.9.0](https://github.com/docker/docker/releases/tag/v1.9.0)**
+### Three arguments form in `docker import`
+**Deprecated In Release: [v0.6.7](https://github.com/docker/docker/releases/tag/v0.6.7)**

 **Removed In Release: [v1.12.0](https://github.com/docker/docker/releases/tag/v1.12.0)**

-Since 1.9, Docker Content Trust Offline key has been renamed to Root key and the Tagging key has been renamed to Repository key. Due to this renaming, we're also changing the corresponding environment variables
+The `docker import` command format `file|URL|- [REPOSITORY [TAG]]` is deprecated since November 2013. It's no more supported.

- DOCKER_CONTENT_TRUST_OFFLINE_PASSPHRASE is now named DOCKER_CONTENT_TRUST_ROOT_PASSPHRASE
- DOCKER_CONTENT_TRUST_TAGGING_PASSPHRASE is now named DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE
-
-### `--api-enable-cors` flag on dockerd
-
-**Deprecated In Release: [v1.6.0](https://github.com/docker/docker/releases/tag/v1.6.0)**
-
-**Removed In Release: [v17.09](https://github.com/docker/docker-ce/releases/tag/v17.09.0-ce)**
-
-The flag `--api-enable-cors` is deprecated since v1.6.0. Use the flag
-`--api-cors-header` instead.
@@ -47,6 +47,7 @@ Options:
                                'host': use the Docker host network stack
                                '<network-name>|<network-id>': connect to a user-defined network
      --no-cache                Do not use cache when building the image
+  -o, --output                  Output destination (format: type=local,dest=path)
      --pull                    Always attempt to pull a newer version of the image
      --progress                Set type of progress output (only if BuildKit enabled) (auto, plain, tty). 
                                Use plain to show container output
@@ -323,7 +324,16 @@ Successfully built 99cc1ad10469
 This example shows the use of the `.dockerignore` file to exclude the `.git`
 directory from the context. Its effect can be seen in the changed size of the
 uploaded context. The builder reference contains detailed information on
-[creating a .dockerignore file](../builder.md#dockerignore-file)
+[creating a .dockerignore file](../builder.md#dockerignore-file).
+
+When using the [BuildKit backend](../builder.md#buildkit), `docker build` searches
+for a `.dockerignore` file relative to the Dockerfile name. For example, running
+`docker build -f myapp.Dockerfile .` will first look for an ignore file named
+`myapp.Dockerfile.dockerignore`. If such a file is not found, the `.dockerignore`
+file is used if present. Using a Dockerfile based `.dockerignore` is useful if a
+project contains multiple Dockerfiles that expect to ignore different sets of
+files.
+

 ### Tag an image (-t)

@@ -489,6 +499,137 @@ FROM alpine AS production-env
 $ docker build -t mybuildimage --target build-env .
 ```

+### Custom build outputs
+
+By default, a local container image is created from the build result. The
+`--output` (or `-o`) flag allows you to override this behavior, and a specify a
+custom exporter. For example, custom exporters allow you to export the build
+artifacts as files on the local filesystem instead of a Docker image, which can
+be useful for generating local binaries, code generation etc.
+
+The value for `--output` is a CSV-formatted string defining the exporter type
+and options. Currently, `local` and `tar` exporters are supported. The `local`
+exporter writes the resulting build files to a directory on the client side. The
+`tar` exporter is similar but writes the files as a single tarball (`.tar`).
+
+If no type is specified, the value defaults to the output directory of the local
+exporter. Use a hyphen (`-`) to write the output tarball to standard output
+(`STDOUT`).
+
+The following example builds an image using the current directory (`.`) as build
+context, and exports the files to a directory named `out` in the current directory.
+If the directory does not exist, Docker creates the directory automatically:
+
+```bash
+$ docker build -o out .
+```
+
+The example above uses the short-hand syntax, omitting the `type` options, and
+thus uses the default (`local`) exporter. The example below shows the equivalent
+using the long-hand CSV syntax, specifying both `type` and `dest` (destination
+path):
+
+```bash
+$ docker build --output type=local,dest=out .
+```
+
+Use the `tar` type to export the files as a `.tar` archive: 
+
+```bash
+$ docker build --output type=tar,dest=out.tar .
+```
+
+The example below shows the equivalent when using the short-hand syntax. In this
+case, `-` is specified as destination, which automatically selects the `tar` type,
+and writes the output tarball to standard output, which is then redirected to
+the `out.tar` file:
+
+```bash
+docker build -o - . > out.tar
+```
+
+The `--output` option exports all files from the target stage. A common pattern
+for exporting only specific files is to do multi-stage builds and to copy the
+desired files to a new scratch stage with [`COPY --from`](../builder.md#copy).
+
+The example `Dockerfile` below uses a separate stage to collect the
+build-artifacts for exporting:
+
+```Dockerfile
+FROM golang AS build-stage
+RUN go get -u github.com/LK4D4/vndr
+
+FROM scratch AS export-stage
+COPY --from=build-stage /go/bin/vndr /
+```
+
+When building the Dockerfile with the `-o` option, only the files from the final
+stage are exported to the `out` directory, in this case, the `vndr` binary:
+
+```bash
+$ docker build -o out .
+
+[+] Building 2.3s (7/7) FINISHED
+ => [internal] load build definition from Dockerfile                                                                          0.1s
+ => => transferring dockerfile: 176B                                                                                          0.0s
+ => [internal] load .dockerignore                                                                                             0.0s
+ => => transferring context: 2B                                                                                               0.0s
+ => [internal] load metadata for docker.io/library/golang:latest                                                              1.6s
+ => [build-stage 1/2] FROM docker.io/library/golang@sha256:2df96417dca0561bf1027742dcc5b446a18957cd28eba6aa79269f23f1846d3f   0.0s
+ => => resolve docker.io/library/golang@sha256:2df96417dca0561bf1027742dcc5b446a18957cd28eba6aa79269f23f1846d3f               0.0s
+ => CACHED [build-stage 2/2] RUN go get -u github.com/LK4D4/vndr                                                              0.0s
+ => [export-stage 1/1] COPY --from=build-stage /go/bin/vndr /                                                                 0.2s
+ => exporting to client                                                                                                       0.4s
+ => => copying files 10.30MB                                                                                                  0.3s
+
+$ ls ./out
+vndr
+```
+
+> **Note**: This feature requires the BuildKit backend. You can either
+> [enable BuildKit](../builder.md#buildkit) or use the [buildx](https://github.com/docker/buildx)
+> plugin which provides more output type options.
+
+### Specifying external cache sources
+
+In addition to local build cache, the builder can reuse the cache generated from
+previous builds with the `--cache-from` flag pointing to an image in the registry.
+
+To use an image as a cache source, cache metadata needs to be written into the
+image on creation. This can be done by setting `--build-arg BUILDKIT_INLINE_CACHE=1`
+when building the image. After that, the built image can be used as a cache source
+for subsequent builds.
+
+Upon importing the cache, the builder will only pull the JSON metadata from the
+registry and determine possible cache hits based on that information. If there
+is a cache hit, the matched layers are pulled into the local environment.
+
+In addition to images, the cache can also be pulled from special cache manifests
+generated by [`buildx`](https://github.com/docker/buildx) or the BuildKit CLI
+(`buildctl`). These manifests (when built with the `type=registry` and `mode=max`
+options) allow pulling layer data for intermediate stages in multi-stage builds.
+
+The following example builds an image with inline-cache metadata and pushes it
+to a registry, then uses the image as a cache source on another machine:
+
+```bash
+$ docker build -t myname/myapp --build-arg BUILDKIT_INLINE_CACHE=1 .
+$ docker push myname/myapp
+```
+
+After pushing the image, the image is used as cache source on another machine.
+BuildKit automatically pulls the image from the registry if needed.
+
+```bash
+# on another machine
+$ docker build --cache-from myname/myapp .
+```
+
+> **Note**: This feature requires the BuildKit backend. You can either
+> [enable BuildKit](../builder.md#buildkit) or use the [buildx](https://github.com/docker/buildx)
+> plugin. The previous builder has limited support for reusing cache from
+> pre-pulled images.
+
 ### Squash an image's layers (--squash) (experimental)

 #### Overview
@@ -1,8 +1,9 @@
 ---
 title: "dockerd"
-aliases: ["/engine/reference/commandline/daemon/"]
 description: "The daemon command description and usage"
 keywords: "container, daemon, runtime"
+redirect_from:
+- /engine/reference/commandline/daemon/
 ---

 <!-- This file is maintained within the docker/cli GitHub
@@ -24,7 +24,7 @@ Options:
  -f, --force     Force the removal of a running container (uses SIGKILL)
      --help      Print usage
  -l, --link      Remove the specified link
-  -v, --volumes   Remove the volumes associated with the container
+  -v, --volumes   Remove anonymous volumes associated with the container
 ```

 ## Examples
@@ -352,6 +352,11 @@ machine. You can also specify `udp` and `sctp` ports.
 The [Docker User Guide](https://docs.docker.com/engine/userguide/networking/default_network/dockerlinks/)
 explains in detail how to manipulate ports in Docker.

+Note that ports which are not bound to the host (i.e., `-p 80:80` instead of
+`-p 127.0.0.1:80:80`) will be accessible from the outside. This also applies if
+you configured UFW to block this specific port, as Docker manages his
+own iptables rules. [Read more](https://docs.docker.com/network/iptables/)
+
 ```bash
 $ docker run --expose 80 ubuntu bash
 ```
@@ -684,7 +689,7 @@ using IPv4 or IPv6 networking in your containers. Use the following
 flags for IPv4 address retrieval for a network device named `eth0`:

 ```bash
-$ HOSTIP=`ip -4 addr show scope global dev eth0 | grep inet | awk '{print \$2}' | cut -d / -f 1`
+$ HOSTIP=`ip -4 addr show scope global dev eth0 | grep inet | awk '{print $2}' | cut -d / -f 1 | sed -n 1p`
 $ docker run  --add-host=docker:${HOSTIP} --rm -it debian
 ```

@@ -98,7 +98,7 @@ be in the range between 1 and 100. The default value of `--limit` is 25.
 ### Filtering

 The filtering flag (`-f` or `--filter`) format is a `key=value` pair. If there is more
-than one filter, then pass multiple flags (e.g. `--filter "foo=bar" --filter "bif=baz"`)
+than one filter, then pass multiple flags (e.g. `--filter is-automated=true --filter stars=3`)

 The currently supported filters are:

@@ -126,7 +126,7 @@ This example displays images with a name containing 'busybox'
 and are automated builds:

 ```bash
-$ docker search --filter is-automated busybox
+$ docker search --filter is-automated=true busybox

 NAME                 DESCRIPTION                                     STARS     OFFICIAL   AUTOMATED
 progrium/busybox                                                     50                   [OK]
@@ -139,7 +139,7 @@ This example displays images with a name containing 'busybox', at least
 3 stars and are official builds:

 ```bash
-$ docker search --filter "is-official=true" --filter "stars=3" busybox
+$ docker search --filter is-official=true --filter stars=3 busybox

 NAME                 DESCRIPTION                                     STARS     OFFICIAL   AUTOMATED
 progrium/busybox                                                     50                   [OK]
@@ -200,12 +200,41 @@ $ docker service create --name redis \

 To grant a service access to multiple secrets, use multiple `--secret` flags.

-Secrets are located in `/run/secrets` in the container.  If no target is
-specified, the name of the secret will be used as the in memory file in the
-container.  If a target is specified, that will be the filename.  In the
-example above, two files will be created: `/run/secrets/ssh` and
+Secrets are located in `/run/secrets` in the container if no target is specified.
+If no target is specified, the name of the secret is used as the in memory file
+in the container. If a target is specified, that is used as the filename. In the
+example above, two files are created: `/run/secrets/ssh` and
 `/run/secrets/app` for each of the secret targets specified.

+### Create a service with configs
+
+Use the `--config` flag to give a container access to a
+[config](config_create.md).
+
+Create a service with a config. The config will be mounted into `redis-config`,
+be owned by the user who runs the command inside the container (often `root`),
+and have file mode `0444` or world-readable. You can specify the `uid` and `gid`
+as numerical IDs or names. When using names, the provided group/user names must
+pre-exist in the container. The `mode` is specified as a 4-number sequence such
+as `0755`.
+
+```bash
+$ docker service create --name=redis --config redis-conf redis:3.0.6
+```
+
+Create a service with a config and specify the target location and file mode:
+
+```bash
+$ docker service create --name redis \
+  --config source=redis-conf,target=/etc/redis/redis.conf,mode=0400 redis:3.0.6
+```
+
+To grant a service access to multiple configs, use multiple `--config` flags.
+
+Configs are located in `/` in the container if no target is specified. If no
+target is specified, the name of the config is used as the name of the file in
+the container. If a target is specified, that is used as the filename.
+
 ### Create a service with a rolling update policy

 ```bash
@@ -638,48 +667,25 @@ $ docker service create \
 ### Specify service constraints (--constraint)

 You can limit the set of nodes where a task can be scheduled by defining
-constraint expressions. Multiple constraints find nodes that satisfy every
+constraint expressions. Constraint expressions can either use a _match_ (`==`)
+or _exclude_ (`!=`) rule. Multiple constraints find nodes that satisfy every
 expression (AND match). Constraints can match node or Docker Engine labels as
 follows:

-
-<table>
-  <tr>
-    <th>node attribute</th>
-    <th>matches</th>
-    <th>example</th>
-  </tr>
-  <tr>
-    <td><tt>node.id</tt></td>
-    <td>Node ID</td>
-    <td><tt>node.id==2ivku8v2gvtg4</tt></td>
-  </tr>
-  <tr>
-    <td><tt>node.hostname</tt></td>
-    <td>Node hostname</td>
-    <td><tt>node.hostname!=node-2</tt></td>
-  </tr>
-  <tr>
-    <td><tt>node.role</tt></td>
-    <td>Node role</td>
-    <td><tt>node.role==manager</tt></td>
-  </tr>
-  <tr>
-    <td><tt>node.labels</tt></td>
-    <td>user defined node labels</td>
-    <td><tt>node.labels.security==high</tt></td>
-  </tr>
-  <tr>
-    <td><tt>engine.labels</tt></td>
-    <td>Docker Engine's labels</td>
-    <td><tt>engine.labels.operatingsystem==ubuntu 14.04</tt></td>
-  </tr>
-</table>
+node attribute       | matches                        | example
+---------------------|--------------------------------|-----------------------------------------------
+`node.id`            | Node ID                        | `node.id==2ivku8v2gvtg4`
+`node.hostname`      | Node hostname                  | `node.hostname!=node-2`
+`node.role`          | Node role (`manager`/`worker`) | `node.role==manager`
+`node.platform.os`   | Node operating system          | `node.platform.os==windows`
+`node.platform.arch` | Node architecture              | `node.platform.arch==x86_64`
+`node.labels`        | User-defined node labels       | `node.labels.security==high`
+`engine.labels`      | Docker Engine's labels         | `engine.labels.operatingsystem==ubuntu-14.04`


-`engine.labels` apply to Docker Engine labels like operating system,
-drivers, etc. Swarm administrators add `node.labels` for operational purposes by
-using the [`docker node update`](node_update.md) command.
+`engine.labels` apply to Docker Engine labels like operating system, drivers,
+etc. Swarm administrators add `node.labels` for operational purposes by using
+the [`docker node update`](node_update.md) command.

 For example, the following limits tasks for the redis service to nodes where the
 node type label equals queue:
@@ -687,10 +693,45 @@ node type label equals queue:
 ```bash
 $ docker service create \
  --name redis_2 \
-  --constraint 'node.labels.type == queue' \
+  --constraint node.platform.os==linux \
+  --constraint node.labels.type==queue \
  redis:3.0.6
 ```

+If the service constraints exclude all nodes in the cluster, a message is printed
+that no suitable node is found, but the scheduler will start a reconciliation
+loop and deploy the service once a suitable node becomes available.
+
+In the example below, no node satisfying the constraint was found, causing the
+service to not reconcile with the desired state:
+
+```bash
+$ docker service create \
+  --name web \
+  --constraint node.labels.region==east \
+  nginx:alpine
+
+lx1wrhhpmbbu0wuk0ybws30bc
+overall progress: 0 out of 1 tasks
+1/1: no suitable node (scheduling constraints not satisfied on 5 nodes)
+
+$ docker service ls
+ID                  NAME     MODE         REPLICAS   IMAGE               PORTS
+b6lww17hrr4e        web      replicated   0/1        nginx:alpine
+```
+
+After adding the `region=east` label to a node in the cluster, the service
+reconciles, and the desired number of replicas are deployed:
+
+```bash
+$ docker node update --label-add region=east yswe2dm4c5fdgtsrli1e8ya5l 
+yswe2dm4c5fdgtsrli1e8ya5l
+
+$ docker service ls
+ID                  NAME     MODE         REPLICAS   IMAGE               PORTS
+b6lww17hrr4e        web      replicated   1/1        nginx:alpine
+```
+
 ### Specify service placement preferences (--placement-pref)

 You can set up the service to divide tasks evenly over different categories of
@@ -701,7 +742,7 @@ of datacenters or availability zones. The example below illustrates this:
 $ docker service create \
  --replicas 9 \
  --name redis_2 \
-  --placement-pref 'spread=node.labels.datacenter' \
+  --placement-pref spread=node.labels.datacenter \
  redis:3.0.6
 ```

@@ -762,6 +803,74 @@ appends a new placement preference after all existing placement preferences.
 `--placement-pref-rm` removes an existing placement preference that matches the
 argument.

+### Specify memory requirements and constraints for a service (--reserve-memory and --limit-memory)
+
+If your service needs a minimum amount of memory in order to run correctly,
+you can use `--reserve-memory` to specify that the service should only be
+scheduled on a node with this much memory available to reserve. If no node is
+available that meets the criteria, the task is not scheduled, but remains in a
+pending state.
+
+The following example requires that 4GB of memory be available and reservable
+on a given node before scheduling the service to run on that node.
+
+```bash
+$ docker service create --reserve-memory=4GB --name=too-big nginx:alpine
+```
+
+The managers won't schedule a set of containers on a single node whose combined
+reservations exceed the memory available on that node.
+
+After a task is scheduled and running, `--reserve-memory` does not enforce a
+memory limit. Use `--limit-memory` to ensure that a task uses no more than a
+given amount of memory on a node. This example limits the amount of memory used
+by the task to 4GB. The task will be scheduled even if each of your nodes has
+only 2GB of memory, because `--limit-memory` is an upper limit.
+
+```bash
+$ docker service create --limit-memory=4GB --name=too-big nginx:alpine
+```
+
+Using `--reserve-memory` and `--limit-memory` does not guarantee that Docker
+will not use more memory on your host than you want. For instance, you could
+create many services, the sum of whose memory usage could exhaust the available
+memory.
+
+You can prevent this scenario from exhausting the available memory by taking
+into account other (non-containerized) software running on the host as well. If
+`--reserve-memory` is greater than or equal to `--limit-memory`, Docker won't
+schedule a service on a host that doesn't have enough memory. `--limit-memory`
+will limit the service's memory to stay within that limit, so if every service
+has a memory-reservation and limit set, Docker services will be less likely to
+saturate the host. Other non-service containers or applications running directly
+on the Docker host could still exhaust memory.
+
+There is a downside to this approach. Reserving memory also means that you may
+not make optimum use of the memory available on the node. Consider a service
+that under normal circumstances uses 100MB of memory, but depending on load can
+"peak" at 500MB. Reserving 500MB for that service (to guarantee can have 500MB
+for those "peaks") results in 400MB of memory being wasted most of the time.
+
+In short, you can take a more conservative or more flexible approach:
+
+- **Conservative**: reserve 500MB, and limit to 500MB. Basically you're now
+  treating the service containers as VMs, and you may be losing a big advantage
+  containers, which is greater density of services per host.
+
+- **Flexible**: limit to 500MB in the assumption that if the service requires
+  more than 500MB, it is malfunctioning. Reserve something between the 100MB
+  "normal" requirement and the 500MB "peak" requirement". This assumes that when
+  this service is at "peak", other services or non-container workloads probably
+  won't be.
+
+The approach you take depends heavily on the memory-usage patterns of your
+workloads. You should test under normal and peak conditions before settling
+on an approach.
+
+On Linux, you can also limit a service's overall memory footprint on a given
+host at the level of the host operating system, using `cgroups` or other
+relevant operating system tools.
+
 ### Specify maximum replicas per node (--replicas-max-per-node)

 Use the `--replicas-max-per-node` flag to set the maximum number of replica tasks that can run on a node.
@@ -79,7 +79,7 @@ github.com/tonistiigi/units                         6950e57a87eaf136bbe44ef2ec8e
 github.com/xeipuuv/gojsonpointer                    4e3ac2762d5f479393488629ee9370b50873b3a6
 github.com/xeipuuv/gojsonreference                  bd5ef7bd5415a7ac448318e64f11a24cd21e594b
 github.com/xeipuuv/gojsonschema                     93e72a773fade158921402d6a24c819b48aba29d
-golang.org/x/crypto                                 88737f569e3a9c7ab309cdc09a07fe7fc87233c3
+golang.org/x/crypto                                 69ecbb4d6d5dab05e49161c6e77ea40a030884e1
 golang.org/x/net                                    eb5bcb51f2a31c7d5141d810b70815c05d9c9146
 golang.org/x/oauth2                                 ef147856a6ddbb60760db74283d2424e98c87bff
 golang.org/x/sync                                   e225da77a7e68af35c70ccbf71af2b83e6acac3c
@@ -87,7 +87,7 @@ golang.org/x/sys                                    4b34438f7a67ee5f45cc6132e2ba
 golang.org/x/text                                   f21a4dfb5e38f5895301dc265a8def02365cc3d0 # v0.3.0
 golang.org/x/time                                   fbb02b2291d28baffd63558aa44b4b56f178d650
 google.golang.org/genproto                          02b4e95473316948020af0b7a4f0f22c73929b0e
-google.golang.org/grpc                              25c4f928eaa6d96443009bd842389fb4fa48664e # v1.20.1
+google.golang.org/grpc                              39e8a7b072a67ca2a75f57fa2e0d50995f5b22f6 # v1.23.1
 gopkg.in/inf.v0                                     d2d2541c53f18d2a059457998ce2876cc8e67cbf # v0.9.1
 gopkg.in/yaml.v2                                    bb4e33bf68bf89cad44d386192cbed201f35b241 # v2.2.3
 gotest.tools                                        1083505acf35a0bd8a696b26837e1fb3187a7a83 # v2.3.0
@@ -0,0 +1,17 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.11
+// +build !gccgo,!appengine
+
+package chacha20
+
+const bufSize = 256
+
+//go:noescape
+func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32)
+
+func (c *Cipher) xorKeyStreamBlocks(dst, src []byte) {
+	xorKeyStreamVX(dst, src, &c.key, &c.nonce, &c.counter)
+}
@@ -0,0 +1,364 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package chacha20 implements the ChaCha20 and XChaCha20 encryption algorithms
+// as specified in RFC 8439 and draft-irtf-cfrg-xchacha-01.
+package chacha20
+
+import (
+	"crypto/cipher"
+	"encoding/binary"
+	"errors"
+	"math/bits"
+
+	"golang.org/x/crypto/internal/subtle"
+)
+
+const (
+	// KeySize is the size of the key used by this cipher, in bytes.
+	KeySize = 32
+
+	// NonceSize is the size of the nonce used with the standard variant of this
+	// cipher, in bytes.
+	//
+	// Note that this is too short to be safely generated at random if the same
+	// key is reused more than 2³² times.
+	NonceSize = 12
+
+	// NonceSizeX is the size of the nonce used with the XChaCha20 variant of
+	// this cipher, in bytes.
+	NonceSizeX = 24
+)
+
+// Cipher is a stateful instance of ChaCha20 or XChaCha20 using a particular key
+// and nonce. A *Cipher implements the cipher.Stream interface.
+type Cipher struct {
+	// The ChaCha20 state is 16 words: 4 constant, 8 of key, 1 of counter
+	// (incremented after each block), and 3 of nonce.
+	key     [8]uint32
+	counter uint32
+	nonce   [3]uint32
+
+	// The last len bytes of buf are leftover key stream bytes from the previous
+	// XORKeyStream invocation. The size of buf depends on how many blocks are
+	// computed at a time.
+	buf [bufSize]byte
+	len int
+
+	// The counter-independent results of the first round are cached after they
+	// are computed the first time.
+	precompDone      bool
+	p1, p5, p9, p13  uint32
+	p2, p6, p10, p14 uint32
+	p3, p7, p11, p15 uint32
+}
+
+var _ cipher.Stream = (*Cipher)(nil)
+
+// NewUnauthenticatedCipher creates a new ChaCha20 stream cipher with the given
+// 32 bytes key and a 12 or 24 bytes nonce. If a nonce of 24 bytes is provided,
+// the XChaCha20 construction will be used. It returns an error if key or nonce
+// have any other length.
+//
+// Note that ChaCha20, like all stream ciphers, is not authenticated and allows
+// attackers to silently tamper with the plaintext. For this reason, it is more
+// appropriate as a building block than as a standalone encryption mechanism.
+// Instead, consider using package golang.org/x/crypto/chacha20poly1305.
+func NewUnauthenticatedCipher(key, nonce []byte) (*Cipher, error) {
+	// This function is split into a wrapper so that the Cipher allocation will
+	// be inlined, and depending on how the caller uses the return value, won't
+	// escape to the heap.
+	c := &Cipher{}
+	return newUnauthenticatedCipher(c, key, nonce)
+}
+
+func newUnauthenticatedCipher(c *Cipher, key, nonce []byte) (*Cipher, error) {
+	if len(key) != KeySize {
+		return nil, errors.New("chacha20: wrong key size")
+	}
+	if len(nonce) == NonceSizeX {
+		// XChaCha20 uses the ChaCha20 core to mix 16 bytes of the nonce into a
+		// derived key, allowing it to operate on a nonce of 24 bytes. See
+		// draft-irtf-cfrg-xchacha-01, Section 2.3.
+		key, _ = HChaCha20(key, nonce[0:16])
+		cNonce := make([]byte, NonceSize)
+		copy(cNonce[4:12], nonce[16:24])
+		nonce = cNonce
+	} else if len(nonce) != NonceSize {
+		return nil, errors.New("chacha20: wrong nonce size")
+	}
+
+	c.key = [8]uint32{
+		binary.LittleEndian.Uint32(key[0:4]),
+		binary.LittleEndian.Uint32(key[4:8]),
+		binary.LittleEndian.Uint32(key[8:12]),
+		binary.LittleEndian.Uint32(key[12:16]),
+		binary.LittleEndian.Uint32(key[16:20]),
+		binary.LittleEndian.Uint32(key[20:24]),
+		binary.LittleEndian.Uint32(key[24:28]),
+		binary.LittleEndian.Uint32(key[28:32]),
+	}
+	c.nonce = [3]uint32{
+		binary.LittleEndian.Uint32(nonce[0:4]),
+		binary.LittleEndian.Uint32(nonce[4:8]),
+		binary.LittleEndian.Uint32(nonce[8:12]),
+	}
+	return c, nil
+}
+
+// The constant first 4 words of the ChaCha20 state.
+const (
+	j0 uint32 = 0x61707865 // expa
+	j1 uint32 = 0x3320646e // nd 3
+	j2 uint32 = 0x79622d32 // 2-by
+	j3 uint32 = 0x6b206574 // te k
+)
+
+const blockSize = 64
+
+// quarterRound is the core of ChaCha20. It shuffles the bits of 4 state words.
+// It's executed 4 times for each of the 20 ChaCha20 rounds, operating on all 16
+// words each round, in columnar or diagonal groups of 4 at a time.
+func quarterRound(a, b, c, d uint32) (uint32, uint32, uint32, uint32) {
+	a += b
+	d ^= a
+	d = bits.RotateLeft32(d, 16)
+	c += d
+	b ^= c
+	b = bits.RotateLeft32(b, 12)
+	a += b
+	d ^= a
+	d = bits.RotateLeft32(d, 8)
+	c += d
+	b ^= c
+	b = bits.RotateLeft32(b, 7)
+	return a, b, c, d
+}
+
+// XORKeyStream XORs each byte in the given slice with a byte from the
+// cipher's key stream. Dst and src must overlap entirely or not at all.
+//
+// If len(dst) < len(src), XORKeyStream will panic. It is acceptable
+// to pass a dst bigger than src, and in that case, XORKeyStream will
+// only update dst[:len(src)] and will not touch the rest of dst.
+//
+// Multiple calls to XORKeyStream behave as if the concatenation of
+// the src buffers was passed in a single run. That is, Cipher
+// maintains state and does not reset at each XORKeyStream call.
+func (s *Cipher) XORKeyStream(dst, src []byte) {
+	if len(src) == 0 {
+		return
+	}
+	if len(dst) < len(src) {
+		panic("chacha20: output smaller than input")
+	}
+	dst = dst[:len(src)]
+	if subtle.InexactOverlap(dst, src) {
+		panic("chacha20: invalid buffer overlap")
+	}
+
+	// First, drain any remaining key stream from a previous XORKeyStream.
+	if s.len != 0 {
+		keyStream := s.buf[bufSize-s.len:]
+		if len(src) < len(keyStream) {
+			keyStream = keyStream[:len(src)]
+		}
+		_ = src[len(keyStream)-1] // bounds check elimination hint
+		for i, b := range keyStream {
+			dst[i] = src[i] ^ b
+		}
+		s.len -= len(keyStream)
+		src = src[len(keyStream):]
+		dst = dst[len(keyStream):]
+	}
+
+	const blocksPerBuf = bufSize / blockSize
+	numBufs := (uint64(len(src)) + bufSize - 1) / bufSize
+	if uint64(s.counter)+numBufs*blocksPerBuf >= 1<<32 {
+		panic("chacha20: counter overflow")
+	}
+
+	// xorKeyStreamBlocks implementations expect input lengths that are a
+	// multiple of bufSize. Platform-specific ones process multiple blocks at a
+	// time, so have bufSizes that are a multiple of blockSize.
+
+	rem := len(src) % bufSize
+	full := len(src) - rem
+
+	if full > 0 {
+		s.xorKeyStreamBlocks(dst[:full], src[:full])
+	}
+
+	// If we have a partial (multi-)block, pad it for xorKeyStreamBlocks, and
+	// keep the leftover keystream for the next XORKeyStream invocation.
+	if rem > 0 {
+		s.buf = [bufSize]byte{}
+		copy(s.buf[:], src[full:])
+		s.xorKeyStreamBlocks(s.buf[:], s.buf[:])
+		s.len = bufSize - copy(dst[full:], s.buf[:])
+	}
+}
+
+func (s *Cipher) xorKeyStreamBlocksGeneric(dst, src []byte) {
+	if len(dst) != len(src) || len(dst)%blockSize != 0 {
+		panic("chacha20: internal error: wrong dst and/or src length")
+	}
+
+	// To generate each block of key stream, the initial cipher state
+	// (represented below) is passed through 20 rounds of shuffling,
+	// alternatively applying quarterRounds by columns (like 1, 5, 9, 13)
+	// or by diagonals (like 1, 6, 11, 12).
+	//
+	//      0:cccccccc   1:cccccccc   2:cccccccc   3:cccccccc
+	//      4:kkkkkkkk   5:kkkkkkkk   6:kkkkkkkk   7:kkkkkkkk
+	//      8:kkkkkkkk   9:kkkkkkkk  10:kkkkkkkk  11:kkkkkkkk
+	//     12:bbbbbbbb  13:nnnnnnnn  14:nnnnnnnn  15:nnnnnnnn
+	//
+	//            c=constant k=key b=blockcount n=nonce
+	var (
+		c0, c1, c2, c3   = j0, j1, j2, j3
+		c4, c5, c6, c7   = s.key[0], s.key[1], s.key[2], s.key[3]
+		c8, c9, c10, c11 = s.key[4], s.key[5], s.key[6], s.key[7]
+		_, c13, c14, c15 = s.counter, s.nonce[0], s.nonce[1], s.nonce[2]
+	)
+
+	// Three quarters of the first round don't depend on the counter, so we can
+	// calculate them here, and reuse them for multiple blocks in the loop, and
+	// for future XORKeyStream invocations.
+	if !s.precompDone {
+		s.p1, s.p5, s.p9, s.p13 = quarterRound(c1, c5, c9, c13)
+		s.p2, s.p6, s.p10, s.p14 = quarterRound(c2, c6, c10, c14)
+		s.p3, s.p7, s.p11, s.p15 = quarterRound(c3, c7, c11, c15)
+		s.precompDone = true
+	}
+
+	for i := 0; i < len(src); i += blockSize {
+		// The remainder of the first column round.
+		fcr0, fcr4, fcr8, fcr12 := quarterRound(c0, c4, c8, s.counter)
+
+		// The second diagonal round.
+		x0, x5, x10, x15 := quarterRound(fcr0, s.p5, s.p10, s.p15)
+		x1, x6, x11, x12 := quarterRound(s.p1, s.p6, s.p11, fcr12)
+		x2, x7, x8, x13 := quarterRound(s.p2, s.p7, fcr8, s.p13)
+		x3, x4, x9, x14 := quarterRound(s.p3, fcr4, s.p9, s.p14)
+
+		// The remaining 18 rounds.
+		for i := 0; i < 9; i++ {
+			// Column round.
+			x0, x4, x8, x12 = quarterRound(x0, x4, x8, x12)
+			x1, x5, x9, x13 = quarterRound(x1, x5, x9, x13)
+			x2, x6, x10, x14 = quarterRound(x2, x6, x10, x14)
+			x3, x7, x11, x15 = quarterRound(x3, x7, x11, x15)
+
+			// Diagonal round.
+			x0, x5, x10, x15 = quarterRound(x0, x5, x10, x15)
+			x1, x6, x11, x12 = quarterRound(x1, x6, x11, x12)
+			x2, x7, x8, x13 = quarterRound(x2, x7, x8, x13)
+			x3, x4, x9, x14 = quarterRound(x3, x4, x9, x14)
+		}
+
+		// Finally, add back the initial state to generate the key stream.
+		x0 += c0
+		x1 += c1
+		x2 += c2
+		x3 += c3
+		x4 += c4
+		x5 += c5
+		x6 += c6
+		x7 += c7
+		x8 += c8
+		x9 += c9
+		x10 += c10
+		x11 += c11
+		x12 += s.counter
+		x13 += c13
+		x14 += c14
+		x15 += c15
+
+		s.counter += 1
+		if s.counter == 0 {
+			panic("chacha20: internal error: counter overflow")
+		}
+
+		in, out := src[i:], dst[i:]
+		in, out = in[:blockSize], out[:blockSize] // bounds check elimination hint
+
+		// XOR the key stream with the source and write out the result.
+		xor(out[0:], in[0:], x0)
+		xor(out[4:], in[4:], x1)
+		xor(out[8:], in[8:], x2)
+		xor(out[12:], in[12:], x3)
+		xor(out[16:], in[16:], x4)
+		xor(out[20:], in[20:], x5)
+		xor(out[24:], in[24:], x6)
+		xor(out[28:], in[28:], x7)
+		xor(out[32:], in[32:], x8)
+		xor(out[36:], in[36:], x9)
+		xor(out[40:], in[40:], x10)
+		xor(out[44:], in[44:], x11)
+		xor(out[48:], in[48:], x12)
+		xor(out[52:], in[52:], x13)
+		xor(out[56:], in[56:], x14)
+		xor(out[60:], in[60:], x15)
+	}
+}
+
+// HChaCha20 uses the ChaCha20 core to generate a derived key from a 32 bytes
+// key and a 16 bytes nonce. It returns an error if key or nonce have any other
+// length. It is used as part of the XChaCha20 construction.
+func HChaCha20(key, nonce []byte) ([]byte, error) {
+	// This function is split into a wrapper so that the slice allocation will
+	// be inlined, and depending on how the caller uses the return value, won't
+	// escape to the heap.
+	out := make([]byte, 32)
+	return hChaCha20(out, key, nonce)
+}
+
+func hChaCha20(out, key, nonce []byte) ([]byte, error) {
+	if len(key) != KeySize {
+		return nil, errors.New("chacha20: wrong HChaCha20 key size")
+	}
+	if len(nonce) != 16 {
+		return nil, errors.New("chacha20: wrong HChaCha20 nonce size")
+	}
+
+	x0, x1, x2, x3 := j0, j1, j2, j3
+	x4 := binary.LittleEndian.Uint32(key[0:4])
+	x5 := binary.LittleEndian.Uint32(key[4:8])
+	x6 := binary.LittleEndian.Uint32(key[8:12])
+	x7 := binary.LittleEndian.Uint32(key[12:16])
+	x8 := binary.LittleEndian.Uint32(key[16:20])
+	x9 := binary.LittleEndian.Uint32(key[20:24])
+	x10 := binary.LittleEndian.Uint32(key[24:28])
+	x11 := binary.LittleEndian.Uint32(key[28:32])
+	x12 := binary.LittleEndian.Uint32(nonce[0:4])
+	x13 := binary.LittleEndian.Uint32(nonce[4:8])
+	x14 := binary.LittleEndian.Uint32(nonce[8:12])
+	x15 := binary.LittleEndian.Uint32(nonce[12:16])
+
+	for i := 0; i < 10; i++ {
+		// Diagonal round.
+		x0, x4, x8, x12 = quarterRound(x0, x4, x8, x12)
+		x1, x5, x9, x13 = quarterRound(x1, x5, x9, x13)
+		x2, x6, x10, x14 = quarterRound(x2, x6, x10, x14)
+		x3, x7, x11, x15 = quarterRound(x3, x7, x11, x15)
+
+		// Column round.
+		x0, x5, x10, x15 = quarterRound(x0, x5, x10, x15)
+		x1, x6, x11, x12 = quarterRound(x1, x6, x11, x12)
+		x2, x7, x8, x13 = quarterRound(x2, x7, x8, x13)
+		x3, x4, x9, x14 = quarterRound(x3, x4, x9, x14)
+	}
+
+	_ = out[31] // bounds check elimination hint
+	binary.LittleEndian.PutUint32(out[0:4], x0)
+	binary.LittleEndian.PutUint32(out[4:8], x1)
+	binary.LittleEndian.PutUint32(out[8:12], x2)
+	binary.LittleEndian.PutUint32(out[12:16], x3)
+	binary.LittleEndian.PutUint32(out[16:20], x12)
+	binary.LittleEndian.PutUint32(out[20:24], x13)
+	binary.LittleEndian.PutUint32(out[24:28], x14)
+	binary.LittleEndian.PutUint32(out[28:32], x15)
+	return out, nil
+}
@@ -0,0 +1,13 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !arm64,!s390x,!ppc64le arm64,!go1.11 gccgo appengine
+
+package chacha20
+
+const bufSize = blockSize
+
+func (s *Cipher) xorKeyStreamBlocks(dst, src []byte) {
+	s.xorKeyStreamBlocksGeneric(dst, src)
+}
@@ -0,0 +1,16 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo,!appengine
+
+package chacha20
+
+const bufSize = 256
+
+//go:noescape
+func chaCha20_ctr32_vsx(out, inp *byte, len int, key *[8]uint32, counter *uint32)
+
+func (c *Cipher) xorKeyStreamBlocks(dst, src []byte) {
+	chaCha20_ctr32_vsx(&dst[0], &src[0], len(src), &c.key, &c.counter)
+}
@@ -0,0 +1,449 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Based on CRYPTOGAMS code with the following comment:
+// # ====================================================================
+// # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+// # project. The module is, however, dual licensed under OpenSSL and
+// # CRYPTOGAMS licenses depending on where you obtain it. For further
+// # details see http://www.openssl.org/~appro/cryptogams/.
+// # ====================================================================
+
+// Code for the perl script that generates the ppc64 assembler
+// can be found in the cryptogams repository at the link below. It is based on
+// the original from openssl.
+
+// https://github.com/dot-asm/cryptogams/commit/a60f5b50ed908e91
+
+// The differences in this and the original implementation are
+// due to the calling conventions and initialization of constants.
+
+// +build !gccgo,!appengine
+
+#include "textflag.h"
+
+#define OUT  R3
+#define INP  R4
+#define LEN  R5
+#define KEY  R6
+#define CNT  R7
+#define TMP  R15
+
+#define CONSTBASE  R16
+#define BLOCKS R17
+
+DATA consts<>+0x00(SB)/8, $0x3320646e61707865
+DATA consts<>+0x08(SB)/8, $0x6b20657479622d32
+DATA consts<>+0x10(SB)/8, $0x0000000000000001
+DATA consts<>+0x18(SB)/8, $0x0000000000000000
+DATA consts<>+0x20(SB)/8, $0x0000000000000004
+DATA consts<>+0x28(SB)/8, $0x0000000000000000
+DATA consts<>+0x30(SB)/8, $0x0a0b08090e0f0c0d
+DATA consts<>+0x38(SB)/8, $0x0203000106070405
+DATA consts<>+0x40(SB)/8, $0x090a0b080d0e0f0c
+DATA consts<>+0x48(SB)/8, $0x0102030005060704
+DATA consts<>+0x50(SB)/8, $0x6170786561707865
+DATA consts<>+0x58(SB)/8, $0x6170786561707865
+DATA consts<>+0x60(SB)/8, $0x3320646e3320646e
+DATA consts<>+0x68(SB)/8, $0x3320646e3320646e
+DATA consts<>+0x70(SB)/8, $0x79622d3279622d32
+DATA consts<>+0x78(SB)/8, $0x79622d3279622d32
+DATA consts<>+0x80(SB)/8, $0x6b2065746b206574
+DATA consts<>+0x88(SB)/8, $0x6b2065746b206574
+DATA consts<>+0x90(SB)/8, $0x0000000100000000
+DATA consts<>+0x98(SB)/8, $0x0000000300000002
+GLOBL consts<>(SB), RODATA, $0xa0
+
+//func chaCha20_ctr32_vsx(out, inp *byte, len int, key *[8]uint32, counter *uint32)
+TEXT ·chaCha20_ctr32_vsx(SB),NOSPLIT,$64-40
+	MOVD out+0(FP), OUT
+	MOVD inp+8(FP), INP
+	MOVD len+16(FP), LEN
+	MOVD key+24(FP), KEY
+	MOVD counter+32(FP), CNT
+
+	// Addressing for constants
+	MOVD $consts<>+0x00(SB), CONSTBASE
+	MOVD $16, R8
+	MOVD $32, R9
+	MOVD $48, R10
+	MOVD $64, R11
+	SRD $6, LEN, BLOCKS
+	// V16
+	LXVW4X (CONSTBASE)(R0), VS48
+	ADD $80,CONSTBASE
+
+	// Load key into V17,V18
+	LXVW4X (KEY)(R0), VS49
+	LXVW4X (KEY)(R8), VS50
+
+	// Load CNT, NONCE into V19
+	LXVW4X (CNT)(R0), VS51
+
+	// Clear V27
+	VXOR V27, V27, V27
+
+	// V28
+	LXVW4X (CONSTBASE)(R11), VS60
+
+	// splat slot from V19 -> V26
+	VSPLTW $0, V19, V26
+
+	VSLDOI $4, V19, V27, V19
+	VSLDOI $12, V27, V19, V19
+
+	VADDUWM V26, V28, V26
+
+	MOVD $10, R14
+	MOVD R14, CTR
+
+loop_outer_vsx:
+	// V0, V1, V2, V3
+	LXVW4X (R0)(CONSTBASE), VS32
+	LXVW4X (R8)(CONSTBASE), VS33
+	LXVW4X (R9)(CONSTBASE), VS34
+	LXVW4X (R10)(CONSTBASE), VS35
+
+	// splat values from V17, V18 into V4-V11
+	VSPLTW $0, V17, V4
+	VSPLTW $1, V17, V5
+	VSPLTW $2, V17, V6
+	VSPLTW $3, V17, V7
+	VSPLTW $0, V18, V8
+	VSPLTW $1, V18, V9
+	VSPLTW $2, V18, V10
+	VSPLTW $3, V18, V11
+
+	// VOR
+	VOR V26, V26, V12
+
+	// splat values from V19 -> V13, V14, V15
+	VSPLTW $1, V19, V13
+	VSPLTW $2, V19, V14
+	VSPLTW $3, V19, V15
+
+	// splat   const values
+	VSPLTISW $-16, V27
+	VSPLTISW $12, V28
+	VSPLTISW $8, V29
+	VSPLTISW $7, V30
+
+loop_vsx:
+	VADDUWM V0, V4, V0
+	VADDUWM V1, V5, V1
+	VADDUWM V2, V6, V2
+	VADDUWM V3, V7, V3
+
+	VXOR V12, V0, V12
+	VXOR V13, V1, V13
+	VXOR V14, V2, V14
+	VXOR V15, V3, V15
+
+	VRLW V12, V27, V12
+	VRLW V13, V27, V13
+	VRLW V14, V27, V14
+	VRLW V15, V27, V15
+
+	VADDUWM V8, V12, V8
+	VADDUWM V9, V13, V9
+	VADDUWM V10, V14, V10
+	VADDUWM V11, V15, V11
+
+	VXOR V4, V8, V4
+	VXOR V5, V9, V5
+	VXOR V6, V10, V6
+	VXOR V7, V11, V7
+
+	VRLW V4, V28, V4
+	VRLW V5, V28, V5
+	VRLW V6, V28, V6
+	VRLW V7, V28, V7
+
+	VADDUWM V0, V4, V0
+	VADDUWM V1, V5, V1
+	VADDUWM V2, V6, V2
+	VADDUWM V3, V7, V3
+
+	VXOR V12, V0, V12
+	VXOR V13, V1, V13
+	VXOR V14, V2, V14
+	VXOR V15, V3, V15
+
+	VRLW V12, V29, V12
+	VRLW V13, V29, V13
+	VRLW V14, V29, V14
+	VRLW V15, V29, V15
+
+	VADDUWM V8, V12, V8
+	VADDUWM V9, V13, V9
+	VADDUWM V10, V14, V10
+	VADDUWM V11, V15, V11
+
+	VXOR V4, V8, V4
+	VXOR V5, V9, V5
+	VXOR V6, V10, V6
+	VXOR V7, V11, V7
+
+	VRLW V4, V30, V4
+	VRLW V5, V30, V5
+	VRLW V6, V30, V6
+	VRLW V7, V30, V7
+
+	VADDUWM V0, V5, V0
+	VADDUWM V1, V6, V1
+	VADDUWM V2, V7, V2
+	VADDUWM V3, V4, V3
+
+	VXOR V15, V0, V15
+	VXOR V12, V1, V12
+	VXOR V13, V2, V13
+	VXOR V14, V3, V14
+
+	VRLW V15, V27, V15
+	VRLW V12, V27, V12
+	VRLW V13, V27, V13
+	VRLW V14, V27, V14
+
+	VADDUWM V10, V15, V10
+	VADDUWM V11, V12, V11
+	VADDUWM V8, V13, V8
+	VADDUWM V9, V14, V9
+
+	VXOR V5, V10, V5
+	VXOR V6, V11, V6
+	VXOR V7, V8, V7
+	VXOR V4, V9, V4
+
+	VRLW V5, V28, V5
+	VRLW V6, V28, V6
+	VRLW V7, V28, V7
+	VRLW V4, V28, V4
+
+	VADDUWM V0, V5, V0
+	VADDUWM V1, V6, V1
+	VADDUWM V2, V7, V2
+	VADDUWM V3, V4, V3
+
+	VXOR V15, V0, V15
+	VXOR V12, V1, V12
+	VXOR V13, V2, V13
+	VXOR V14, V3, V14
+
+	VRLW V15, V29, V15
+	VRLW V12, V29, V12
+	VRLW V13, V29, V13
+	VRLW V14, V29, V14
+
+	VADDUWM V10, V15, V10
+	VADDUWM V11, V12, V11
+	VADDUWM V8, V13, V8
+	VADDUWM V9, V14, V9
+
+	VXOR V5, V10, V5
+	VXOR V6, V11, V6
+	VXOR V7, V8, V7
+	VXOR V4, V9, V4
+
+	VRLW V5, V30, V5
+	VRLW V6, V30, V6
+	VRLW V7, V30, V7
+	VRLW V4, V30, V4
+	BC   16, LT, loop_vsx
+
+	VADDUWM V12, V26, V12
+
+	WORD $0x13600F8C		// VMRGEW V0, V1, V27
+	WORD $0x13821F8C		// VMRGEW V2, V3, V28
+
+	WORD $0x10000E8C		// VMRGOW V0, V1, V0
+	WORD $0x10421E8C		// VMRGOW V2, V3, V2
+
+	WORD $0x13A42F8C		// VMRGEW V4, V5, V29
+	WORD $0x13C63F8C		// VMRGEW V6, V7, V30
+
+	XXPERMDI VS32, VS34, $0, VS33
+	XXPERMDI VS32, VS34, $3, VS35
+	XXPERMDI VS59, VS60, $0, VS32
+	XXPERMDI VS59, VS60, $3, VS34
+
+	WORD $0x10842E8C		// VMRGOW V4, V5, V4
+	WORD $0x10C63E8C		// VMRGOW V6, V7, V6
+
+	WORD $0x13684F8C		// VMRGEW V8, V9, V27
+	WORD $0x138A5F8C		// VMRGEW V10, V11, V28
+
+	XXPERMDI VS36, VS38, $0, VS37
+	XXPERMDI VS36, VS38, $3, VS39
+	XXPERMDI VS61, VS62, $0, VS36
+	XXPERMDI VS61, VS62, $3, VS38
+
+	WORD $0x11084E8C		// VMRGOW V8, V9, V8
+	WORD $0x114A5E8C		// VMRGOW V10, V11, V10
+
+	WORD $0x13AC6F8C		// VMRGEW V12, V13, V29
+	WORD $0x13CE7F8C		// VMRGEW V14, V15, V30
+
+	XXPERMDI VS40, VS42, $0, VS41
+	XXPERMDI VS40, VS42, $3, VS43
+	XXPERMDI VS59, VS60, $0, VS40
+	XXPERMDI VS59, VS60, $3, VS42
+
+	WORD $0x118C6E8C		// VMRGOW V12, V13, V12
+	WORD $0x11CE7E8C		// VMRGOW V14, V15, V14
+
+	VSPLTISW $4, V27
+	VADDUWM V26, V27, V26
+
+	XXPERMDI VS44, VS46, $0, VS45
+	XXPERMDI VS44, VS46, $3, VS47
+	XXPERMDI VS61, VS62, $0, VS44
+	XXPERMDI VS61, VS62, $3, VS46
+
+	VADDUWM V0, V16, V0
+	VADDUWM V4, V17, V4
+	VADDUWM V8, V18, V8
+	VADDUWM V12, V19, V12
+
+	CMPU LEN, $64
+	BLT tail_vsx
+
+	// Bottom of loop
+	LXVW4X (INP)(R0), VS59
+	LXVW4X (INP)(R8), VS60
+	LXVW4X (INP)(R9), VS61
+	LXVW4X (INP)(R10), VS62
+
+	VXOR V27, V0, V27
+	VXOR V28, V4, V28
+	VXOR V29, V8, V29
+	VXOR V30, V12, V30
+
+	STXVW4X VS59, (OUT)(R0)
+	STXVW4X VS60, (OUT)(R8)
+	ADD     $64, INP
+	STXVW4X VS61, (OUT)(R9)
+	ADD     $-64, LEN
+	STXVW4X VS62, (OUT)(R10)
+	ADD     $64, OUT
+	BEQ     done_vsx
+
+	VADDUWM V1, V16, V0
+	VADDUWM V5, V17, V4
+	VADDUWM V9, V18, V8
+	VADDUWM V13, V19, V12
+
+	CMPU  LEN, $64
+	BLT   tail_vsx
+
+	LXVW4X (INP)(R0), VS59
+	LXVW4X (INP)(R8), VS60
+	LXVW4X (INP)(R9), VS61
+	LXVW4X (INP)(R10), VS62
+	VXOR   V27, V0, V27
+
+	VXOR V28, V4, V28
+	VXOR V29, V8, V29
+	VXOR V30, V12, V30
+
+	STXVW4X VS59, (OUT)(R0)
+	STXVW4X VS60, (OUT)(R8)
+	ADD     $64, INP
+	STXVW4X VS61, (OUT)(R9)
+	ADD     $-64, LEN
+	STXVW4X VS62, (OUT)(V10)
+	ADD     $64, OUT
+	BEQ     done_vsx
+
+	VADDUWM V2, V16, V0
+	VADDUWM V6, V17, V4
+	VADDUWM V10, V18, V8
+	VADDUWM V14, V19, V12
+
+	CMPU LEN, $64
+	BLT  tail_vsx
+
+	LXVW4X (INP)(R0), VS59
+	LXVW4X (INP)(R8), VS60
+	LXVW4X (INP)(R9), VS61
+	LXVW4X (INP)(R10), VS62
+
+	VXOR V27, V0, V27
+	VXOR V28, V4, V28
+	VXOR V29, V8, V29
+	VXOR V30, V12, V30
+
+	STXVW4X VS59, (OUT)(R0)
+	STXVW4X VS60, (OUT)(R8)
+	ADD     $64, INP
+	STXVW4X VS61, (OUT)(R9)
+	ADD     $-64, LEN
+	STXVW4X VS62, (OUT)(R10)
+	ADD     $64, OUT
+	BEQ     done_vsx
+
+	VADDUWM V3, V16, V0
+	VADDUWM V7, V17, V4
+	VADDUWM V11, V18, V8
+	VADDUWM V15, V19, V12
+
+	CMPU  LEN, $64
+	BLT   tail_vsx
+
+	LXVW4X (INP)(R0), VS59
+	LXVW4X (INP)(R8), VS60
+	LXVW4X (INP)(R9), VS61
+	LXVW4X (INP)(R10), VS62
+
+	VXOR V27, V0, V27
+	VXOR V28, V4, V28
+	VXOR V29, V8, V29
+	VXOR V30, V12, V30
+
+	STXVW4X VS59, (OUT)(R0)
+	STXVW4X VS60, (OUT)(R8)
+	ADD     $64, INP
+	STXVW4X VS61, (OUT)(R9)
+	ADD     $-64, LEN
+	STXVW4X VS62, (OUT)(R10)
+	ADD     $64, OUT
+
+	MOVD $10, R14
+	MOVD R14, CTR
+	BNE  loop_outer_vsx
+
+done_vsx:
+	// Increment counter by number of 64 byte blocks
+	MOVD (CNT), R14
+	ADD  BLOCKS, R14
+	MOVD R14, (CNT)
+	RET
+
+tail_vsx:
+	ADD  $32, R1, R11
+	MOVD LEN, CTR
+
+	// Save values on stack to copy from
+	STXVW4X VS32, (R11)(R0)
+	STXVW4X VS36, (R11)(R8)
+	STXVW4X VS40, (R11)(R9)
+	STXVW4X VS44, (R11)(R10)
+	ADD $-1, R11, R12
+	ADD $-1, INP
+	ADD $-1, OUT
+
+looptail_vsx:
+	// Copying the result to OUT
+	// in bytes.
+	MOVBZU 1(R12), KEY
+	MOVBZU 1(INP), TMP
+	XOR    KEY, TMP, KEY
+	MOVBU  KEY, 1(OUT)
+	BC     16, LT, looptail_vsx
+
+	// Clear the stack values
+	STXVW4X VS48, (R11)(R0)
+	STXVW4X VS48, (R11)(R8)
+	STXVW4X VS48, (R11)(R9)
+	STXVW4X VS48, (R11)(R10)
+	BR      done_vsx
@@ -0,0 +1,26 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo,!appengine
+
+package chacha20
+
+import "golang.org/x/sys/cpu"
+
+var haveAsm = cpu.S390X.HasVX
+
+const bufSize = 256
+
+// xorKeyStreamVX is an assembly implementation of XORKeyStream. It must only
+// be called when the vector facility is available. Implementation in asm_s390x.s.
+//go:noescape
+func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32)
+
+func (c *Cipher) xorKeyStreamBlocks(dst, src []byte) {
+	if cpu.S390X.HasVX {
+		xorKeyStreamVX(dst, src, &c.key, &c.nonce, &c.counter)
+	} else {
+		c.xorKeyStreamBlocksGeneric(dst, src)
+	}
+}
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

-// +build s390x,!gccgo,!appengine
+// +build !gccgo,!appengine

 #include "go_asm.h"
 #include "textflag.h"
@@ -24,15 +24,6 @@ DATA ·constants<>+0x14(SB)/4, $0x3320646e
 DATA ·constants<>+0x18(SB)/4, $0x79622d32
 DATA ·constants<>+0x1c(SB)/4, $0x6b206574

-// EXRL targets:
-TEXT ·mvcSrcToBuf(SB), NOFRAME|NOSPLIT, $0
-	MVC $1, (R1), (R8)
-	RET
-
-TEXT ·mvcBufToDst(SB), NOFRAME|NOSPLIT, $0
-	MVC $1, (R8), (R9)
-	RET
-
 #define BSWAP V5
 #define J0    V6
 #define KEY0  V7
@@ -144,7 +135,7 @@ TEXT ·mvcBufToDst(SB), NOFRAME|NOSPLIT, $0
 	VMRHF v, w, c \ // c = {a[2], b[2], c[2], d[2]}
 	VMRLF v, w, d // d = {a[3], b[3], c[3], d[3]}

-// func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32, buf *[256]byte, len *int)
+// func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32)
 TEXT ·xorKeyStreamVX(SB), NOSPLIT, $0
 	MOVD $·constants<>(SB), R1
 	MOVD dst+0(FP), R2         // R2=&dst[0]
@@ -152,25 +143,10 @@ TEXT ·xorKeyStreamVX(SB), NOSPLIT, $0
 	MOVD key+48(FP), R5        // R5=key
 	MOVD nonce+56(FP), R6      // R6=nonce
 	MOVD counter+64(FP), R7    // R7=counter
-	MOVD buf+72(FP), R8        // R8=buf
-	MOVD len+80(FP), R9        // R9=len

 	// load BSWAP and J0
 	VLM (R1), BSWAP, J0

-	// set up tail buffer
-	ADD     $-1, R4, R12
-	MOVBZ   R12, R12
-	CMPUBEQ R12, $255, aligned
-	MOVD    R4, R1
-	AND     $~255, R1
-	MOVD    $(R3)(R1*1), R1
-	EXRL    $·mvcSrcToBuf(SB), R12
-	MOVD    $255, R0
-	SUB     R12, R0
-	MOVD    R0, (R9)               // update len
-
-aligned:
 	// setup
 	MOVD  $95, R0
 	VLM   (R5), KEY0, KEY1
@@ -217,9 +193,7 @@ loop:

 	// decrement length
 	ADD $-256, R4
-	BLT tail

-continue:
 	// rearrange vectors
 	SHUFFLE(X0, X1, X2, X3, M0, M1, M2, M3)
 	ADDV(J0, X0, X1, X2, X3)
@@ -245,16 +219,6 @@ continue:
 	MOVD $256(R3), R3

 	CMPBNE  R4, $0, chacha
-	CMPUBEQ R12, $255, return
-	EXRL    $·mvcBufToDst(SB), R12 // len was updated during setup

-return:
 	VSTEF $0, CTR, (R7)
 	RET
-
-tail:
-	MOVD R2, R9
-	MOVD R8, R2
-	MOVD R8, R3
-	MOVD $0, R4
-	JMP  continue
@@ -4,9 +4,7 @@

 package chacha20

-import (
-	"runtime"
-)
+import "runtime"

 // Platforms that have fast unaligned 32-bit little endian accesses.
 const unaligned = runtime.GOARCH == "386" ||
@@ -1,8 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This code was translated into a form compatible with 6a from the public
-// domain sources in SUPERCOP: https://bench.cr.yp.to/supercop.html
-
-#define REDMASK51     0x0007FFFFFFFFFFFF
@@ -1,20 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This code was translated into a form compatible with 6a from the public
-// domain sources in SUPERCOP: https://bench.cr.yp.to/supercop.html
-
-// +build amd64,!gccgo,!appengine
-
-// These constants cannot be encoded in non-MOVQ immediates.
-// We access them directly from memory instead.
-
-DATA ·_121666_213(SB)/8, $996687872
-GLOBL ·_121666_213(SB), 8, $8
-
-DATA ·_2P0(SB)/8, $0xFFFFFFFFFFFDA
-GLOBL ·_2P0(SB), 8, $8
-
-DATA ·_2P1234(SB)/8, $0xFFFFFFFFFFFFE
-GLOBL ·_2P1234(SB), 8, $8
@@ -1,65 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build amd64,!gccgo,!appengine
-
-// func cswap(inout *[4][5]uint64, v uint64)
-TEXT ·cswap(SB),7,$0
-	MOVQ inout+0(FP),DI
-	MOVQ v+8(FP),SI
-
-	SUBQ $1, SI
-	NOTQ SI
-	MOVQ SI, X15
-	PSHUFD $0x44, X15, X15
-
-	MOVOU 0(DI), X0
-	MOVOU 16(DI), X2
-	MOVOU 32(DI), X4
-	MOVOU 48(DI), X6
-	MOVOU 64(DI), X8
-	MOVOU 80(DI), X1
-	MOVOU 96(DI), X3
-	MOVOU 112(DI), X5
-	MOVOU 128(DI), X7
-	MOVOU 144(DI), X9
-
-	MOVO X1, X10
-	MOVO X3, X11
-	MOVO X5, X12
-	MOVO X7, X13
-	MOVO X9, X14
-
-	PXOR X0, X10
-	PXOR X2, X11
-	PXOR X4, X12
-	PXOR X6, X13
-	PXOR X8, X14
-	PAND X15, X10
-	PAND X15, X11
-	PAND X15, X12
-	PAND X15, X13
-	PAND X15, X14
-	PXOR X10, X0
-	PXOR X10, X1
-	PXOR X11, X2
-	PXOR X11, X3
-	PXOR X12, X4
-	PXOR X12, X5
-	PXOR X13, X6
-	PXOR X13, X7
-	PXOR X14, X8
-	PXOR X14, X9
-
-	MOVOU X0, 0(DI)
-	MOVOU X2, 16(DI)
-	MOVOU X4, 32(DI)
-	MOVOU X6, 48(DI)
-	MOVOU X8, 64(DI)
-	MOVOU X1, 80(DI)
-	MOVOU X3, 96(DI)
-	MOVOU X5, 112(DI)
-	MOVOU X7, 128(DI)
-	MOVOU X9, 144(DI)
-	RET
@@ -1,834 +1,95 @@
-// Copyright 2013 The Go Authors. All rights reserved.
+// Copyright 2019 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

-// We have an implementation in amd64 assembly so this code is only run on
-// non-amd64 platforms. The amd64 assembly does not support gccgo.
-// +build !amd64 gccgo appengine
-
-package curve25519
+// Package curve25519 provides an implementation of the X25519 function, which
+// performs scalar multiplication on the elliptic curve known as Curve25519.
+// See RFC 7748.
+package curve25519 // import "golang.org/x/crypto/curve25519"

 import (
-	"encoding/binary"
+	"crypto/subtle"
+	"fmt"
 )

-// This code is a port of the public domain, "ref10" implementation of
-// curve25519 from SUPERCOP 20130419 by D. J. Bernstein.
+// ScalarMult sets dst to the product scalar * point.
+//
+// Deprecated: when provided a low-order point, ScalarMult will set dst to all
+// zeroes, irrespective of the scalar. Instead, use the X25519 function, which
+// will return an error.
+func ScalarMult(dst, scalar, point *[32]byte) {
+	scalarMult(dst, scalar, point)
+}

-// fieldElement represents an element of the field GF(2^255 - 19). An element
-// t, entries t[0]...t[9], represents the integer t[0]+2^26 t[1]+2^51 t[2]+2^77
-// t[3]+2^102 t[4]+...+2^230 t[9]. Bounds on each t[i] vary depending on
-// context.
-type fieldElement [10]int32
+// ScalarBaseMult sets dst to the product scalar * base where base is the
+// standard generator.
+//
+// It is recommended to use the X25519 function with Basepoint instead, as
+// copying into fixed size arrays can lead to unexpected bugs.
+func ScalarBaseMult(dst, scalar *[32]byte) {
+	ScalarMult(dst, scalar, &basePoint)
+}

-func feZero(fe *fieldElement) {
-	for i := range fe {
-		fe[i] = 0
+const (
+	// ScalarSize is the size of the scalar input to X25519.
+	ScalarSize = 32
+	// PointSize is the size of the point input to X25519.
+	PointSize = 32
+)
+
+// Basepoint is the canonical Curve25519 generator.
+var Basepoint []byte
+
+var basePoint = [32]byte{9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
+
+func init() { Basepoint = basePoint[:] }
+
+func checkBasepoint() {
+	if subtle.ConstantTimeCompare(Basepoint, []byte{
+		0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	}) != 1 {
+		panic("curve25519: global Basepoint value was modified")
 	}
 }

-func feOne(fe *fieldElement) {
-	feZero(fe)
-	fe[0] = 1
+// X25519 returns the result of the scalar multiplication (scalar * point),
+// according to RFC 7748, Section 5. scalar, point and the return value are
+// slices of 32 bytes.
+//
+// scalar can be generated at random, for example with crypto/rand. point should
+// be either Basepoint or the output of another X25519 call.
+//
+// If point is Basepoint (but not if it's a different slice with the same
+// contents) a precomputed implementation might be used for performance.
+func X25519(scalar, point []byte) ([]byte, error) {
+	// Outline the body of function, to let the allocation be inlined in the
+	// caller, and possibly avoid escaping to the heap.
+	var dst [32]byte
+	return x25519(&dst, scalar, point)
 }

-func feAdd(dst, a, b *fieldElement) {
-	for i := range dst {
-		dst[i] = a[i] + b[i]
+func x25519(dst *[32]byte, scalar, point []byte) ([]byte, error) {
+	var in [32]byte
+	if l := len(scalar); l != 32 {
+		return nil, fmt.Errorf("bad scalar length: %d, expected %d", l, 32)
 	}
-}
-
-func feSub(dst, a, b *fieldElement) {
-	for i := range dst {
-		dst[i] = a[i] - b[i]
-	}
-}
-
-func feCopy(dst, src *fieldElement) {
-	for i := range dst {
-		dst[i] = src[i]
-	}
-}
-
-// feCSwap replaces (f,g) with (g,f) if b == 1; replaces (f,g) with (f,g) if b == 0.
-//
-// Preconditions: b in {0,1}.
-func feCSwap(f, g *fieldElement, b int32) {
-	b = -b
-	for i := range f {
-		t := b & (f[i] ^ g[i])
-		f[i] ^= t
-		g[i] ^= t
-	}
-}
-
-// load3 reads a 24-bit, little-endian value from in.
-func load3(in []byte) int64 {
-	var r int64
-	r = int64(in[0])
-	r |= int64(in[1]) << 8
-	r |= int64(in[2]) << 16
-	return r
-}
-
-// load4 reads a 32-bit, little-endian value from in.
-func load4(in []byte) int64 {
-	return int64(binary.LittleEndian.Uint32(in))
-}
-
-func feFromBytes(dst *fieldElement, src *[32]byte) {
-	h0 := load4(src[:])
-	h1 := load3(src[4:]) << 6
-	h2 := load3(src[7:]) << 5
-	h3 := load3(src[10:]) << 3
-	h4 := load3(src[13:]) << 2
-	h5 := load4(src[16:])
-	h6 := load3(src[20:]) << 7
-	h7 := load3(src[23:]) << 5
-	h8 := load3(src[26:]) << 4
-	h9 := (load3(src[29:]) & 0x7fffff) << 2
-
-	var carry [10]int64
-	carry[9] = (h9 + 1<<24) >> 25
-	h0 += carry[9] * 19
-	h9 -= carry[9] << 25
-	carry[1] = (h1 + 1<<24) >> 25
-	h2 += carry[1]
-	h1 -= carry[1] << 25
-	carry[3] = (h3 + 1<<24) >> 25
-	h4 += carry[3]
-	h3 -= carry[3] << 25
-	carry[5] = (h5 + 1<<24) >> 25
-	h6 += carry[5]
-	h5 -= carry[5] << 25
-	carry[7] = (h7 + 1<<24) >> 25
-	h8 += carry[7]
-	h7 -= carry[7] << 25
-
-	carry[0] = (h0 + 1<<25) >> 26
-	h1 += carry[0]
-	h0 -= carry[0] << 26
-	carry[2] = (h2 + 1<<25) >> 26
-	h3 += carry[2]
-	h2 -= carry[2] << 26
-	carry[4] = (h4 + 1<<25) >> 26
-	h5 += carry[4]
-	h4 -= carry[4] << 26
-	carry[6] = (h6 + 1<<25) >> 26
-	h7 += carry[6]
-	h6 -= carry[6] << 26
-	carry[8] = (h8 + 1<<25) >> 26
-	h9 += carry[8]
-	h8 -= carry[8] << 26
-
-	dst[0] = int32(h0)
-	dst[1] = int32(h1)
-	dst[2] = int32(h2)
-	dst[3] = int32(h3)
-	dst[4] = int32(h4)
-	dst[5] = int32(h5)
-	dst[6] = int32(h6)
-	dst[7] = int32(h7)
-	dst[8] = int32(h8)
-	dst[9] = int32(h9)
-}
-
-// feToBytes marshals h to s.
-// Preconditions:
-//   |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc.
-//
-// Write p=2^255-19; q=floor(h/p).
-// Basic claim: q = floor(2^(-255)(h + 19 2^(-25)h9 + 2^(-1))).
-//
-// Proof:
-//   Have |h|<=p so |q|<=1 so |19^2 2^(-255) q|<1/4.
-//   Also have |h-2^230 h9|<2^230 so |19 2^(-255)(h-2^230 h9)|<1/4.
-//
-//   Write y=2^(-1)-19^2 2^(-255)q-19 2^(-255)(h-2^230 h9).
-//   Then 0<y<1.
-//
-//   Write r=h-pq.
-//   Have 0<=r<=p-1=2^255-20.
-//   Thus 0<=r+19(2^-255)r<r+19(2^-255)2^255<=2^255-1.
-//
-//   Write x=r+19(2^-255)r+y.
-//   Then 0<x<2^255 so floor(2^(-255)x) = 0 so floor(q+2^(-255)x) = q.
-//
-//   Have q+2^(-255)x = 2^(-255)(h + 19 2^(-25) h9 + 2^(-1))
-//   so floor(2^(-255)(h + 19 2^(-25) h9 + 2^(-1))) = q.
-func feToBytes(s *[32]byte, h *fieldElement) {
-	var carry [10]int32
-
-	q := (19*h[9] + (1 << 24)) >> 25
-	q = (h[0] + q) >> 26
-	q = (h[1] + q) >> 25
-	q = (h[2] + q) >> 26
-	q = (h[3] + q) >> 25
-	q = (h[4] + q) >> 26
-	q = (h[5] + q) >> 25
-	q = (h[6] + q) >> 26
-	q = (h[7] + q) >> 25
-	q = (h[8] + q) >> 26
-	q = (h[9] + q) >> 25
-
-	// Goal: Output h-(2^255-19)q, which is between 0 and 2^255-20.
-	h[0] += 19 * q
-	// Goal: Output h-2^255 q, which is between 0 and 2^255-20.
-
-	carry[0] = h[0] >> 26
-	h[1] += carry[0]
-	h[0] -= carry[0] << 26
-	carry[1] = h[1] >> 25
-	h[2] += carry[1]
-	h[1] -= carry[1] << 25
-	carry[2] = h[2] >> 26
-	h[3] += carry[2]
-	h[2] -= carry[2] << 26
-	carry[3] = h[3] >> 25
-	h[4] += carry[3]
-	h[3] -= carry[3] << 25
-	carry[4] = h[4] >> 26
-	h[5] += carry[4]
-	h[4] -= carry[4] << 26
-	carry[5] = h[5] >> 25
-	h[6] += carry[5]
-	h[5] -= carry[5] << 25
-	carry[6] = h[6] >> 26
-	h[7] += carry[6]
-	h[6] -= carry[6] << 26
-	carry[7] = h[7] >> 25
-	h[8] += carry[7]
-	h[7] -= carry[7] << 25
-	carry[8] = h[8] >> 26
-	h[9] += carry[8]
-	h[8] -= carry[8] << 26
-	carry[9] = h[9] >> 25
-	h[9] -= carry[9] << 25
-	// h10 = carry9
-
-	// Goal: Output h[0]+...+2^255 h10-2^255 q, which is between 0 and 2^255-20.
-	// Have h[0]+...+2^230 h[9] between 0 and 2^255-1;
-	// evidently 2^255 h10-2^255 q = 0.
-	// Goal: Output h[0]+...+2^230 h[9].
-
-	s[0] = byte(h[0] >> 0)
-	s[1] = byte(h[0] >> 8)
-	s[2] = byte(h[0] >> 16)
-	s[3] = byte((h[0] >> 24) | (h[1] << 2))
-	s[4] = byte(h[1] >> 6)
-	s[5] = byte(h[1] >> 14)
-	s[6] = byte((h[1] >> 22) | (h[2] << 3))
-	s[7] = byte(h[2] >> 5)
-	s[8] = byte(h[2] >> 13)
-	s[9] = byte((h[2] >> 21) | (h[3] << 5))
-	s[10] = byte(h[3] >> 3)
-	s[11] = byte(h[3] >> 11)
-	s[12] = byte((h[3] >> 19) | (h[4] << 6))
-	s[13] = byte(h[4] >> 2)
-	s[14] = byte(h[4] >> 10)
-	s[15] = byte(h[4] >> 18)
-	s[16] = byte(h[5] >> 0)
-	s[17] = byte(h[5] >> 8)
-	s[18] = byte(h[5] >> 16)
-	s[19] = byte((h[5] >> 24) | (h[6] << 1))
-	s[20] = byte(h[6] >> 7)
-	s[21] = byte(h[6] >> 15)
-	s[22] = byte((h[6] >> 23) | (h[7] << 3))
-	s[23] = byte(h[7] >> 5)
-	s[24] = byte(h[7] >> 13)
-	s[25] = byte((h[7] >> 21) | (h[8] << 4))
-	s[26] = byte(h[8] >> 4)
-	s[27] = byte(h[8] >> 12)
-	s[28] = byte((h[8] >> 20) | (h[9] << 6))
-	s[29] = byte(h[9] >> 2)
-	s[30] = byte(h[9] >> 10)
-	s[31] = byte(h[9] >> 18)
-}
-
-// feMul calculates h = f * g
-// Can overlap h with f or g.
-//
-// Preconditions:
-//    |f| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc.
-//    |g| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc.
-//
-// Postconditions:
-//    |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc.
-//
-// Notes on implementation strategy:
-//
-// Using schoolbook multiplication.
-// Karatsuba would save a little in some cost models.
-//
-// Most multiplications by 2 and 19 are 32-bit precomputations;
-// cheaper than 64-bit postcomputations.
-//
-// There is one remaining multiplication by 19 in the carry chain;
-// one *19 precomputation can be merged into this,
-// but the resulting data flow is considerably less clean.
-//
-// There are 12 carries below.
-// 10 of them are 2-way parallelizable and vectorizable.
-// Can get away with 11 carries, but then data flow is much deeper.
-//
-// With tighter constraints on inputs can squeeze carries into int32.
-func feMul(h, f, g *fieldElement) {
-	f0 := f[0]
-	f1 := f[1]
-	f2 := f[2]
-	f3 := f[3]
-	f4 := f[4]
-	f5 := f[5]
-	f6 := f[6]
-	f7 := f[7]
-	f8 := f[8]
-	f9 := f[9]
-	g0 := g[0]
-	g1 := g[1]
-	g2 := g[2]
-	g3 := g[3]
-	g4 := g[4]
-	g5 := g[5]
-	g6 := g[6]
-	g7 := g[7]
-	g8 := g[8]
-	g9 := g[9]
-	g1_19 := 19 * g1 // 1.4*2^29
-	g2_19 := 19 * g2 // 1.4*2^30; still ok
-	g3_19 := 19 * g3
-	g4_19 := 19 * g4
-	g5_19 := 19 * g5
-	g6_19 := 19 * g6
-	g7_19 := 19 * g7
-	g8_19 := 19 * g8
-	g9_19 := 19 * g9
-	f1_2 := 2 * f1
-	f3_2 := 2 * f3
-	f5_2 := 2 * f5
-	f7_2 := 2 * f7
-	f9_2 := 2 * f9
-	f0g0 := int64(f0) * int64(g0)
-	f0g1 := int64(f0) * int64(g1)
-	f0g2 := int64(f0) * int64(g2)
-	f0g3 := int64(f0) * int64(g3)
-	f0g4 := int64(f0) * int64(g4)
-	f0g5 := int64(f0) * int64(g5)
-	f0g6 := int64(f0) * int64(g6)
-	f0g7 := int64(f0) * int64(g7)
-	f0g8 := int64(f0) * int64(g8)
-	f0g9 := int64(f0) * int64(g9)
-	f1g0 := int64(f1) * int64(g0)
-	f1g1_2 := int64(f1_2) * int64(g1)
-	f1g2 := int64(f1) * int64(g2)
-	f1g3_2 := int64(f1_2) * int64(g3)
-	f1g4 := int64(f1) * int64(g4)
-	f1g5_2 := int64(f1_2) * int64(g5)
-	f1g6 := int64(f1) * int64(g6)
-	f1g7_2 := int64(f1_2) * int64(g7)
-	f1g8 := int64(f1) * int64(g8)
-	f1g9_38 := int64(f1_2) * int64(g9_19)
-	f2g0 := int64(f2) * int64(g0)
-	f2g1 := int64(f2) * int64(g1)
-	f2g2 := int64(f2) * int64(g2)
-	f2g3 := int64(f2) * int64(g3)
-	f2g4 := int64(f2) * int64(g4)
-	f2g5 := int64(f2) * int64(g5)
-	f2g6 := int64(f2) * int64(g6)
-	f2g7 := int64(f2) * int64(g7)
-	f2g8_19 := int64(f2) * int64(g8_19)
-	f2g9_19 := int64(f2) * int64(g9_19)
-	f3g0 := int64(f3) * int64(g0)
-	f3g1_2 := int64(f3_2) * int64(g1)
-	f3g2 := int64(f3) * int64(g2)
-	f3g3_2 := int64(f3_2) * int64(g3)
-	f3g4 := int64(f3) * int64(g4)
-	f3g5_2 := int64(f3_2) * int64(g5)
-	f3g6 := int64(f3) * int64(g6)
-	f3g7_38 := int64(f3_2) * int64(g7_19)
-	f3g8_19 := int64(f3) * int64(g8_19)
-	f3g9_38 := int64(f3_2) * int64(g9_19)
-	f4g0 := int64(f4) * int64(g0)
-	f4g1 := int64(f4) * int64(g1)
-	f4g2 := int64(f4) * int64(g2)
-	f4g3 := int64(f4) * int64(g3)
-	f4g4 := int64(f4) * int64(g4)
-	f4g5 := int64(f4) * int64(g5)
-	f4g6_19 := int64(f4) * int64(g6_19)
-	f4g7_19 := int64(f4) * int64(g7_19)
-	f4g8_19 := int64(f4) * int64(g8_19)
-	f4g9_19 := int64(f4) * int64(g9_19)
-	f5g0 := int64(f5) * int64(g0)
-	f5g1_2 := int64(f5_2) * int64(g1)
-	f5g2 := int64(f5) * int64(g2)
-	f5g3_2 := int64(f5_2) * int64(g3)
-	f5g4 := int64(f5) * int64(g4)
-	f5g5_38 := int64(f5_2) * int64(g5_19)
-	f5g6_19 := int64(f5) * int64(g6_19)
-	f5g7_38 := int64(f5_2) * int64(g7_19)
-	f5g8_19 := int64(f5) * int64(g8_19)
-	f5g9_38 := int64(f5_2) * int64(g9_19)
-	f6g0 := int64(f6) * int64(g0)
-	f6g1 := int64(f6) * int64(g1)
-	f6g2 := int64(f6) * int64(g2)
-	f6g3 := int64(f6) * int64(g3)
-	f6g4_19 := int64(f6) * int64(g4_19)
-	f6g5_19 := int64(f6) * int64(g5_19)
-	f6g6_19 := int64(f6) * int64(g6_19)
-	f6g7_19 := int64(f6) * int64(g7_19)
-	f6g8_19 := int64(f6) * int64(g8_19)
-	f6g9_19 := int64(f6) * int64(g9_19)
-	f7g0 := int64(f7) * int64(g0)
-	f7g1_2 := int64(f7_2) * int64(g1)
-	f7g2 := int64(f7) * int64(g2)
-	f7g3_38 := int64(f7_2) * int64(g3_19)
-	f7g4_19 := int64(f7) * int64(g4_19)
-	f7g5_38 := int64(f7_2) * int64(g5_19)
-	f7g6_19 := int64(f7) * int64(g6_19)
-	f7g7_38 := int64(f7_2) * int64(g7_19)
-	f7g8_19 := int64(f7) * int64(g8_19)
-	f7g9_38 := int64(f7_2) * int64(g9_19)
-	f8g0 := int64(f8) * int64(g0)
-	f8g1 := int64(f8) * int64(g1)
-	f8g2_19 := int64(f8) * int64(g2_19)
-	f8g3_19 := int64(f8) * int64(g3_19)
-	f8g4_19 := int64(f8) * int64(g4_19)
-	f8g5_19 := int64(f8) * int64(g5_19)
-	f8g6_19 := int64(f8) * int64(g6_19)
-	f8g7_19 := int64(f8) * int64(g7_19)
-	f8g8_19 := int64(f8) * int64(g8_19)
-	f8g9_19 := int64(f8) * int64(g9_19)
-	f9g0 := int64(f9) * int64(g0)
-	f9g1_38 := int64(f9_2) * int64(g1_19)
-	f9g2_19 := int64(f9) * int64(g2_19)
-	f9g3_38 := int64(f9_2) * int64(g3_19)
-	f9g4_19 := int64(f9) * int64(g4_19)
-	f9g5_38 := int64(f9_2) * int64(g5_19)
-	f9g6_19 := int64(f9) * int64(g6_19)
-	f9g7_38 := int64(f9_2) * int64(g7_19)
-	f9g8_19 := int64(f9) * int64(g8_19)
-	f9g9_38 := int64(f9_2) * int64(g9_19)
-	h0 := f0g0 + f1g9_38 + f2g8_19 + f3g7_38 + f4g6_19 + f5g5_38 + f6g4_19 + f7g3_38 + f8g2_19 + f9g1_38
-	h1 := f0g1 + f1g0 + f2g9_19 + f3g8_19 + f4g7_19 + f5g6_19 + f6g5_19 + f7g4_19 + f8g3_19 + f9g2_19
-	h2 := f0g2 + f1g1_2 + f2g0 + f3g9_38 + f4g8_19 + f5g7_38 + f6g6_19 + f7g5_38 + f8g4_19 + f9g3_38
-	h3 := f0g3 + f1g2 + f2g1 + f3g0 + f4g9_19 + f5g8_19 + f6g7_19 + f7g6_19 + f8g5_19 + f9g4_19
-	h4 := f0g4 + f1g3_2 + f2g2 + f3g1_2 + f4g0 + f5g9_38 + f6g8_19 + f7g7_38 + f8g6_19 + f9g5_38
-	h5 := f0g5 + f1g4 + f2g3 + f3g2 + f4g1 + f5g0 + f6g9_19 + f7g8_19 + f8g7_19 + f9g6_19
-	h6 := f0g6 + f1g5_2 + f2g4 + f3g3_2 + f4g2 + f5g1_2 + f6g0 + f7g9_38 + f8g8_19 + f9g7_38
-	h7 := f0g7 + f1g6 + f2g5 + f3g4 + f4g3 + f5g2 + f6g1 + f7g0 + f8g9_19 + f9g8_19
-	h8 := f0g8 + f1g7_2 + f2g6 + f3g5_2 + f4g4 + f5g3_2 + f6g2 + f7g1_2 + f8g0 + f9g9_38
-	h9 := f0g9 + f1g8 + f2g7 + f3g6 + f4g5 + f5g4 + f6g3 + f7g2 + f8g1 + f9g0
-	var carry [10]int64
-
-	// |h0| <= (1.1*1.1*2^52*(1+19+19+19+19)+1.1*1.1*2^50*(38+38+38+38+38))
-	//   i.e. |h0| <= 1.2*2^59; narrower ranges for h2, h4, h6, h8
-	// |h1| <= (1.1*1.1*2^51*(1+1+19+19+19+19+19+19+19+19))
-	//   i.e. |h1| <= 1.5*2^58; narrower ranges for h3, h5, h7, h9
-
-	carry[0] = (h0 + (1 << 25)) >> 26
-	h1 += carry[0]
-	h0 -= carry[0] << 26
-	carry[4] = (h4 + (1 << 25)) >> 26
-	h5 += carry[4]
-	h4 -= carry[4] << 26
-	// |h0| <= 2^25
-	// |h4| <= 2^25
-	// |h1| <= 1.51*2^58
-	// |h5| <= 1.51*2^58
-
-	carry[1] = (h1 + (1 << 24)) >> 25
-	h2 += carry[1]
-	h1 -= carry[1] << 25
-	carry[5] = (h5 + (1 << 24)) >> 25
-	h6 += carry[5]
-	h5 -= carry[5] << 25
-	// |h1| <= 2^24; from now on fits into int32
-	// |h5| <= 2^24; from now on fits into int32
-	// |h2| <= 1.21*2^59
-	// |h6| <= 1.21*2^59
-
-	carry[2] = (h2 + (1 << 25)) >> 26
-	h3 += carry[2]
-	h2 -= carry[2] << 26
-	carry[6] = (h6 + (1 << 25)) >> 26
-	h7 += carry[6]
-	h6 -= carry[6] << 26
-	// |h2| <= 2^25; from now on fits into int32 unchanged
-	// |h6| <= 2^25; from now on fits into int32 unchanged
-	// |h3| <= 1.51*2^58
-	// |h7| <= 1.51*2^58
-
-	carry[3] = (h3 + (1 << 24)) >> 25
-	h4 += carry[3]
-	h3 -= carry[3] << 25
-	carry[7] = (h7 + (1 << 24)) >> 25
-	h8 += carry[7]
-	h7 -= carry[7] << 25
-	// |h3| <= 2^24; from now on fits into int32 unchanged
-	// |h7| <= 2^24; from now on fits into int32 unchanged
-	// |h4| <= 1.52*2^33
-	// |h8| <= 1.52*2^33
-
-	carry[4] = (h4 + (1 << 25)) >> 26
-	h5 += carry[4]
-	h4 -= carry[4] << 26
-	carry[8] = (h8 + (1 << 25)) >> 26
-	h9 += carry[8]
-	h8 -= carry[8] << 26
-	// |h4| <= 2^25; from now on fits into int32 unchanged
-	// |h8| <= 2^25; from now on fits into int32 unchanged
-	// |h5| <= 1.01*2^24
-	// |h9| <= 1.51*2^58
-
-	carry[9] = (h9 + (1 << 24)) >> 25
-	h0 += carry[9] * 19
-	h9 -= carry[9] << 25
-	// |h9| <= 2^24; from now on fits into int32 unchanged
-	// |h0| <= 1.8*2^37
-
-	carry[0] = (h0 + (1 << 25)) >> 26
-	h1 += carry[0]
-	h0 -= carry[0] << 26
-	// |h0| <= 2^25; from now on fits into int32 unchanged
-	// |h1| <= 1.01*2^24
-
-	h[0] = int32(h0)
-	h[1] = int32(h1)
-	h[2] = int32(h2)
-	h[3] = int32(h3)
-	h[4] = int32(h4)
-	h[5] = int32(h5)
-	h[6] = int32(h6)
-	h[7] = int32(h7)
-	h[8] = int32(h8)
-	h[9] = int32(h9)
-}
-
-// feSquare calculates h = f*f. Can overlap h with f.
-//
-// Preconditions:
-//    |f| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc.
-//
-// Postconditions:
-//    |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc.
-func feSquare(h, f *fieldElement) {
-	f0 := f[0]
-	f1 := f[1]
-	f2 := f[2]
-	f3 := f[3]
-	f4 := f[4]
-	f5 := f[5]
-	f6 := f[6]
-	f7 := f[7]
-	f8 := f[8]
-	f9 := f[9]
-	f0_2 := 2 * f0
-	f1_2 := 2 * f1
-	f2_2 := 2 * f2
-	f3_2 := 2 * f3
-	f4_2 := 2 * f4
-	f5_2 := 2 * f5
-	f6_2 := 2 * f6
-	f7_2 := 2 * f7
-	f5_38 := 38 * f5 // 1.31*2^30
-	f6_19 := 19 * f6 // 1.31*2^30
-	f7_38 := 38 * f7 // 1.31*2^30
-	f8_19 := 19 * f8 // 1.31*2^30
-	f9_38 := 38 * f9 // 1.31*2^30
-	f0f0 := int64(f0) * int64(f0)
-	f0f1_2 := int64(f0_2) * int64(f1)
-	f0f2_2 := int64(f0_2) * int64(f2)
-	f0f3_2 := int64(f0_2) * int64(f3)
-	f0f4_2 := int64(f0_2) * int64(f4)
-	f0f5_2 := int64(f0_2) * int64(f5)
-	f0f6_2 := int64(f0_2) * int64(f6)
-	f0f7_2 := int64(f0_2) * int64(f7)
-	f0f8_2 := int64(f0_2) * int64(f8)
-	f0f9_2 := int64(f0_2) * int64(f9)
-	f1f1_2 := int64(f1_2) * int64(f1)
-	f1f2_2 := int64(f1_2) * int64(f2)
-	f1f3_4 := int64(f1_2) * int64(f3_2)
-	f1f4_2 := int64(f1_2) * int64(f4)
-	f1f5_4 := int64(f1_2) * int64(f5_2)
-	f1f6_2 := int64(f1_2) * int64(f6)
-	f1f7_4 := int64(f1_2) * int64(f7_2)
-	f1f8_2 := int64(f1_2) * int64(f8)
-	f1f9_76 := int64(f1_2) * int64(f9_38)
-	f2f2 := int64(f2) * int64(f2)
-	f2f3_2 := int64(f2_2) * int64(f3)
-	f2f4_2 := int64(f2_2) * int64(f4)
-	f2f5_2 := int64(f2_2) * int64(f5)
-	f2f6_2 := int64(f2_2) * int64(f6)
-	f2f7_2 := int64(f2_2) * int64(f7)
-	f2f8_38 := int64(f2_2) * int64(f8_19)
-	f2f9_38 := int64(f2) * int64(f9_38)
-	f3f3_2 := int64(f3_2) * int64(f3)
-	f3f4_2 := int64(f3_2) * int64(f4)
-	f3f5_4 := int64(f3_2) * int64(f5_2)
-	f3f6_2 := int64(f3_2) * int64(f6)
-	f3f7_76 := int64(f3_2) * int64(f7_38)
-	f3f8_38 := int64(f3_2) * int64(f8_19)
-	f3f9_76 := int64(f3_2) * int64(f9_38)
-	f4f4 := int64(f4) * int64(f4)
-	f4f5_2 := int64(f4_2) * int64(f5)
-	f4f6_38 := int64(f4_2) * int64(f6_19)
-	f4f7_38 := int64(f4) * int64(f7_38)
-	f4f8_38 := int64(f4_2) * int64(f8_19)
-	f4f9_38 := int64(f4) * int64(f9_38)
-	f5f5_38 := int64(f5) * int64(f5_38)
-	f5f6_38 := int64(f5_2) * int64(f6_19)
-	f5f7_76 := int64(f5_2) * int64(f7_38)
-	f5f8_38 := int64(f5_2) * int64(f8_19)
-	f5f9_76 := int64(f5_2) * int64(f9_38)
-	f6f6_19 := int64(f6) * int64(f6_19)
-	f6f7_38 := int64(f6) * int64(f7_38)
-	f6f8_38 := int64(f6_2) * int64(f8_19)
-	f6f9_38 := int64(f6) * int64(f9_38)
-	f7f7_38 := int64(f7) * int64(f7_38)
-	f7f8_38 := int64(f7_2) * int64(f8_19)
-	f7f9_76 := int64(f7_2) * int64(f9_38)
-	f8f8_19 := int64(f8) * int64(f8_19)
-	f8f9_38 := int64(f8) * int64(f9_38)
-	f9f9_38 := int64(f9) * int64(f9_38)
-	h0 := f0f0 + f1f9_76 + f2f8_38 + f3f7_76 + f4f6_38 + f5f5_38
-	h1 := f0f1_2 + f2f9_38 + f3f8_38 + f4f7_38 + f5f6_38
-	h2 := f0f2_2 + f1f1_2 + f3f9_76 + f4f8_38 + f5f7_76 + f6f6_19
-	h3 := f0f3_2 + f1f2_2 + f4f9_38 + f5f8_38 + f6f7_38
-	h4 := f0f4_2 + f1f3_4 + f2f2 + f5f9_76 + f6f8_38 + f7f7_38
-	h5 := f0f5_2 + f1f4_2 + f2f3_2 + f6f9_38 + f7f8_38
-	h6 := f0f6_2 + f1f5_4 + f2f4_2 + f3f3_2 + f7f9_76 + f8f8_19
-	h7 := f0f7_2 + f1f6_2 + f2f5_2 + f3f4_2 + f8f9_38
-	h8 := f0f8_2 + f1f7_4 + f2f6_2 + f3f5_4 + f4f4 + f9f9_38
-	h9 := f0f9_2 + f1f8_2 + f2f7_2 + f3f6_2 + f4f5_2
-	var carry [10]int64
-
-	carry[0] = (h0 + (1 << 25)) >> 26
-	h1 += carry[0]
-	h0 -= carry[0] << 26
-	carry[4] = (h4 + (1 << 25)) >> 26
-	h5 += carry[4]
-	h4 -= carry[4] << 26
-
-	carry[1] = (h1 + (1 << 24)) >> 25
-	h2 += carry[1]
-	h1 -= carry[1] << 25
-	carry[5] = (h5 + (1 << 24)) >> 25
-	h6 += carry[5]
-	h5 -= carry[5] << 25
-
-	carry[2] = (h2 + (1 << 25)) >> 26
-	h3 += carry[2]
-	h2 -= carry[2] << 26
-	carry[6] = (h6 + (1 << 25)) >> 26
-	h7 += carry[6]
-	h6 -= carry[6] << 26
-
-	carry[3] = (h3 + (1 << 24)) >> 25
-	h4 += carry[3]
-	h3 -= carry[3] << 25
-	carry[7] = (h7 + (1 << 24)) >> 25
-	h8 += carry[7]
-	h7 -= carry[7] << 25
-
-	carry[4] = (h4 + (1 << 25)) >> 26
-	h5 += carry[4]
-	h4 -= carry[4] << 26
-	carry[8] = (h8 + (1 << 25)) >> 26
-	h9 += carry[8]
-	h8 -= carry[8] << 26
-
-	carry[9] = (h9 + (1 << 24)) >> 25
-	h0 += carry[9] * 19
-	h9 -= carry[9] << 25
-
-	carry[0] = (h0 + (1 << 25)) >> 26
-	h1 += carry[0]
-	h0 -= carry[0] << 26
-
-	h[0] = int32(h0)
-	h[1] = int32(h1)
-	h[2] = int32(h2)
-	h[3] = int32(h3)
-	h[4] = int32(h4)
-	h[5] = int32(h5)
-	h[6] = int32(h6)
-	h[7] = int32(h7)
-	h[8] = int32(h8)
-	h[9] = int32(h9)
-}
-
-// feMul121666 calculates h = f * 121666. Can overlap h with f.
-//
-// Preconditions:
-//    |f| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc.
-//
-// Postconditions:
-//    |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc.
-func feMul121666(h, f *fieldElement) {
-	h0 := int64(f[0]) * 121666
-	h1 := int64(f[1]) * 121666
-	h2 := int64(f[2]) * 121666
-	h3 := int64(f[3]) * 121666
-	h4 := int64(f[4]) * 121666
-	h5 := int64(f[5]) * 121666
-	h6 := int64(f[6]) * 121666
-	h7 := int64(f[7]) * 121666
-	h8 := int64(f[8]) * 121666
-	h9 := int64(f[9]) * 121666
-	var carry [10]int64
-
-	carry[9] = (h9 + (1 << 24)) >> 25
-	h0 += carry[9] * 19
-	h9 -= carry[9] << 25
-	carry[1] = (h1 + (1 << 24)) >> 25
-	h2 += carry[1]
-	h1 -= carry[1] << 25
-	carry[3] = (h3 + (1 << 24)) >> 25
-	h4 += carry[3]
-	h3 -= carry[3] << 25
-	carry[5] = (h5 + (1 << 24)) >> 25
-	h6 += carry[5]
-	h5 -= carry[5] << 25
-	carry[7] = (h7 + (1 << 24)) >> 25
-	h8 += carry[7]
-	h7 -= carry[7] << 25
-
-	carry[0] = (h0 + (1 << 25)) >> 26
-	h1 += carry[0]
-	h0 -= carry[0] << 26
-	carry[2] = (h2 + (1 << 25)) >> 26
-	h3 += carry[2]
-	h2 -= carry[2] << 26
-	carry[4] = (h4 + (1 << 25)) >> 26
-	h5 += carry[4]
-	h4 -= carry[4] << 26
-	carry[6] = (h6 + (1 << 25)) >> 26
-	h7 += carry[6]
-	h6 -= carry[6] << 26
-	carry[8] = (h8 + (1 << 25)) >> 26
-	h9 += carry[8]
-	h8 -= carry[8] << 26
-
-	h[0] = int32(h0)
-	h[1] = int32(h1)
-	h[2] = int32(h2)
-	h[3] = int32(h3)
-	h[4] = int32(h4)
-	h[5] = int32(h5)
-	h[6] = int32(h6)
-	h[7] = int32(h7)
-	h[8] = int32(h8)
-	h[9] = int32(h9)
-}
-
-// feInvert sets out = z^-1.
-func feInvert(out, z *fieldElement) {
-	var t0, t1, t2, t3 fieldElement
-	var i int
-
-	feSquare(&t0, z)
-	for i = 1; i < 1; i++ {
-		feSquare(&t0, &t0)
-	}
-	feSquare(&t1, &t0)
-	for i = 1; i < 2; i++ {
-		feSquare(&t1, &t1)
-	}
-	feMul(&t1, z, &t1)
-	feMul(&t0, &t0, &t1)
-	feSquare(&t2, &t0)
-	for i = 1; i < 1; i++ {
-		feSquare(&t2, &t2)
-	}
-	feMul(&t1, &t1, &t2)
-	feSquare(&t2, &t1)
-	for i = 1; i < 5; i++ {
-		feSquare(&t2, &t2)
-	}
-	feMul(&t1, &t2, &t1)
-	feSquare(&t2, &t1)
-	for i = 1; i < 10; i++ {
-		feSquare(&t2, &t2)
-	}
-	feMul(&t2, &t2, &t1)
-	feSquare(&t3, &t2)
-	for i = 1; i < 20; i++ {
-		feSquare(&t3, &t3)
-	}
-	feMul(&t2, &t3, &t2)
-	feSquare(&t2, &t2)
-	for i = 1; i < 10; i++ {
-		feSquare(&t2, &t2)
-	}
-	feMul(&t1, &t2, &t1)
-	feSquare(&t2, &t1)
-	for i = 1; i < 50; i++ {
-		feSquare(&t2, &t2)
-	}
-	feMul(&t2, &t2, &t1)
-	feSquare(&t3, &t2)
-	for i = 1; i < 100; i++ {
-		feSquare(&t3, &t3)
-	}
-	feMul(&t2, &t3, &t2)
-	feSquare(&t2, &t2)
-	for i = 1; i < 50; i++ {
-		feSquare(&t2, &t2)
-	}
-	feMul(&t1, &t2, &t1)
-	feSquare(&t1, &t1)
-	for i = 1; i < 5; i++ {
-		feSquare(&t1, &t1)
-	}
-	feMul(out, &t1, &t0)
-}
-
-func scalarMult(out, in, base *[32]byte) {
-	var e [32]byte
-
-	copy(e[:], in[:])
-	e[0] &= 248
-	e[31] &= 127
-	e[31] |= 64
-
-	var x1, x2, z2, x3, z3, tmp0, tmp1 fieldElement
-	feFromBytes(&x1, base)
-	feOne(&x2)
-	feCopy(&x3, &x1)
-	feOne(&z3)
-
-	swap := int32(0)
-	for pos := 254; pos >= 0; pos-- {
-		b := e[pos/8] >> uint(pos&7)
-		b &= 1
-		swap ^= int32(b)
-		feCSwap(&x2, &x3, swap)
-		feCSwap(&z2, &z3, swap)
-		swap = int32(b)
-
-		feSub(&tmp0, &x3, &z3)
-		feSub(&tmp1, &x2, &z2)
-		feAdd(&x2, &x2, &z2)
-		feAdd(&z2, &x3, &z3)
-		feMul(&z3, &tmp0, &x2)
-		feMul(&z2, &z2, &tmp1)
-		feSquare(&tmp0, &tmp1)
-		feSquare(&tmp1, &x2)
-		feAdd(&x3, &z3, &z2)
-		feSub(&z2, &z3, &z2)
-		feMul(&x2, &tmp1, &tmp0)
-		feSub(&tmp1, &tmp1, &tmp0)
-		feSquare(&z2, &z2)
-		feMul121666(&z3, &tmp1)
-		feSquare(&x3, &x3)
-		feAdd(&tmp0, &tmp0, &z3)
-		feMul(&z3, &x1, &z2)
-		feMul(&z2, &tmp1, &tmp0)
-	}
-
-	feCSwap(&x2, &x3, swap)
-	feCSwap(&z2, &z3, swap)
-
-	feInvert(&z2, &z2)
-	feMul(&x2, &x2, &z2)
-	feToBytes(out, &x2)
+	if l := len(point); l != 32 {
+		return nil, fmt.Errorf("bad point length: %d, expected %d", l, 32)
+	}
+	copy(in[:], scalar)
+	if &point[0] == &Basepoint[0] {
+		checkBasepoint()
+		ScalarBaseMult(dst, &in)
+	} else {
+		var base, zero [32]byte
+		copy(base[:], point)
+		ScalarMult(dst, &in, &base)
+		if subtle.ConstantTimeCompare(dst[:], zero[:]) == 1 {
+			return nil, fmt.Errorf("bad input point: low order point")
+		}
+	}
+	return dst[:], nil
 }
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

-// +build amd64,!gccgo,!appengine
+// +build amd64,!gccgo,!appengine,!purego

 package curve25519

@@ -5,9 +5,84 @@
 // This code was translated into a form compatible with 6a from the public
 // domain sources in SUPERCOP: https://bench.cr.yp.to/supercop.html

-// +build amd64,!gccgo,!appengine
+// +build amd64,!gccgo,!appengine,!purego

-#include "const_amd64.h"
+#define REDMASK51     0x0007FFFFFFFFFFFF
+
+// These constants cannot be encoded in non-MOVQ immediates.
+// We access them directly from memory instead.
+
+DATA ·_121666_213(SB)/8, $996687872
+GLOBL ·_121666_213(SB), 8, $8
+
+DATA ·_2P0(SB)/8, $0xFFFFFFFFFFFDA
+GLOBL ·_2P0(SB), 8, $8
+
+DATA ·_2P1234(SB)/8, $0xFFFFFFFFFFFFE
+GLOBL ·_2P1234(SB), 8, $8
+
+// func freeze(inout *[5]uint64)
+TEXT ·freeze(SB),7,$0-8
+	MOVQ inout+0(FP), DI
+
+	MOVQ 0(DI),SI
+	MOVQ 8(DI),DX
+	MOVQ 16(DI),CX
+	MOVQ 24(DI),R8
+	MOVQ 32(DI),R9
+	MOVQ $REDMASK51,AX
+	MOVQ AX,R10
+	SUBQ $18,R10
+	MOVQ $3,R11
+REDUCELOOP:
+	MOVQ SI,R12
+	SHRQ $51,R12
+	ANDQ AX,SI
+	ADDQ R12,DX
+	MOVQ DX,R12
+	SHRQ $51,R12
+	ANDQ AX,DX
+	ADDQ R12,CX
+	MOVQ CX,R12
+	SHRQ $51,R12
+	ANDQ AX,CX
+	ADDQ R12,R8
+	MOVQ R8,R12
+	SHRQ $51,R12
+	ANDQ AX,R8
+	ADDQ R12,R9
+	MOVQ R9,R12
+	SHRQ $51,R12
+	ANDQ AX,R9
+	IMUL3Q $19,R12,R12
+	ADDQ R12,SI
+	SUBQ $1,R11
+	JA REDUCELOOP
+	MOVQ $1,R12
+	CMPQ R10,SI
+	CMOVQLT R11,R12
+	CMPQ AX,DX
+	CMOVQNE R11,R12
+	CMPQ AX,CX
+	CMOVQNE R11,R12
+	CMPQ AX,R8
+	CMOVQNE R11,R12
+	CMPQ AX,R9
+	CMOVQNE R11,R12
+	NEGQ R12
+	ANDQ R12,AX
+	ANDQ R12,R10
+	SUBQ R10,SI
+	SUBQ AX,DX
+	SUBQ AX,CX
+	SUBQ AX,R8
+	SUBQ AX,R9
+	MOVQ SI,0(DI)
+	MOVQ DX,8(DI)
+	MOVQ CX,16(DI)
+	MOVQ R8,24(DI)
+	MOVQ R9,32(DI)
+	RET

 // func ladderstep(inout *[5][5]uint64)
 TEXT ·ladderstep(SB),0,$296-8
@@ -121,18 +196,18 @@ TEXT ·ladderstep(SB),0,$296-8
 	ADDQ AX,R12
 	ADCQ DX,R13
 	MOVQ $REDMASK51,DX
-	SHLQ $13,CX:SI
+	SHLQ $13,SI,CX
 	ANDQ DX,SI
-	SHLQ $13,R9:R8
+	SHLQ $13,R8,R9
 	ANDQ DX,R8
 	ADDQ CX,R8
-	SHLQ $13,R11:R10
+	SHLQ $13,R10,R11
 	ANDQ DX,R10
 	ADDQ R9,R10
-	SHLQ $13,R13:R12
+	SHLQ $13,R12,R13
 	ANDQ DX,R12
 	ADDQ R11,R12
-	SHLQ $13,R15:R14
+	SHLQ $13,R14,R15
 	ANDQ DX,R14
 	ADDQ R13,R14
 	IMUL3Q $19,R15,CX
@@ -236,18 +311,18 @@ TEXT ·ladderstep(SB),0,$296-8
 	ADDQ AX,R12
 	ADCQ DX,R13
 	MOVQ $REDMASK51,DX
-	SHLQ $13,CX:SI
+	SHLQ $13,SI,CX
 	ANDQ DX,SI
-	SHLQ $13,R9:R8
+	SHLQ $13,R8,R9
 	ANDQ DX,R8
 	ADDQ CX,R8
-	SHLQ $13,R11:R10
+	SHLQ $13,R10,R11
 	ANDQ DX,R10
 	ADDQ R9,R10
-	SHLQ $13,R13:R12
+	SHLQ $13,R12,R13
 	ANDQ DX,R12
 	ADDQ R11,R12
-	SHLQ $13,R15:R14
+	SHLQ $13,R14,R15
 	ANDQ DX,R14
 	ADDQ R13,R14
 	IMUL3Q $19,R15,CX
@@ -441,18 +516,18 @@ TEXT ·ladderstep(SB),0,$296-8
 	ADDQ AX,R12
 	ADCQ DX,R13
 	MOVQ $REDMASK51,DX
-	SHLQ $13,CX:SI
+	SHLQ $13,SI,CX
 	ANDQ DX,SI
-	SHLQ $13,R9:R8
+	SHLQ $13,R8,R9
 	ANDQ DX,R8
 	ADDQ CX,R8
-	SHLQ $13,R11:R10
+	SHLQ $13,R10,R11
 	ANDQ DX,R10
 	ADDQ R9,R10
-	SHLQ $13,R13:R12
+	SHLQ $13,R12,R13
 	ANDQ DX,R12
 	ADDQ R11,R12
-	SHLQ $13,R15:R14
+	SHLQ $13,R14,R15
 	ANDQ DX,R14
 	ADDQ R13,R14
 	IMUL3Q $19,R15,CX
@@ -591,18 +666,18 @@ TEXT ·ladderstep(SB),0,$296-8
 	ADDQ AX,R12
 	ADCQ DX,R13
 	MOVQ $REDMASK51,DX
-	SHLQ $13,CX:SI
+	SHLQ $13,SI,CX
 	ANDQ DX,SI
-	SHLQ $13,R9:R8
+	SHLQ $13,R8,R9
 	ANDQ DX,R8
 	ADDQ CX,R8
-	SHLQ $13,R11:R10
+	SHLQ $13,R10,R11
 	ANDQ DX,R10
 	ADDQ R9,R10
-	SHLQ $13,R13:R12
+	SHLQ $13,R12,R13
 	ANDQ DX,R12
 	ADDQ R11,R12
-	SHLQ $13,R15:R14
+	SHLQ $13,R14,R15
 	ANDQ DX,R14
 	ADDQ R13,R14
 	IMUL3Q $19,R15,CX
@@ -731,18 +806,18 @@ TEXT ·ladderstep(SB),0,$296-8
 	ADDQ AX,R12
 	ADCQ DX,R13
 	MOVQ $REDMASK51,DX
-	SHLQ $13,CX:SI
+	SHLQ $13,SI,CX
 	ANDQ DX,SI
-	SHLQ $13,R9:R8
+	SHLQ $13,R8,R9
 	ANDQ DX,R8
 	ADDQ CX,R8
-	SHLQ $13,R11:R10
+	SHLQ $13,R10,R11
 	ANDQ DX,R10
 	ADDQ R9,R10
-	SHLQ $13,R13:R12
+	SHLQ $13,R12,R13
 	ANDQ DX,R12
 	ADDQ R11,R12
-	SHLQ $13,R15:R14
+	SHLQ $13,R14,R15
 	ANDQ DX,R14
 	ADDQ R13,R14
 	IMUL3Q $19,R15,CX
@@ -846,18 +921,18 @@ TEXT ·ladderstep(SB),0,$296-8
 	ADDQ AX,R12
 	ADCQ DX,R13
 	MOVQ $REDMASK51,DX
-	SHLQ $13,CX:SI
+	SHLQ $13,SI,CX
 	ANDQ DX,SI
-	SHLQ $13,R9:R8
+	SHLQ $13,R8,R9
 	ANDQ DX,R8
 	ADDQ CX,R8
-	SHLQ $13,R11:R10
+	SHLQ $13,R10,R11
 	ANDQ DX,R10
 	ADDQ R9,R10
-	SHLQ $13,R13:R12
+	SHLQ $13,R12,R13
 	ANDQ DX,R12
 	ADDQ R11,R12
-	SHLQ $13,R15:R14
+	SHLQ $13,R14,R15
 	ANDQ DX,R14
 	ADDQ R13,R14
 	IMUL3Q $19,R15,CX
@@ -996,18 +1071,18 @@ TEXT ·ladderstep(SB),0,$296-8
 	ADDQ AX,R12
 	ADCQ DX,R13
 	MOVQ $REDMASK51,DX
-	SHLQ $13,CX:SI
+	SHLQ $13,SI,CX
 	ANDQ DX,SI
-	SHLQ $13,R9:R8
+	SHLQ $13,R8,R9
 	ANDQ DX,R8
 	ADDQ CX,R8
-	SHLQ $13,R11:R10
+	SHLQ $13,R10,R11
 	ANDQ DX,R10
 	ADDQ R9,R10
-	SHLQ $13,R13:R12
+	SHLQ $13,R12,R13
 	ANDQ DX,R12
 	ADDQ R11,R12
-	SHLQ $13,R15:R14
+	SHLQ $13,R14,R15
 	ANDQ DX,R14
 	ADDQ R13,R14
 	IMUL3Q $19,R15,CX
@@ -1146,18 +1221,18 @@ TEXT ·ladderstep(SB),0,$296-8
 	ADDQ AX,R12
 	ADCQ DX,R13
 	MOVQ $REDMASK51,DX
-	SHLQ $13,CX:SI
+	SHLQ $13,SI,CX
 	ANDQ DX,SI
-	SHLQ $13,R9:R8
+	SHLQ $13,R8,R9
 	ANDQ DX,R8
 	ADDQ CX,R8
-	SHLQ $13,R11:R10
+	SHLQ $13,R10,R11
 	ANDQ DX,R10
 	ADDQ R9,R10
-	SHLQ $13,R13:R12
+	SHLQ $13,R12,R13
 	ANDQ DX,R12
 	ADDQ R11,R12
-	SHLQ $13,R15:R14
+	SHLQ $13,R14,R15
 	ANDQ DX,R14
 	ADDQ R13,R14
 	IMUL3Q $19,R15,CX
@@ -1332,18 +1407,18 @@ TEXT ·ladderstep(SB),0,$296-8
 	ADDQ AX,R12
 	ADCQ DX,R13
 	MOVQ $REDMASK51,DX
-	SHLQ $13,CX:SI
+	SHLQ $13,SI,CX
 	ANDQ DX,SI
-	SHLQ $13,R9:R8
+	SHLQ $13,R8,R9
 	ANDQ DX,R8
 	ADDQ CX,R8
-	SHLQ $13,R11:R10
+	SHLQ $13,R10,R11
 	ANDQ DX,R10
 	ADDQ R9,R10
-	SHLQ $13,R13:R12
+	SHLQ $13,R12,R13
 	ANDQ DX,R12
 	ADDQ R11,R12
-	SHLQ $13,R15:R14
+	SHLQ $13,R14,R15
 	ANDQ DX,R14
 	ADDQ R13,R14
 	IMUL3Q $19,R15,CX
@@ -1375,3 +1450,344 @@ TEXT ·ladderstep(SB),0,$296-8
 	MOVQ AX,104(DI)
 	MOVQ R10,112(DI)
 	RET
+
+// func cswap(inout *[4][5]uint64, v uint64)
+TEXT ·cswap(SB),7,$0
+	MOVQ inout+0(FP),DI
+	MOVQ v+8(FP),SI
+
+	SUBQ $1, SI
+	NOTQ SI
+	MOVQ SI, X15
+	PSHUFD $0x44, X15, X15
+
+	MOVOU 0(DI), X0
+	MOVOU 16(DI), X2
+	MOVOU 32(DI), X4
+	MOVOU 48(DI), X6
+	MOVOU 64(DI), X8
+	MOVOU 80(DI), X1
+	MOVOU 96(DI), X3
+	MOVOU 112(DI), X5
+	MOVOU 128(DI), X7
+	MOVOU 144(DI), X9
+
+	MOVO X1, X10
+	MOVO X3, X11
+	MOVO X5, X12
+	MOVO X7, X13
+	MOVO X9, X14
+
+	PXOR X0, X10
+	PXOR X2, X11
+	PXOR X4, X12
+	PXOR X6, X13
+	PXOR X8, X14
+	PAND X15, X10
+	PAND X15, X11
+	PAND X15, X12
+	PAND X15, X13
+	PAND X15, X14
+	PXOR X10, X0
+	PXOR X10, X1
+	PXOR X11, X2
+	PXOR X11, X3
+	PXOR X12, X4
+	PXOR X12, X5
+	PXOR X13, X6
+	PXOR X13, X7
+	PXOR X14, X8
+	PXOR X14, X9
+
+	MOVOU X0, 0(DI)
+	MOVOU X2, 16(DI)
+	MOVOU X4, 32(DI)
+	MOVOU X6, 48(DI)
+	MOVOU X8, 64(DI)
+	MOVOU X1, 80(DI)
+	MOVOU X3, 96(DI)
+	MOVOU X5, 112(DI)
+	MOVOU X7, 128(DI)
+	MOVOU X9, 144(DI)
+	RET
+
+// func mul(dest, a, b *[5]uint64)
+TEXT ·mul(SB),0,$16-24
+	MOVQ dest+0(FP), DI
+	MOVQ a+8(FP), SI
+	MOVQ b+16(FP), DX
+
+	MOVQ DX,CX
+	MOVQ 24(SI),DX
+	IMUL3Q $19,DX,AX
+	MOVQ AX,0(SP)
+	MULQ 16(CX)
+	MOVQ AX,R8
+	MOVQ DX,R9
+	MOVQ 32(SI),DX
+	IMUL3Q $19,DX,AX
+	MOVQ AX,8(SP)
+	MULQ 8(CX)
+	ADDQ AX,R8
+	ADCQ DX,R9
+	MOVQ 0(SI),AX
+	MULQ 0(CX)
+	ADDQ AX,R8
+	ADCQ DX,R9
+	MOVQ 0(SI),AX
+	MULQ 8(CX)
+	MOVQ AX,R10
+	MOVQ DX,R11
+	MOVQ 0(SI),AX
+	MULQ 16(CX)
+	MOVQ AX,R12
+	MOVQ DX,R13
+	MOVQ 0(SI),AX
+	MULQ 24(CX)
+	MOVQ AX,R14
+	MOVQ DX,R15
+	MOVQ 0(SI),AX
+	MULQ 32(CX)
+	MOVQ AX,BX
+	MOVQ DX,BP
+	MOVQ 8(SI),AX
+	MULQ 0(CX)
+	ADDQ AX,R10
+	ADCQ DX,R11
+	MOVQ 8(SI),AX
+	MULQ 8(CX)
+	ADDQ AX,R12
+	ADCQ DX,R13
+	MOVQ 8(SI),AX
+	MULQ 16(CX)
+	ADDQ AX,R14
+	ADCQ DX,R15
+	MOVQ 8(SI),AX
+	MULQ 24(CX)
+	ADDQ AX,BX
+	ADCQ DX,BP
+	MOVQ 8(SI),DX
+	IMUL3Q $19,DX,AX
+	MULQ 32(CX)
+	ADDQ AX,R8
+	ADCQ DX,R9
+	MOVQ 16(SI),AX
+	MULQ 0(CX)
+	ADDQ AX,R12
+	ADCQ DX,R13
+	MOVQ 16(SI),AX
+	MULQ 8(CX)
+	ADDQ AX,R14
+	ADCQ DX,R15
+	MOVQ 16(SI),AX
+	MULQ 16(CX)
+	ADDQ AX,BX
+	ADCQ DX,BP
+	MOVQ 16(SI),DX
+	IMUL3Q $19,DX,AX
+	MULQ 24(CX)
+	ADDQ AX,R8
+	ADCQ DX,R9
+	MOVQ 16(SI),DX
+	IMUL3Q $19,DX,AX
+	MULQ 32(CX)
+	ADDQ AX,R10
+	ADCQ DX,R11
+	MOVQ 24(SI),AX
+	MULQ 0(CX)
+	ADDQ AX,R14
+	ADCQ DX,R15
+	MOVQ 24(SI),AX
+	MULQ 8(CX)
+	ADDQ AX,BX
+	ADCQ DX,BP
+	MOVQ 0(SP),AX
+	MULQ 24(CX)
+	ADDQ AX,R10
+	ADCQ DX,R11
+	MOVQ 0(SP),AX
+	MULQ 32(CX)
+	ADDQ AX,R12
+	ADCQ DX,R13
+	MOVQ 32(SI),AX
+	MULQ 0(CX)
+	ADDQ AX,BX
+	ADCQ DX,BP
+	MOVQ 8(SP),AX
+	MULQ 16(CX)
+	ADDQ AX,R10
+	ADCQ DX,R11
+	MOVQ 8(SP),AX
+	MULQ 24(CX)
+	ADDQ AX,R12
+	ADCQ DX,R13
+	MOVQ 8(SP),AX
+	MULQ 32(CX)
+	ADDQ AX,R14
+	ADCQ DX,R15
+	MOVQ $REDMASK51,SI
+	SHLQ $13,R8,R9
+	ANDQ SI,R8
+	SHLQ $13,R10,R11
+	ANDQ SI,R10
+	ADDQ R9,R10
+	SHLQ $13,R12,R13
+	ANDQ SI,R12
+	ADDQ R11,R12
+	SHLQ $13,R14,R15
+	ANDQ SI,R14
+	ADDQ R13,R14
+	SHLQ $13,BX,BP
+	ANDQ SI,BX
+	ADDQ R15,BX
+	IMUL3Q $19,BP,DX
+	ADDQ DX,R8
+	MOVQ R8,DX
+	SHRQ $51,DX
+	ADDQ R10,DX
+	MOVQ DX,CX
+	SHRQ $51,DX
+	ANDQ SI,R8
+	ADDQ R12,DX
+	MOVQ DX,R9
+	SHRQ $51,DX
+	ANDQ SI,CX
+	ADDQ R14,DX
+	MOVQ DX,AX
+	SHRQ $51,DX
+	ANDQ SI,R9
+	ADDQ BX,DX
+	MOVQ DX,R10
+	SHRQ $51,DX
+	ANDQ SI,AX
+	IMUL3Q $19,DX,DX
+	ADDQ DX,R8
+	ANDQ SI,R10
+	MOVQ R8,0(DI)
+	MOVQ CX,8(DI)
+	MOVQ R9,16(DI)
+	MOVQ AX,24(DI)
+	MOVQ R10,32(DI)
+	RET
+
+// func square(out, in *[5]uint64)
+TEXT ·square(SB),7,$0-16
+	MOVQ out+0(FP), DI
+	MOVQ in+8(FP), SI
+
+	MOVQ 0(SI),AX
+	MULQ 0(SI)
+	MOVQ AX,CX
+	MOVQ DX,R8
+	MOVQ 0(SI),AX
+	SHLQ $1,AX
+	MULQ 8(SI)
+	MOVQ AX,R9
+	MOVQ DX,R10
+	MOVQ 0(SI),AX
+	SHLQ $1,AX
+	MULQ 16(SI)
+	MOVQ AX,R11
+	MOVQ DX,R12
+	MOVQ 0(SI),AX
+	SHLQ $1,AX
+	MULQ 24(SI)
+	MOVQ AX,R13
+	MOVQ DX,R14
+	MOVQ 0(SI),AX
+	SHLQ $1,AX
+	MULQ 32(SI)
+	MOVQ AX,R15
+	MOVQ DX,BX
+	MOVQ 8(SI),AX
+	MULQ 8(SI)
+	ADDQ AX,R11
+	ADCQ DX,R12
+	MOVQ 8(SI),AX
+	SHLQ $1,AX
+	MULQ 16(SI)
+	ADDQ AX,R13
+	ADCQ DX,R14
+	MOVQ 8(SI),AX
+	SHLQ $1,AX
+	MULQ 24(SI)
+	ADDQ AX,R15
+	ADCQ DX,BX
+	MOVQ 8(SI),DX
+	IMUL3Q $38,DX,AX
+	MULQ 32(SI)
+	ADDQ AX,CX
+	ADCQ DX,R8
+	MOVQ 16(SI),AX
+	MULQ 16(SI)
+	ADDQ AX,R15
+	ADCQ DX,BX
+	MOVQ 16(SI),DX
+	IMUL3Q $38,DX,AX
+	MULQ 24(SI)
+	ADDQ AX,CX
+	ADCQ DX,R8
+	MOVQ 16(SI),DX
+	IMUL3Q $38,DX,AX
+	MULQ 32(SI)
+	ADDQ AX,R9
+	ADCQ DX,R10
+	MOVQ 24(SI),DX
+	IMUL3Q $19,DX,AX
+	MULQ 24(SI)
+	ADDQ AX,R9
+	ADCQ DX,R10
+	MOVQ 24(SI),DX
+	IMUL3Q $38,DX,AX
+	MULQ 32(SI)
+	ADDQ AX,R11
+	ADCQ DX,R12
+	MOVQ 32(SI),DX
+	IMUL3Q $19,DX,AX
+	MULQ 32(SI)
+	ADDQ AX,R13
+	ADCQ DX,R14
+	MOVQ $REDMASK51,SI
+	SHLQ $13,CX,R8
+	ANDQ SI,CX
+	SHLQ $13,R9,R10
+	ANDQ SI,R9
+	ADDQ R8,R9
+	SHLQ $13,R11,R12
+	ANDQ SI,R11
+	ADDQ R10,R11
+	SHLQ $13,R13,R14
+	ANDQ SI,R13
+	ADDQ R12,R13
+	SHLQ $13,R15,BX
+	ANDQ SI,R15
+	ADDQ R14,R15
+	IMUL3Q $19,BX,DX
+	ADDQ DX,CX
+	MOVQ CX,DX
+	SHRQ $51,DX
+	ADDQ R9,DX
+	ANDQ SI,CX
+	MOVQ DX,R8
+	SHRQ $51,DX
+	ADDQ R11,DX
+	ANDQ SI,R8
+	MOVQ DX,R9
+	SHRQ $51,DX
+	ADDQ R13,DX
+	ANDQ SI,R9
+	MOVQ DX,AX
+	SHRQ $51,DX
+	ADDQ R15,DX
+	ANDQ SI,AX
+	MOVQ DX,R10
+	SHRQ $51,DX
+	IMUL3Q $19,DX,DX
+	ADDQ DX,CX
+	ANDQ SI,R10
+	MOVQ CX,0(DI)
+	MOVQ R8,8(DI)
+	MOVQ R9,16(DI)
+	MOVQ AX,24(DI)
+	MOVQ R10,32(DI)
+	RET
@@ -0,0 +1,828 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package curve25519
+
+import "encoding/binary"
+
+// This code is a port of the public domain, "ref10" implementation of
+// curve25519 from SUPERCOP 20130419 by D. J. Bernstein.
+
+// fieldElement represents an element of the field GF(2^255 - 19). An element
+// t, entries t[0]...t[9], represents the integer t[0]+2^26 t[1]+2^51 t[2]+2^77
+// t[3]+2^102 t[4]+...+2^230 t[9]. Bounds on each t[i] vary depending on
+// context.
+type fieldElement [10]int32
+
+func feZero(fe *fieldElement) {
+	for i := range fe {
+		fe[i] = 0
+	}
+}
+
+func feOne(fe *fieldElement) {
+	feZero(fe)
+	fe[0] = 1
+}
+
+func feAdd(dst, a, b *fieldElement) {
+	for i := range dst {
+		dst[i] = a[i] + b[i]
+	}
+}
+
+func feSub(dst, a, b *fieldElement) {
+	for i := range dst {
+		dst[i] = a[i] - b[i]
+	}
+}
+
+func feCopy(dst, src *fieldElement) {
+	for i := range dst {
+		dst[i] = src[i]
+	}
+}
+
+// feCSwap replaces (f,g) with (g,f) if b == 1; replaces (f,g) with (f,g) if b == 0.
+//
+// Preconditions: b in {0,1}.
+func feCSwap(f, g *fieldElement, b int32) {
+	b = -b
+	for i := range f {
+		t := b & (f[i] ^ g[i])
+		f[i] ^= t
+		g[i] ^= t
+	}
+}
+
+// load3 reads a 24-bit, little-endian value from in.
+func load3(in []byte) int64 {
+	var r int64
+	r = int64(in[0])
+	r |= int64(in[1]) << 8
+	r |= int64(in[2]) << 16
+	return r
+}
+
+// load4 reads a 32-bit, little-endian value from in.
+func load4(in []byte) int64 {
+	return int64(binary.LittleEndian.Uint32(in))
+}
+
+func feFromBytes(dst *fieldElement, src *[32]byte) {
+	h0 := load4(src[:])
+	h1 := load3(src[4:]) << 6
+	h2 := load3(src[7:]) << 5
+	h3 := load3(src[10:]) << 3
+	h4 := load3(src[13:]) << 2
+	h5 := load4(src[16:])
+	h6 := load3(src[20:]) << 7
+	h7 := load3(src[23:]) << 5
+	h8 := load3(src[26:]) << 4
+	h9 := (load3(src[29:]) & 0x7fffff) << 2
+
+	var carry [10]int64
+	carry[9] = (h9 + 1<<24) >> 25
+	h0 += carry[9] * 19
+	h9 -= carry[9] << 25
+	carry[1] = (h1 + 1<<24) >> 25
+	h2 += carry[1]
+	h1 -= carry[1] << 25
+	carry[3] = (h3 + 1<<24) >> 25
+	h4 += carry[3]
+	h3 -= carry[3] << 25
+	carry[5] = (h5 + 1<<24) >> 25
+	h6 += carry[5]
+	h5 -= carry[5] << 25
+	carry[7] = (h7 + 1<<24) >> 25
+	h8 += carry[7]
+	h7 -= carry[7] << 25
+
+	carry[0] = (h0 + 1<<25) >> 26
+	h1 += carry[0]
+	h0 -= carry[0] << 26
+	carry[2] = (h2 + 1<<25) >> 26
+	h3 += carry[2]
+	h2 -= carry[2] << 26
+	carry[4] = (h4 + 1<<25) >> 26
+	h5 += carry[4]
+	h4 -= carry[4] << 26
+	carry[6] = (h6 + 1<<25) >> 26
+	h7 += carry[6]
+	h6 -= carry[6] << 26
+	carry[8] = (h8 + 1<<25) >> 26
+	h9 += carry[8]
+	h8 -= carry[8] << 26
+
+	dst[0] = int32(h0)
+	dst[1] = int32(h1)
+	dst[2] = int32(h2)
+	dst[3] = int32(h3)
+	dst[4] = int32(h4)
+	dst[5] = int32(h5)
+	dst[6] = int32(h6)
+	dst[7] = int32(h7)
+	dst[8] = int32(h8)
+	dst[9] = int32(h9)
+}
+
+// feToBytes marshals h to s.
+// Preconditions:
+//   |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc.
+//
+// Write p=2^255-19; q=floor(h/p).
+// Basic claim: q = floor(2^(-255)(h + 19 2^(-25)h9 + 2^(-1))).
+//
+// Proof:
+//   Have |h|<=p so |q|<=1 so |19^2 2^(-255) q|<1/4.
+//   Also have |h-2^230 h9|<2^230 so |19 2^(-255)(h-2^230 h9)|<1/4.
+//
+//   Write y=2^(-1)-19^2 2^(-255)q-19 2^(-255)(h-2^230 h9).
+//   Then 0<y<1.
+//
+//   Write r=h-pq.
+//   Have 0<=r<=p-1=2^255-20.
+//   Thus 0<=r+19(2^-255)r<r+19(2^-255)2^255<=2^255-1.
+//
+//   Write x=r+19(2^-255)r+y.
+//   Then 0<x<2^255 so floor(2^(-255)x) = 0 so floor(q+2^(-255)x) = q.
+//
+//   Have q+2^(-255)x = 2^(-255)(h + 19 2^(-25) h9 + 2^(-1))
+//   so floor(2^(-255)(h + 19 2^(-25) h9 + 2^(-1))) = q.
+func feToBytes(s *[32]byte, h *fieldElement) {
+	var carry [10]int32
+
+	q := (19*h[9] + (1 << 24)) >> 25
+	q = (h[0] + q) >> 26
+	q = (h[1] + q) >> 25
+	q = (h[2] + q) >> 26
+	q = (h[3] + q) >> 25
+	q = (h[4] + q) >> 26
+	q = (h[5] + q) >> 25
+	q = (h[6] + q) >> 26
+	q = (h[7] + q) >> 25
+	q = (h[8] + q) >> 26
+	q = (h[9] + q) >> 25
+
+	// Goal: Output h-(2^255-19)q, which is between 0 and 2^255-20.
+	h[0] += 19 * q
+	// Goal: Output h-2^255 q, which is between 0 and 2^255-20.
+
+	carry[0] = h[0] >> 26
+	h[1] += carry[0]
+	h[0] -= carry[0] << 26
+	carry[1] = h[1] >> 25
+	h[2] += carry[1]
+	h[1] -= carry[1] << 25
+	carry[2] = h[2] >> 26
+	h[3] += carry[2]
+	h[2] -= carry[2] << 26
+	carry[3] = h[3] >> 25
+	h[4] += carry[3]
+	h[3] -= carry[3] << 25
+	carry[4] = h[4] >> 26
+	h[5] += carry[4]
+	h[4] -= carry[4] << 26
+	carry[5] = h[5] >> 25
+	h[6] += carry[5]
+	h[5] -= carry[5] << 25
+	carry[6] = h[6] >> 26
+	h[7] += carry[6]
+	h[6] -= carry[6] << 26
+	carry[7] = h[7] >> 25
+	h[8] += carry[7]
+	h[7] -= carry[7] << 25
+	carry[8] = h[8] >> 26
+	h[9] += carry[8]
+	h[8] -= carry[8] << 26
+	carry[9] = h[9] >> 25
+	h[9] -= carry[9] << 25
+	// h10 = carry9
+
+	// Goal: Output h[0]+...+2^255 h10-2^255 q, which is between 0 and 2^255-20.
+	// Have h[0]+...+2^230 h[9] between 0 and 2^255-1;
+	// evidently 2^255 h10-2^255 q = 0.
+	// Goal: Output h[0]+...+2^230 h[9].
+
+	s[0] = byte(h[0] >> 0)
+	s[1] = byte(h[0] >> 8)
+	s[2] = byte(h[0] >> 16)
+	s[3] = byte((h[0] >> 24) | (h[1] << 2))
+	s[4] = byte(h[1] >> 6)
+	s[5] = byte(h[1] >> 14)
+	s[6] = byte((h[1] >> 22) | (h[2] << 3))
+	s[7] = byte(h[2] >> 5)
+	s[8] = byte(h[2] >> 13)
+	s[9] = byte((h[2] >> 21) | (h[3] << 5))
+	s[10] = byte(h[3] >> 3)
+	s[11] = byte(h[3] >> 11)
+	s[12] = byte((h[3] >> 19) | (h[4] << 6))
+	s[13] = byte(h[4] >> 2)
+	s[14] = byte(h[4] >> 10)
+	s[15] = byte(h[4] >> 18)
+	s[16] = byte(h[5] >> 0)
+	s[17] = byte(h[5] >> 8)
+	s[18] = byte(h[5] >> 16)
+	s[19] = byte((h[5] >> 24) | (h[6] << 1))
+	s[20] = byte(h[6] >> 7)
+	s[21] = byte(h[6] >> 15)
+	s[22] = byte((h[6] >> 23) | (h[7] << 3))
+	s[23] = byte(h[7] >> 5)
+	s[24] = byte(h[7] >> 13)
+	s[25] = byte((h[7] >> 21) | (h[8] << 4))
+	s[26] = byte(h[8] >> 4)
+	s[27] = byte(h[8] >> 12)
+	s[28] = byte((h[8] >> 20) | (h[9] << 6))
+	s[29] = byte(h[9] >> 2)
+	s[30] = byte(h[9] >> 10)
+	s[31] = byte(h[9] >> 18)
+}
+
+// feMul calculates h = f * g
+// Can overlap h with f or g.
+//
+// Preconditions:
+//    |f| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc.
+//    |g| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc.
+//
+// Postconditions:
+//    |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc.
+//
+// Notes on implementation strategy:
+//
+// Using schoolbook multiplication.
+// Karatsuba would save a little in some cost models.
+//
+// Most multiplications by 2 and 19 are 32-bit precomputations;
+// cheaper than 64-bit postcomputations.
+//
+// There is one remaining multiplication by 19 in the carry chain;
+// one *19 precomputation can be merged into this,
+// but the resulting data flow is considerably less clean.
+//
+// There are 12 carries below.
+// 10 of them are 2-way parallelizable and vectorizable.
+// Can get away with 11 carries, but then data flow is much deeper.
+//
+// With tighter constraints on inputs can squeeze carries into int32.
+func feMul(h, f, g *fieldElement) {
+	f0 := f[0]
+	f1 := f[1]
+	f2 := f[2]
+	f3 := f[3]
+	f4 := f[4]
+	f5 := f[5]
+	f6 := f[6]
+	f7 := f[7]
+	f8 := f[8]
+	f9 := f[9]
+	g0 := g[0]
+	g1 := g[1]
+	g2 := g[2]
+	g3 := g[3]
+	g4 := g[4]
+	g5 := g[5]
+	g6 := g[6]
+	g7 := g[7]
+	g8 := g[8]
+	g9 := g[9]
+	g1_19 := 19 * g1 // 1.4*2^29
+	g2_19 := 19 * g2 // 1.4*2^30; still ok
+	g3_19 := 19 * g3
+	g4_19 := 19 * g4
+	g5_19 := 19 * g5
+	g6_19 := 19 * g6
+	g7_19 := 19 * g7
+	g8_19 := 19 * g8
+	g9_19 := 19 * g9
+	f1_2 := 2 * f1
+	f3_2 := 2 * f3
+	f5_2 := 2 * f5
+	f7_2 := 2 * f7
+	f9_2 := 2 * f9
+	f0g0 := int64(f0) * int64(g0)
+	f0g1 := int64(f0) * int64(g1)
+	f0g2 := int64(f0) * int64(g2)
+	f0g3 := int64(f0) * int64(g3)
+	f0g4 := int64(f0) * int64(g4)
+	f0g5 := int64(f0) * int64(g5)
+	f0g6 := int64(f0) * int64(g6)
+	f0g7 := int64(f0) * int64(g7)
+	f0g8 := int64(f0) * int64(g8)
+	f0g9 := int64(f0) * int64(g9)
+	f1g0 := int64(f1) * int64(g0)
+	f1g1_2 := int64(f1_2) * int64(g1)
+	f1g2 := int64(f1) * int64(g2)
+	f1g3_2 := int64(f1_2) * int64(g3)
+	f1g4 := int64(f1) * int64(g4)
+	f1g5_2 := int64(f1_2) * int64(g5)
+	f1g6 := int64(f1) * int64(g6)
+	f1g7_2 := int64(f1_2) * int64(g7)
+	f1g8 := int64(f1) * int64(g8)
+	f1g9_38 := int64(f1_2) * int64(g9_19)
+	f2g0 := int64(f2) * int64(g0)
+	f2g1 := int64(f2) * int64(g1)
+	f2g2 := int64(f2) * int64(g2)
+	f2g3 := int64(f2) * int64(g3)
+	f2g4 := int64(f2) * int64(g4)
+	f2g5 := int64(f2) * int64(g5)
+	f2g6 := int64(f2) * int64(g6)
+	f2g7 := int64(f2) * int64(g7)
+	f2g8_19 := int64(f2) * int64(g8_19)
+	f2g9_19 := int64(f2) * int64(g9_19)
+	f3g0 := int64(f3) * int64(g0)
+	f3g1_2 := int64(f3_2) * int64(g1)
+	f3g2 := int64(f3) * int64(g2)
+	f3g3_2 := int64(f3_2) * int64(g3)
+	f3g4 := int64(f3) * int64(g4)
+	f3g5_2 := int64(f3_2) * int64(g5)
+	f3g6 := int64(f3) * int64(g6)
+	f3g7_38 := int64(f3_2) * int64(g7_19)
+	f3g8_19 := int64(f3) * int64(g8_19)
+	f3g9_38 := int64(f3_2) * int64(g9_19)
+	f4g0 := int64(f4) * int64(g0)
+	f4g1 := int64(f4) * int64(g1)
+	f4g2 := int64(f4) * int64(g2)
+	f4g3 := int64(f4) * int64(g3)
+	f4g4 := int64(f4) * int64(g4)
+	f4g5 := int64(f4) * int64(g5)
+	f4g6_19 := int64(f4) * int64(g6_19)
+	f4g7_19 := int64(f4) * int64(g7_19)
+	f4g8_19 := int64(f4) * int64(g8_19)
+	f4g9_19 := int64(f4) * int64(g9_19)
+	f5g0 := int64(f5) * int64(g0)
+	f5g1_2 := int64(f5_2) * int64(g1)
+	f5g2 := int64(f5) * int64(g2)
+	f5g3_2 := int64(f5_2) * int64(g3)
+	f5g4 := int64(f5) * int64(g4)
+	f5g5_38 := int64(f5_2) * int64(g5_19)
+	f5g6_19 := int64(f5) * int64(g6_19)
+	f5g7_38 := int64(f5_2) * int64(g7_19)
+	f5g8_19 := int64(f5) * int64(g8_19)
+	f5g9_38 := int64(f5_2) * int64(g9_19)
+	f6g0 := int64(f6) * int64(g0)
+	f6g1 := int64(f6) * int64(g1)
+	f6g2 := int64(f6) * int64(g2)
+	f6g3 := int64(f6) * int64(g3)
+	f6g4_19 := int64(f6) * int64(g4_19)
+	f6g5_19 := int64(f6) * int64(g5_19)
+	f6g6_19 := int64(f6) * int64(g6_19)
+	f6g7_19 := int64(f6) * int64(g7_19)
+	f6g8_19 := int64(f6) * int64(g8_19)
+	f6g9_19 := int64(f6) * int64(g9_19)
+	f7g0 := int64(f7) * int64(g0)
+	f7g1_2 := int64(f7_2) * int64(g1)
+	f7g2 := int64(f7) * int64(g2)
+	f7g3_38 := int64(f7_2) * int64(g3_19)
+	f7g4_19 := int64(f7) * int64(g4_19)
+	f7g5_38 := int64(f7_2) * int64(g5_19)
+	f7g6_19 := int64(f7) * int64(g6_19)
+	f7g7_38 := int64(f7_2) * int64(g7_19)
+	f7g8_19 := int64(f7) * int64(g8_19)
+	f7g9_38 := int64(f7_2) * int64(g9_19)
+	f8g0 := int64(f8) * int64(g0)
+	f8g1 := int64(f8) * int64(g1)
+	f8g2_19 := int64(f8) * int64(g2_19)
+	f8g3_19 := int64(f8) * int64(g3_19)
+	f8g4_19 := int64(f8) * int64(g4_19)
+	f8g5_19 := int64(f8) * int64(g5_19)
+	f8g6_19 := int64(f8) * int64(g6_19)
+	f8g7_19 := int64(f8) * int64(g7_19)
+	f8g8_19 := int64(f8) * int64(g8_19)
+	f8g9_19 := int64(f8) * int64(g9_19)
+	f9g0 := int64(f9) * int64(g0)
+	f9g1_38 := int64(f9_2) * int64(g1_19)
+	f9g2_19 := int64(f9) * int64(g2_19)
+	f9g3_38 := int64(f9_2) * int64(g3_19)
+	f9g4_19 := int64(f9) * int64(g4_19)
+	f9g5_38 := int64(f9_2) * int64(g5_19)
+	f9g6_19 := int64(f9) * int64(g6_19)
+	f9g7_38 := int64(f9_2) * int64(g7_19)
+	f9g8_19 := int64(f9) * int64(g8_19)
+	f9g9_38 := int64(f9_2) * int64(g9_19)
+	h0 := f0g0 + f1g9_38 + f2g8_19 + f3g7_38 + f4g6_19 + f5g5_38 + f6g4_19 + f7g3_38 + f8g2_19 + f9g1_38
+	h1 := f0g1 + f1g0 + f2g9_19 + f3g8_19 + f4g7_19 + f5g6_19 + f6g5_19 + f7g4_19 + f8g3_19 + f9g2_19
+	h2 := f0g2 + f1g1_2 + f2g0 + f3g9_38 + f4g8_19 + f5g7_38 + f6g6_19 + f7g5_38 + f8g4_19 + f9g3_38
+	h3 := f0g3 + f1g2 + f2g1 + f3g0 + f4g9_19 + f5g8_19 + f6g7_19 + f7g6_19 + f8g5_19 + f9g4_19
+	h4 := f0g4 + f1g3_2 + f2g2 + f3g1_2 + f4g0 + f5g9_38 + f6g8_19 + f7g7_38 + f8g6_19 + f9g5_38
+	h5 := f0g5 + f1g4 + f2g3 + f3g2 + f4g1 + f5g0 + f6g9_19 + f7g8_19 + f8g7_19 + f9g6_19
+	h6 := f0g6 + f1g5_2 + f2g4 + f3g3_2 + f4g2 + f5g1_2 + f6g0 + f7g9_38 + f8g8_19 + f9g7_38
+	h7 := f0g7 + f1g6 + f2g5 + f3g4 + f4g3 + f5g2 + f6g1 + f7g0 + f8g9_19 + f9g8_19
+	h8 := f0g8 + f1g7_2 + f2g6 + f3g5_2 + f4g4 + f5g3_2 + f6g2 + f7g1_2 + f8g0 + f9g9_38
+	h9 := f0g9 + f1g8 + f2g7 + f3g6 + f4g5 + f5g4 + f6g3 + f7g2 + f8g1 + f9g0
+	var carry [10]int64
+
+	// |h0| <= (1.1*1.1*2^52*(1+19+19+19+19)+1.1*1.1*2^50*(38+38+38+38+38))
+	//   i.e. |h0| <= 1.2*2^59; narrower ranges for h2, h4, h6, h8
+	// |h1| <= (1.1*1.1*2^51*(1+1+19+19+19+19+19+19+19+19))
+	//   i.e. |h1| <= 1.5*2^58; narrower ranges for h3, h5, h7, h9
+
+	carry[0] = (h0 + (1 << 25)) >> 26
+	h1 += carry[0]
+	h0 -= carry[0] << 26
+	carry[4] = (h4 + (1 << 25)) >> 26
+	h5 += carry[4]
+	h4 -= carry[4] << 26
+	// |h0| <= 2^25
+	// |h4| <= 2^25
+	// |h1| <= 1.51*2^58
+	// |h5| <= 1.51*2^58
+
+	carry[1] = (h1 + (1 << 24)) >> 25
+	h2 += carry[1]
+	h1 -= carry[1] << 25
+	carry[5] = (h5 + (1 << 24)) >> 25
+	h6 += carry[5]
+	h5 -= carry[5] << 25
+	// |h1| <= 2^24; from now on fits into int32
+	// |h5| <= 2^24; from now on fits into int32
+	// |h2| <= 1.21*2^59
+	// |h6| <= 1.21*2^59
+
+	carry[2] = (h2 + (1 << 25)) >> 26
+	h3 += carry[2]
+	h2 -= carry[2] << 26
+	carry[6] = (h6 + (1 << 25)) >> 26
+	h7 += carry[6]
+	h6 -= carry[6] << 26
+	// |h2| <= 2^25; from now on fits into int32 unchanged
+	// |h6| <= 2^25; from now on fits into int32 unchanged
+	// |h3| <= 1.51*2^58
+	// |h7| <= 1.51*2^58
+
+	carry[3] = (h3 + (1 << 24)) >> 25
+	h4 += carry[3]
+	h3 -= carry[3] << 25
+	carry[7] = (h7 + (1 << 24)) >> 25
+	h8 += carry[7]
+	h7 -= carry[7] << 25
+	// |h3| <= 2^24; from now on fits into int32 unchanged
+	// |h7| <= 2^24; from now on fits into int32 unchanged
+	// |h4| <= 1.52*2^33
+	// |h8| <= 1.52*2^33
+
+	carry[4] = (h4 + (1 << 25)) >> 26
+	h5 += carry[4]
+	h4 -= carry[4] << 26
+	carry[8] = (h8 + (1 << 25)) >> 26
+	h9 += carry[8]
+	h8 -= carry[8] << 26
+	// |h4| <= 2^25; from now on fits into int32 unchanged
+	// |h8| <= 2^25; from now on fits into int32 unchanged
+	// |h5| <= 1.01*2^24
+	// |h9| <= 1.51*2^58
+
+	carry[9] = (h9 + (1 << 24)) >> 25
+	h0 += carry[9] * 19
+	h9 -= carry[9] << 25
+	// |h9| <= 2^24; from now on fits into int32 unchanged
+	// |h0| <= 1.8*2^37
+
+	carry[0] = (h0 + (1 << 25)) >> 26
+	h1 += carry[0]
+	h0 -= carry[0] << 26
+	// |h0| <= 2^25; from now on fits into int32 unchanged
+	// |h1| <= 1.01*2^24
+
+	h[0] = int32(h0)
+	h[1] = int32(h1)
+	h[2] = int32(h2)
+	h[3] = int32(h3)
+	h[4] = int32(h4)
+	h[5] = int32(h5)
+	h[6] = int32(h6)
+	h[7] = int32(h7)
+	h[8] = int32(h8)
+	h[9] = int32(h9)
+}
+
+// feSquare calculates h = f*f. Can overlap h with f.
+//
+// Preconditions:
+//    |f| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc.
+//
+// Postconditions:
+//    |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc.
+func feSquare(h, f *fieldElement) {
+	f0 := f[0]
+	f1 := f[1]
+	f2 := f[2]
+	f3 := f[3]
+	f4 := f[4]
+	f5 := f[5]
+	f6 := f[6]
+	f7 := f[7]
+	f8 := f[8]
+	f9 := f[9]
+	f0_2 := 2 * f0
+	f1_2 := 2 * f1
+	f2_2 := 2 * f2
+	f3_2 := 2 * f3
+	f4_2 := 2 * f4
+	f5_2 := 2 * f5
+	f6_2 := 2 * f6
+	f7_2 := 2 * f7
+	f5_38 := 38 * f5 // 1.31*2^30
+	f6_19 := 19 * f6 // 1.31*2^30
+	f7_38 := 38 * f7 // 1.31*2^30
+	f8_19 := 19 * f8 // 1.31*2^30
+	f9_38 := 38 * f9 // 1.31*2^30
+	f0f0 := int64(f0) * int64(f0)
+	f0f1_2 := int64(f0_2) * int64(f1)
+	f0f2_2 := int64(f0_2) * int64(f2)
+	f0f3_2 := int64(f0_2) * int64(f3)
+	f0f4_2 := int64(f0_2) * int64(f4)
+	f0f5_2 := int64(f0_2) * int64(f5)
+	f0f6_2 := int64(f0_2) * int64(f6)
+	f0f7_2 := int64(f0_2) * int64(f7)
+	f0f8_2 := int64(f0_2) * int64(f8)
+	f0f9_2 := int64(f0_2) * int64(f9)
+	f1f1_2 := int64(f1_2) * int64(f1)
+	f1f2_2 := int64(f1_2) * int64(f2)
+	f1f3_4 := int64(f1_2) * int64(f3_2)
+	f1f4_2 := int64(f1_2) * int64(f4)
+	f1f5_4 := int64(f1_2) * int64(f5_2)
+	f1f6_2 := int64(f1_2) * int64(f6)
+	f1f7_4 := int64(f1_2) * int64(f7_2)
+	f1f8_2 := int64(f1_2) * int64(f8)
+	f1f9_76 := int64(f1_2) * int64(f9_38)
+	f2f2 := int64(f2) * int64(f2)
+	f2f3_2 := int64(f2_2) * int64(f3)
+	f2f4_2 := int64(f2_2) * int64(f4)
+	f2f5_2 := int64(f2_2) * int64(f5)
+	f2f6_2 := int64(f2_2) * int64(f6)
+	f2f7_2 := int64(f2_2) * int64(f7)
+	f2f8_38 := int64(f2_2) * int64(f8_19)
+	f2f9_38 := int64(f2) * int64(f9_38)
+	f3f3_2 := int64(f3_2) * int64(f3)
+	f3f4_2 := int64(f3_2) * int64(f4)
+	f3f5_4 := int64(f3_2) * int64(f5_2)
+	f3f6_2 := int64(f3_2) * int64(f6)
+	f3f7_76 := int64(f3_2) * int64(f7_38)
+	f3f8_38 := int64(f3_2) * int64(f8_19)
+	f3f9_76 := int64(f3_2) * int64(f9_38)
+	f4f4 := int64(f4) * int64(f4)
+	f4f5_2 := int64(f4_2) * int64(f5)
+	f4f6_38 := int64(f4_2) * int64(f6_19)
+	f4f7_38 := int64(f4) * int64(f7_38)
+	f4f8_38 := int64(f4_2) * int64(f8_19)
+	f4f9_38 := int64(f4) * int64(f9_38)
+	f5f5_38 := int64(f5) * int64(f5_38)
+	f5f6_38 := int64(f5_2) * int64(f6_19)
+	f5f7_76 := int64(f5_2) * int64(f7_38)
+	f5f8_38 := int64(f5_2) * int64(f8_19)
+	f5f9_76 := int64(f5_2) * int64(f9_38)
+	f6f6_19 := int64(f6) * int64(f6_19)
+	f6f7_38 := int64(f6) * int64(f7_38)
+	f6f8_38 := int64(f6_2) * int64(f8_19)
+	f6f9_38 := int64(f6) * int64(f9_38)
+	f7f7_38 := int64(f7) * int64(f7_38)
+	f7f8_38 := int64(f7_2) * int64(f8_19)
+	f7f9_76 := int64(f7_2) * int64(f9_38)
+	f8f8_19 := int64(f8) * int64(f8_19)
+	f8f9_38 := int64(f8) * int64(f9_38)
+	f9f9_38 := int64(f9) * int64(f9_38)
+	h0 := f0f0 + f1f9_76 + f2f8_38 + f3f7_76 + f4f6_38 + f5f5_38
+	h1 := f0f1_2 + f2f9_38 + f3f8_38 + f4f7_38 + f5f6_38
+	h2 := f0f2_2 + f1f1_2 + f3f9_76 + f4f8_38 + f5f7_76 + f6f6_19
+	h3 := f0f3_2 + f1f2_2 + f4f9_38 + f5f8_38 + f6f7_38
+	h4 := f0f4_2 + f1f3_4 + f2f2 + f5f9_76 + f6f8_38 + f7f7_38
+	h5 := f0f5_2 + f1f4_2 + f2f3_2 + f6f9_38 + f7f8_38
+	h6 := f0f6_2 + f1f5_4 + f2f4_2 + f3f3_2 + f7f9_76 + f8f8_19
+	h7 := f0f7_2 + f1f6_2 + f2f5_2 + f3f4_2 + f8f9_38
+	h8 := f0f8_2 + f1f7_4 + f2f6_2 + f3f5_4 + f4f4 + f9f9_38
+	h9 := f0f9_2 + f1f8_2 + f2f7_2 + f3f6_2 + f4f5_2
+	var carry [10]int64
+
+	carry[0] = (h0 + (1 << 25)) >> 26
+	h1 += carry[0]
+	h0 -= carry[0] << 26
+	carry[4] = (h4 + (1 << 25)) >> 26
+	h5 += carry[4]
+	h4 -= carry[4] << 26
+
+	carry[1] = (h1 + (1 << 24)) >> 25
+	h2 += carry[1]
+	h1 -= carry[1] << 25
+	carry[5] = (h5 + (1 << 24)) >> 25
+	h6 += carry[5]
+	h5 -= carry[5] << 25
+
+	carry[2] = (h2 + (1 << 25)) >> 26
+	h3 += carry[2]
+	h2 -= carry[2] << 26
+	carry[6] = (h6 + (1 << 25)) >> 26
+	h7 += carry[6]
+	h6 -= carry[6] << 26
+
+	carry[3] = (h3 + (1 << 24)) >> 25
+	h4 += carry[3]
+	h3 -= carry[3] << 25
+	carry[7] = (h7 + (1 << 24)) >> 25
+	h8 += carry[7]
+	h7 -= carry[7] << 25
+
+	carry[4] = (h4 + (1 << 25)) >> 26
+	h5 += carry[4]
+	h4 -= carry[4] << 26
+	carry[8] = (h8 + (1 << 25)) >> 26
+	h9 += carry[8]
+	h8 -= carry[8] << 26
+
+	carry[9] = (h9 + (1 << 24)) >> 25
+	h0 += carry[9] * 19
+	h9 -= carry[9] << 25
+
+	carry[0] = (h0 + (1 << 25)) >> 26
+	h1 += carry[0]
+	h0 -= carry[0] << 26
+
+	h[0] = int32(h0)
+	h[1] = int32(h1)
+	h[2] = int32(h2)
+	h[3] = int32(h3)
+	h[4] = int32(h4)
+	h[5] = int32(h5)
+	h[6] = int32(h6)
+	h[7] = int32(h7)
+	h[8] = int32(h8)
+	h[9] = int32(h9)
+}
+
+// feMul121666 calculates h = f * 121666. Can overlap h with f.
+//
+// Preconditions:
+//    |f| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc.
+//
+// Postconditions:
+//    |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc.
+func feMul121666(h, f *fieldElement) {
+	h0 := int64(f[0]) * 121666
+	h1 := int64(f[1]) * 121666
+	h2 := int64(f[2]) * 121666
+	h3 := int64(f[3]) * 121666
+	h4 := int64(f[4]) * 121666
+	h5 := int64(f[5]) * 121666
+	h6 := int64(f[6]) * 121666
+	h7 := int64(f[7]) * 121666
+	h8 := int64(f[8]) * 121666
+	h9 := int64(f[9]) * 121666
+	var carry [10]int64
+
+	carry[9] = (h9 + (1 << 24)) >> 25
+	h0 += carry[9] * 19
+	h9 -= carry[9] << 25
+	carry[1] = (h1 + (1 << 24)) >> 25
+	h2 += carry[1]
+	h1 -= carry[1] << 25
+	carry[3] = (h3 + (1 << 24)) >> 25
+	h4 += carry[3]
+	h3 -= carry[3] << 25
+	carry[5] = (h5 + (1 << 24)) >> 25
+	h6 += carry[5]
+	h5 -= carry[5] << 25
+	carry[7] = (h7 + (1 << 24)) >> 25
+	h8 += carry[7]
+	h7 -= carry[7] << 25
+
+	carry[0] = (h0 + (1 << 25)) >> 26
+	h1 += carry[0]
+	h0 -= carry[0] << 26
+	carry[2] = (h2 + (1 << 25)) >> 26
+	h3 += carry[2]
+	h2 -= carry[2] << 26
+	carry[4] = (h4 + (1 << 25)) >> 26
+	h5 += carry[4]
+	h4 -= carry[4] << 26
+	carry[6] = (h6 + (1 << 25)) >> 26
+	h7 += carry[6]
+	h6 -= carry[6] << 26
+	carry[8] = (h8 + (1 << 25)) >> 26
+	h9 += carry[8]
+	h8 -= carry[8] << 26
+
+	h[0] = int32(h0)
+	h[1] = int32(h1)
+	h[2] = int32(h2)
+	h[3] = int32(h3)
+	h[4] = int32(h4)
+	h[5] = int32(h5)
+	h[6] = int32(h6)
+	h[7] = int32(h7)
+	h[8] = int32(h8)
+	h[9] = int32(h9)
+}
+
+// feInvert sets out = z^-1.
+func feInvert(out, z *fieldElement) {
+	var t0, t1, t2, t3 fieldElement
+	var i int
+
+	feSquare(&t0, z)
+	for i = 1; i < 1; i++ {
+		feSquare(&t0, &t0)
+	}
+	feSquare(&t1, &t0)
+	for i = 1; i < 2; i++ {
+		feSquare(&t1, &t1)
+	}
+	feMul(&t1, z, &t1)
+	feMul(&t0, &t0, &t1)
+	feSquare(&t2, &t0)
+	for i = 1; i < 1; i++ {
+		feSquare(&t2, &t2)
+	}
+	feMul(&t1, &t1, &t2)
+	feSquare(&t2, &t1)
+	for i = 1; i < 5; i++ {
+		feSquare(&t2, &t2)
+	}
+	feMul(&t1, &t2, &t1)
+	feSquare(&t2, &t1)
+	for i = 1; i < 10; i++ {
+		feSquare(&t2, &t2)
+	}
+	feMul(&t2, &t2, &t1)
+	feSquare(&t3, &t2)
+	for i = 1; i < 20; i++ {
+		feSquare(&t3, &t3)
+	}
+	feMul(&t2, &t3, &t2)
+	feSquare(&t2, &t2)
+	for i = 1; i < 10; i++ {
+		feSquare(&t2, &t2)
+	}
+	feMul(&t1, &t2, &t1)
+	feSquare(&t2, &t1)
+	for i = 1; i < 50; i++ {
+		feSquare(&t2, &t2)
+	}
+	feMul(&t2, &t2, &t1)
+	feSquare(&t3, &t2)
+	for i = 1; i < 100; i++ {
+		feSquare(&t3, &t3)
+	}
+	feMul(&t2, &t3, &t2)
+	feSquare(&t2, &t2)
+	for i = 1; i < 50; i++ {
+		feSquare(&t2, &t2)
+	}
+	feMul(&t1, &t2, &t1)
+	feSquare(&t1, &t1)
+	for i = 1; i < 5; i++ {
+		feSquare(&t1, &t1)
+	}
+	feMul(out, &t1, &t0)
+}
+
+func scalarMultGeneric(out, in, base *[32]byte) {
+	var e [32]byte
+
+	copy(e[:], in[:])
+	e[0] &= 248
+	e[31] &= 127
+	e[31] |= 64
+
+	var x1, x2, z2, x3, z3, tmp0, tmp1 fieldElement
+	feFromBytes(&x1, base)
+	feOne(&x2)
+	feCopy(&x3, &x1)
+	feOne(&z3)
+
+	swap := int32(0)
+	for pos := 254; pos >= 0; pos-- {
+		b := e[pos/8] >> uint(pos&7)
+		b &= 1
+		swap ^= int32(b)
+		feCSwap(&x2, &x3, swap)
+		feCSwap(&z2, &z3, swap)
+		swap = int32(b)
+
+		feSub(&tmp0, &x3, &z3)
+		feSub(&tmp1, &x2, &z2)
+		feAdd(&x2, &x2, &z2)
+		feAdd(&z2, &x3, &z3)
+		feMul(&z3, &tmp0, &x2)
+		feMul(&z2, &z2, &tmp1)
+		feSquare(&tmp0, &tmp1)
+		feSquare(&tmp1, &x2)
+		feAdd(&x3, &z3, &z2)
+		feSub(&z2, &z3, &z2)
+		feMul(&x2, &tmp1, &tmp0)
+		feSub(&tmp1, &tmp1, &tmp0)
+		feSquare(&z2, &z2)
+		feMul121666(&z3, &tmp1)
+		feSquare(&x3, &x3)
+		feAdd(&tmp0, &tmp0, &z3)
+		feMul(&z3, &x1, &z2)
+		feMul(&z2, &tmp1, &tmp0)
+	}
+
+	feCSwap(&x2, &x3, swap)
+	feCSwap(&z2, &z3, swap)
+
+	feInvert(&z2, &z2)
+	feMul(&x2, &x2, &z2)
+	feToBytes(out, &x2)
+}
@@ -0,0 +1,11 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !amd64 gccgo appengine purego
+
+package curve25519
+
+func scalarMult(out, in, base *[32]byte) {
+	scalarMultGeneric(out, in, base)
+}
@@ -1,23 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package curve25519 provides an implementation of scalar multiplication on
-// the elliptic curve known as curve25519. See https://cr.yp.to/ecdh.html
-package curve25519 // import "golang.org/x/crypto/curve25519"
-
-// basePoint is the x coordinate of the generator of the curve.
-var basePoint = [32]byte{9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
-
-// ScalarMult sets dst to the product in*base where dst and base are the x
-// coordinates of group points and all values are in little-endian form.
-func ScalarMult(dst, in, base *[32]byte) {
-	scalarMult(dst, in, base)
-}
-
-// ScalarBaseMult sets dst to the product in*base where dst and base are the x
-// coordinates of group points, base is the standard generator and all values
-// are in little-endian form.
-func ScalarBaseMult(dst, in *[32]byte) {
-	ScalarMult(dst, in, &basePoint)
-}
@@ -1,73 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This code was translated into a form compatible with 6a from the public
-// domain sources in SUPERCOP: https://bench.cr.yp.to/supercop.html
-
-// +build amd64,!gccgo,!appengine
-
-#include "const_amd64.h"
-
-// func freeze(inout *[5]uint64)
-TEXT ·freeze(SB),7,$0-8
-	MOVQ inout+0(FP), DI
-
-	MOVQ 0(DI),SI
-	MOVQ 8(DI),DX
-	MOVQ 16(DI),CX
-	MOVQ 24(DI),R8
-	MOVQ 32(DI),R9
-	MOVQ $REDMASK51,AX
-	MOVQ AX,R10
-	SUBQ $18,R10
-	MOVQ $3,R11
-REDUCELOOP:
-	MOVQ SI,R12
-	SHRQ $51,R12
-	ANDQ AX,SI
-	ADDQ R12,DX
-	MOVQ DX,R12
-	SHRQ $51,R12
-	ANDQ AX,DX
-	ADDQ R12,CX
-	MOVQ CX,R12
-	SHRQ $51,R12
-	ANDQ AX,CX
-	ADDQ R12,R8
-	MOVQ R8,R12
-	SHRQ $51,R12
-	ANDQ AX,R8
-	ADDQ R12,R9
-	MOVQ R9,R12
-	SHRQ $51,R12
-	ANDQ AX,R9
-	IMUL3Q $19,R12,R12
-	ADDQ R12,SI
-	SUBQ $1,R11
-	JA REDUCELOOP
-	MOVQ $1,R12
-	CMPQ R10,SI
-	CMOVQLT R11,R12
-	CMPQ AX,DX
-	CMOVQNE R11,R12
-	CMPQ AX,CX
-	CMOVQNE R11,R12
-	CMPQ AX,R8
-	CMOVQNE R11,R12
-	CMPQ AX,R9
-	CMOVQNE R11,R12
-	NEGQ R12
-	ANDQ R12,AX
-	ANDQ R12,R10
-	SUBQ R10,SI
-	SUBQ AX,DX
-	SUBQ AX,CX
-	SUBQ AX,R8
-	SUBQ AX,R9
-	MOVQ SI,0(DI)
-	MOVQ DX,8(DI)
-	MOVQ CX,16(DI)
-	MOVQ R8,24(DI)
-	MOVQ R9,32(DI)
-	RET
@@ -1,169 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This code was translated into a form compatible with 6a from the public
-// domain sources in SUPERCOP: https://bench.cr.yp.to/supercop.html
-
-// +build amd64,!gccgo,!appengine
-
-#include "const_amd64.h"
-
-// func mul(dest, a, b *[5]uint64)
-TEXT ·mul(SB),0,$16-24
-	MOVQ dest+0(FP), DI
-	MOVQ a+8(FP), SI
-	MOVQ b+16(FP), DX
-
-	MOVQ DX,CX
-	MOVQ 24(SI),DX
-	IMUL3Q $19,DX,AX
-	MOVQ AX,0(SP)
-	MULQ 16(CX)
-	MOVQ AX,R8
-	MOVQ DX,R9
-	MOVQ 32(SI),DX
-	IMUL3Q $19,DX,AX
-	MOVQ AX,8(SP)
-	MULQ 8(CX)
-	ADDQ AX,R8
-	ADCQ DX,R9
-	MOVQ 0(SI),AX
-	MULQ 0(CX)
-	ADDQ AX,R8
-	ADCQ DX,R9
-	MOVQ 0(SI),AX
-	MULQ 8(CX)
-	MOVQ AX,R10
-	MOVQ DX,R11
-	MOVQ 0(SI),AX
-	MULQ 16(CX)
-	MOVQ AX,R12
-	MOVQ DX,R13
-	MOVQ 0(SI),AX
-	MULQ 24(CX)
-	MOVQ AX,R14
-	MOVQ DX,R15
-	MOVQ 0(SI),AX
-	MULQ 32(CX)
-	MOVQ AX,BX
-	MOVQ DX,BP
-	MOVQ 8(SI),AX
-	MULQ 0(CX)
-	ADDQ AX,R10
-	ADCQ DX,R11
-	MOVQ 8(SI),AX
-	MULQ 8(CX)
-	ADDQ AX,R12
-	ADCQ DX,R13
-	MOVQ 8(SI),AX
-	MULQ 16(CX)
-	ADDQ AX,R14
-	ADCQ DX,R15
-	MOVQ 8(SI),AX
-	MULQ 24(CX)
-	ADDQ AX,BX
-	ADCQ DX,BP
-	MOVQ 8(SI),DX
-	IMUL3Q $19,DX,AX
-	MULQ 32(CX)
-	ADDQ AX,R8
-	ADCQ DX,R9
-	MOVQ 16(SI),AX
-	MULQ 0(CX)
-	ADDQ AX,R12
-	ADCQ DX,R13
-	MOVQ 16(SI),AX
-	MULQ 8(CX)
-	ADDQ AX,R14
-	ADCQ DX,R15
-	MOVQ 16(SI),AX
-	MULQ 16(CX)
-	ADDQ AX,BX
-	ADCQ DX,BP
-	MOVQ 16(SI),DX
-	IMUL3Q $19,DX,AX
-	MULQ 24(CX)
-	ADDQ AX,R8
-	ADCQ DX,R9
-	MOVQ 16(SI),DX
-	IMUL3Q $19,DX,AX
-	MULQ 32(CX)
-	ADDQ AX,R10
-	ADCQ DX,R11
-	MOVQ 24(SI),AX
-	MULQ 0(CX)
-	ADDQ AX,R14
-	ADCQ DX,R15
-	MOVQ 24(SI),AX
-	MULQ 8(CX)
-	ADDQ AX,BX
-	ADCQ DX,BP
-	MOVQ 0(SP),AX
-	MULQ 24(CX)
-	ADDQ AX,R10
-	ADCQ DX,R11
-	MOVQ 0(SP),AX
-	MULQ 32(CX)
-	ADDQ AX,R12
-	ADCQ DX,R13
-	MOVQ 32(SI),AX
-	MULQ 0(CX)
-	ADDQ AX,BX
-	ADCQ DX,BP
-	MOVQ 8(SP),AX
-	MULQ 16(CX)
-	ADDQ AX,R10
-	ADCQ DX,R11
-	MOVQ 8(SP),AX
-	MULQ 24(CX)
-	ADDQ AX,R12
-	ADCQ DX,R13
-	MOVQ 8(SP),AX
-	MULQ 32(CX)
-	ADDQ AX,R14
-	ADCQ DX,R15
-	MOVQ $REDMASK51,SI
-	SHLQ $13,R9:R8
-	ANDQ SI,R8
-	SHLQ $13,R11:R10
-	ANDQ SI,R10
-	ADDQ R9,R10
-	SHLQ $13,R13:R12
-	ANDQ SI,R12
-	ADDQ R11,R12
-	SHLQ $13,R15:R14
-	ANDQ SI,R14
-	ADDQ R13,R14
-	SHLQ $13,BP:BX
-	ANDQ SI,BX
-	ADDQ R15,BX
-	IMUL3Q $19,BP,DX
-	ADDQ DX,R8
-	MOVQ R8,DX
-	SHRQ $51,DX
-	ADDQ R10,DX
-	MOVQ DX,CX
-	SHRQ $51,DX
-	ANDQ SI,R8
-	ADDQ R12,DX
-	MOVQ DX,R9
-	SHRQ $51,DX
-	ANDQ SI,CX
-	ADDQ R14,DX
-	MOVQ DX,AX
-	SHRQ $51,DX
-	ANDQ SI,R9
-	ADDQ BX,DX
-	MOVQ DX,R10
-	SHRQ $51,DX
-	ANDQ SI,AX
-	IMUL3Q $19,DX,DX
-	ADDQ DX,R8
-	ANDQ SI,R10
-	MOVQ R8,0(DI)
-	MOVQ CX,8(DI)
-	MOVQ R9,16(DI)
-	MOVQ AX,24(DI)
-	MOVQ R10,32(DI)
-	RET
@@ -1,132 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This code was translated into a form compatible with 6a from the public
-// domain sources in SUPERCOP: https://bench.cr.yp.to/supercop.html
-
-// +build amd64,!gccgo,!appengine
-
-#include "const_amd64.h"
-
-// func square(out, in *[5]uint64)
-TEXT ·square(SB),7,$0-16
-	MOVQ out+0(FP), DI
-	MOVQ in+8(FP), SI
-
-	MOVQ 0(SI),AX
-	MULQ 0(SI)
-	MOVQ AX,CX
-	MOVQ DX,R8
-	MOVQ 0(SI),AX
-	SHLQ $1,AX
-	MULQ 8(SI)
-	MOVQ AX,R9
-	MOVQ DX,R10
-	MOVQ 0(SI),AX
-	SHLQ $1,AX
-	MULQ 16(SI)
-	MOVQ AX,R11
-	MOVQ DX,R12
-	MOVQ 0(SI),AX
-	SHLQ $1,AX
-	MULQ 24(SI)
-	MOVQ AX,R13
-	MOVQ DX,R14
-	MOVQ 0(SI),AX
-	SHLQ $1,AX
-	MULQ 32(SI)
-	MOVQ AX,R15
-	MOVQ DX,BX
-	MOVQ 8(SI),AX
-	MULQ 8(SI)
-	ADDQ AX,R11
-	ADCQ DX,R12
-	MOVQ 8(SI),AX
-	SHLQ $1,AX
-	MULQ 16(SI)
-	ADDQ AX,R13
-	ADCQ DX,R14
-	MOVQ 8(SI),AX
-	SHLQ $1,AX
-	MULQ 24(SI)
-	ADDQ AX,R15
-	ADCQ DX,BX
-	MOVQ 8(SI),DX
-	IMUL3Q $38,DX,AX
-	MULQ 32(SI)
-	ADDQ AX,CX
-	ADCQ DX,R8
-	MOVQ 16(SI),AX
-	MULQ 16(SI)
-	ADDQ AX,R15
-	ADCQ DX,BX
-	MOVQ 16(SI),DX
-	IMUL3Q $38,DX,AX
-	MULQ 24(SI)
-	ADDQ AX,CX
-	ADCQ DX,R8
-	MOVQ 16(SI),DX
-	IMUL3Q $38,DX,AX
-	MULQ 32(SI)
-	ADDQ AX,R9
-	ADCQ DX,R10
-	MOVQ 24(SI),DX
-	IMUL3Q $19,DX,AX
-	MULQ 24(SI)
-	ADDQ AX,R9
-	ADCQ DX,R10
-	MOVQ 24(SI),DX
-	IMUL3Q $38,DX,AX
-	MULQ 32(SI)
-	ADDQ AX,R11
-	ADCQ DX,R12
-	MOVQ 32(SI),DX
-	IMUL3Q $19,DX,AX
-	MULQ 32(SI)
-	ADDQ AX,R13
-	ADCQ DX,R14
-	MOVQ $REDMASK51,SI
-	SHLQ $13,R8:CX
-	ANDQ SI,CX
-	SHLQ $13,R10:R9
-	ANDQ SI,R9
-	ADDQ R8,R9
-	SHLQ $13,R12:R11
-	ANDQ SI,R11
-	ADDQ R10,R11
-	SHLQ $13,R14:R13
-	ANDQ SI,R13
-	ADDQ R12,R13
-	SHLQ $13,BX:R15
-	ANDQ SI,R15
-	ADDQ R14,R15
-	IMUL3Q $19,BX,DX
-	ADDQ DX,CX
-	MOVQ CX,DX
-	SHRQ $51,DX
-	ADDQ R9,DX
-	ANDQ SI,CX
-	MOVQ DX,R8
-	SHRQ $51,DX
-	ADDQ R11,DX
-	ANDQ SI,R8
-	MOVQ DX,R9
-	SHRQ $51,DX
-	ADDQ R13,DX
-	ANDQ SI,R9
-	MOVQ DX,AX
-	SHRQ $51,DX
-	ADDQ R15,DX
-	ANDQ SI,AX
-	MOVQ DX,R10
-	SHRQ $51,DX
-	IMUL3Q $19,DX,DX
-	ADDQ DX,CX
-	ANDQ SI,R10
-	MOVQ CX,0(DI)
-	MOVQ R8,8(DI)
-	MOVQ R9,16(DI)
-	MOVQ AX,24(DI)
-	MOVQ R10,32(DI)
-	RET
@@ -2,6 +2,11 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

+// In Go 1.13, the ed25519 package was promoted to the standard library as
+// crypto/ed25519, and this package became a wrapper for the standard library one.
+//
+// +build !go1.13
+
 // Package ed25519 implements the Ed25519 signature algorithm. See
 // https://ed25519.cr.yp.to/.
 //
@@ -0,0 +1,73 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.13
+
+// Package ed25519 implements the Ed25519 signature algorithm. See
+// https://ed25519.cr.yp.to/.
+//
+// These functions are also compatible with the “Ed25519” function defined in
+// RFC 8032. However, unlike RFC 8032's formulation, this package's private key
+// representation includes a public key suffix to make multiple signing
+// operations with the same key more efficient. This package refers to the RFC
+// 8032 private key as the “seed”.
+//
+// Beginning with Go 1.13, the functionality of this package was moved to the
+// standard library as crypto/ed25519. This package only acts as a compatibility
+// wrapper.
+package ed25519
+
+import (
+	"crypto/ed25519"
+	"io"
+)
+
+const (
+	// PublicKeySize is the size, in bytes, of public keys as used in this package.
+	PublicKeySize = 32
+	// PrivateKeySize is the size, in bytes, of private keys as used in this package.
+	PrivateKeySize = 64
+	// SignatureSize is the size, in bytes, of signatures generated and verified by this package.
+	SignatureSize = 64
+	// SeedSize is the size, in bytes, of private key seeds. These are the private key representations used by RFC 8032.
+	SeedSize = 32
+)
+
+// PublicKey is the type of Ed25519 public keys.
+//
+// This type is an alias for crypto/ed25519's PublicKey type.
+// See the crypto/ed25519 package for the methods on this type.
+type PublicKey = ed25519.PublicKey
+
+// PrivateKey is the type of Ed25519 private keys. It implements crypto.Signer.
+//
+// This type is an alias for crypto/ed25519's PrivateKey type.
+// See the crypto/ed25519 package for the methods on this type.
+type PrivateKey = ed25519.PrivateKey
+
+// GenerateKey generates a public/private key pair using entropy from rand.
+// If rand is nil, crypto/rand.Reader will be used.
+func GenerateKey(rand io.Reader) (PublicKey, PrivateKey, error) {
+	return ed25519.GenerateKey(rand)
+}
+
+// NewKeyFromSeed calculates a private key from a seed. It will panic if
+// len(seed) is not SeedSize. This function is provided for interoperability
+// with RFC 8032. RFC 8032's private keys correspond to seeds in this
+// package.
+func NewKeyFromSeed(seed []byte) PrivateKey {
+	return ed25519.NewKeyFromSeed(seed)
+}
+
+// Sign signs the message with privateKey and returns a signature. It will
+// panic if len(privateKey) is not PrivateKeySize.
+func Sign(privateKey PrivateKey, message []byte) []byte {
+	return ed25519.Sign(privateKey, message)
+}
+
+// Verify reports whether sig is a valid signature of message by publicKey. It
+// will panic if len(publicKey) is not PublicKeySize.
+func Verify(publicKey PublicKey, message, sig []byte) bool {
+	return ed25519.Verify(publicKey, message, sig)
+}
@@ -1,3 +1,8 @@
 module golang.org/x/crypto

-require golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e
+go 1.11
+
+require (
+	golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3
+	golang.org/x/sys v0.0.0-20190412213103-97732733099d
+)
@@ -1,31 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build go1.11
-// +build !gccgo
-
-package chacha20
-
-const (
-	haveAsm = true
-	bufSize = 256
-)
-
-//go:noescape
-func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32)
-
-func (c *Cipher) xorKeyStreamAsm(dst, src []byte) {
-
-	if len(src) >= bufSize {
-		xorKeyStreamVX(dst, src, &c.key, &c.nonce, &c.counter)
-	}
-
-	if len(src)%bufSize != 0 {
-		i := len(src) - len(src)%bufSize
-		c.buf = [bufSize]byte{}
-		copy(c.buf[:], src[i:])
-		xorKeyStreamVX(c.buf[:], c.buf[:], &c.key, &c.nonce, &c.counter)
-		c.len = bufSize - copy(dst[i:], c.buf[:len(src)%bufSize])
-	}
-}
@@ -1,264 +0,0 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package ChaCha20 implements the core ChaCha20 function as specified
-// in https://tools.ietf.org/html/rfc7539#section-2.3.
-package chacha20
-
-import (
-	"crypto/cipher"
-	"encoding/binary"
-
-	"golang.org/x/crypto/internal/subtle"
-)
-
-// assert that *Cipher implements cipher.Stream
-var _ cipher.Stream = (*Cipher)(nil)
-
-// Cipher is a stateful instance of ChaCha20 using a particular key
-// and nonce. A *Cipher implements the cipher.Stream interface.
-type Cipher struct {
-	key     [8]uint32
-	counter uint32 // incremented after each block
-	nonce   [3]uint32
-	buf     [bufSize]byte // buffer for unused keystream bytes
-	len     int           // number of unused keystream bytes at end of buf
-}
-
-// New creates a new ChaCha20 stream cipher with the given key and nonce.
-// The initial counter value is set to 0.
-func New(key [8]uint32, nonce [3]uint32) *Cipher {
-	return &Cipher{key: key, nonce: nonce}
-}
-
-// ChaCha20 constants spelling "expand 32-byte k"
-const (
-	j0 uint32 = 0x61707865
-	j1 uint32 = 0x3320646e
-	j2 uint32 = 0x79622d32
-	j3 uint32 = 0x6b206574
-)
-
-func quarterRound(a, b, c, d uint32) (uint32, uint32, uint32, uint32) {
-	a += b
-	d ^= a
-	d = (d << 16) | (d >> 16)
-	c += d
-	b ^= c
-	b = (b << 12) | (b >> 20)
-	a += b
-	d ^= a
-	d = (d << 8) | (d >> 24)
-	c += d
-	b ^= c
-	b = (b << 7) | (b >> 25)
-	return a, b, c, d
-}
-
-// XORKeyStream XORs each byte in the given slice with a byte from the
-// cipher's key stream. Dst and src must overlap entirely or not at all.
-//
-// If len(dst) < len(src), XORKeyStream will panic. It is acceptable
-// to pass a dst bigger than src, and in that case, XORKeyStream will
-// only update dst[:len(src)] and will not touch the rest of dst.
-//
-// Multiple calls to XORKeyStream behave as if the concatenation of
-// the src buffers was passed in a single run. That is, Cipher
-// maintains state and does not reset at each XORKeyStream call.
-func (s *Cipher) XORKeyStream(dst, src []byte) {
-	if len(dst) < len(src) {
-		panic("chacha20: output smaller than input")
-	}
-	if subtle.InexactOverlap(dst[:len(src)], src) {
-		panic("chacha20: invalid buffer overlap")
-	}
-
-	// xor src with buffered keystream first
-	if s.len != 0 {
-		buf := s.buf[len(s.buf)-s.len:]
-		if len(src) < len(buf) {
-			buf = buf[:len(src)]
-		}
-		td, ts := dst[:len(buf)], src[:len(buf)] // BCE hint
-		for i, b := range buf {
-			td[i] = ts[i] ^ b
-		}
-		s.len -= len(buf)
-		if s.len != 0 {
-			return
-		}
-		s.buf = [len(s.buf)]byte{} // zero the empty buffer
-		src = src[len(buf):]
-		dst = dst[len(buf):]
-	}
-
-	if len(src) == 0 {
-		return
-	}
-	if haveAsm {
-		if uint64(len(src))+uint64(s.counter)*64 > (1<<38)-64 {
-			panic("chacha20: counter overflow")
-		}
-		s.xorKeyStreamAsm(dst, src)
-		return
-	}
-
-	// set up a 64-byte buffer to pad out the final block if needed
-	// (hoisted out of the main loop to avoid spills)
-	rem := len(src) % 64  // length of final block
-	fin := len(src) - rem // index of final block
-	if rem > 0 {
-		copy(s.buf[len(s.buf)-64:], src[fin:])
-	}
-
-	// pre-calculate most of the first round
-	s1, s5, s9, s13 := quarterRound(j1, s.key[1], s.key[5], s.nonce[0])
-	s2, s6, s10, s14 := quarterRound(j2, s.key[2], s.key[6], s.nonce[1])
-	s3, s7, s11, s15 := quarterRound(j3, s.key[3], s.key[7], s.nonce[2])
-
-	n := len(src)
-	src, dst = src[:n:n], dst[:n:n] // BCE hint
-	for i := 0; i < n; i += 64 {
-		// calculate the remainder of the first round
-		s0, s4, s8, s12 := quarterRound(j0, s.key[0], s.key[4], s.counter)
-
-		// execute the second round
-		x0, x5, x10, x15 := quarterRound(s0, s5, s10, s15)
-		x1, x6, x11, x12 := quarterRound(s1, s6, s11, s12)
-		x2, x7, x8, x13 := quarterRound(s2, s7, s8, s13)
-		x3, x4, x9, x14 := quarterRound(s3, s4, s9, s14)
-
-		// execute the remaining 18 rounds
-		for i := 0; i < 9; i++ {
-			x0, x4, x8, x12 = quarterRound(x0, x4, x8, x12)
-			x1, x5, x9, x13 = quarterRound(x1, x5, x9, x13)
-			x2, x6, x10, x14 = quarterRound(x2, x6, x10, x14)
-			x3, x7, x11, x15 = quarterRound(x3, x7, x11, x15)
-
-			x0, x5, x10, x15 = quarterRound(x0, x5, x10, x15)
-			x1, x6, x11, x12 = quarterRound(x1, x6, x11, x12)
-			x2, x7, x8, x13 = quarterRound(x2, x7, x8, x13)
-			x3, x4, x9, x14 = quarterRound(x3, x4, x9, x14)
-		}
-
-		x0 += j0
-		x1 += j1
-		x2 += j2
-		x3 += j3
-
-		x4 += s.key[0]
-		x5 += s.key[1]
-		x6 += s.key[2]
-		x7 += s.key[3]
-		x8 += s.key[4]
-		x9 += s.key[5]
-		x10 += s.key[6]
-		x11 += s.key[7]
-
-		x12 += s.counter
-		x13 += s.nonce[0]
-		x14 += s.nonce[1]
-		x15 += s.nonce[2]
-
-		// increment the counter
-		s.counter += 1
-		if s.counter == 0 {
-			panic("chacha20: counter overflow")
-		}
-
-		// pad to 64 bytes if needed
-		in, out := src[i:], dst[i:]
-		if i == fin {
-			// src[fin:] has already been copied into s.buf before
-			// the main loop
-			in, out = s.buf[len(s.buf)-64:], s.buf[len(s.buf)-64:]
-		}
-		in, out = in[:64], out[:64] // BCE hint
-
-		// XOR the key stream with the source and write out the result
-		xor(out[0:], in[0:], x0)
-		xor(out[4:], in[4:], x1)
-		xor(out[8:], in[8:], x2)
-		xor(out[12:], in[12:], x3)
-		xor(out[16:], in[16:], x4)
-		xor(out[20:], in[20:], x5)
-		xor(out[24:], in[24:], x6)
-		xor(out[28:], in[28:], x7)
-		xor(out[32:], in[32:], x8)
-		xor(out[36:], in[36:], x9)
-		xor(out[40:], in[40:], x10)
-		xor(out[44:], in[44:], x11)
-		xor(out[48:], in[48:], x12)
-		xor(out[52:], in[52:], x13)
-		xor(out[56:], in[56:], x14)
-		xor(out[60:], in[60:], x15)
-	}
-	// copy any trailing bytes out of the buffer and into dst
-	if rem != 0 {
-		s.len = 64 - rem
-		copy(dst[fin:], s.buf[len(s.buf)-64:])
-	}
-}
-
-// Advance discards bytes in the key stream until the next 64 byte block
-// boundary is reached and updates the counter accordingly. If the key
-// stream is already at a block boundary no bytes will be discarded and
-// the counter will be unchanged.
-func (s *Cipher) Advance() {
-	s.len -= s.len % 64
-	if s.len == 0 {
-		s.buf = [len(s.buf)]byte{}
-	}
-}
-
-// XORKeyStream crypts bytes from in to out using the given key and counters.
-// In and out must overlap entirely or not at all. Counter contains the raw
-// ChaCha20 counter bytes (i.e. block counter followed by nonce).
-func XORKeyStream(out, in []byte, counter *[16]byte, key *[32]byte) {
-	s := Cipher{
-		key: [8]uint32{
-			binary.LittleEndian.Uint32(key[0:4]),
-			binary.LittleEndian.Uint32(key[4:8]),
-			binary.LittleEndian.Uint32(key[8:12]),
-			binary.LittleEndian.Uint32(key[12:16]),
-			binary.LittleEndian.Uint32(key[16:20]),
-			binary.LittleEndian.Uint32(key[20:24]),
-			binary.LittleEndian.Uint32(key[24:28]),
-			binary.LittleEndian.Uint32(key[28:32]),
-		},
-		nonce: [3]uint32{
-			binary.LittleEndian.Uint32(counter[4:8]),
-			binary.LittleEndian.Uint32(counter[8:12]),
-			binary.LittleEndian.Uint32(counter[12:16]),
-		},
-		counter: binary.LittleEndian.Uint32(counter[0:4]),
-	}
-	s.XORKeyStream(out, in)
-}
-
-// HChaCha20 uses the ChaCha20 core to generate a derived key from a key and a
-// nonce. It should only be used as part of the XChaCha20 construction.
-func HChaCha20(key *[8]uint32, nonce *[4]uint32) [8]uint32 {
-	x0, x1, x2, x3 := j0, j1, j2, j3
-	x4, x5, x6, x7 := key[0], key[1], key[2], key[3]
-	x8, x9, x10, x11 := key[4], key[5], key[6], key[7]
-	x12, x13, x14, x15 := nonce[0], nonce[1], nonce[2], nonce[3]
-
-	for i := 0; i < 10; i++ {
-		x0, x4, x8, x12 = quarterRound(x0, x4, x8, x12)
-		x1, x5, x9, x13 = quarterRound(x1, x5, x9, x13)
-		x2, x6, x10, x14 = quarterRound(x2, x6, x10, x14)
-		x3, x7, x11, x15 = quarterRound(x3, x7, x11, x15)
-
-		x0, x5, x10, x15 = quarterRound(x0, x5, x10, x15)
-		x1, x6, x11, x12 = quarterRound(x1, x6, x11, x12)
-		x2, x7, x8, x13 = quarterRound(x2, x7, x8, x13)
-		x3, x4, x9, x14 = quarterRound(x3, x4, x9, x14)
-	}
-
-	var out [8]uint32
-	out[0], out[1], out[2], out[3] = x0, x1, x2, x3
-	out[4], out[5], out[6], out[7] = x12, x13, x14, x15
-	return out
-}
@@ -1,16 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build !arm64,!s390x arm64,!go1.11 gccgo appengine
-
-package chacha20
-
-const (
-	bufSize = 64
-	haveAsm = false
-)
-
-func (*Cipher) xorKeyStreamAsm(dst, src []byte) {
-	panic("not implemented")
-}
@@ -1,29 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build s390x,!gccgo,!appengine
-
-package chacha20
-
-import (
-	"golang.org/x/sys/cpu"
-)
-
-var haveAsm = cpu.S390X.HasVX
-
-const bufSize = 256
-
-// xorKeyStreamVX is an assembly implementation of XORKeyStream. It must only
-// be called when the vector facility is available.
-// Implementation in asm_s390x.s.
-//go:noescape
-func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32, buf *[256]byte, len *int)
-
-func (c *Cipher) xorKeyStreamAsm(dst, src []byte) {
-	xorKeyStreamVX(dst, src, &c.key, &c.nonce, &c.counter, &c.buf, &c.len)
-}
-
-// EXRL targets, DO NOT CALL!
-func mvcSrcToBuf()
-func mvcBufToDst()
@@ -0,0 +1,39 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !go1.13
+
+package poly1305
+
+// Generic fallbacks for the math/bits intrinsics, copied from
+// src/math/bits/bits.go. They were added in Go 1.12, but Add64 and Sum64 had
+// variable time fallbacks until Go 1.13.
+
+func bitsAdd64(x, y, carry uint64) (sum, carryOut uint64) {
+	sum = x + y + carry
+	carryOut = ((x & y) | ((x | y) &^ sum)) >> 63
+	return
+}
+
+func bitsSub64(x, y, borrow uint64) (diff, borrowOut uint64) {
+	diff = x - y - borrow
+	borrowOut = ((^x & y) | (^(x ^ y) & diff)) >> 63
+	return
+}
+
+func bitsMul64(x, y uint64) (hi, lo uint64) {
+	const mask32 = 1<<32 - 1
+	x0 := x & mask32
+	x1 := x >> 32
+	y0 := y & mask32
+	y1 := y >> 32
+	w0 := x0 * y0
+	t := x1*y0 + w0>>32
+	w1 := t & mask32
+	w2 := t >> 32
+	w1 += x0 * y1
+	hi = x1*y1 + w2 + w1>>32
+	lo = x * y
+	return
+}
@@ -0,0 +1,21 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.13
+
+package poly1305
+
+import "math/bits"
+
+func bitsAdd64(x, y, carry uint64) (sum, carryOut uint64) {
+	return bits.Add64(x, y, carry)
+}
+
+func bitsSub64(x, y, borrow uint64) (diff, borrowOut uint64) {
+	return bits.Sub64(x, y, borrow)
+}
+
+func bitsMul64(x, y uint64) (hi, lo uint64) {
+	return bits.Mul64(x, y)
+}
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

-// +build !amd64 gccgo appengine
+// +build !amd64,!ppc64le gccgo appengine

 package poly1305

@@ -22,8 +22,14 @@ import "crypto/subtle"
 // TagSize is the size, in bytes, of a poly1305 authenticator.
 const TagSize = 16

-// Verify returns true if mac is a valid authenticator for m with the given
-// key.
+// Sum generates an authenticator for msg using a one-time key and puts the
+// 16-byte result into out. Authenticating two different messages with the same
+// key allows an attacker to forge messages at will.
+func Sum(out *[16]byte, m []byte, key *[32]byte) {
+	sum(out, m, key)
+}
+
+// Verify returns true if mac is a valid authenticator for m with the given key.
 func Verify(mac *[16]byte, m []byte, key *[32]byte) bool {
 	var tmp [16]byte
 	Sum(&tmp, m, key)
@@ -7,62 +7,52 @@
 package poly1305

 //go:noescape
-func initialize(state *[7]uint64, key *[32]byte)
+func update(state *macState, msg []byte)

-//go:noescape
-func update(state *[7]uint64, msg []byte)
-
-//go:noescape
-func finalize(tag *[TagSize]byte, state *[7]uint64)
-
-// Sum generates an authenticator for m using a one-time key and puts the
-// 16-byte result into out. Authenticating two different messages with the same
-// key allows an attacker to forge messages at will.
-func Sum(out *[16]byte, m []byte, key *[32]byte) {
+func sum(out *[16]byte, m []byte, key *[32]byte) {
 	h := newMAC(key)
 	h.Write(m)
 	h.Sum(out)
 }

 func newMAC(key *[32]byte) (h mac) {
-	initialize(&h.state, key)
+	initialize(key, &h.r, &h.s)
 	return
 }

-type mac struct {
-	state [7]uint64 // := uint64{ h0, h1, h2, r0, r1, pad0, pad1 }
+// mac is a wrapper for macGeneric that redirects calls that would have gone to
+// updateGeneric to update.
+//
+// Its Write and Sum methods are otherwise identical to the macGeneric ones, but
+// using function pointers would carry a major performance cost.
+type mac struct{ macGeneric }

-	buffer [TagSize]byte
-	offset int
-}
-
-func (h *mac) Write(p []byte) (n int, err error) {
-	n = len(p)
+func (h *mac) Write(p []byte) (int, error) {
+	nn := len(p)
 	if h.offset > 0 {
-		remaining := TagSize - h.offset
-		if n < remaining {
-			h.offset += copy(h.buffer[h.offset:], p)
-			return n, nil
+		n := copy(h.buffer[h.offset:], p)
+		if h.offset+n < TagSize {
+			h.offset += n
+			return nn, nil
 		}
-		copy(h.buffer[h.offset:], p[:remaining])
-		p = p[remaining:]
+		p = p[n:]
 		h.offset = 0
-		update(&h.state, h.buffer[:])
+		update(&h.macState, h.buffer[:])
 	}
-	if nn := len(p) - (len(p) % TagSize); nn > 0 {
-		update(&h.state, p[:nn])
-		p = p[nn:]
+	if n := len(p) - (len(p) % TagSize); n > 0 {
+		update(&h.macState, p[:n])
+		p = p[n:]
 	}
 	if len(p) > 0 {
 		h.offset += copy(h.buffer[h.offset:], p)
 	}
-	return n, nil
+	return nn, nil
 }

 func (h *mac) Sum(out *[16]byte) {
-	state := h.state
+	state := h.macState
 	if h.offset > 0 {
 		update(&state, h.buffer[:h.offset])
 	}
-	finalize(out, &state)
+	finalize(out, &state.h, &state.s)
 }
@@ -54,10 +54,6 @@
 	ADCQ  t3, h1;                  \
 	ADCQ  $0, h2

-DATA ·poly1305Mask<>+0x00(SB)/8, $0x0FFFFFFC0FFFFFFF
-DATA ·poly1305Mask<>+0x08(SB)/8, $0x0FFFFFFC0FFFFFFC
-GLOBL ·poly1305Mask<>(SB), RODATA, $16
-
 // func update(state *[7]uint64, msg []byte)
 TEXT ·update(SB), $0-32
 	MOVQ state+0(FP), DI
@@ -110,39 +106,3 @@ done:
 	MOVQ R9, 8(DI)
 	MOVQ R10, 16(DI)
 	RET
-
-// func initialize(state *[7]uint64, key *[32]byte)
-TEXT ·initialize(SB), $0-16
-	MOVQ state+0(FP), DI
-	MOVQ key+8(FP), SI
-
-	// state[0...7] is initialized with zero
-	MOVOU 0(SI), X0
-	MOVOU 16(SI), X1
-	MOVOU ·poly1305Mask<>(SB), X2
-	PAND  X2, X0
-	MOVOU X0, 24(DI)
-	MOVOU X1, 40(DI)
-	RET
-
-// func finalize(tag *[TagSize]byte, state *[7]uint64)
-TEXT ·finalize(SB), $0-16
-	MOVQ tag+0(FP), DI
-	MOVQ state+8(FP), SI
-
-	MOVQ    0(SI), AX
-	MOVQ    8(SI), BX
-	MOVQ    16(SI), CX
-	MOVQ    AX, R8
-	MOVQ    BX, R9
-	SUBQ    $0xFFFFFFFFFFFFFFFB, AX
-	SBBQ    $0xFFFFFFFFFFFFFFFF, BX
-	SBBQ    $3, CX
-	CMOVQCS R8, AX
-	CMOVQCS R9, BX
-	ADDQ    40(SI), AX
-	ADCQ    48(SI), BX
-
-	MOVQ AX, 0(DI)
-	MOVQ BX, 8(DI)
-	RET
@@ -1,22 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build arm,!gccgo,!appengine,!nacl
-
-package poly1305
-
-// This function is implemented in sum_arm.s
-//go:noescape
-func poly1305_auth_armv6(out *[16]byte, m *byte, mlen uint32, key *[32]byte)
-
-// Sum generates an authenticator for m using a one-time key and puts the
-// 16-byte result into out. Authenticating two different messages with the same
-// key allows an attacker to forge messages at will.
-func Sum(out *[16]byte, m []byte, key *[32]byte) {
-	var mPtr *byte
-	if len(m) > 0 {
-		mPtr = &m[0]
-	}
-	poly1305_auth_armv6(out, mPtr, uint32(len(m)), key)
-}
@@ -1,427 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build arm,!gccgo,!appengine,!nacl
-
-#include "textflag.h"
-
-// This code was translated into a form compatible with 5a from the public
-// domain source by Andrew Moon: github.com/floodyberry/poly1305-opt/blob/master/app/extensions/poly1305.
-
-DATA ·poly1305_init_constants_armv6<>+0x00(SB)/4, $0x3ffffff
-DATA ·poly1305_init_constants_armv6<>+0x04(SB)/4, $0x3ffff03
-DATA ·poly1305_init_constants_armv6<>+0x08(SB)/4, $0x3ffc0ff
-DATA ·poly1305_init_constants_armv6<>+0x0c(SB)/4, $0x3f03fff
-DATA ·poly1305_init_constants_armv6<>+0x10(SB)/4, $0x00fffff
-GLOBL ·poly1305_init_constants_armv6<>(SB), 8, $20
-
-// Warning: the linker may use R11 to synthesize certain instructions. Please
-// take care and verify that no synthetic instructions use it.
-
-TEXT poly1305_init_ext_armv6<>(SB), NOSPLIT, $0
-	// Needs 16 bytes of stack and 64 bytes of space pointed to by R0.  (It
-	// might look like it's only 60 bytes of space but the final four bytes
-	// will be written by another function.) We need to skip over four
-	// bytes of stack because that's saving the value of 'g'.
-	ADD       $4, R13, R8
-	MOVM.IB   [R4-R7], (R8)
-	MOVM.IA.W (R1), [R2-R5]
-	MOVW      $·poly1305_init_constants_armv6<>(SB), R7
-	MOVW      R2, R8
-	MOVW      R2>>26, R9
-	MOVW      R3>>20, g
-	MOVW      R4>>14, R11
-	MOVW      R5>>8, R12
-	ORR       R3<<6, R9, R9
-	ORR       R4<<12, g, g
-	ORR       R5<<18, R11, R11
-	MOVM.IA   (R7), [R2-R6]
-	AND       R8, R2, R2
-	AND       R9, R3, R3
-	AND       g, R4, R4
-	AND       R11, R5, R5
-	AND       R12, R6, R6
-	MOVM.IA.W [R2-R6], (R0)
-	EOR       R2, R2, R2
-	EOR       R3, R3, R3
-	EOR       R4, R4, R4
-	EOR       R5, R5, R5
-	EOR       R6, R6, R6
-	MOVM.IA.W [R2-R6], (R0)
-	MOVM.IA.W (R1), [R2-R5]
-	MOVM.IA   [R2-R6], (R0)
-	ADD       $20, R13, R0
-	MOVM.DA   (R0), [R4-R7]
-	RET
-
-#define MOVW_UNALIGNED(Rsrc, Rdst, Rtmp, offset) \
-	MOVBU (offset+0)(Rsrc), Rtmp; \
-	MOVBU Rtmp, (offset+0)(Rdst); \
-	MOVBU (offset+1)(Rsrc), Rtmp; \
-	MOVBU Rtmp, (offset+1)(Rdst); \
-	MOVBU (offset+2)(Rsrc), Rtmp; \
-	MOVBU Rtmp, (offset+2)(Rdst); \
-	MOVBU (offset+3)(Rsrc), Rtmp; \
-	MOVBU Rtmp, (offset+3)(Rdst)
-
-TEXT poly1305_blocks_armv6<>(SB), NOSPLIT, $0
-	// Needs 24 bytes of stack for saved registers and then 88 bytes of
-	// scratch space after that. We assume that 24 bytes at (R13) have
-	// already been used: four bytes for the link register saved in the
-	// prelude of poly1305_auth_armv6, four bytes for saving the value of g
-	// in that function and 16 bytes of scratch space used around
-	// poly1305_finish_ext_armv6_skip1.
-	ADD     $24, R13, R12
-	MOVM.IB [R4-R8, R14], (R12)
-	MOVW    R0, 88(R13)
-	MOVW    R1, 92(R13)
-	MOVW    R2, 96(R13)
-	MOVW    R1, R14
-	MOVW    R2, R12
-	MOVW    56(R0), R8
-	WORD    $0xe1180008                // TST R8, R8 not working see issue 5921
-	EOR     R6, R6, R6
-	MOVW.EQ $(1<<24), R6
-	MOVW    R6, 84(R13)
-	ADD     $116, R13, g
-	MOVM.IA (R0), [R0-R9]
-	MOVM.IA [R0-R4], (g)
-	CMP     $16, R12
-	BLO     poly1305_blocks_armv6_done
-
-poly1305_blocks_armv6_mainloop:
-	WORD    $0xe31e0003                            // TST R14, #3 not working see issue 5921
-	BEQ     poly1305_blocks_armv6_mainloop_aligned
-	ADD     $100, R13, g
-	MOVW_UNALIGNED(R14, g, R0, 0)
-	MOVW_UNALIGNED(R14, g, R0, 4)
-	MOVW_UNALIGNED(R14, g, R0, 8)
-	MOVW_UNALIGNED(R14, g, R0, 12)
-	MOVM.IA (g), [R0-R3]
-	ADD     $16, R14
-	B       poly1305_blocks_armv6_mainloop_loaded
-
-poly1305_blocks_armv6_mainloop_aligned:
-	MOVM.IA.W (R14), [R0-R3]
-
-poly1305_blocks_armv6_mainloop_loaded:
-	MOVW    R0>>26, g
-	MOVW    R1>>20, R11
-	MOVW    R2>>14, R12
-	MOVW    R14, 92(R13)
-	MOVW    R3>>8, R4
-	ORR     R1<<6, g, g
-	ORR     R2<<12, R11, R11
-	ORR     R3<<18, R12, R12
-	BIC     $0xfc000000, R0, R0
-	BIC     $0xfc000000, g, g
-	MOVW    84(R13), R3
-	BIC     $0xfc000000, R11, R11
-	BIC     $0xfc000000, R12, R12
-	ADD     R0, R5, R5
-	ADD     g, R6, R6
-	ORR     R3, R4, R4
-	ADD     R11, R7, R7
-	ADD     $116, R13, R14
-	ADD     R12, R8, R8
-	ADD     R4, R9, R9
-	MOVM.IA (R14), [R0-R4]
-	MULLU   R4, R5, (R11, g)
-	MULLU   R3, R5, (R14, R12)
-	MULALU  R3, R6, (R11, g)
-	MULALU  R2, R6, (R14, R12)
-	MULALU  R2, R7, (R11, g)
-	MULALU  R1, R7, (R14, R12)
-	ADD     R4<<2, R4, R4
-	ADD     R3<<2, R3, R3
-	MULALU  R1, R8, (R11, g)
-	MULALU  R0, R8, (R14, R12)
-	MULALU  R0, R9, (R11, g)
-	MULALU  R4, R9, (R14, R12)
-	MOVW    g, 76(R13)
-	MOVW    R11, 80(R13)
-	MOVW    R12, 68(R13)
-	MOVW    R14, 72(R13)
-	MULLU   R2, R5, (R11, g)
-	MULLU   R1, R5, (R14, R12)
-	MULALU  R1, R6, (R11, g)
-	MULALU  R0, R6, (R14, R12)
-	MULALU  R0, R7, (R11, g)
-	MULALU  R4, R7, (R14, R12)
-	ADD     R2<<2, R2, R2
-	ADD     R1<<2, R1, R1
-	MULALU  R4, R8, (R11, g)
-	MULALU  R3, R8, (R14, R12)
-	MULALU  R3, R9, (R11, g)
-	MULALU  R2, R9, (R14, R12)
-	MOVW    g, 60(R13)
-	MOVW    R11, 64(R13)
-	MOVW    R12, 52(R13)
-	MOVW    R14, 56(R13)
-	MULLU   R0, R5, (R11, g)
-	MULALU  R4, R6, (R11, g)
-	MULALU  R3, R7, (R11, g)
-	MULALU  R2, R8, (R11, g)
-	MULALU  R1, R9, (R11, g)
-	ADD     $52, R13, R0
-	MOVM.IA (R0), [R0-R7]
-	MOVW    g>>26, R12
-	MOVW    R4>>26, R14
-	ORR     R11<<6, R12, R12
-	ORR     R5<<6, R14, R14
-	BIC     $0xfc000000, g, g
-	BIC     $0xfc000000, R4, R4
-	ADD.S   R12, R0, R0
-	ADC     $0, R1, R1
-	ADD.S   R14, R6, R6
-	ADC     $0, R7, R7
-	MOVW    R0>>26, R12
-	MOVW    R6>>26, R14
-	ORR     R1<<6, R12, R12
-	ORR     R7<<6, R14, R14
-	BIC     $0xfc000000, R0, R0
-	BIC     $0xfc000000, R6, R6
-	ADD     R14<<2, R14, R14
-	ADD.S   R12, R2, R2
-	ADC     $0, R3, R3
-	ADD     R14, g, g
-	MOVW    R2>>26, R12
-	MOVW    g>>26, R14
-	ORR     R3<<6, R12, R12
-	BIC     $0xfc000000, g, R5
-	BIC     $0xfc000000, R2, R7
-	ADD     R12, R4, R4
-	ADD     R14, R0, R0
-	MOVW    R4>>26, R12
-	BIC     $0xfc000000, R4, R8
-	ADD     R12, R6, R9
-	MOVW    96(R13), R12
-	MOVW    92(R13), R14
-	MOVW    R0, R6
-	CMP     $32, R12
-	SUB     $16, R12, R12
-	MOVW    R12, 96(R13)
-	BHS     poly1305_blocks_armv6_mainloop
-
-poly1305_blocks_armv6_done:
-	MOVW    88(R13), R12
-	MOVW    R5, 20(R12)
-	MOVW    R6, 24(R12)
-	MOVW    R7, 28(R12)
-	MOVW    R8, 32(R12)
-	MOVW    R9, 36(R12)
-	ADD     $48, R13, R0
-	MOVM.DA (R0), [R4-R8, R14]
-	RET
-
-#define MOVHUP_UNALIGNED(Rsrc, Rdst, Rtmp) \
-	MOVBU.P 1(Rsrc), Rtmp; \
-	MOVBU.P Rtmp, 1(Rdst); \
-	MOVBU.P 1(Rsrc), Rtmp; \
-	MOVBU.P Rtmp, 1(Rdst)
-
-#define MOVWP_UNALIGNED(Rsrc, Rdst, Rtmp) \
-	MOVHUP_UNALIGNED(Rsrc, Rdst, Rtmp); \
-	MOVHUP_UNALIGNED(Rsrc, Rdst, Rtmp)
-
-// func poly1305_auth_armv6(out *[16]byte, m *byte, mlen uint32, key *[32]key)
-TEXT ·poly1305_auth_armv6(SB), $196-16
-	// The value 196, just above, is the sum of 64 (the size of the context
-	// structure) and 132 (the amount of stack needed).
-	//
-	// At this point, the stack pointer (R13) has been moved down. It
-	// points to the saved link register and there's 196 bytes of free
-	// space above it.
-	//
-	// The stack for this function looks like:
-	//
-	// +---------------------
-	// |
-	// | 64 bytes of context structure
-	// |
-	// +---------------------
-	// |
-	// | 112 bytes for poly1305_blocks_armv6
-	// |
-	// +---------------------
-	// | 16 bytes of final block, constructed at
-	// | poly1305_finish_ext_armv6_skip8
-	// +---------------------
-	// | four bytes of saved 'g'
-	// +---------------------
-	// | lr, saved by prelude    <- R13 points here
-	// +---------------------
-	MOVW g, 4(R13)
-
-	MOVW out+0(FP), R4
-	MOVW m+4(FP), R5
-	MOVW mlen+8(FP), R6
-	MOVW key+12(FP), R7
-
-	ADD  $136, R13, R0 // 136 = 4 + 4 + 16 + 112
-	MOVW R7, R1
-
-	// poly1305_init_ext_armv6 will write to the stack from R13+4, but
-	// that's ok because none of the other values have been written yet.
-	BL    poly1305_init_ext_armv6<>(SB)
-	BIC.S $15, R6, R2
-	BEQ   poly1305_auth_armv6_noblocks
-	ADD   $136, R13, R0
-	MOVW  R5, R1
-	ADD   R2, R5, R5
-	SUB   R2, R6, R6
-	BL    poly1305_blocks_armv6<>(SB)
-
-poly1305_auth_armv6_noblocks:
-	ADD  $136, R13, R0
-	MOVW R5, R1
-	MOVW R6, R2
-	MOVW R4, R3
-
-	MOVW  R0, R5
-	MOVW  R1, R6
-	MOVW  R2, R7
-	MOVW  R3, R8
-	AND.S R2, R2, R2
-	BEQ   poly1305_finish_ext_armv6_noremaining
-	EOR   R0, R0
-	ADD   $8, R13, R9                           // 8 = offset to 16 byte scratch space
-	MOVW  R0, (R9)
-	MOVW  R0, 4(R9)
-	MOVW  R0, 8(R9)
-	MOVW  R0, 12(R9)
-	WORD  $0xe3110003                           // TST R1, #3 not working see issue 5921
-	BEQ   poly1305_finish_ext_armv6_aligned
-	WORD  $0xe3120008                           // TST R2, #8 not working see issue 5921
-	BEQ   poly1305_finish_ext_armv6_skip8
-	MOVWP_UNALIGNED(R1, R9, g)
-	MOVWP_UNALIGNED(R1, R9, g)
-
-poly1305_finish_ext_armv6_skip8:
-	WORD $0xe3120004                     // TST $4, R2 not working see issue 5921
-	BEQ  poly1305_finish_ext_armv6_skip4
-	MOVWP_UNALIGNED(R1, R9, g)
-
-poly1305_finish_ext_armv6_skip4:
-	WORD $0xe3120002                     // TST $2, R2 not working see issue 5921
-	BEQ  poly1305_finish_ext_armv6_skip2
-	MOVHUP_UNALIGNED(R1, R9, g)
-	B    poly1305_finish_ext_armv6_skip2
-
-poly1305_finish_ext_armv6_aligned:
-	WORD      $0xe3120008                             // TST R2, #8 not working see issue 5921
-	BEQ       poly1305_finish_ext_armv6_skip8_aligned
-	MOVM.IA.W (R1), [g-R11]
-	MOVM.IA.W [g-R11], (R9)
-
-poly1305_finish_ext_armv6_skip8_aligned:
-	WORD   $0xe3120004                             // TST $4, R2 not working see issue 5921
-	BEQ    poly1305_finish_ext_armv6_skip4_aligned
-	MOVW.P 4(R1), g
-	MOVW.P g, 4(R9)
-
-poly1305_finish_ext_armv6_skip4_aligned:
-	WORD    $0xe3120002                     // TST $2, R2 not working see issue 5921
-	BEQ     poly1305_finish_ext_armv6_skip2
-	MOVHU.P 2(R1), g
-	MOVH.P  g, 2(R9)
-
-poly1305_finish_ext_armv6_skip2:
-	WORD    $0xe3120001                     // TST $1, R2 not working see issue 5921
-	BEQ     poly1305_finish_ext_armv6_skip1
-	MOVBU.P 1(R1), g
-	MOVBU.P g, 1(R9)
-
-poly1305_finish_ext_armv6_skip1:
-	MOVW  $1, R11
-	MOVBU R11, 0(R9)
-	MOVW  R11, 56(R5)
-	MOVW  R5, R0
-	ADD   $8, R13, R1
-	MOVW  $16, R2
-	BL    poly1305_blocks_armv6<>(SB)
-
-poly1305_finish_ext_armv6_noremaining:
-	MOVW      20(R5), R0
-	MOVW      24(R5), R1
-	MOVW      28(R5), R2
-	MOVW      32(R5), R3
-	MOVW      36(R5), R4
-	MOVW      R4>>26, R12
-	BIC       $0xfc000000, R4, R4
-	ADD       R12<<2, R12, R12
-	ADD       R12, R0, R0
-	MOVW      R0>>26, R12
-	BIC       $0xfc000000, R0, R0
-	ADD       R12, R1, R1
-	MOVW      R1>>26, R12
-	BIC       $0xfc000000, R1, R1
-	ADD       R12, R2, R2
-	MOVW      R2>>26, R12
-	BIC       $0xfc000000, R2, R2
-	ADD       R12, R3, R3
-	MOVW      R3>>26, R12
-	BIC       $0xfc000000, R3, R3
-	ADD       R12, R4, R4
-	ADD       $5, R0, R6
-	MOVW      R6>>26, R12
-	BIC       $0xfc000000, R6, R6
-	ADD       R12, R1, R7
-	MOVW      R7>>26, R12
-	BIC       $0xfc000000, R7, R7
-	ADD       R12, R2, g
-	MOVW      g>>26, R12
-	BIC       $0xfc000000, g, g
-	ADD       R12, R3, R11
-	MOVW      $-(1<<26), R12
-	ADD       R11>>26, R12, R12
-	BIC       $0xfc000000, R11, R11
-	ADD       R12, R4, R9
-	MOVW      R9>>31, R12
-	SUB       $1, R12
-	AND       R12, R6, R6
-	AND       R12, R7, R7
-	AND       R12, g, g
-	AND       R12, R11, R11
-	AND       R12, R9, R9
-	MVN       R12, R12
-	AND       R12, R0, R0
-	AND       R12, R1, R1
-	AND       R12, R2, R2
-	AND       R12, R3, R3
-	AND       R12, R4, R4
-	ORR       R6, R0, R0
-	ORR       R7, R1, R1
-	ORR       g, R2, R2
-	ORR       R11, R3, R3
-	ORR       R9, R4, R4
-	ORR       R1<<26, R0, R0
-	MOVW      R1>>6, R1
-	ORR       R2<<20, R1, R1
-	MOVW      R2>>12, R2
-	ORR       R3<<14, R2, R2
-	MOVW      R3>>18, R3
-	ORR       R4<<8, R3, R3
-	MOVW      40(R5), R6
-	MOVW      44(R5), R7
-	MOVW      48(R5), g
-	MOVW      52(R5), R11
-	ADD.S     R6, R0, R0
-	ADC.S     R7, R1, R1
-	ADC.S     g, R2, R2
-	ADC.S     R11, R3, R3
-	MOVM.IA   [R0-R3], (R8)
-	MOVW      R5, R12
-	EOR       R0, R0, R0
-	EOR       R1, R1, R1
-	EOR       R2, R2, R2
-	EOR       R3, R3, R3
-	EOR       R4, R4, R4
-	EOR       R5, R5, R5
-	EOR       R6, R6, R6
-	EOR       R7, R7, R7
-	MOVM.IA.W [R0-R7], (R12)
-	MOVM.IA   [R0-R7], (R12)
-	MOVW      4(R13), g
-	RET
@@ -2,18 +2,29 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

+// This file provides the generic implementation of Sum and MAC. Other files
+// might provide optimized assembly implementations of some of this code.
+
 package poly1305

 import "encoding/binary"

-const (
-	msgBlock   = uint32(1 << 24)
-	finalBlock = uint32(0)
-)
+// Poly1305 [RFC 7539] is a relatively simple algorithm: the authentication tag
+// for a 64 bytes message is approximately
+//
+//     s + m[0:16] * r⁴ + m[16:32] * r³ + m[32:48] * r² + m[48:64] * r  mod  2¹³⁰ - 5
+//
+// for some secret r and s. It can be computed sequentially like
+//
+//     for len(msg) > 0:
+//         h += read(msg, 16)
+//         h *= r
+//         h %= 2¹³⁰ - 5
+//     return h + s
+//
+// All the complexity is about doing performant constant-time math on numbers
+// larger than any available numeric type.

-// sumGeneric generates an authenticator for msg using a one-time key and
-// puts the 16-byte result into out. This is the generic implementation of
-// Sum and should be called if no assembly implementation is available.
 func sumGeneric(out *[TagSize]byte, msg []byte, key *[32]byte) {
 	h := newMACGeneric(key)
 	h.Write(msg)
@@ -21,152 +32,276 @@ func sumGeneric(out *[TagSize]byte, msg []byte, key *[32]byte) {
 }

 func newMACGeneric(key *[32]byte) (h macGeneric) {
-	h.r[0] = binary.LittleEndian.Uint32(key[0:]) & 0x3ffffff
-	h.r[1] = (binary.LittleEndian.Uint32(key[3:]) >> 2) & 0x3ffff03
-	h.r[2] = (binary.LittleEndian.Uint32(key[6:]) >> 4) & 0x3ffc0ff
-	h.r[3] = (binary.LittleEndian.Uint32(key[9:]) >> 6) & 0x3f03fff
-	h.r[4] = (binary.LittleEndian.Uint32(key[12:]) >> 8) & 0x00fffff
-
-	h.s[0] = binary.LittleEndian.Uint32(key[16:])
-	h.s[1] = binary.LittleEndian.Uint32(key[20:])
-	h.s[2] = binary.LittleEndian.Uint32(key[24:])
-	h.s[3] = binary.LittleEndian.Uint32(key[28:])
+	initialize(key, &h.r, &h.s)
 	return
 }

+// macState holds numbers in saturated 64-bit little-endian limbs. That is,
+// the value of [x0, x1, x2] is x[0] + x[1] * 2⁶⁴ + x[2] * 2¹²⁸.
+type macState struct {
+	// h is the main accumulator. It is to be interpreted modulo 2¹³⁰ - 5, but
+	// can grow larger during and after rounds.
+	h [3]uint64
+	// r and s are the private key components.
+	r [2]uint64
+	s [2]uint64
+}
+
 type macGeneric struct {
-	h, r [5]uint32
-	s    [4]uint32
+	macState

 	buffer [TagSize]byte
 	offset int
 }

-func (h *macGeneric) Write(p []byte) (n int, err error) {
-	n = len(p)
+// Write splits the incoming message into TagSize chunks, and passes them to
+// update. It buffers incomplete chunks.
+func (h *macGeneric) Write(p []byte) (int, error) {
+	nn := len(p)
 	if h.offset > 0 {
-		remaining := TagSize - h.offset
-		if n < remaining {
-			h.offset += copy(h.buffer[h.offset:], p)
-			return n, nil
+		n := copy(h.buffer[h.offset:], p)
+		if h.offset+n < TagSize {
+			h.offset += n
+			return nn, nil
 		}
-		copy(h.buffer[h.offset:], p[:remaining])
-		p = p[remaining:]
+		p = p[n:]
 		h.offset = 0
-		updateGeneric(h.buffer[:], msgBlock, &(h.h), &(h.r))
+		updateGeneric(&h.macState, h.buffer[:])
 	}
-	if nn := len(p) - (len(p) % TagSize); nn > 0 {
-		updateGeneric(p, msgBlock, &(h.h), &(h.r))
-		p = p[nn:]
+	if n := len(p) - (len(p) % TagSize); n > 0 {
+		updateGeneric(&h.macState, p[:n])
+		p = p[n:]
 	}
 	if len(p) > 0 {
 		h.offset += copy(h.buffer[h.offset:], p)
 	}
-	return n, nil
+	return nn, nil
 }

-func (h *macGeneric) Sum(out *[16]byte) {
-	H, R := h.h, h.r
+// Sum flushes the last incomplete chunk from the buffer, if any, and generates
+// the MAC output. It does not modify its state, in order to allow for multiple
+// calls to Sum, even if no Write is allowed after Sum.
+func (h *macGeneric) Sum(out *[TagSize]byte) {
+	state := h.macState
 	if h.offset > 0 {
-		var buffer [TagSize]byte
-		copy(buffer[:], h.buffer[:h.offset])
-		buffer[h.offset] = 1 // invariant: h.offset < TagSize
-		updateGeneric(buffer[:], finalBlock, &H, &R)
+		updateGeneric(&state, h.buffer[:h.offset])
 	}
-	finalizeGeneric(out, &H, &(h.s))
+	finalize(out, &state.h, &state.s)
 }

-func updateGeneric(msg []byte, flag uint32, h, r *[5]uint32) {
-	h0, h1, h2, h3, h4 := h[0], h[1], h[2], h[3], h[4]
-	r0, r1, r2, r3, r4 := uint64(r[0]), uint64(r[1]), uint64(r[2]), uint64(r[3]), uint64(r[4])
-	R1, R2, R3, R4 := r1*5, r2*5, r3*5, r4*5
+// [rMask0, rMask1] is the specified Poly1305 clamping mask in little-endian. It
+// clears some bits of the secret coefficient to make it possible to implement
+// multiplication more efficiently.
+const (
+	rMask0 = 0x0FFFFFFC0FFFFFFF
+	rMask1 = 0x0FFFFFFC0FFFFFFC
+)

-	for len(msg) >= TagSize {
-		// h += msg
-		h0 += binary.LittleEndian.Uint32(msg[0:]) & 0x3ffffff
-		h1 += (binary.LittleEndian.Uint32(msg[3:]) >> 2) & 0x3ffffff
-		h2 += (binary.LittleEndian.Uint32(msg[6:]) >> 4) & 0x3ffffff
-		h3 += (binary.LittleEndian.Uint32(msg[9:]) >> 6) & 0x3ffffff
-		h4 += (binary.LittleEndian.Uint32(msg[12:]) >> 8) | flag
+func initialize(key *[32]byte, r, s *[2]uint64) {
+	r[0] = binary.LittleEndian.Uint64(key[0:8]) & rMask0
+	r[1] = binary.LittleEndian.Uint64(key[8:16]) & rMask1
+	s[0] = binary.LittleEndian.Uint64(key[16:24])
+	s[1] = binary.LittleEndian.Uint64(key[24:32])
+}

-		// h *= r
-		d0 := (uint64(h0) * r0) + (uint64(h1) * R4) + (uint64(h2) * R3) + (uint64(h3) * R2) + (uint64(h4) * R1)
-		d1 := (d0 >> 26) + (uint64(h0) * r1) + (uint64(h1) * r0) + (uint64(h2) * R4) + (uint64(h3) * R3) + (uint64(h4) * R2)
-		d2 := (d1 >> 26) + (uint64(h0) * r2) + (uint64(h1) * r1) + (uint64(h2) * r0) + (uint64(h3) * R4) + (uint64(h4) * R3)
-		d3 := (d2 >> 26) + (uint64(h0) * r3) + (uint64(h1) * r2) + (uint64(h2) * r1) + (uint64(h3) * r0) + (uint64(h4) * R4)
-		d4 := (d3 >> 26) + (uint64(h0) * r4) + (uint64(h1) * r3) + (uint64(h2) * r2) + (uint64(h3) * r1) + (uint64(h4) * r0)
+// uint128 holds a 128-bit number as two 64-bit limbs, for use with the
+// bits.Mul64 and bits.Add64 intrinsics.
+type uint128 struct {
+	lo, hi uint64
+}

-		// h %= p
-		h0 = uint32(d0) & 0x3ffffff
-		h1 = uint32(d1) & 0x3ffffff
-		h2 = uint32(d2) & 0x3ffffff
-		h3 = uint32(d3) & 0x3ffffff
-		h4 = uint32(d4) & 0x3ffffff
+func mul64(a, b uint64) uint128 {
+	hi, lo := bitsMul64(a, b)
+	return uint128{lo, hi}
+}

-		h0 += uint32(d4>>26) * 5
-		h1 += h0 >> 26
-		h0 = h0 & 0x3ffffff
+func add128(a, b uint128) uint128 {
+	lo, c := bitsAdd64(a.lo, b.lo, 0)
+	hi, c := bitsAdd64(a.hi, b.hi, c)
+	if c != 0 {
+		panic("poly1305: unexpected overflow")
+	}
+	return uint128{lo, hi}
+}

-		msg = msg[TagSize:]
+func shiftRightBy2(a uint128) uint128 {
+	a.lo = a.lo>>2 | (a.hi&3)<<62
+	a.hi = a.hi >> 2
+	return a
+}
+
+// updateGeneric absorbs msg into the state.h accumulator. For each chunk m of
+// 128 bits of message, it computes
+//
+//     h₊ = (h + m) * r  mod  2¹³⁰ - 5
+//
+// If the msg length is not a multiple of TagSize, it assumes the last
+// incomplete chunk is the final one.
+func updateGeneric(state *macState, msg []byte) {
+	h0, h1, h2 := state.h[0], state.h[1], state.h[2]
+	r0, r1 := state.r[0], state.r[1]
+
+	for len(msg) > 0 {
+		var c uint64
+
+		// For the first step, h + m, we use a chain of bits.Add64 intrinsics.
+		// The resulting value of h might exceed 2¹³⁰ - 5, but will be partially
+		// reduced at the end of the multiplication below.
+		//
+		// The spec requires us to set a bit just above the message size, not to
+		// hide leading zeroes. For full chunks, that's 1 << 128, so we can just
+		// add 1 to the most significant (2¹²⁸) limb, h2.
+		if len(msg) >= TagSize {
+			h0, c = bitsAdd64(h0, binary.LittleEndian.Uint64(msg[0:8]), 0)
+			h1, c = bitsAdd64(h1, binary.LittleEndian.Uint64(msg[8:16]), c)
+			h2 += c + 1
+
+			msg = msg[TagSize:]
+		} else {
+			var buf [TagSize]byte
+			copy(buf[:], msg)
+			buf[len(msg)] = 1
+
+			h0, c = bitsAdd64(h0, binary.LittleEndian.Uint64(buf[0:8]), 0)
+			h1, c = bitsAdd64(h1, binary.LittleEndian.Uint64(buf[8:16]), c)
+			h2 += c
+
+			msg = nil
+		}
+
+		// Multiplication of big number limbs is similar to elementary school
+		// columnar multiplication. Instead of digits, there are 64-bit limbs.
+		//
+		// We are multiplying a 3 limbs number, h, by a 2 limbs number, r.
+		//
+		//                        h2    h1    h0  x
+		//                              r1    r0  =
+		//                       ----------------
+		//                      h2r0  h1r0  h0r0     <-- individual 128-bit products
+		//            +   h2r1  h1r1  h0r1
+		//               ------------------------
+		//                 m3    m2    m1    m0      <-- result in 128-bit overlapping limbs
+		//               ------------------------
+		//         m3.hi m2.hi m1.hi m0.hi           <-- carry propagation
+		//     +         m3.lo m2.lo m1.lo m0.lo
+		//        -------------------------------
+		//           t4    t3    t2    t1    t0      <-- final result in 64-bit limbs
+		//
+		// The main difference from pen-and-paper multiplication is that we do
+		// carry propagation in a separate step, as if we wrote two digit sums
+		// at first (the 128-bit limbs), and then carried the tens all at once.
+
+		h0r0 := mul64(h0, r0)
+		h1r0 := mul64(h1, r0)
+		h2r0 := mul64(h2, r0)
+		h0r1 := mul64(h0, r1)
+		h1r1 := mul64(h1, r1)
+		h2r1 := mul64(h2, r1)
+
+		// Since h2 is known to be at most 7 (5 + 1 + 1), and r0 and r1 have their
+		// top 4 bits cleared by rMask{0,1}, we know that their product is not going
+		// to overflow 64 bits, so we can ignore the high part of the products.
+		//
+		// This also means that the product doesn't have a fifth limb (t4).
+		if h2r0.hi != 0 {
+			panic("poly1305: unexpected overflow")
+		}
+		if h2r1.hi != 0 {
+			panic("poly1305: unexpected overflow")
+		}
+
+		m0 := h0r0
+		m1 := add128(h1r0, h0r1) // These two additions don't overflow thanks again
+		m2 := add128(h2r0, h1r1) // to the 4 masked bits at the top of r0 and r1.
+		m3 := h2r1
+
+		t0 := m0.lo
+		t1, c := bitsAdd64(m1.lo, m0.hi, 0)
+		t2, c := bitsAdd64(m2.lo, m1.hi, c)
+		t3, _ := bitsAdd64(m3.lo, m2.hi, c)
+
+		// Now we have the result as 4 64-bit limbs, and we need to reduce it
+		// modulo 2¹³⁰ - 5. The special shape of this Crandall prime lets us do
+		// a cheap partial reduction according to the reduction identity
+		//
+		//     c * 2¹³⁰ + n  =  c * 5 + n  mod  2¹³⁰ - 5
+		//
+		// because 2¹³⁰ = 5 mod 2¹³⁰ - 5. Partial reduction since the result is
+		// likely to be larger than 2¹³⁰ - 5, but still small enough to fit the
+		// assumptions we make about h in the rest of the code.
+		//
+		// See also https://speakerdeck.com/gtank/engineering-prime-numbers?slide=23
+
+		// We split the final result at the 2¹³⁰ mark into h and cc, the carry.
+		// Note that the carry bits are effectively shifted left by 2, in other
+		// words, cc = c * 4 for the c in the reduction identity.
+		h0, h1, h2 = t0, t1, t2&maskLow2Bits
+		cc := uint128{t2 & maskNotLow2Bits, t3}
+
+		// To add c * 5 to h, we first add cc = c * 4, and then add (cc >> 2) = c.
+
+		h0, c = bitsAdd64(h0, cc.lo, 0)
+		h1, c = bitsAdd64(h1, cc.hi, c)
+		h2 += c
+
+		cc = shiftRightBy2(cc)
+
+		h0, c = bitsAdd64(h0, cc.lo, 0)
+		h1, c = bitsAdd64(h1, cc.hi, c)
+		h2 += c
+
+		// h2 is at most 3 + 1 + 1 = 5, making the whole of h at most
+		//
+		//     5 * 2¹²⁸ + (2¹²⁸ - 1) = 6 * 2¹²⁸ - 1
 	}

-	h[0], h[1], h[2], h[3], h[4] = h0, h1, h2, h3, h4
+	state.h[0], state.h[1], state.h[2] = h0, h1, h2
 }

-func finalizeGeneric(out *[TagSize]byte, h *[5]uint32, s *[4]uint32) {
-	h0, h1, h2, h3, h4 := h[0], h[1], h[2], h[3], h[4]
+const (
+	maskLow2Bits    uint64 = 0x0000000000000003
+	maskNotLow2Bits uint64 = ^maskLow2Bits
+)

-	// h %= p reduction
-	h2 += h1 >> 26
-	h1 &= 0x3ffffff
-	h3 += h2 >> 26
-	h2 &= 0x3ffffff
-	h4 += h3 >> 26
-	h3 &= 0x3ffffff
-	h0 += 5 * (h4 >> 26)
-	h4 &= 0x3ffffff
-	h1 += h0 >> 26
-	h0 &= 0x3ffffff
+// select64 returns x if v == 1 and y if v == 0, in constant time.
+func select64(v, x, y uint64) uint64 { return ^(v-1)&x | (v-1)&y }

-	// h - p
-	t0 := h0 + 5
-	t1 := h1 + (t0 >> 26)
-	t2 := h2 + (t1 >> 26)
-	t3 := h3 + (t2 >> 26)
-	t4 := h4 + (t3 >> 26) - (1 << 26)
-	t0 &= 0x3ffffff
-	t1 &= 0x3ffffff
-	t2 &= 0x3ffffff
-	t3 &= 0x3ffffff
+// [p0, p1, p2] is 2¹³⁰ - 5 in little endian order.
+const (
+	p0 = 0xFFFFFFFFFFFFFFFB
+	p1 = 0xFFFFFFFFFFFFFFFF
+	p2 = 0x0000000000000003
+)

-	// select h if h < p else h - p
-	t_mask := (t4 >> 31) - 1
-	h_mask := ^t_mask
-	h0 = (h0 & h_mask) | (t0 & t_mask)
-	h1 = (h1 & h_mask) | (t1 & t_mask)
-	h2 = (h2 & h_mask) | (t2 & t_mask)
-	h3 = (h3 & h_mask) | (t3 & t_mask)
-	h4 = (h4 & h_mask) | (t4 & t_mask)
+// finalize completes the modular reduction of h and computes
+//
+//     out = h + s  mod  2¹²⁸
+//
+func finalize(out *[TagSize]byte, h *[3]uint64, s *[2]uint64) {
+	h0, h1, h2 := h[0], h[1], h[2]

-	// h %= 2^128
-	h0 |= h1 << 26
-	h1 = ((h1 >> 6) | (h2 << 20))
-	h2 = ((h2 >> 12) | (h3 << 14))
-	h3 = ((h3 >> 18) | (h4 << 8))
+	// After the partial reduction in updateGeneric, h might be more than
+	// 2¹³⁰ - 5, but will be less than 2 * (2¹³⁰ - 5). To complete the reduction
+	// in constant time, we compute t = h - (2¹³⁰ - 5), and select h as the
+	// result if the subtraction underflows, and t otherwise.

-	// s: the s part of the key
-	// tag = (h + s) % (2^128)
-	t := uint64(h0) + uint64(s[0])
-	h0 = uint32(t)
-	t = uint64(h1) + uint64(s[1]) + (t >> 32)
-	h1 = uint32(t)
-	t = uint64(h2) + uint64(s[2]) + (t >> 32)
-	h2 = uint32(t)
-	t = uint64(h3) + uint64(s[3]) + (t >> 32)
-	h3 = uint32(t)
+	hMinusP0, b := bitsSub64(h0, p0, 0)
+	hMinusP1, b := bitsSub64(h1, p1, b)
+	_, b = bitsSub64(h2, p2, b)

-	binary.LittleEndian.PutUint32(out[0:], h0)
-	binary.LittleEndian.PutUint32(out[4:], h1)
-	binary.LittleEndian.PutUint32(out[8:], h2)
-	binary.LittleEndian.PutUint32(out[12:], h3)
+	// h = h if h < p else h - p
+	h0 = select64(b, h0, hMinusP0)
+	h1 = select64(b, h1, hMinusP1)
+
+	// Finally, we compute the last Poly1305 step
+	//
+	//     tag = h + s  mod  2¹²⁸
+	//
+	// by just doing a wide addition with the 128 low bits of h and discarding
+	// the overflow.
+	h0, c := bitsAdd64(h0, s[0], 0)
+	h1, _ = bitsAdd64(h1, s[1], c)
+
+	binary.LittleEndian.PutUint64(out[0:8], h0)
+	binary.LittleEndian.PutUint64(out[8:16], h1)
 }
@@ -2,14 +2,11 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

-// +build s390x,!go1.11 !arm,!amd64,!s390x gccgo appengine nacl
+// +build s390x,!go1.11 !amd64,!s390x,!ppc64le gccgo appengine nacl

 package poly1305

-// Sum generates an authenticator for msg using a one-time key and puts the
-// 16-byte result into out. Authenticating two different messages with the same
-// key allows an attacker to forge messages at will.
-func Sum(out *[TagSize]byte, msg []byte, key *[32]byte) {
+func sum(out *[TagSize]byte, msg []byte, key *[32]byte) {
 	h := newMAC(key)
 	h.Write(msg)
 	h.Sum(out)
@@ -0,0 +1,58 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ppc64le,!gccgo,!appengine
+
+package poly1305
+
+//go:noescape
+func update(state *macState, msg []byte)
+
+func sum(out *[16]byte, m []byte, key *[32]byte) {
+	h := newMAC(key)
+	h.Write(m)
+	h.Sum(out)
+}
+
+func newMAC(key *[32]byte) (h mac) {
+	initialize(key, &h.r, &h.s)
+	return
+}
+
+// mac is a wrapper for macGeneric that redirects calls that would have gone to
+// updateGeneric to update.
+//
+// Its Write and Sum methods are otherwise identical to the macGeneric ones, but
+// using function pointers would carry a major performance cost.
+type mac struct{ macGeneric }
+
+func (h *mac) Write(p []byte) (int, error) {
+	nn := len(p)
+	if h.offset > 0 {
+		n := copy(h.buffer[h.offset:], p)
+		if h.offset+n < TagSize {
+			h.offset += n
+			return nn, nil
+		}
+		p = p[n:]
+		h.offset = 0
+		update(&h.macState, h.buffer[:])
+	}
+	if n := len(p) - (len(p) % TagSize); n > 0 {
+		update(&h.macState, p[:n])
+		p = p[n:]
+	}
+	if len(p) > 0 {
+		h.offset += copy(h.buffer[h.offset:], p)
+	}
+	return nn, nil
+}
+
+func (h *mac) Sum(out *[16]byte) {
+	state := h.macState
+	if h.offset > 0 {
+		update(&state, h.buffer[:h.offset])
+	}
+	finalize(out, &state.h, &state.s)
+}
@@ -0,0 +1,181 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ppc64le,!gccgo,!appengine
+
+#include "textflag.h"
+
+// This was ported from the amd64 implementation.
+
+#define POLY1305_ADD(msg, h0, h1, h2, t0, t1, t2) \
+	MOVD (msg), t0;  \
+	MOVD 8(msg), t1; \
+	MOVD $1, t2;     \
+	ADDC t0, h0, h0; \
+	ADDE t1, h1, h1; \
+	ADDE t2, h2;     \
+	ADD  $16, msg
+
+#define POLY1305_MUL(h0, h1, h2, r0, r1, t0, t1, t2, t3, t4, t5) \
+	MULLD  r0, h0, t0;  \
+	MULLD  r0, h1, t4;  \
+	MULHDU r0, h0, t1;  \
+	MULHDU r0, h1, t5;  \
+	ADDC   t4, t1, t1;  \
+	MULLD  r0, h2, t2;  \
+	ADDZE  t5;          \
+	MULHDU r1, h0, t4;  \
+	MULLD  r1, h0, h0;  \
+	ADD    t5, t2, t2;  \
+	ADDC   h0, t1, t1;  \
+	MULLD  h2, r1, t3;  \
+	ADDZE  t4, h0;      \
+	MULHDU r1, h1, t5;  \
+	MULLD  r1, h1, t4;  \
+	ADDC   t4, t2, t2;  \
+	ADDE   t5, t3, t3;  \
+	ADDC   h0, t2, t2;  \
+	MOVD   $-4, t4;     \
+	MOVD   t0, h0;      \
+	MOVD   t1, h1;      \
+	ADDZE  t3;          \
+	ANDCC  $3, t2, h2;  \
+	AND    t2, t4, t0;  \
+	ADDC   t0, h0, h0;  \
+	ADDE   t3, h1, h1;  \
+	SLD    $62, t3, t4; \
+	SRD    $2, t2;      \
+	ADDZE  h2;          \
+	OR     t4, t2, t2;  \
+	SRD    $2, t3;      \
+	ADDC   t2, h0, h0;  \
+	ADDE   t3, h1, h1;  \
+	ADDZE  h2
+
+DATA ·poly1305Mask<>+0x00(SB)/8, $0x0FFFFFFC0FFFFFFF
+DATA ·poly1305Mask<>+0x08(SB)/8, $0x0FFFFFFC0FFFFFFC
+GLOBL ·poly1305Mask<>(SB), RODATA, $16
+
+// func update(state *[7]uint64, msg []byte)
+TEXT ·update(SB), $0-32
+	MOVD state+0(FP), R3
+	MOVD msg_base+8(FP), R4
+	MOVD msg_len+16(FP), R5
+
+	MOVD 0(R3), R8   // h0
+	MOVD 8(R3), R9   // h1
+	MOVD 16(R3), R10 // h2
+	MOVD 24(R3), R11 // r0
+	MOVD 32(R3), R12 // r1
+
+	CMP R5, $16
+	BLT bytes_between_0_and_15
+
+loop:
+	POLY1305_ADD(R4, R8, R9, R10, R20, R21, R22)
+
+multiply:
+	POLY1305_MUL(R8, R9, R10, R11, R12, R16, R17, R18, R14, R20, R21)
+	ADD $-16, R5
+	CMP R5, $16
+	BGE loop
+
+bytes_between_0_and_15:
+	CMP  $0, R5
+	BEQ  done
+	MOVD $0, R16 // h0
+	MOVD $0, R17 // h1
+
+flush_buffer:
+	CMP R5, $8
+	BLE just1
+
+	MOVD $8, R21
+	SUB  R21, R5, R21
+
+	// Greater than 8 -- load the rightmost remaining bytes in msg
+	// and put into R17 (h1)
+	MOVD (R4)(R21), R17
+	MOVD $16, R22
+
+	// Find the offset to those bytes
+	SUB R5, R22, R22
+	SLD $3, R22
+
+	// Shift to get only the bytes in msg
+	SRD R22, R17, R17
+
+	// Put 1 at high end
+	MOVD $1, R23
+	SLD  $3, R21
+	SLD  R21, R23, R23
+	OR   R23, R17, R17
+
+	// Remainder is 8
+	MOVD $8, R5
+
+just1:
+	CMP R5, $8
+	BLT less8
+
+	// Exactly 8
+	MOVD (R4), R16
+
+	CMP $0, R17
+
+	// Check if we've already set R17; if not
+	// set 1 to indicate end of msg.
+	BNE  carry
+	MOVD $1, R17
+	BR   carry
+
+less8:
+	MOVD  $0, R16   // h0
+	MOVD  $0, R22   // shift count
+	CMP   R5, $4
+	BLT   less4
+	MOVWZ (R4), R16
+	ADD   $4, R4
+	ADD   $-4, R5
+	MOVD  $32, R22
+
+less4:
+	CMP   R5, $2
+	BLT   less2
+	MOVHZ (R4), R21
+	SLD   R22, R21, R21
+	OR    R16, R21, R16
+	ADD   $16, R22
+	ADD   $-2, R5
+	ADD   $2, R4
+
+less2:
+	CMP   $0, R5
+	BEQ   insert1
+	MOVBZ (R4), R21
+	SLD   R22, R21, R21
+	OR    R16, R21, R16
+	ADD   $8, R22
+
+insert1:
+	// Insert 1 at end of msg
+	MOVD $1, R21
+	SLD  R22, R21, R21
+	OR   R16, R21, R16
+
+carry:
+	// Add new values to h0, h1, h2
+	ADDC R16, R8
+	ADDE R17, R9
+	ADDE $0, R10
+	MOVD $16, R5
+	ADD  R5, R4
+	BR   multiply
+
+done:
+	// Save h0, h1, h2 in state
+	MOVD R8, 0(R3)
+	MOVD R9, 8(R3)
+	MOVD R10, 16(R3)
+	RET
@@ -22,10 +22,7 @@ func poly1305vx(out *[16]byte, m *byte, mlen uint64, key *[32]byte)
 //go:noescape
 func poly1305vmsl(out *[16]byte, m *byte, mlen uint64, key *[32]byte)

-// Sum generates an authenticator for m using a one-time key and puts the
-// 16-byte result into out. Authenticating two different messages with the same
-// key allows an attacker to forge messages at will.
-func Sum(out *[16]byte, m []byte, key *[32]byte) {
+func sum(out *[16]byte, m []byte, key *[32]byte) {
 	if cpu.S390X.HasVX {
 		var mPtr *byte
 		if len(m) > 0 {
@@ -17,12 +17,14 @@ import (
 // These constants from [PROTOCOL.certkeys] represent the algorithm names
 // for certificate types supported by this package.
 const (
-	CertAlgoRSAv01      = "ssh-rsa-cert-v01@openssh.com"
-	CertAlgoDSAv01      = "ssh-dss-cert-v01@openssh.com"
-	CertAlgoECDSA256v01 = "ecdsa-sha2-nistp256-cert-v01@openssh.com"
-	CertAlgoECDSA384v01 = "ecdsa-sha2-nistp384-cert-v01@openssh.com"
-	CertAlgoECDSA521v01 = "ecdsa-sha2-nistp521-cert-v01@openssh.com"
-	CertAlgoED25519v01  = "ssh-ed25519-cert-v01@openssh.com"
+	CertAlgoRSAv01        = "ssh-rsa-cert-v01@openssh.com"
+	CertAlgoDSAv01        = "ssh-dss-cert-v01@openssh.com"
+	CertAlgoECDSA256v01   = "ecdsa-sha2-nistp256-cert-v01@openssh.com"
+	CertAlgoECDSA384v01   = "ecdsa-sha2-nistp384-cert-v01@openssh.com"
+	CertAlgoECDSA521v01   = "ecdsa-sha2-nistp521-cert-v01@openssh.com"
+	CertAlgoSKECDSA256v01 = "sk-ecdsa-sha2-nistp256-cert-v01@openssh.com"
+	CertAlgoED25519v01    = "ssh-ed25519-cert-v01@openssh.com"
+	CertAlgoSKED25519v01  = "sk-ssh-ed25519-cert-v01@openssh.com"
 )

 // Certificate types distinguish between host and user
@@ -37,6 +39,7 @@ const (
 type Signature struct {
 	Format string
 	Blob   []byte
+	Rest   []byte `ssh:"rest"`
 }

 // CertTimeInfinity can be used for OpenSSHCertV01.ValidBefore to indicate that
@@ -429,12 +432,14 @@ func (c *Certificate) SignCert(rand io.Reader, authority Signer) error {
 }

 var certAlgoNames = map[string]string{
-	KeyAlgoRSA:      CertAlgoRSAv01,
-	KeyAlgoDSA:      CertAlgoDSAv01,
-	KeyAlgoECDSA256: CertAlgoECDSA256v01,
-	KeyAlgoECDSA384: CertAlgoECDSA384v01,
-	KeyAlgoECDSA521: CertAlgoECDSA521v01,
-	KeyAlgoED25519:  CertAlgoED25519v01,
+	KeyAlgoRSA:        CertAlgoRSAv01,
+	KeyAlgoDSA:        CertAlgoDSAv01,
+	KeyAlgoECDSA256:   CertAlgoECDSA256v01,
+	KeyAlgoECDSA384:   CertAlgoECDSA384v01,
+	KeyAlgoECDSA521:   CertAlgoECDSA521v01,
+	KeyAlgoSKECDSA256: CertAlgoSKECDSA256v01,
+	KeyAlgoED25519:    CertAlgoED25519v01,
+	KeyAlgoSKED25519:  CertAlgoSKED25519v01,
 }

 // certToPrivAlgo returns the underlying algorithm for a certificate algorithm.
@@ -518,6 +523,12 @@ func parseSignatureBody(in []byte) (out *Signature, rest []byte, ok bool) {
 		return
 	}

+	switch out.Format {
+	case KeyAlgoSKECDSA256, CertAlgoSKECDSA256v01, KeyAlgoSKED25519, CertAlgoSKED25519v01:
+		out.Rest = in
+		return out, nil, ok
+	}
+
 	return out, in, ok
 }

@@ -16,9 +16,8 @@ import (
 	"hash"
 	"io"
 	"io/ioutil"
-	"math/bits"

-	"golang.org/x/crypto/internal/chacha20"
+	"golang.org/x/crypto/chacha20"
 	"golang.org/x/crypto/poly1305"
 )

@@ -642,8 +641,8 @@ const chacha20Poly1305ID = "chacha20-poly1305@openssh.com"
 // the methods here also implement padding, which RFC4253 Section 6
 // also requires of stream ciphers.
 type chacha20Poly1305Cipher struct {
-	lengthKey  [8]uint32
-	contentKey [8]uint32
+	lengthKey  [32]byte
+	contentKey [32]byte
 	buf        []byte
 }

@@ -656,21 +655,21 @@ func newChaCha20Cipher(key, unusedIV, unusedMACKey []byte, unusedAlgs directionA
 		buf: make([]byte, 256),
 	}

-	for i := range c.contentKey {
-		c.contentKey[i] = binary.LittleEndian.Uint32(key[i*4 : (i+1)*4])
-	}
-	for i := range c.lengthKey {
-		c.lengthKey[i] = binary.LittleEndian.Uint32(key[(i+8)*4 : (i+9)*4])
-	}
+	copy(c.contentKey[:], key[:32])
+	copy(c.lengthKey[:], key[32:])
 	return c, nil
 }

 func (c *chacha20Poly1305Cipher) readCipherPacket(seqNum uint32, r io.Reader) ([]byte, error) {
-	nonce := [3]uint32{0, 0, bits.ReverseBytes32(seqNum)}
-	s := chacha20.New(c.contentKey, nonce)
-	var polyKey [32]byte
+	nonce := make([]byte, 12)
+	binary.BigEndian.PutUint32(nonce[8:], seqNum)
+	s, err := chacha20.NewUnauthenticatedCipher(c.contentKey[:], nonce)
+	if err != nil {
+		return nil, err
+	}
+	var polyKey, discardBuf [32]byte
 	s.XORKeyStream(polyKey[:], polyKey[:])
-	s.Advance() // skip next 32 bytes
+	s.XORKeyStream(discardBuf[:], discardBuf[:]) // skip the next 32 bytes

 	encryptedLength := c.buf[:4]
 	if _, err := io.ReadFull(r, encryptedLength); err != nil {
@@ -678,7 +677,11 @@ func (c *chacha20Poly1305Cipher) readCipherPacket(seqNum uint32, r io.Reader) ([
 	}

 	var lenBytes [4]byte
-	chacha20.New(c.lengthKey, nonce).XORKeyStream(lenBytes[:], encryptedLength)
+	ls, err := chacha20.NewUnauthenticatedCipher(c.lengthKey[:], nonce)
+	if err != nil {
+		return nil, err
+	}
+	ls.XORKeyStream(lenBytes[:], encryptedLength)

 	length := binary.BigEndian.Uint32(lenBytes[:])
 	if length > maxPacket {
@@ -724,11 +727,15 @@ func (c *chacha20Poly1305Cipher) readCipherPacket(seqNum uint32, r io.Reader) ([
 }

 func (c *chacha20Poly1305Cipher) writeCipherPacket(seqNum uint32, w io.Writer, rand io.Reader, payload []byte) error {
-	nonce := [3]uint32{0, 0, bits.ReverseBytes32(seqNum)}
-	s := chacha20.New(c.contentKey, nonce)
-	var polyKey [32]byte
+	nonce := make([]byte, 12)
+	binary.BigEndian.PutUint32(nonce[8:], seqNum)
+	s, err := chacha20.NewUnauthenticatedCipher(c.contentKey[:], nonce)
+	if err != nil {
+		return err
+	}
+	var polyKey, discardBuf [32]byte
 	s.XORKeyStream(polyKey[:], polyKey[:])
-	s.Advance() // skip next 32 bytes
+	s.XORKeyStream(discardBuf[:], discardBuf[:]) // skip the next 32 bytes

 	// There is no blocksize, so fall back to multiple of 8 byte
 	// padding, as described in RFC 4253, Sec 6.
@@ -748,7 +755,11 @@ func (c *chacha20Poly1305Cipher) writeCipherPacket(seqNum uint32, w io.Writer, r
 	}

 	binary.BigEndian.PutUint32(c.buf, uint32(1+len(payload)+padding))
-	chacha20.New(c.lengthKey, nonce).XORKeyStream(c.buf, c.buf[:4])
+	ls, err := chacha20.NewUnauthenticatedCipher(c.lengthKey[:], nonce)
+	if err != nil {
+		return err
+	}
+	ls.XORKeyStream(c.buf, c.buf[:4])
 	c.buf[4] = byte(padding)
 	copy(c.buf[5:], payload)
 	packetEnd := 5 + len(payload) + padding
@@ -523,3 +523,117 @@ func (r *retryableAuthMethod) method() string {
 func RetryableAuthMethod(auth AuthMethod, maxTries int) AuthMethod {
 	return &retryableAuthMethod{authMethod: auth, maxTries: maxTries}
 }
+
+// GSSAPIWithMICAuthMethod is an AuthMethod with "gssapi-with-mic" authentication.
+// See RFC 4462 section 3
+// gssAPIClient is implementation of the GSSAPIClient interface, see the definition of the interface for details.
+// target is the server host you want to log in to.
+func GSSAPIWithMICAuthMethod(gssAPIClient GSSAPIClient, target string) AuthMethod {
+	if gssAPIClient == nil {
+		panic("gss-api client must be not nil with enable gssapi-with-mic")
+	}
+	return &gssAPIWithMICCallback{gssAPIClient: gssAPIClient, target: target}
+}
+
+type gssAPIWithMICCallback struct {
+	gssAPIClient GSSAPIClient
+	target       string
+}
+
+func (g *gssAPIWithMICCallback) auth(session []byte, user string, c packetConn, rand io.Reader) (authResult, []string, error) {
+	m := &userAuthRequestMsg{
+		User:    user,
+		Service: serviceSSH,
+		Method:  g.method(),
+	}
+	// The GSS-API authentication method is initiated when the client sends an SSH_MSG_USERAUTH_REQUEST.
+	// See RFC 4462 section 3.2.
+	m.Payload = appendU32(m.Payload, 1)
+	m.Payload = appendString(m.Payload, string(krb5OID))
+	if err := c.writePacket(Marshal(m)); err != nil {
+		return authFailure, nil, err
+	}
+	// The server responds to the SSH_MSG_USERAUTH_REQUEST with either an
+	// SSH_MSG_USERAUTH_FAILURE if none of the mechanisms are supported or
+	// with an SSH_MSG_USERAUTH_GSSAPI_RESPONSE.
+	// See RFC 4462 section 3.3.
+	// OpenSSH supports Kerberos V5 mechanism only for GSS-API authentication,so I don't want to check
+	// selected mech if it is valid.
+	packet, err := c.readPacket()
+	if err != nil {
+		return authFailure, nil, err
+	}
+	userAuthGSSAPIResp := &userAuthGSSAPIResponse{}
+	if err := Unmarshal(packet, userAuthGSSAPIResp); err != nil {
+		return authFailure, nil, err
+	}
+	// Start the loop into the exchange token.
+	// See RFC 4462 section 3.4.
+	var token []byte
+	defer g.gssAPIClient.DeleteSecContext()
+	for {
+		// Initiates the establishment of a security context between the application and a remote peer.
+		nextToken, needContinue, err := g.gssAPIClient.InitSecContext("host@"+g.target, token, false)
+		if err != nil {
+			return authFailure, nil, err
+		}
+		if len(nextToken) > 0 {
+			if err := c.writePacket(Marshal(&userAuthGSSAPIToken{
+				Token: nextToken,
+			})); err != nil {
+				return authFailure, nil, err
+			}
+		}
+		if !needContinue {
+			break
+		}
+		packet, err = c.readPacket()
+		if err != nil {
+			return authFailure, nil, err
+		}
+		switch packet[0] {
+		case msgUserAuthFailure:
+			var msg userAuthFailureMsg
+			if err := Unmarshal(packet, &msg); err != nil {
+				return authFailure, nil, err
+			}
+			if msg.PartialSuccess {
+				return authPartialSuccess, msg.Methods, nil
+			}
+			return authFailure, msg.Methods, nil
+		case msgUserAuthGSSAPIError:
+			userAuthGSSAPIErrorResp := &userAuthGSSAPIError{}
+			if err := Unmarshal(packet, userAuthGSSAPIErrorResp); err != nil {
+				return authFailure, nil, err
+			}
+			return authFailure, nil, fmt.Errorf("GSS-API Error:\n"+
+				"Major Status: %d\n"+
+				"Minor Status: %d\n"+
+				"Error Message: %s\n", userAuthGSSAPIErrorResp.MajorStatus, userAuthGSSAPIErrorResp.MinorStatus,
+				userAuthGSSAPIErrorResp.Message)
+		case msgUserAuthGSSAPIToken:
+			userAuthGSSAPITokenReq := &userAuthGSSAPIToken{}
+			if err := Unmarshal(packet, userAuthGSSAPITokenReq); err != nil {
+				return authFailure, nil, err
+			}
+			token = userAuthGSSAPITokenReq.Token
+		}
+	}
+	// Binding Encryption Keys.
+	// See RFC 4462 section 3.5.
+	micField := buildMIC(string(session), user, "ssh-connection", "gssapi-with-mic")
+	micToken, err := g.gssAPIClient.GetMIC(micField)
+	if err != nil {
+		return authFailure, nil, err
+	}
+	if err := c.writePacket(Marshal(&userAuthGSSAPIMIC{
+		MIC: micToken,
+	})); err != nil {
+		return authFailure, nil, err
+	}
+	return handleAuthResponse(c)
+}
+
+func (g *gssAPIWithMICCallback) method() string {
+	return "gssapi-with-mic"
+}
@@ -51,6 +51,21 @@ var supportedKexAlgos = []string{
 	kexAlgoDH14SHA1, kexAlgoDH1SHA1,
 }

+// serverForbiddenKexAlgos contains key exchange algorithms, that are forbidden
+// for the server half.
+var serverForbiddenKexAlgos = map[string]struct{}{
+	kexAlgoDHGEXSHA1:   {}, // server half implementation is only minimal to satisfy the automated tests
+	kexAlgoDHGEXSHA256: {}, // server half implementation is only minimal to satisfy the automated tests
+}
+
+// preferredKexAlgos specifies the default preference for key-exchange algorithms
+// in preference order.
+var preferredKexAlgos = []string{
+	kexAlgoCurve25519SHA256,
+	kexAlgoECDH256, kexAlgoECDH384, kexAlgoECDH521,
+	kexAlgoDH14SHA1,
+}
+
 // supportedHostKeyAlgos specifies the supported host-key algorithms (i.e. methods
 // of authenticating servers) in preference order.
 var supportedHostKeyAlgos = []string{
@@ -109,6 +124,7 @@ func findCommon(what string, client []string, server []string) (common string, e
 	return "", fmt.Errorf("ssh: no common algorithm for %s; client offered: %v, server offered: %v", what, client, server)
 }

+// directionAlgorithms records algorithm choices in one direction (either read or write)
 type directionAlgorithms struct {
 	Cipher      string
 	MAC         string
@@ -137,7 +153,7 @@ type algorithms struct {
 	r       directionAlgorithms
 }

-func findAgreedAlgorithms(clientKexInit, serverKexInit *kexInitMsg) (algs *algorithms, err error) {
+func findAgreedAlgorithms(isClient bool, clientKexInit, serverKexInit *kexInitMsg) (algs *algorithms, err error) {
 	result := &algorithms{}

 	result.kex, err = findCommon("key exchange", clientKexInit.KexAlgos, serverKexInit.KexAlgos)
@@ -150,32 +166,37 @@ func findAgreedAlgorithms(clientKexInit, serverKexInit *kexInitMsg) (algs *algor
 		return
 	}

-	result.w.Cipher, err = findCommon("client to server cipher", clientKexInit.CiphersClientServer, serverKexInit.CiphersClientServer)
+	stoc, ctos := &result.w, &result.r
+	if isClient {
+		ctos, stoc = stoc, ctos
+	}
+
+	ctos.Cipher, err = findCommon("client to server cipher", clientKexInit.CiphersClientServer, serverKexInit.CiphersClientServer)
 	if err != nil {
 		return
 	}

-	result.r.Cipher, err = findCommon("server to client cipher", clientKexInit.CiphersServerClient, serverKexInit.CiphersServerClient)
+	stoc.Cipher, err = findCommon("server to client cipher", clientKexInit.CiphersServerClient, serverKexInit.CiphersServerClient)
 	if err != nil {
 		return
 	}

-	result.w.MAC, err = findCommon("client to server MAC", clientKexInit.MACsClientServer, serverKexInit.MACsClientServer)
+	ctos.MAC, err = findCommon("client to server MAC", clientKexInit.MACsClientServer, serverKexInit.MACsClientServer)
 	if err != nil {
 		return
 	}

-	result.r.MAC, err = findCommon("server to client MAC", clientKexInit.MACsServerClient, serverKexInit.MACsServerClient)
+	stoc.MAC, err = findCommon("server to client MAC", clientKexInit.MACsServerClient, serverKexInit.MACsServerClient)
 	if err != nil {
 		return
 	}

-	result.w.Compression, err = findCommon("client to server compression", clientKexInit.CompressionClientServer, serverKexInit.CompressionClientServer)
+	ctos.Compression, err = findCommon("client to server compression", clientKexInit.CompressionClientServer, serverKexInit.CompressionClientServer)
 	if err != nil {
 		return
 	}

-	result.r.Compression, err = findCommon("server to client compression", clientKexInit.CompressionServerClient, serverKexInit.CompressionServerClient)
+	stoc.Compression, err = findCommon("server to client compression", clientKexInit.CompressionServerClient, serverKexInit.CompressionServerClient)
 	if err != nil {
 		return
 	}
@@ -233,7 +254,7 @@ func (c *Config) SetDefaults() {
 	c.Ciphers = ciphers

 	if c.KeyExchanges == nil {
-		c.KeyExchanges = supportedKexAlgos
+		c.KeyExchanges = preferredKexAlgos
 	}

 	if c.MACs == nil {
@@ -543,7 +543,8 @@ func (t *handshakeTransport) enterKeyExchange(otherInitPacket []byte) error {

 	clientInit := otherInit
 	serverInit := t.sentInitMsg
-	if len(t.hostKeys) == 0 {
+	isClient := len(t.hostKeys) == 0
+	if isClient {
 		clientInit, serverInit = serverInit, clientInit

 		magics.clientKexInit = t.sentInitPacket
@@ -551,7 +552,7 @@ func (t *handshakeTransport) enterKeyExchange(otherInitPacket []byte) error {
 	}

 	var err error
-	t.algorithms, err = findAgreedAlgorithms(clientInit, serverInit)
+	t.algorithms, err = findAgreedAlgorithms(isClient, clientInit, serverInit)
 	if err != nil {
 		return err
 	}
@@ -10,7 +10,9 @@ import (
 	"crypto/elliptic"
 	"crypto/rand"
 	"crypto/subtle"
+	"encoding/binary"
 	"errors"
+	"fmt"
 	"io"
 	"math/big"

@@ -24,6 +26,12 @@ const (
 	kexAlgoECDH384          = "ecdh-sha2-nistp384"
 	kexAlgoECDH521          = "ecdh-sha2-nistp521"
 	kexAlgoCurve25519SHA256 = "curve25519-sha256@libssh.org"
+
+	// For the following kex only the client half contains a production
+	// ready implementation. The server half only consists of a minimal
+	// implementation to satisfy the automated tests.
+	kexAlgoDHGEXSHA1   = "diffie-hellman-group-exchange-sha1"
+	kexAlgoDHGEXSHA256 = "diffie-hellman-group-exchange-sha256"
 )

 // kexResult captures the outcome of a key exchange.
@@ -204,7 +212,7 @@ func (group *dhGroup) Server(c packetConn, randSource io.Reader, magics *handsha
 		HostKey:   hostKeyBytes,
 		Signature: sig,
 		Hash:      crypto.SHA1,
-	}, nil
+	}, err
 }

 // ecdh performs Elliptic Curve Diffie-Hellman key exchange as
@@ -402,6 +410,8 @@ func init() {
 	kexAlgoMap[kexAlgoECDH384] = &ecdh{elliptic.P384()}
 	kexAlgoMap[kexAlgoECDH256] = &ecdh{elliptic.P256()}
 	kexAlgoMap[kexAlgoCurve25519SHA256] = &curve25519sha256{}
+	kexAlgoMap[kexAlgoDHGEXSHA1] = &dhGEXSHA{hashFunc: crypto.SHA1}
+	kexAlgoMap[kexAlgoDHGEXSHA256] = &dhGEXSHA{hashFunc: crypto.SHA256}
 }

 // curve25519sha256 implements the curve25519-sha256@libssh.org key
@@ -538,3 +548,242 @@ func (kex *curve25519sha256) Server(c packetConn, rand io.Reader, magics *handsh
 		Hash:      crypto.SHA256,
 	}, nil
 }
+
+// dhGEXSHA implements the diffie-hellman-group-exchange-sha1 and
+// diffie-hellman-group-exchange-sha256 key agreement protocols,
+// as described in RFC 4419
+type dhGEXSHA struct {
+	g, p     *big.Int
+	hashFunc crypto.Hash
+}
+
+const numMRTests = 64
+
+const (
+	dhGroupExchangeMinimumBits   = 2048
+	dhGroupExchangePreferredBits = 2048
+	dhGroupExchangeMaximumBits   = 8192
+)
+
+func (gex *dhGEXSHA) diffieHellman(theirPublic, myPrivate *big.Int) (*big.Int, error) {
+	if theirPublic.Sign() <= 0 || theirPublic.Cmp(gex.p) >= 0 {
+		return nil, fmt.Errorf("ssh: DH parameter out of bounds")
+	}
+	return new(big.Int).Exp(theirPublic, myPrivate, gex.p), nil
+}
+
+func (gex *dhGEXSHA) Client(c packetConn, randSource io.Reader, magics *handshakeMagics) (*kexResult, error) {
+	// Send GexRequest
+	kexDHGexRequest := kexDHGexRequestMsg{
+		MinBits:      dhGroupExchangeMinimumBits,
+		PreferedBits: dhGroupExchangePreferredBits,
+		MaxBits:      dhGroupExchangeMaximumBits,
+	}
+	if err := c.writePacket(Marshal(&kexDHGexRequest)); err != nil {
+		return nil, err
+	}
+
+	// Receive GexGroup
+	packet, err := c.readPacket()
+	if err != nil {
+		return nil, err
+	}
+
+	var kexDHGexGroup kexDHGexGroupMsg
+	if err = Unmarshal(packet, &kexDHGexGroup); err != nil {
+		return nil, err
+	}
+
+	// reject if p's bit length < dhGroupExchangeMinimumBits or > dhGroupExchangeMaximumBits
+	if kexDHGexGroup.P.BitLen() < dhGroupExchangeMinimumBits || kexDHGexGroup.P.BitLen() > dhGroupExchangeMaximumBits {
+		return nil, fmt.Errorf("ssh: server-generated gex p is out of range (%d bits)", kexDHGexGroup.P.BitLen())
+	}
+
+	gex.p = kexDHGexGroup.P
+	gex.g = kexDHGexGroup.G
+
+	// Check if p is safe by verifing that p and (p-1)/2 are primes
+	one := big.NewInt(1)
+	var pHalf = &big.Int{}
+	pHalf.Rsh(gex.p, 1)
+	if !gex.p.ProbablyPrime(numMRTests) || !pHalf.ProbablyPrime(numMRTests) {
+		return nil, fmt.Errorf("ssh: server provided gex p is not safe")
+	}
+
+	// Check if g is safe by verifing that g > 1 and g < p - 1
+	var pMinusOne = &big.Int{}
+	pMinusOne.Sub(gex.p, one)
+	if gex.g.Cmp(one) != 1 && gex.g.Cmp(pMinusOne) != -1 {
+		return nil, fmt.Errorf("ssh: server provided gex g is not safe")
+	}
+
+	// Send GexInit
+	x, err := rand.Int(randSource, pHalf)
+	if err != nil {
+		return nil, err
+	}
+	X := new(big.Int).Exp(gex.g, x, gex.p)
+	kexDHGexInit := kexDHGexInitMsg{
+		X: X,
+	}
+	if err := c.writePacket(Marshal(&kexDHGexInit)); err != nil {
+		return nil, err
+	}
+
+	// Receive GexReply
+	packet, err = c.readPacket()
+	if err != nil {
+		return nil, err
+	}
+
+	var kexDHGexReply kexDHGexReplyMsg
+	if err = Unmarshal(packet, &kexDHGexReply); err != nil {
+		return nil, err
+	}
+
+	kInt, err := gex.diffieHellman(kexDHGexReply.Y, x)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check if k is safe by verifing that k > 1 and k < p - 1
+	if kInt.Cmp(one) != 1 && kInt.Cmp(pMinusOne) != -1 {
+		return nil, fmt.Errorf("ssh: derived k is not safe")
+	}
+
+	h := gex.hashFunc.New()
+	magics.write(h)
+	writeString(h, kexDHGexReply.HostKey)
+	binary.Write(h, binary.BigEndian, uint32(dhGroupExchangeMinimumBits))
+	binary.Write(h, binary.BigEndian, uint32(dhGroupExchangePreferredBits))
+	binary.Write(h, binary.BigEndian, uint32(dhGroupExchangeMaximumBits))
+	writeInt(h, gex.p)
+	writeInt(h, gex.g)
+	writeInt(h, X)
+	writeInt(h, kexDHGexReply.Y)
+	K := make([]byte, intLength(kInt))
+	marshalInt(K, kInt)
+	h.Write(K)
+
+	return &kexResult{
+		H:         h.Sum(nil),
+		K:         K,
+		HostKey:   kexDHGexReply.HostKey,
+		Signature: kexDHGexReply.Signature,
+		Hash:      gex.hashFunc,
+	}, nil
+}
+
+// Server half implementation of the Diffie Hellman Key Exchange with SHA1 and SHA256.
+//
+// This is a minimal implementation to satisfy the automated tests.
+func (gex *dhGEXSHA) Server(c packetConn, randSource io.Reader, magics *handshakeMagics, priv Signer) (result *kexResult, err error) {
+	// Receive GexRequest
+	packet, err := c.readPacket()
+	if err != nil {
+		return
+	}
+	var kexDHGexRequest kexDHGexRequestMsg
+	if err = Unmarshal(packet, &kexDHGexRequest); err != nil {
+		return
+	}
+
+	// smoosh the user's preferred size into our own limits
+	if kexDHGexRequest.PreferedBits > dhGroupExchangeMaximumBits {
+		kexDHGexRequest.PreferedBits = dhGroupExchangeMaximumBits
+	}
+	if kexDHGexRequest.PreferedBits < dhGroupExchangeMinimumBits {
+		kexDHGexRequest.PreferedBits = dhGroupExchangeMinimumBits
+	}
+	// fix min/max if they're inconsistent.  technically, we could just pout
+	// and hang up, but there's no harm in giving them the benefit of the
+	// doubt and just picking a bitsize for them.
+	if kexDHGexRequest.MinBits > kexDHGexRequest.PreferedBits {
+		kexDHGexRequest.MinBits = kexDHGexRequest.PreferedBits
+	}
+	if kexDHGexRequest.MaxBits < kexDHGexRequest.PreferedBits {
+		kexDHGexRequest.MaxBits = kexDHGexRequest.PreferedBits
+	}
+
+	// Send GexGroup
+	// This is the group called diffie-hellman-group14-sha1 in RFC
+	// 4253 and Oakley Group 14 in RFC 3526.
+	p, _ := new(big.Int).SetString("FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3DC2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F83655D23DCA3AD961C62F356208552BB9ED529077096966D670C354E4ABC9804F1746C08CA18217C32905E462E36CE3BE39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9DE2BCBF6955817183995497CEA956AE515D2261898FA051015728E5A8AACAA68FFFFFFFFFFFFFFFF", 16)
+	gex.p = p
+	gex.g = big.NewInt(2)
+
+	kexDHGexGroup := kexDHGexGroupMsg{
+		P: gex.p,
+		G: gex.g,
+	}
+	if err := c.writePacket(Marshal(&kexDHGexGroup)); err != nil {
+		return nil, err
+	}
+
+	// Receive GexInit
+	packet, err = c.readPacket()
+	if err != nil {
+		return
+	}
+	var kexDHGexInit kexDHGexInitMsg
+	if err = Unmarshal(packet, &kexDHGexInit); err != nil {
+		return
+	}
+
+	var pHalf = &big.Int{}
+	pHalf.Rsh(gex.p, 1)
+
+	y, err := rand.Int(randSource, pHalf)
+	if err != nil {
+		return
+	}
+
+	Y := new(big.Int).Exp(gex.g, y, gex.p)
+	kInt, err := gex.diffieHellman(kexDHGexInit.X, y)
+	if err != nil {
+		return nil, err
+	}
+
+	hostKeyBytes := priv.PublicKey().Marshal()
+
+	h := gex.hashFunc.New()
+	magics.write(h)
+	writeString(h, hostKeyBytes)
+	binary.Write(h, binary.BigEndian, uint32(dhGroupExchangeMinimumBits))
+	binary.Write(h, binary.BigEndian, uint32(dhGroupExchangePreferredBits))
+	binary.Write(h, binary.BigEndian, uint32(dhGroupExchangeMaximumBits))
+	writeInt(h, gex.p)
+	writeInt(h, gex.g)
+	writeInt(h, kexDHGexInit.X)
+	writeInt(h, Y)
+
+	K := make([]byte, intLength(kInt))
+	marshalInt(K, kInt)
+	h.Write(K)
+
+	H := h.Sum(nil)
+
+	// H is already a hash, but the hostkey signing will apply its
+	// own key-specific hash algorithm.
+	sig, err := signAndMarshal(priv, randSource, H)
+	if err != nil {
+		return nil, err
+	}
+
+	kexDHGexReply := kexDHGexReplyMsg{
+		HostKey:   hostKeyBytes,
+		Y:         Y,
+		Signature: sig,
+	}
+	packet = Marshal(&kexDHGexReply)
+
+	err = c.writePacket(packet)
+
+	return &kexResult{
+		H:         H,
+		K:         K,
+		HostKey:   hostKeyBytes,
+		Signature: sig,
+		Hash:      gex.hashFunc,
+	}, err
+}
@@ -30,12 +30,14 @@ import (
 // These constants represent the algorithm names for key types supported by this
 // package.
 const (
-	KeyAlgoRSA      = "ssh-rsa"
-	KeyAlgoDSA      = "ssh-dss"
-	KeyAlgoECDSA256 = "ecdsa-sha2-nistp256"
-	KeyAlgoECDSA384 = "ecdsa-sha2-nistp384"
-	KeyAlgoECDSA521 = "ecdsa-sha2-nistp521"
-	KeyAlgoED25519  = "ssh-ed25519"
+	KeyAlgoRSA        = "ssh-rsa"
+	KeyAlgoDSA        = "ssh-dss"
+	KeyAlgoECDSA256   = "ecdsa-sha2-nistp256"
+	KeyAlgoSKECDSA256 = "sk-ecdsa-sha2-nistp256@openssh.com"
+	KeyAlgoECDSA384   = "ecdsa-sha2-nistp384"
+	KeyAlgoECDSA521   = "ecdsa-sha2-nistp521"
+	KeyAlgoED25519    = "ssh-ed25519"
+	KeyAlgoSKED25519  = "sk-ssh-ed25519@openssh.com"
 )

 // These constants represent non-default signature algorithms that are supported
@@ -58,9 +60,13 @@ func parsePubKey(in []byte, algo string) (pubKey PublicKey, rest []byte, err err
 		return parseDSA(in)
 	case KeyAlgoECDSA256, KeyAlgoECDSA384, KeyAlgoECDSA521:
 		return parseECDSA(in)
+	case KeyAlgoSKECDSA256:
+		return parseSKECDSA(in)
 	case KeyAlgoED25519:
 		return parseED25519(in)
-	case CertAlgoRSAv01, CertAlgoDSAv01, CertAlgoECDSA256v01, CertAlgoECDSA384v01, CertAlgoECDSA521v01, CertAlgoED25519v01:
+	case KeyAlgoSKED25519:
+		return parseSKEd25519(in)
+	case CertAlgoRSAv01, CertAlgoDSAv01, CertAlgoECDSA256v01, CertAlgoECDSA384v01, CertAlgoECDSA521v01, CertAlgoSKECDSA256v01, CertAlgoED25519v01, CertAlgoSKED25519v01:
 		cert, err := parseCert(in, certToPrivAlgo(algo))
 		if err != nil {
 			return nil, nil, err
@@ -685,6 +691,218 @@ func (k *ecdsaPublicKey) CryptoPublicKey() crypto.PublicKey {
 	return (*ecdsa.PublicKey)(k)
 }

+// skFields holds the additional fields present in U2F/FIDO2 signatures.
+// See openssh/PROTOCOL.u2f 'SSH U2F Signatures' for details.
+type skFields struct {
+	// Flags contains U2F/FIDO2 flags such as 'user present'
+	Flags byte
+	// Counter is a monotonic signature counter which can be
+	// used to detect concurrent use of a private key, should
+	// it be extracted from hardware.
+	Counter uint32
+}
+
+type skECDSAPublicKey struct {
+	// application is a URL-like string, typically "ssh:" for SSH.
+	// see openssh/PROTOCOL.u2f for details.
+	application string
+	ecdsa.PublicKey
+}
+
+func (k *skECDSAPublicKey) Type() string {
+	return KeyAlgoSKECDSA256
+}
+
+func (k *skECDSAPublicKey) nistID() string {
+	return "nistp256"
+}
+
+func parseSKECDSA(in []byte) (out PublicKey, rest []byte, err error) {
+	var w struct {
+		Curve       string
+		KeyBytes    []byte
+		Application string
+		Rest        []byte `ssh:"rest"`
+	}
+
+	if err := Unmarshal(in, &w); err != nil {
+		return nil, nil, err
+	}
+
+	key := new(skECDSAPublicKey)
+	key.application = w.Application
+
+	if w.Curve != "nistp256" {
+		return nil, nil, errors.New("ssh: unsupported curve")
+	}
+	key.Curve = elliptic.P256()
+
+	key.X, key.Y = elliptic.Unmarshal(key.Curve, w.KeyBytes)
+	if key.X == nil || key.Y == nil {
+		return nil, nil, errors.New("ssh: invalid curve point")
+	}
+
+	return key, w.Rest, nil
+}
+
+func (k *skECDSAPublicKey) Marshal() []byte {
+	// See RFC 5656, section 3.1.
+	keyBytes := elliptic.Marshal(k.Curve, k.X, k.Y)
+	w := struct {
+		Name        string
+		ID          string
+		Key         []byte
+		Application string
+	}{
+		k.Type(),
+		k.nistID(),
+		keyBytes,
+		k.application,
+	}
+
+	return Marshal(&w)
+}
+
+func (k *skECDSAPublicKey) Verify(data []byte, sig *Signature) error {
+	if sig.Format != k.Type() {
+		return fmt.Errorf("ssh: signature type %s for key type %s", sig.Format, k.Type())
+	}
+
+	h := ecHash(k.Curve).New()
+	h.Write([]byte(k.application))
+	appDigest := h.Sum(nil)
+
+	h.Reset()
+	h.Write(data)
+	dataDigest := h.Sum(nil)
+
+	var ecSig struct {
+		R *big.Int
+		S *big.Int
+	}
+	if err := Unmarshal(sig.Blob, &ecSig); err != nil {
+		return err
+	}
+
+	var skf skFields
+	if err := Unmarshal(sig.Rest, &skf); err != nil {
+		return err
+	}
+
+	blob := struct {
+		ApplicationDigest []byte `ssh:"rest"`
+		Flags             byte
+		Counter           uint32
+		MessageDigest     []byte `ssh:"rest"`
+	}{
+		appDigest,
+		skf.Flags,
+		skf.Counter,
+		dataDigest,
+	}
+
+	original := Marshal(blob)
+
+	h.Reset()
+	h.Write(original)
+	digest := h.Sum(nil)
+
+	if ecdsa.Verify((*ecdsa.PublicKey)(&k.PublicKey), digest, ecSig.R, ecSig.S) {
+		return nil
+	}
+	return errors.New("ssh: signature did not verify")
+}
+
+type skEd25519PublicKey struct {
+	// application is a URL-like string, typically "ssh:" for SSH.
+	// see openssh/PROTOCOL.u2f for details.
+	application string
+	ed25519.PublicKey
+}
+
+func (k *skEd25519PublicKey) Type() string {
+	return KeyAlgoSKED25519
+}
+
+func parseSKEd25519(in []byte) (out PublicKey, rest []byte, err error) {
+	var w struct {
+		KeyBytes    []byte
+		Application string
+		Rest        []byte `ssh:"rest"`
+	}
+
+	if err := Unmarshal(in, &w); err != nil {
+		return nil, nil, err
+	}
+
+	key := new(skEd25519PublicKey)
+	key.application = w.Application
+	key.PublicKey = ed25519.PublicKey(w.KeyBytes)
+
+	return key, w.Rest, nil
+}
+
+func (k *skEd25519PublicKey) Marshal() []byte {
+	w := struct {
+		Name        string
+		KeyBytes    []byte
+		Application string
+	}{
+		KeyAlgoSKED25519,
+		[]byte(k.PublicKey),
+		k.application,
+	}
+	return Marshal(&w)
+}
+
+func (k *skEd25519PublicKey) Verify(data []byte, sig *Signature) error {
+	if sig.Format != k.Type() {
+		return fmt.Errorf("ssh: signature type %s for key type %s", sig.Format, k.Type())
+	}
+
+	h := sha256.New()
+	h.Write([]byte(k.application))
+	appDigest := h.Sum(nil)
+
+	h.Reset()
+	h.Write(data)
+	dataDigest := h.Sum(nil)
+
+	var edSig struct {
+		Signature []byte `ssh:"rest"`
+	}
+
+	if err := Unmarshal(sig.Blob, &edSig); err != nil {
+		return err
+	}
+
+	var skf skFields
+	if err := Unmarshal(sig.Rest, &skf); err != nil {
+		return err
+	}
+
+	blob := struct {
+		ApplicationDigest []byte `ssh:"rest"`
+		Flags             byte
+		Counter           uint32
+		MessageDigest     []byte `ssh:"rest"`
+	}{
+		appDigest,
+		skf.Flags,
+		skf.Counter,
+		dataDigest,
+	}
+
+	original := Marshal(blob)
+
+	edKey := (ed25519.PublicKey)(k.PublicKey)
+	if ok := ed25519.Verify(edKey, original, edSig.Signature); !ok {
+		return errors.New("ssh: signature did not verify")
+	}
+
+	return nil
+}
+
 // NewSignerFromKey takes an *rsa.PrivateKey, *dsa.PrivateKey,
 // *ecdsa.PrivateKey or any other crypto.Signer and returns a
 // corresponding Signer instance. ECDSA keys must use P-256, P-384 or
@@ -837,7 +1055,8 @@ func NewPublicKey(key interface{}) (PublicKey, error) {
 }

 // ParsePrivateKey returns a Signer from a PEM encoded private key. It supports
-// the same keys as ParseRawPrivateKey.
+// the same keys as ParseRawPrivateKey. If the private key is encrypted, it
+// will return a PassphraseMissingError.
 func ParsePrivateKey(pemBytes []byte) (Signer, error) {
 	key, err := ParseRawPrivateKey(pemBytes)
 	if err != nil {
@@ -850,8 +1069,8 @@ func ParsePrivateKey(pemBytes []byte) (Signer, error) {
 // ParsePrivateKeyWithPassphrase returns a Signer from a PEM encoded private
 // key and passphrase. It supports the same keys as
 // ParseRawPrivateKeyWithPassphrase.
-func ParsePrivateKeyWithPassphrase(pemBytes, passPhrase []byte) (Signer, error) {
-	key, err := ParseRawPrivateKeyWithPassphrase(pemBytes, passPhrase)
+func ParsePrivateKeyWithPassphrase(pemBytes, passphrase []byte) (Signer, error) {
+	key, err := ParseRawPrivateKeyWithPassphrase(pemBytes, passphrase)
 	if err != nil {
 		return nil, err
 	}
@@ -867,8 +1086,21 @@ func encryptedBlock(block *pem.Block) bool {
 	return strings.Contains(block.Headers["Proc-Type"], "ENCRYPTED")
 }

+// A PassphraseMissingError indicates that parsing this private key requires a
+// passphrase. Use ParsePrivateKeyWithPassphrase.
+type PassphraseMissingError struct {
+	// PublicKey will be set if the private key format includes an unencrypted
+	// public key along with the encrypted private key.
+	PublicKey PublicKey
+}
+
+func (*PassphraseMissingError) Error() string {
+	return "ssh: this private key is passphrase protected"
+}
+
 // ParseRawPrivateKey returns a private key from a PEM encoded private key. It
-// supports RSA (PKCS#1), PKCS#8, DSA (OpenSSL), and ECDSA private keys.
+// supports RSA (PKCS#1), PKCS#8, DSA (OpenSSL), and ECDSA private keys. If the
+// private key is encrypted, it will return a PassphraseMissingError.
 func ParseRawPrivateKey(pemBytes []byte) (interface{}, error) {
 	block, _ := pem.Decode(pemBytes)
 	if block == nil {
@@ -876,7 +1108,7 @@ func ParseRawPrivateKey(pemBytes []byte) (interface{}, error) {
 	}

 	if encryptedBlock(block) {
-		return nil, errors.New("ssh: cannot decode encrypted private keys")
+		return nil, &PassphraseMissingError{}
 	}

 	switch block.Type {
@@ -899,24 +1131,22 @@ func ParseRawPrivateKey(pemBytes []byte) (interface{}, error) {
 // ParseRawPrivateKeyWithPassphrase returns a private key decrypted with
 // passphrase from a PEM encoded private key. If wrong passphrase, return
 // x509.IncorrectPasswordError.
-func ParseRawPrivateKeyWithPassphrase(pemBytes, passPhrase []byte) (interface{}, error) {
+func ParseRawPrivateKeyWithPassphrase(pemBytes, passphrase []byte) (interface{}, error) {
 	block, _ := pem.Decode(pemBytes)
 	if block == nil {
 		return nil, errors.New("ssh: no key found")
 	}
-	buf := block.Bytes

-	if encryptedBlock(block) {
-		if x509.IsEncryptedPEMBlock(block) {
-			var err error
-			buf, err = x509.DecryptPEMBlock(block, passPhrase)
-			if err != nil {
-				if err == x509.IncorrectPasswordError {
-					return nil, err
-				}
-				return nil, fmt.Errorf("ssh: cannot decode encrypted private keys: %v", err)
-			}
+	if !encryptedBlock(block) || !x509.IsEncryptedPEMBlock(block) {
+		return nil, errors.New("ssh: not an encrypted key")
+	}
+
+	buf, err := x509.DecryptPEMBlock(block, passphrase)
+	if err != nil {
+		if err == x509.IncorrectPasswordError {
+			return nil, err
 		}
+		return nil, fmt.Errorf("ssh: cannot decode encrypted private keys: %v", err)
 	}

 	switch block.Type {
@@ -926,8 +1156,6 @@ func ParseRawPrivateKeyWithPassphrase(pemBytes, passPhrase []byte) (interface{},
 		return x509.ParseECPrivateKey(buf)
 	case "DSA PRIVATE KEY":
 		return ParseDSAPrivateKey(buf)
-	case "OPENSSH PRIVATE KEY":
-		return parseOpenSSHPrivateKey(buf)
 	default:
 		return nil, fmt.Errorf("ssh: unsupported key type %q", block.Type)
 	}
@@ -97,6 +97,36 @@ type kexDHReplyMsg struct {
 	Signature []byte
 }

+// See RFC 4419, section 5.
+const msgKexDHGexGroup = 31
+
+type kexDHGexGroupMsg struct {
+	P *big.Int `sshtype:"31"`
+	G *big.Int
+}
+
+const msgKexDHGexInit = 32
+
+type kexDHGexInitMsg struct {
+	X *big.Int `sshtype:"32"`
+}
+
+const msgKexDHGexReply = 33
+
+type kexDHGexReplyMsg struct {
+	HostKey   []byte `sshtype:"33"`
+	Y         *big.Int
+	Signature []byte
+}
+
+const msgKexDHGexRequest = 34
+
+type kexDHGexRequestMsg struct {
+	MinBits      uint32 `sshtype:"34"`
+	PreferedBits uint32
+	MaxBits      uint32
+}
+
 // See RFC 4253, section 10.
 const msgServiceRequest = 5

@@ -275,6 +305,42 @@ type userAuthPubKeyOkMsg struct {
 	PubKey []byte
 }

+// See RFC 4462, section 3
+const msgUserAuthGSSAPIResponse = 60
+
+type userAuthGSSAPIResponse struct {
+	SupportMech []byte `sshtype:"60"`
+}
+
+const msgUserAuthGSSAPIToken = 61
+
+type userAuthGSSAPIToken struct {
+	Token []byte `sshtype:"61"`
+}
+
+const msgUserAuthGSSAPIMIC = 66
+
+type userAuthGSSAPIMIC struct {
+	MIC []byte `sshtype:"66"`
+}
+
+// See RFC 4462, section 3.9
+const msgUserAuthGSSAPIErrTok = 64
+
+type userAuthGSSAPIErrTok struct {
+	ErrorToken []byte `sshtype:"64"`
+}
+
+// See RFC 4462, section 3.8
+const msgUserAuthGSSAPIError = 65
+
+type userAuthGSSAPIError struct {
+	MajorStatus uint32 `sshtype:"65"`
+	MinorStatus uint32
+	Message     string
+	LanguageTag string
+}
+
 // typeTags returns the possible type bytes for the given reflect.Type, which
 // should be a struct. The possible values are separated by a '|' character.
 func typeTags(structType reflect.Type) (tags []byte) {
@@ -756,6 +822,14 @@ func decode(packet []byte) (interface{}, error) {
 		msg = new(channelRequestSuccessMsg)
 	case msgChannelFailure:
 		msg = new(channelRequestFailureMsg)
+	case msgUserAuthGSSAPIToken:
+		msg = new(userAuthGSSAPIToken)
+	case msgUserAuthGSSAPIMIC:
+		msg = new(userAuthGSSAPIMIC)
+	case msgUserAuthGSSAPIErrTok:
+		msg = new(userAuthGSSAPIErrTok)
+	case msgUserAuthGSSAPIError:
+		msg = new(userAuthGSSAPIError)
 	default:
 		return nil, unexpectedMessageError(0, packet[0])
 	}
@@ -45,6 +45,20 @@ type Permissions struct {
 	Extensions map[string]string
 }

+type GSSAPIWithMICConfig struct {
+	// AllowLogin, must be set, is called when gssapi-with-mic
+	// authentication is selected (RFC 4462 section 3). The srcName is from the
+	// results of the GSS-API authentication. The format is username@DOMAIN.
+	// GSSAPI just guarantees to the server who the user is, but not if they can log in, and with what permissions.
+	// This callback is called after the user identity is established with GSSAPI to decide if the user can login with
+	// which permissions. If the user is allowed to login, it should return a nil error.
+	AllowLogin func(conn ConnMetadata, srcName string) (*Permissions, error)
+
+	// Server must be set. It's the implementation
+	// of the GSSAPIServer interface. See GSSAPIServer interface for details.
+	Server GSSAPIServer
+}
+
 // ServerConfig holds server specific configuration data.
 type ServerConfig struct {
 	// Config contains configuration shared between client and server.
@@ -99,6 +113,10 @@ type ServerConfig struct {
 	// BannerCallback, if present, is called and the return string is sent to
 	// the client after key exchange completed but before authentication.
 	BannerCallback func(conn ConnMetadata) string
+
+	// GSSAPIWithMICConfig includes gssapi server and callback, which if both non-nil, is used
+	// when gssapi-with-mic authentication is selected (RFC 4462 section 3).
+	GSSAPIWithMICConfig *GSSAPIWithMICConfig
 }

 // AddHostKey adds a private key as a host key. If an existing host
@@ -175,6 +193,12 @@ func NewServerConn(c net.Conn, config *ServerConfig) (*ServerConn, <-chan NewCha
 	if fullConf.MaxAuthTries == 0 {
 		fullConf.MaxAuthTries = 6
 	}
+	// Check if the config contains any unsupported key exchanges
+	for _, kex := range fullConf.KeyExchanges {
+		if _, ok := serverForbiddenKexAlgos[kex]; ok {
+			return nil, nil, nil, fmt.Errorf("ssh: unsupported key exchange %s for server", kex)
+		}
+	}

 	s := &connection{
 		sshConn: sshConn{conn: c},
@@ -204,7 +228,9 @@ func (s *connection) serverHandshake(config *ServerConfig) (*Permissions, error)
 		return nil, errors.New("ssh: server has no host keys")
 	}

-	if !config.NoClientAuth && config.PasswordCallback == nil && config.PublicKeyCallback == nil && config.KeyboardInteractiveCallback == nil {
+	if !config.NoClientAuth && config.PasswordCallback == nil && config.PublicKeyCallback == nil &&
+		config.KeyboardInteractiveCallback == nil && (config.GSSAPIWithMICConfig == nil ||
+		config.GSSAPIWithMICConfig.AllowLogin == nil || config.GSSAPIWithMICConfig.Server == nil) {
 		return nil, errors.New("ssh: no authentication methods configured but NoClientAuth is also false")
 	}

@@ -258,8 +284,8 @@ func (s *connection) serverHandshake(config *ServerConfig) (*Permissions, error)

 func isAcceptableAlgo(algo string) bool {
 	switch algo {
-	case KeyAlgoRSA, KeyAlgoDSA, KeyAlgoECDSA256, KeyAlgoECDSA384, KeyAlgoECDSA521, KeyAlgoED25519,
-		CertAlgoRSAv01, CertAlgoDSAv01, CertAlgoECDSA256v01, CertAlgoECDSA384v01, CertAlgoECDSA521v01, CertAlgoED25519v01:
+	case KeyAlgoRSA, KeyAlgoDSA, KeyAlgoECDSA256, KeyAlgoECDSA384, KeyAlgoECDSA521, KeyAlgoSKECDSA256, KeyAlgoED25519, KeyAlgoSKED25519,
+		CertAlgoRSAv01, CertAlgoDSAv01, CertAlgoECDSA256v01, CertAlgoECDSA384v01, CertAlgoECDSA521v01, CertAlgoSKECDSA256v01, CertAlgoED25519v01, CertAlgoSKED25519v01:
 		return true
 	}
 	return false
@@ -295,6 +321,55 @@ func checkSourceAddress(addr net.Addr, sourceAddrs string) error {
 	return fmt.Errorf("ssh: remote address %v is not allowed because of source-address restriction", addr)
 }

+func gssExchangeToken(gssapiConfig *GSSAPIWithMICConfig, firstToken []byte, s *connection,
+	sessionID []byte, userAuthReq userAuthRequestMsg) (authErr error, perms *Permissions, err error) {
+	gssAPIServer := gssapiConfig.Server
+	defer gssAPIServer.DeleteSecContext()
+	var srcName string
+	for {
+		var (
+			outToken     []byte
+			needContinue bool
+		)
+		outToken, srcName, needContinue, err = gssAPIServer.AcceptSecContext(firstToken)
+		if err != nil {
+			return err, nil, nil
+		}
+		if len(outToken) != 0 {
+			if err := s.transport.writePacket(Marshal(&userAuthGSSAPIToken{
+				Token: outToken,
+			})); err != nil {
+				return nil, nil, err
+			}
+		}
+		if !needContinue {
+			break
+		}
+		packet, err := s.transport.readPacket()
+		if err != nil {
+			return nil, nil, err
+		}
+		userAuthGSSAPITokenReq := &userAuthGSSAPIToken{}
+		if err := Unmarshal(packet, userAuthGSSAPITokenReq); err != nil {
+			return nil, nil, err
+		}
+	}
+	packet, err := s.transport.readPacket()
+	if err != nil {
+		return nil, nil, err
+	}
+	userAuthGSSAPIMICReq := &userAuthGSSAPIMIC{}
+	if err := Unmarshal(packet, userAuthGSSAPIMICReq); err != nil {
+		return nil, nil, err
+	}
+	mic := buildMIC(string(sessionID), userAuthReq.User, userAuthReq.Service, userAuthReq.Method)
+	if err := gssAPIServer.VerifyMIC(mic, userAuthGSSAPIMICReq.MIC); err != nil {
+		return err, nil, nil
+	}
+	perms, authErr = gssapiConfig.AllowLogin(s, srcName)
+	return authErr, perms, nil
+}
+
 // ServerAuthError represents server authentication errors and is
 // sometimes returned by NewServerConn. It appends any authentication
 // errors that may occur, and is returned if all of the authentication
@@ -496,6 +571,49 @@ userAuthLoop:
 				authErr = candidate.result
 				perms = candidate.perms
 			}
+		case "gssapi-with-mic":
+			gssapiConfig := config.GSSAPIWithMICConfig
+			userAuthRequestGSSAPI, err := parseGSSAPIPayload(userAuthReq.Payload)
+			if err != nil {
+				return nil, parseError(msgUserAuthRequest)
+			}
+			// OpenSSH supports Kerberos V5 mechanism only for GSS-API authentication.
+			if userAuthRequestGSSAPI.N == 0 {
+				authErr = fmt.Errorf("ssh: Mechanism negotiation is not supported")
+				break
+			}
+			var i uint32
+			present := false
+			for i = 0; i < userAuthRequestGSSAPI.N; i++ {
+				if userAuthRequestGSSAPI.OIDS[i].Equal(krb5Mesh) {
+					present = true
+					break
+				}
+			}
+			if !present {
+				authErr = fmt.Errorf("ssh: GSSAPI authentication must use the Kerberos V5 mechanism")
+				break
+			}
+			// Initial server response, see RFC 4462 section 3.3.
+			if err := s.transport.writePacket(Marshal(&userAuthGSSAPIResponse{
+				SupportMech: krb5OID,
+			})); err != nil {
+				return nil, err
+			}
+			// Exchange token, see RFC 4462 section 3.4.
+			packet, err := s.transport.readPacket()
+			if err != nil {
+				return nil, err
+			}
+			userAuthGSSAPITokenReq := &userAuthGSSAPIToken{}
+			if err := Unmarshal(packet, userAuthGSSAPITokenReq); err != nil {
+				return nil, err
+			}
+			authErr, perms, err = gssExchangeToken(gssapiConfig, userAuthGSSAPITokenReq.Token, s, sessionID,
+				userAuthReq)
+			if err != nil {
+				return nil, err
+			}
 		default:
 			authErr = fmt.Errorf("ssh: unknown method %q", userAuthReq.Method)
 		}
@@ -522,6 +640,10 @@ userAuthLoop:
 		if config.KeyboardInteractiveCallback != nil {
 			failureMsg.Methods = append(failureMsg.Methods, "keyboard-interactive")
 		}
+		if config.GSSAPIWithMICConfig != nil && config.GSSAPIWithMICConfig.Server != nil &&
+			config.GSSAPIWithMICConfig.AllowLogin != nil {
+			failureMsg.Methods = append(failureMsg.Methods, "gssapi-with-mic")
+		}

 		if len(failureMsg.Methods) == 0 {
 			return nil, errors.New("ssh: no authentication methods configured but NoClientAuth is also false")
@@ -0,0 +1,139 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ssh
+
+import (
+	"encoding/asn1"
+	"errors"
+)
+
+var krb5OID []byte
+
+func init() {
+	krb5OID, _ = asn1.Marshal(krb5Mesh)
+}
+
+// GSSAPIClient provides the API to plug-in GSSAPI authentication for client logins.
+type GSSAPIClient interface {
+	// InitSecContext initiates the establishment of a security context for GSS-API between the
+	// ssh client and ssh server. Initially the token parameter should be specified as nil.
+	// The routine may return a outputToken which should be transferred to
+	// the ssh server, where the ssh server will present it to
+	// AcceptSecContext. If no token need be sent, InitSecContext will indicate this by setting
+	// needContinue to false. To complete the context
+	// establishment, one or more reply tokens may be required from the ssh
+	// server;if so, InitSecContext will return a needContinue which is true.
+	// In this case, InitSecContext should be called again when the
+	// reply token is received from the ssh server, passing the reply
+	// token to InitSecContext via the token parameters.
+	// See RFC 2743 section 2.2.1 and RFC 4462 section 3.4.
+	InitSecContext(target string, token []byte, isGSSDelegCreds bool) (outputToken []byte, needContinue bool, err error)
+	// GetMIC generates a cryptographic MIC for the SSH2 message, and places
+	// the MIC in a token for transfer to the ssh server.
+	// The contents of the MIC field are obtained by calling GSS_GetMIC()
+	// over the following, using the GSS-API context that was just
+	// established:
+	//  string    session identifier
+	//  byte      SSH_MSG_USERAUTH_REQUEST
+	//  string    user name
+	//  string    service
+	//  string    "gssapi-with-mic"
+	// See RFC 2743 section 2.3.1 and RFC 4462 3.5.
+	GetMIC(micFiled []byte) ([]byte, error)
+	// Whenever possible, it should be possible for
+	// DeleteSecContext() calls to be successfully processed even
+	// if other calls cannot succeed, thereby enabling context-related
+	// resources to be released.
+	// In addition to deleting established security contexts,
+	// gss_delete_sec_context must also be able to delete "half-built"
+	// security contexts resulting from an incomplete sequence of
+	// InitSecContext()/AcceptSecContext() calls.
+	// See RFC 2743 section 2.2.3.
+	DeleteSecContext() error
+}
+
+// GSSAPIServer provides the API to plug in GSSAPI authentication for server logins.
+type GSSAPIServer interface {
+	// AcceptSecContext allows a remotely initiated security context between the application
+	// and a remote peer to be established by the ssh client. The routine may return a
+	// outputToken which should be transferred to the ssh client,
+	// where the ssh client will present it to InitSecContext.
+	// If no token need be sent, AcceptSecContext will indicate this
+	// by setting the needContinue to false. To
+	// complete the context establishment, one or more reply tokens may be
+	// required from the ssh client. if so, AcceptSecContext
+	// will return a needContinue which is true, in which case it
+	// should be called again when the reply token is received from the ssh
+	// client, passing the token to AcceptSecContext via the
+	// token parameters.
+	// The srcName return value is the authenticated username.
+	// See RFC 2743 section 2.2.2 and RFC 4462 section 3.4.
+	AcceptSecContext(token []byte) (outputToken []byte, srcName string, needContinue bool, err error)
+	// VerifyMIC verifies that a cryptographic MIC, contained in the token parameter,
+	// fits the supplied message is received from the ssh client.
+	// See RFC 2743 section 2.3.2.
+	VerifyMIC(micField []byte, micToken []byte) error
+	// Whenever possible, it should be possible for
+	// DeleteSecContext() calls to be successfully processed even
+	// if other calls cannot succeed, thereby enabling context-related
+	// resources to be released.
+	// In addition to deleting established security contexts,
+	// gss_delete_sec_context must also be able to delete "half-built"
+	// security contexts resulting from an incomplete sequence of
+	// InitSecContext()/AcceptSecContext() calls.
+	// See RFC 2743 section 2.2.3.
+	DeleteSecContext() error
+}
+
+var (
+	// OpenSSH supports Kerberos V5 mechanism only for GSS-API authentication,
+	// so we also support the krb5 mechanism only.
+	// See RFC 1964 section 1.
+	krb5Mesh = asn1.ObjectIdentifier{1, 2, 840, 113554, 1, 2, 2}
+)
+
+// The GSS-API authentication method is initiated when the client sends an SSH_MSG_USERAUTH_REQUEST
+// See RFC 4462 section 3.2.
+type userAuthRequestGSSAPI struct {
+	N    uint32
+	OIDS []asn1.ObjectIdentifier
+}
+
+func parseGSSAPIPayload(payload []byte) (*userAuthRequestGSSAPI, error) {
+	n, rest, ok := parseUint32(payload)
+	if !ok {
+		return nil, errors.New("parse uint32 failed")
+	}
+	s := &userAuthRequestGSSAPI{
+		N:    n,
+		OIDS: make([]asn1.ObjectIdentifier, n),
+	}
+	for i := 0; i < int(n); i++ {
+		var (
+			desiredMech []byte
+			err         error
+		)
+		desiredMech, rest, ok = parseString(rest)
+		if !ok {
+			return nil, errors.New("parse string failed")
+		}
+		if rest, err = asn1.Unmarshal(desiredMech, &s.OIDS[i]); err != nil {
+			return nil, err
+		}
+
+	}
+	return s, nil
+}
+
+// See RFC 4462 section 3.6.
+func buildMIC(sessionID string, username string, service string, authMethod string) []byte {
+	out := make([]byte, 0, 0)
+	out = appendString(out, sessionID)
+	out = append(out, msgUserAuthRequest)
+	out = appendString(out, username)
+	out = appendString(out, service)
+	out = appendString(out, authMethod)
+	return out
+}
@@ -947,6 +947,10 @@ func readPasswordLine(reader io.Reader) ([]byte, error) {
 		n, err := reader.Read(buf[:])
 		if n > 0 {
 			switch buf[0] {
+			case '\b':
+				if len(ret) > 0 {
+					ret = ret[:len(ret)-1]
+				}
 			case '\n':
 				return ret, nil
 			case '\r':
@@ -85,8 +85,8 @@ func ReadPassword(fd int) ([]byte, error) {
 	}
 	old := st

-	st &^= (windows.ENABLE_ECHO_INPUT)
-	st |= (windows.ENABLE_PROCESSED_INPUT | windows.ENABLE_LINE_INPUT | windows.ENABLE_PROCESSED_OUTPUT)
+	st &^= (windows.ENABLE_ECHO_INPUT | windows.ENABLE_LINE_INPUT)
+	st |= (windows.ENABLE_PROCESSED_OUTPUT | windows.ENABLE_PROCESSED_INPUT)
 	if err := windows.SetConsoleMode(windows.Handle(fd), st); err != nil {
 		return nil, err
 	}
@@ -1,42 +1,96 @@
 # gRPC-Go

-[![Build Status](https://travis-ci.org/grpc/grpc-go.svg)](https://travis-ci.org/grpc/grpc-go) [![GoDoc](https://godoc.org/google.golang.org/grpc?status.svg)](https://godoc.org/google.golang.org/grpc) [![GoReportCard](https://goreportcard.com/badge/grpc/grpc-go)](https://goreportcard.com/report/github.com/grpc/grpc-go)
+[![Build Status](https://travis-ci.org/grpc/grpc-go.svg)](https://travis-ci.org/grpc/grpc-go)
+[![GoDoc](https://godoc.org/google.golang.org/grpc?status.svg)](https://godoc.org/google.golang.org/grpc)
+[![GoReportCard](https://goreportcard.com/badge/grpc/grpc-go)](https://goreportcard.com/report/github.com/grpc/grpc-go)

-The Go implementation of [gRPC](https://grpc.io/): A high performance, open source, general RPC framework that puts mobile and HTTP/2 first. For more information see the [gRPC Quick Start: Go](https://grpc.io/docs/quickstart/go.html) guide.
+The Go implementation of [gRPC](https://grpc.io/): A high performance, open
+source, general RPC framework that puts mobile and HTTP/2 first. For more
+information see the [gRPC Quick Start:
+Go](https://grpc.io/docs/quickstart/go.html) guide.

 Installation
 ------------

-To install this package, you need to install Go and setup your Go workspace on your computer. The simplest way to install the library is to run:
+To install this package, you need to install Go and setup your Go workspace on
+your computer. The simplest way to install the library is to run:

 ```
 $ go get -u google.golang.org/grpc
 ```

+With Go module support (Go 1.11+), simply `import "google.golang.org/grpc"` in
+your source code and `go [build|run|test]` will automatically download the
+necessary dependencies ([Go modules
+ref](https://github.com/golang/go/wiki/Modules)).
+
+If you are trying to access grpc-go from within China, please see the
+[FAQ](#FAQ) below.
+
 Prerequisites
 -------------
-
 gRPC-Go requires Go 1.9 or later.

-Constraints
-----------
-The grpc package should only depend on standard Go packages and a small number of exceptions. If your contribution introduces new dependencies which are NOT in the [list](https://godoc.org/google.golang.org/grpc?imports), you need a discussion with gRPC-Go authors and consultants.
-
 Documentation
 -------------
-See [API documentation](https://godoc.org/google.golang.org/grpc) for package and API descriptions and find examples in the [examples directory](examples/).
+- See [godoc](https://godoc.org/google.golang.org/grpc) for package and API
+  descriptions.
+- Documentation on specific topics can be found in the [Documentation
+  directory](Documentation/).
+- Examples can be found in the [examples directory](examples/).

 Performance
 -----------
-See the current benchmarks for some of the languages supported in [this dashboard](https://performance-dot-grpc-testing.appspot.com/explore?dashboard=5652536396611584&widget=490377658&container=1286539696).
+Performance benchmark data for grpc-go and other languages is maintained in
+[this
+dashboard](https://performance-dot-grpc-testing.appspot.com/explore?dashboard=5652536396611584&widget=490377658&container=1286539696).

 Status
 ------
-General Availability [Google Cloud Platform Launch Stages](https://cloud.google.com/terms/launch-stages).
+General Availability [Google Cloud Platform Launch
+Stages](https://cloud.google.com/terms/launch-stages).

 FAQ
 ---

+#### I/O Timeout Errors
+
+The `golang.org` domain may be blocked from some countries.  `go get` usually
+produces an error like the following when this happens:
+
+```
+$ go get -u google.golang.org/grpc
+package google.golang.org/grpc: unrecognized import path "google.golang.org/grpc" (https fetch: Get https://google.golang.org/grpc?go-get=1: dial tcp 216.239.37.1:443: i/o timeout)
+```
+
+To build Go code, there are several options:
+
+- Set up a VPN and access google.golang.org through that.
+
+- Without Go module support: `git clone` the repo manually:
+
+  ```
+  git clone https://github.com/grpc/grpc-go.git $GOPATH/src/google.golang.org/grpc
+  ```
+
+  You will need to do the same for all of grpc's dependencies in `golang.org`,
+  e.g. `golang.org/x/net`.
+
+- With Go module support: it is possible to use the `replace` feature of `go
+  mod` to create aliases for golang.org packages.  In your project's directory:
+
+  ```
+  go mod edit -replace=google.golang.org/grpc=github.com/grpc/grpc-go@latest
+  go mod tidy
+  go mod vendor
+  go build -mod=vendor
+  ```
+
+  Again, this will need to be done for all transitive dependencies hosted on
+  golang.org as well.  Please refer to [this
+  issue](https://github.com/golang/go/issues/28652) in the golang repo regarding
+  this concern.
+
 #### Compiling error, undefined: grpc.SupportPackageIsVersion

 Please update proto package, gRPC package and rebuild the proto files:
@@ -43,7 +43,7 @@ type Address struct {

 // BalancerConfig specifies the configurations for Balancer.
 //
-// Deprecated: please use package balancer.
+// Deprecated: please use package balancer.  May be removed in a future 1.x release.
 type BalancerConfig struct {
 	// DialCreds is the transport credential the Balancer implementation can
 	// use to dial to a remote load balancer server. The Balancer implementations
@@ -57,7 +57,7 @@ type BalancerConfig struct {

 // BalancerGetOptions configures a Get call.
 //
-// Deprecated: please use package balancer.
+// Deprecated: please use package balancer.  May be removed in a future 1.x release.
 type BalancerGetOptions struct {
 	// BlockingWait specifies whether Get should block when there is no
 	// connected address.
@@ -66,7 +66,7 @@ type BalancerGetOptions struct {

 // Balancer chooses network addresses for RPCs.
 //
-// Deprecated: please use package balancer.
+// Deprecated: please use package balancer.  May be removed in a future 1.x release.
 type Balancer interface {
 	// Start does the initialization work to bootstrap a Balancer. For example,
 	// this function may start the name resolution and watch the updates. It will
@@ -120,7 +120,7 @@ type Balancer interface {
 // RoundRobin returns a Balancer that selects addresses round-robin. It uses r to watch
 // the name resolution updates and updates the addresses available correspondingly.
 //
-// Deprecated: please use package balancer/roundrobin.
+// Deprecated: please use package balancer/roundrobin. May be removed in a future 1.x release.
 func RoundRobin(r naming.Resolver) Balancer {
 	return &roundRobin{r: r}
 }
@@ -22,6 +22,7 @@ package balancer

 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"net"
 	"strings"
@@ -31,6 +32,7 @@ import (
 	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
 )

 var (
@@ -39,7 +41,10 @@ var (
 )

 // Register registers the balancer builder to the balancer map. b.Name
-// (lowercased) will be used as the name registered with this builder.
+// (lowercased) will be used as the name registered with this builder.  If the
+// Builder implements ConfigParser, ParseConfig will be called when new service
+// configs are received by the resolver, and the result will be provided to the
+// Balancer in UpdateClientConnState.
 //
 // NOTE: this function must only be called during initialization time (i.e. in
 // an init() function), and is not thread-safe. If multiple Balancers are
@@ -138,6 +143,8 @@ type ClientConn interface {
 	ResolveNow(resolver.ResolveNowOption)

 	// Target returns the dial target for this ClientConn.
+	//
+	// Deprecated: Use the Target field in the BuildOptions instead.
 	Target() string
 }

@@ -155,6 +162,10 @@ type BuildOptions struct {
 	Dialer func(context.Context, string) (net.Conn, error)
 	// ChannelzParentID is the entity parent's channelz unique identification number.
 	ChannelzParentID int64
+	// Target contains the parsed address info of the dial target. It is the same resolver.Target as
+	// passed to the resolver.
+	// See the documentation for the resolver.Target type for details about what it contains.
+	Target resolver.Target
 }

 // Builder creates a balancer.
@@ -166,6 +177,14 @@ type Builder interface {
 	Name() string
 }

+// ConfigParser parses load balancer configs.
+type ConfigParser interface {
+	// ParseConfig parses the JSON load balancer config provided into an
+	// internal form or returns an error if the config is invalid.  For future
+	// compatibility reasons, unknown fields in the config should be ignored.
+	ParseConfig(LoadBalancingConfigJSON json.RawMessage) (serviceconfig.LoadBalancingConfig, error)
+}
+
 // PickOptions contains addition information for the Pick operation.
 type PickOptions struct {
 	// FullMethodName is the method name that NewClientStream() is called
@@ -264,7 +283,7 @@ type Balancer interface {
 	// non-nil error to gRPC.
 	//
 	// Deprecated: if V2Balancer is implemented by the Balancer,
-	// UpdateResolverState will be called instead.
+	// UpdateClientConnState will be called instead.
 	HandleResolvedAddrs([]resolver.Address, error)
 	// Close closes the balancer. The balancer is not required to call
 	// ClientConn.RemoveSubConn for its existing SubConns.
@@ -277,14 +296,23 @@ type SubConnState struct {
 	// TODO: add last connection error
 }

+// ClientConnState describes the state of a ClientConn relevant to the
+// balancer.
+type ClientConnState struct {
+	ResolverState resolver.State
+	// The parsed load balancing configuration returned by the builder's
+	// ParseConfig method, if implemented.
+	BalancerConfig serviceconfig.LoadBalancingConfig
+}
+
 // V2Balancer is defined for documentation purposes.  If a Balancer also
-// implements V2Balancer, its UpdateResolverState method will be called instead
-// of HandleResolvedAddrs and its UpdateSubConnState will be called instead of
-// HandleSubConnStateChange.
+// implements V2Balancer, its UpdateClientConnState method will be called
+// instead of HandleResolvedAddrs and its UpdateSubConnState will be called
+// instead of HandleSubConnStateChange.
 type V2Balancer interface {
-	// UpdateResolverState is called by gRPC when the state of the resolver
+	// UpdateClientConnState is called by gRPC when the state of the ClientConn
 	// changes.
-	UpdateResolverState(resolver.State)
+	UpdateClientConnState(ClientConnState)
 	// UpdateSubConnState is called by gRPC when the state of a SubConn
 	// changes.
 	UpdateSubConnState(SubConn, SubConnState)
@@ -70,13 +70,15 @@ func (b *baseBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error)
 	panic("not implemented")
 }

-func (b *baseBalancer) UpdateResolverState(s resolver.State) {
-	// TODO: handle s.Err (log if not nil) once implemented.
-	// TODO: handle s.ServiceConfig?
-	grpclog.Infoln("base.baseBalancer: got new resolver state: ", s)
+func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) {
+	// TODO: handle s.ResolverState.Err (log if not nil) once implemented.
+	// TODO: handle s.ResolverState.ServiceConfig?
+	if grpclog.V(2) {
+		grpclog.Infoln("base.baseBalancer: got new ClientConn state: ", s)
+	}
 	// addrsSet is the set converted from addrs, it's used for quick lookup of an address.
 	addrsSet := make(map[resolver.Address]struct{})
-	for _, a := range s.Addresses {
+	for _, a := range s.ResolverState.Addresses {
 		addrsSet[a] = struct{}{}
 		if _, ok := b.subConns[a]; !ok {
 			// a is a new address (not existing in b.subConns).
@@ -127,10 +129,14 @@ func (b *baseBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectiv

 func (b *baseBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
 	s := state.ConnectivityState
-	grpclog.Infof("base.baseBalancer: handle SubConn state change: %p, %v", sc, s)
+	if grpclog.V(2) {
+		grpclog.Infof("base.baseBalancer: handle SubConn state change: %p, %v", sc, s)
+	}
 	oldS, ok := b.scStates[sc]
 	if !ok {
-		grpclog.Infof("base.baseBalancer: got state changes for an unknown SubConn: %p, %v", sc, s)
+		if grpclog.V(2) {
+			grpclog.Infof("base.baseBalancer: got state changes for an unknown SubConn: %p, %v", sc, s)
+		}
 		return
 	}
 	b.scStates[sc] = s
@@ -88,7 +88,7 @@ type ccBalancerWrapper struct {
 	cc               *ClientConn
 	balancer         balancer.Balancer
 	stateChangeQueue *scStateUpdateBuffer
-	resolverUpdateCh chan *resolver.State
+	ccUpdateCh       chan *balancer.ClientConnState
 	done             chan struct{}

 	mu       sync.Mutex
@@ -99,7 +99,7 @@ func newCCBalancerWrapper(cc *ClientConn, b balancer.Builder, bopts balancer.Bui
 	ccb := &ccBalancerWrapper{
 		cc:               cc,
 		stateChangeQueue: newSCStateUpdateBuffer(),
-		resolverUpdateCh: make(chan *resolver.State, 1),
+		ccUpdateCh:       make(chan *balancer.ClientConnState, 1),
 		done:             make(chan struct{}),
 		subConns:         make(map[*acBalancerWrapper]struct{}),
 	}
@@ -126,7 +126,7 @@ func (ccb *ccBalancerWrapper) watcher() {
 			} else {
 				ccb.balancer.HandleSubConnStateChange(t.sc, t.state)
 			}
-		case s := <-ccb.resolverUpdateCh:
+		case s := <-ccb.ccUpdateCh:
 			select {
 			case <-ccb.done:
 				ccb.balancer.Close()
@@ -134,9 +134,9 @@ func (ccb *ccBalancerWrapper) watcher() {
 			default:
 			}
 			if ub, ok := ccb.balancer.(balancer.V2Balancer); ok {
-				ub.UpdateResolverState(*s)
+				ub.UpdateClientConnState(*s)
 			} else {
-				ccb.balancer.HandleResolvedAddrs(s.Addresses, nil)
+				ccb.balancer.HandleResolvedAddrs(s.ResolverState.Addresses, nil)
 			}
 		case <-ccb.done:
 		}
@@ -151,9 +151,11 @@ func (ccb *ccBalancerWrapper) watcher() {
 			for acbw := range scs {
 				ccb.cc.removeAddrConn(acbw.getAddrConn(), errConnDrain)
 			}
+			ccb.UpdateBalancerState(connectivity.Connecting, nil)
 			return
 		default:
 		}
+		ccb.cc.firstResolveEvent.Fire()
 	}
 }

@@ -178,9 +180,10 @@ func (ccb *ccBalancerWrapper) handleSubConnStateChange(sc balancer.SubConn, s co
 	})
 }

-func (ccb *ccBalancerWrapper) updateResolverState(s resolver.State) {
+func (ccb *ccBalancerWrapper) updateClientConnState(ccs *balancer.ClientConnState) {
 	if ccb.cc.curBalancerName != grpclbName {
 		// Filter any grpclb addresses since we don't have the grpclb balancer.
+		s := &ccs.ResolverState
 		for i := 0; i < len(s.Addresses); {
 			if s.Addresses[i].Type == resolver.GRPCLB {
 				copy(s.Addresses[i:], s.Addresses[i+1:])
@@ -191,10 +194,10 @@ func (ccb *ccBalancerWrapper) updateResolverState(s resolver.State) {
 		}
 	}
 	select {
-	case <-ccb.resolverUpdateCh:
+	case <-ccb.ccUpdateCh:
 	default:
 	}
-	ccb.resolverUpdateCh <- &s
+	ccb.ccUpdateCh <- ccs
 }

 func (ccb *ccBalancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
@@ -20,7 +20,6 @@ package grpc

 import (
 	"context"
-	"strings"
 	"sync"

 	"google.golang.org/grpc/balancer"
@@ -34,13 +33,7 @@ type balancerWrapperBuilder struct {
 }

 func (bwb *balancerWrapperBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
-	targetAddr := cc.Target()
-	targetSplitted := strings.Split(targetAddr, ":///")
-	if len(targetSplitted) >= 2 {
-		targetAddr = targetSplitted[1]
-	}
-
-	bwb.b.Start(targetAddr, BalancerConfig{
+	bwb.b.Start(opts.Target.Endpoint, BalancerConfig{
 		DialCreds: opts.DialCreds,
 		Dialer:    opts.Dialer,
 	})
@@ -49,7 +42,7 @@ func (bwb *balancerWrapperBuilder) Build(cc balancer.ClientConn, opts balancer.B
 		balancer:   bwb.b,
 		pickfirst:  pickfirst,
 		cc:         cc,
-		targetAddr: targetAddr,
+		targetAddr: opts.Target.Endpoint,
 		startCh:    make(chan struct{}),
 		conns:      make(map[resolver.Address]balancer.SubConn),
 		connSt:     make(map[balancer.SubConn]*scState),
@@ -120,7 +113,7 @@ func (bw *balancerWrapper) lbWatcher() {
 	}

 	for addrs := range notifyCh {
-		grpclog.Infof("balancerWrapper: got update addr from Notify: %v\n", addrs)
+		grpclog.Infof("balancerWrapper: got update addr from Notify: %v", addrs)
 		if bw.pickfirst {
 			var (
 				oldA  resolver.Address
@@ -38,13 +38,13 @@ import (
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/internal/backoff"
 	"google.golang.org/grpc/internal/channelz"
-	"google.golang.org/grpc/internal/envconfig"
 	"google.golang.org/grpc/internal/grpcsync"
 	"google.golang.org/grpc/internal/transport"
 	"google.golang.org/grpc/keepalive"
 	"google.golang.org/grpc/resolver"
 	_ "google.golang.org/grpc/resolver/dns"         // To register dns resolver.
 	_ "google.golang.org/grpc/resolver/passthrough" // To register passthrough resolver.
+	"google.golang.org/grpc/serviceconfig"
 	"google.golang.org/grpc/status"
 )

@@ -137,6 +137,9 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 		opt.apply(&cc.dopts)
 	}

+	chainUnaryClientInterceptors(cc)
+	chainStreamClientInterceptors(cc)
+
 	defer func() {
 		if err != nil {
 			cc.Close()
@@ -290,6 +293,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 		CredsBundle:      cc.dopts.copts.CredsBundle,
 		Dialer:           cc.dopts.copts.Dialer,
 		ChannelzParentID: cc.channelzID,
+		Target:           cc.parsedTarget,
 	}

 	// Build the resolver.
@@ -327,6 +331,68 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
 	return cc, nil
 }

+// chainUnaryClientInterceptors chains all unary client interceptors into one.
+func chainUnaryClientInterceptors(cc *ClientConn) {
+	interceptors := cc.dopts.chainUnaryInts
+	// Prepend dopts.unaryInt to the chaining interceptors if it exists, since unaryInt will
+	// be executed before any other chained interceptors.
+	if cc.dopts.unaryInt != nil {
+		interceptors = append([]UnaryClientInterceptor{cc.dopts.unaryInt}, interceptors...)
+	}
+	var chainedInt UnaryClientInterceptor
+	if len(interceptors) == 0 {
+		chainedInt = nil
+	} else if len(interceptors) == 1 {
+		chainedInt = interceptors[0]
+	} else {
+		chainedInt = func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error {
+			return interceptors[0](ctx, method, req, reply, cc, getChainUnaryInvoker(interceptors, 0, invoker), opts...)
+		}
+	}
+	cc.dopts.unaryInt = chainedInt
+}
+
+// getChainUnaryInvoker recursively generate the chained unary invoker.
+func getChainUnaryInvoker(interceptors []UnaryClientInterceptor, curr int, finalInvoker UnaryInvoker) UnaryInvoker {
+	if curr == len(interceptors)-1 {
+		return finalInvoker
+	}
+	return func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, opts ...CallOption) error {
+		return interceptors[curr+1](ctx, method, req, reply, cc, getChainUnaryInvoker(interceptors, curr+1, finalInvoker), opts...)
+	}
+}
+
+// chainStreamClientInterceptors chains all stream client interceptors into one.
+func chainStreamClientInterceptors(cc *ClientConn) {
+	interceptors := cc.dopts.chainStreamInts
+	// Prepend dopts.streamInt to the chaining interceptors if it exists, since streamInt will
+	// be executed before any other chained interceptors.
+	if cc.dopts.streamInt != nil {
+		interceptors = append([]StreamClientInterceptor{cc.dopts.streamInt}, interceptors...)
+	}
+	var chainedInt StreamClientInterceptor
+	if len(interceptors) == 0 {
+		chainedInt = nil
+	} else if len(interceptors) == 1 {
+		chainedInt = interceptors[0]
+	} else {
+		chainedInt = func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, streamer Streamer, opts ...CallOption) (ClientStream, error) {
+			return interceptors[0](ctx, desc, cc, method, getChainStreamer(interceptors, 0, streamer), opts...)
+		}
+	}
+	cc.dopts.streamInt = chainedInt
+}
+
+// getChainStreamer recursively generate the chained client stream constructor.
+func getChainStreamer(interceptors []StreamClientInterceptor, curr int, finalStreamer Streamer) Streamer {
+	if curr == len(interceptors)-1 {
+		return finalStreamer
+	}
+	return func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (ClientStream, error) {
+		return interceptors[curr+1](ctx, desc, cc, method, getChainStreamer(interceptors, curr+1, finalStreamer), opts...)
+	}
+}
+
 // connectivityStateManager keeps the connectivity.State of ClientConn.
 // This struct will eventually be exported so the balancers can access it.
 type connectivityStateManager struct {
@@ -466,24 +532,6 @@ func (cc *ClientConn) waitForResolvedAddrs(ctx context.Context) error {
 	}
 }

-// gRPC should resort to default service config when:
-// * resolver service config is disabled
-// * or, resolver does not return a service config or returns an invalid one.
-func (cc *ClientConn) fallbackToDefaultServiceConfig(sc string) bool {
-	if cc.dopts.disableServiceConfig {
-		return true
-	}
-	// The logic below is temporary, will be removed once we change the resolver.State ServiceConfig field type.
-	// Right now, we assume that empty service config string means resolver does not return a config.
-	if sc == "" {
-		return true
-	}
-	// TODO: the logic below is temporary. Once we finish the logic to validate service config
-	// in resolver, we will replace the logic below.
-	_, err := parseServiceConfig(sc)
-	return err != nil
-}
-
 func (cc *ClientConn) updateResolverState(s resolver.State) error {
 	cc.mu.Lock()
 	defer cc.mu.Unlock()
@@ -494,54 +542,47 @@ func (cc *ClientConn) updateResolverState(s resolver.State) error {
 		return nil
 	}

-	if cc.fallbackToDefaultServiceConfig(s.ServiceConfig) {
+	if cc.dopts.disableServiceConfig || s.ServiceConfig == nil {
 		if cc.dopts.defaultServiceConfig != nil && cc.sc == nil {
 			cc.applyServiceConfig(cc.dopts.defaultServiceConfig)
 		}
-	} else {
-		// TODO: the parsing logic below will be moved inside resolver.
-		sc, err := parseServiceConfig(s.ServiceConfig)
-		if err != nil {
-			return err
-		}
-		if cc.sc == nil || cc.sc.rawJSONString != s.ServiceConfig {
-			cc.applyServiceConfig(sc)
-		}
-	}
-
-	// update the service config that will be sent to balancer.
-	if cc.sc != nil {
-		s.ServiceConfig = cc.sc.rawJSONString
+	} else if sc, ok := s.ServiceConfig.(*ServiceConfig); ok {
+		cc.applyServiceConfig(sc)
 	}

+	var balCfg serviceconfig.LoadBalancingConfig
 	if cc.dopts.balancerBuilder == nil {
 		// Only look at balancer types and switch balancer if balancer dial
 		// option is not set.
-		var isGRPCLB bool
-		for _, a := range s.Addresses {
-			if a.Type == resolver.GRPCLB {
-				isGRPCLB = true
-				break
-			}
-		}
 		var newBalancerName string
-		// TODO: use new loadBalancerConfig field with appropriate priority.
-		if isGRPCLB {
-			newBalancerName = grpclbName
-		} else if cc.sc != nil && cc.sc.LB != nil {
-			newBalancerName = *cc.sc.LB
+		if cc.sc != nil && cc.sc.lbConfig != nil {
+			newBalancerName = cc.sc.lbConfig.name
+			balCfg = cc.sc.lbConfig.cfg
 		} else {
-			newBalancerName = PickFirstBalancerName
+			var isGRPCLB bool
+			for _, a := range s.Addresses {
+				if a.Type == resolver.GRPCLB {
+					isGRPCLB = true
+					break
+				}
+			}
+			if isGRPCLB {
+				newBalancerName = grpclbName
+			} else if cc.sc != nil && cc.sc.LB != nil {
+				newBalancerName = *cc.sc.LB
+			} else {
+				newBalancerName = PickFirstBalancerName
+			}
 		}
 		cc.switchBalancer(newBalancerName)
 	} else if cc.balancerWrapper == nil {
 		// Balancer dial option was set, and this is the first time handling
 		// resolved addresses. Build a balancer with dopts.balancerBuilder.
+		cc.curBalancerName = cc.dopts.balancerBuilder.Name()
 		cc.balancerWrapper = newCCBalancerWrapper(cc, cc.dopts.balancerBuilder, cc.balancerBuildOpts)
 	}

-	cc.balancerWrapper.updateResolverState(s)
-	cc.firstResolveEvent.Fire()
+	cc.balancerWrapper.updateClientConnState(&balancer.ClientConnState{ResolverState: s, BalancerConfig: balCfg})
 	return nil
 }

@@ -554,7 +595,7 @@ func (cc *ClientConn) updateResolverState(s resolver.State) error {
 //
 // Caller must hold cc.mu.
 func (cc *ClientConn) switchBalancer(name string) {
-	if strings.ToLower(cc.curBalancerName) == strings.ToLower(name) {
+	if strings.EqualFold(cc.curBalancerName, name) {
 		return
 	}

@@ -693,6 +734,8 @@ func (ac *addrConn) connect() error {
 		ac.mu.Unlock()
 		return nil
 	}
+	// Update connectivity state within the lock to prevent subsequent or
+	// concurrent calls from resetting the transport more than once.
 	ac.updateConnectivityState(connectivity.Connecting)
 	ac.mu.Unlock()

@@ -703,7 +746,16 @@ func (ac *addrConn) connect() error {

 // tryUpdateAddrs tries to update ac.addrs with the new addresses list.
 //
-// It checks whether current connected address of ac is in the new addrs list.
+// If ac is Connecting, it returns false. The caller should tear down the ac and
+// create a new one. Note that the backoff will be reset when this happens.
+//
+// If ac is TransientFailure, it updates ac.addrs and returns true. The updated
+// addresses will be picked up by retry in the next iteration after backoff.
+//
+// If ac is Shutdown or Idle, it updates ac.addrs and returns true.
+//
+// If ac is Ready, it checks whether current connected address of ac is in the
+// new addrs list.
 //  - If true, it updates ac.addrs and returns true. The ac will keep using
 //    the existing connection.
 //  - If false, it does nothing and returns false.
@@ -711,17 +763,18 @@ func (ac *addrConn) tryUpdateAddrs(addrs []resolver.Address) bool {
 	ac.mu.Lock()
 	defer ac.mu.Unlock()
 	grpclog.Infof("addrConn: tryUpdateAddrs curAddr: %v, addrs: %v", ac.curAddr, addrs)
-	if ac.state == connectivity.Shutdown {
+	if ac.state == connectivity.Shutdown ||
+		ac.state == connectivity.TransientFailure ||
+		ac.state == connectivity.Idle {
 		ac.addrs = addrs
 		return true
 	}

-	// Unless we're busy reconnecting already, let's reconnect from the top of
-	// the list.
-	if ac.state != connectivity.Ready {
+	if ac.state == connectivity.Connecting {
 		return false
 	}

+	// ac.state is Ready, try to find the connected address.
 	var curAddrFound bool
 	for _, a := range addrs {
 		if reflect.DeepEqual(ac.curAddr, a) {
@@ -970,6 +1023,9 @@ func (ac *addrConn) resetTransport() {
 		// The spec doesn't mention what should be done for multiple addresses.
 		// https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md#proposed-backoff-algorithm
 		connectDeadline := time.Now().Add(dialDuration)
+
+		ac.updateConnectivityState(connectivity.Connecting)
+		ac.transport = nil
 		ac.mu.Unlock()

 		newTr, addr, reconnect, err := ac.tryAllAddrs(addrs, connectDeadline)
@@ -1004,55 +1060,32 @@ func (ac *addrConn) resetTransport() {

 		ac.mu.Lock()
 		if ac.state == connectivity.Shutdown {
-			newTr.Close()
 			ac.mu.Unlock()
+			newTr.Close()
 			return
 		}
 		ac.curAddr = addr
 		ac.transport = newTr
 		ac.backoffIdx = 0

-		healthCheckConfig := ac.cc.healthCheckConfig()
-		// LB channel health checking is only enabled when all the four requirements below are met:
-		// 1. it is not disabled by the user with the WithDisableHealthCheck DialOption,
-		// 2. the internal.HealthCheckFunc is set by importing the grpc/healthcheck package,
-		// 3. a service config with non-empty healthCheckConfig field is provided,
-		// 4. the current load balancer allows it.
 		hctx, hcancel := context.WithCancel(ac.ctx)
-		healthcheckManagingState := false
-		if !ac.cc.dopts.disableHealthCheck && healthCheckConfig != nil && ac.scopts.HealthCheckEnabled {
-			if ac.cc.dopts.healthCheckFunc == nil {
-				// TODO: add a link to the health check doc in the error message.
-				grpclog.Error("the client side LB channel health check function has not been set.")
-			} else {
-				// TODO(deklerk) refactor to just return transport
-				go ac.startHealthCheck(hctx, newTr, addr, healthCheckConfig.ServiceName)
-				healthcheckManagingState = true
-			}
-		}
-		if !healthcheckManagingState {
-			ac.updateConnectivityState(connectivity.Ready)
-		}
+		ac.startHealthCheck(hctx)
 		ac.mu.Unlock()

 		// Block until the created transport is down. And when this happens,
 		// we restart from the top of the addr list.
 		<-reconnect.Done()
 		hcancel()
-
-		// Need to reconnect after a READY, the addrConn enters
-		// TRANSIENT_FAILURE.
+		// restart connecting - the top of the loop will set state to
+		// CONNECTING.  This is against the current connectivity semantics doc,
+		// however it allows for graceful behavior for RPCs not yet dispatched
+		// - unfortunate timing would otherwise lead to the RPC failing even
+		// though the TRANSIENT_FAILURE state (called for by the doc) would be
+		// instantaneous.
 		//
-		// This will set addrConn to TRANSIENT_FAILURE for a very short period
-		// of time, and turns CONNECTING. It seems reasonable to skip this, but
-		// READY-CONNECTING is not a valid transition.
-		ac.mu.Lock()
-		if ac.state == connectivity.Shutdown {
-			ac.mu.Unlock()
-			return
-		}
-		ac.updateConnectivityState(connectivity.TransientFailure)
-		ac.mu.Unlock()
+		// Ideally we should transition to Idle here and block until there is
+		// RPC activity that leads to the balancer requesting a reconnect of
+		// the associated SubConn.
 	}
 }

@@ -1066,8 +1099,6 @@ func (ac *addrConn) tryAllAddrs(addrs []resolver.Address, connectDeadline time.T
 			ac.mu.Unlock()
 			return nil, resolver.Address{}, nil, errConnClosing
 		}
-		ac.updateConnectivityState(connectivity.Connecting)
-		ac.transport = nil

 		ac.cc.mu.RLock()
 		ac.dopts.copts.KeepaliveParams = ac.cc.mkp
@@ -1111,14 +1142,35 @@ func (ac *addrConn) createTransport(addr resolver.Address, copts transport.Conne
 		Authority: ac.cc.authority,
 	}

+	once := sync.Once{}
 	onGoAway := func(r transport.GoAwayReason) {
 		ac.mu.Lock()
 		ac.adjustParams(r)
+		once.Do(func() {
+			if ac.state == connectivity.Ready {
+				// Prevent this SubConn from being used for new RPCs by setting its
+				// state to Connecting.
+				//
+				// TODO: this should be Idle when grpc-go properly supports it.
+				ac.updateConnectivityState(connectivity.Connecting)
+			}
+		})
 		ac.mu.Unlock()
 		reconnect.Fire()
 	}

 	onClose := func() {
+		ac.mu.Lock()
+		once.Do(func() {
+			if ac.state == connectivity.Ready {
+				// Prevent this SubConn from being used for new RPCs by setting its
+				// state to Connecting.
+				//
+				// TODO: this should be Idle when grpc-go properly supports it.
+				ac.updateConnectivityState(connectivity.Connecting)
+			}
+		})
+		ac.mu.Unlock()
 		close(onCloseCalled)
 		reconnect.Fire()
 	}
@@ -1140,60 +1192,99 @@ func (ac *addrConn) createTransport(addr resolver.Address, copts transport.Conne
 		return nil, nil, err
 	}

-	if ac.dopts.reqHandshake == envconfig.RequireHandshakeOn {
-		select {
-		case <-time.After(connectDeadline.Sub(time.Now())):
-			// We didn't get the preface in time.
-			newTr.Close()
-			grpclog.Warningf("grpc: addrConn.createTransport failed to connect to %v: didn't receive server preface in time. Reconnecting...", addr)
-			return nil, nil, errors.New("timed out waiting for server handshake")
-		case <-prefaceReceived:
-			// We got the preface - huzzah! things are good.
-		case <-onCloseCalled:
-			// The transport has already closed - noop.
-			return nil, nil, errors.New("connection closed")
-			// TODO(deklerk) this should bail on ac.ctx.Done(). Add a test and fix.
-		}
+	select {
+	case <-time.After(connectDeadline.Sub(time.Now())):
+		// We didn't get the preface in time.
+		newTr.Close()
+		grpclog.Warningf("grpc: addrConn.createTransport failed to connect to %v: didn't receive server preface in time. Reconnecting...", addr)
+		return nil, nil, errors.New("timed out waiting for server handshake")
+	case <-prefaceReceived:
+		// We got the preface - huzzah! things are good.
+	case <-onCloseCalled:
+		// The transport has already closed - noop.
+		return nil, nil, errors.New("connection closed")
+		// TODO(deklerk) this should bail on ac.ctx.Done(). Add a test and fix.
 	}
 	return newTr, reconnect, nil
 }

-func (ac *addrConn) startHealthCheck(ctx context.Context, newTr transport.ClientTransport, addr resolver.Address, serviceName string) {
-	// Set up the health check helper functions
-	newStream := func() (interface{}, error) {
-		return ac.newClientStream(ctx, &StreamDesc{ServerStreams: true}, "/grpc.health.v1.Health/Watch", newTr)
+// startHealthCheck starts the health checking stream (RPC) to watch the health
+// stats of this connection if health checking is requested and configured.
+//
+// LB channel health checking is enabled when all requirements below are met:
+// 1. it is not disabled by the user with the WithDisableHealthCheck DialOption
+// 2. internal.HealthCheckFunc is set by importing the grpc/healthcheck package
+// 3. a service config with non-empty healthCheckConfig field is provided
+// 4. the load balancer requests it
+//
+// It sets addrConn to READY if the health checking stream is not started.
+//
+// Caller must hold ac.mu.
+func (ac *addrConn) startHealthCheck(ctx context.Context) {
+	var healthcheckManagingState bool
+	defer func() {
+		if !healthcheckManagingState {
+			ac.updateConnectivityState(connectivity.Ready)
+		}
+	}()
+
+	if ac.cc.dopts.disableHealthCheck {
+		return
 	}
-	firstReady := true
-	reportHealth := func(ok bool) {
+	healthCheckConfig := ac.cc.healthCheckConfig()
+	if healthCheckConfig == nil {
+		return
+	}
+	if !ac.scopts.HealthCheckEnabled {
+		return
+	}
+	healthCheckFunc := ac.cc.dopts.healthCheckFunc
+	if healthCheckFunc == nil {
+		// The health package is not imported to set health check function.
+		//
+		// TODO: add a link to the health check doc in the error message.
+		grpclog.Error("Health check is requested but health check function is not set.")
+		return
+	}
+
+	healthcheckManagingState = true
+
+	// Set up the health check helper functions.
+	currentTr := ac.transport
+	newStream := func(method string) (interface{}, error) {
+		ac.mu.Lock()
+		if ac.transport != currentTr {
+			ac.mu.Unlock()
+			return nil, status.Error(codes.Canceled, "the provided transport is no longer valid to use")
+		}
+		ac.mu.Unlock()
+		return newNonRetryClientStream(ctx, &StreamDesc{ServerStreams: true}, method, currentTr, ac)
+	}
+	setConnectivityState := func(s connectivity.State) {
 		ac.mu.Lock()
 		defer ac.mu.Unlock()
-		if ac.transport != newTr {
+		if ac.transport != currentTr {
 			return
 		}
-		if ok {
-			if firstReady {
-				firstReady = false
-				ac.curAddr = addr
-			}
-			ac.updateConnectivityState(connectivity.Ready)
-		} else {
-			ac.updateConnectivityState(connectivity.TransientFailure)
-		}
+		ac.updateConnectivityState(s)
 	}
-	err := ac.cc.dopts.healthCheckFunc(ctx, newStream, reportHealth, serviceName)
-	if err != nil {
-		if status.Code(err) == codes.Unimplemented {
-			if channelz.IsOn() {
-				channelz.AddTraceEvent(ac.channelzID, &channelz.TraceEventDesc{
-					Desc:     "Subchannel health check is unimplemented at server side, thus health check is disabled",
-					Severity: channelz.CtError,
-				})
+	// Start the health checking stream.
+	go func() {
+		err := ac.cc.dopts.healthCheckFunc(ctx, newStream, setConnectivityState, healthCheckConfig.ServiceName)
+		if err != nil {
+			if status.Code(err) == codes.Unimplemented {
+				if channelz.IsOn() {
+					channelz.AddTraceEvent(ac.channelzID, &channelz.TraceEventDesc{
+						Desc:     "Subchannel health check is unimplemented at server side, thus health check is disabled",
+						Severity: channelz.CtError,
+					})
+				}
+				grpclog.Error("Subchannel health check is unimplemented at server side, thus health check is disabled")
+			} else {
+				grpclog.Errorf("HealthCheckFunc exits with unexpected error %v", err)
 			}
-			grpclog.Error("Subchannel health check is unimplemented at server side, thus health check is disabled")
-		} else {
-			grpclog.Errorf("HealthCheckFunc exits with unexpected error %v", err)
 		}
-	}
+	}()
 }

 func (ac *addrConn) resetConnectBackoff() {
@@ -132,7 +132,8 @@ const (

 	// Unavailable indicates the service is currently unavailable.
 	// This is a most likely a transient condition and may be corrected
-	// by retrying with a backoff.
+	// by retrying with a backoff. Note that it is not always safe to retry
+	// non-idempotent operations.
 	//
 	// See litmus test above for deciding between FailedPrecondition,
 	// Aborted, and Unavailable.
@@ -278,24 +278,22 @@ type ChannelzSecurityValue interface {
 // TLSChannelzSecurityValue defines the struct that TLS protocol should return
 // from GetSecurityValue(), containing security info like cipher and certificate used.
 type TLSChannelzSecurityValue struct {
+	ChannelzSecurityValue
 	StandardName      string
 	LocalCertificate  []byte
 	RemoteCertificate []byte
 }

-func (*TLSChannelzSecurityValue) isChannelzSecurityValue() {}
-
 // OtherChannelzSecurityValue defines the struct that non-TLS protocol should return
 // from GetSecurityValue(), which contains protocol specific security info. Note
 // the Value field will be sent to users of channelz requesting channel info, and
 // thus sensitive info should better be avoided.
 type OtherChannelzSecurityValue struct {
+	ChannelzSecurityValue
 	Name  string
 	Value proto.Message
 }

-func (*OtherChannelzSecurityValue) isChannelzSecurityValue() {}
-
 var cipherSuiteLookup = map[uint16]string{
 	tls.TLS_RSA_WITH_RC4_128_SHA:                "TLS_RSA_WITH_RC4_128_SHA",
 	tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA:           "TLS_RSA_WITH_3DES_EDE_CBC_SHA",
@@ -39,8 +39,12 @@ import (
 // dialOptions configure a Dial call. dialOptions are set by the DialOption
 // values passed to Dial.
 type dialOptions struct {
-	unaryInt    UnaryClientInterceptor
-	streamInt   StreamClientInterceptor
+	unaryInt  UnaryClientInterceptor
+	streamInt StreamClientInterceptor
+
+	chainUnaryInts  []UnaryClientInterceptor
+	chainStreamInts []StreamClientInterceptor
+
 	cp          Compressor
 	dc          Decompressor
 	bs          backoff.Strategy
@@ -56,7 +60,6 @@ type dialOptions struct {
 	balancerBuilder balancer.Builder
 	// This is to support grpclb.
 	resolverBuilder             resolver.Builder
-	reqHandshake                envconfig.RequireHandshakeSetting
 	channelzParentID            int64
 	disableServiceConfig        bool
 	disableRetry                bool
@@ -96,17 +99,6 @@ func newFuncDialOption(f func(*dialOptions)) *funcDialOption {
 	}
 }

-// WithWaitForHandshake blocks until the initial settings frame is received from
-// the server before assigning RPCs to the connection.
-//
-// Deprecated: this is the default behavior, and this option will be removed
-// after the 1.18 release.
-func WithWaitForHandshake() DialOption {
-	return newFuncDialOption(func(o *dialOptions) {
-		o.reqHandshake = envconfig.RequireHandshakeOn
-	})
-}
-
 // WithWriteBufferSize determines how much data can be batched before doing a
 // write on the wire. The corresponding memory allocation for this buffer will
 // be twice the size to keep syscalls low. The default value for this buffer is
@@ -152,7 +144,8 @@ func WithInitialConnWindowSize(s int32) DialOption {
 // WithMaxMsgSize returns a DialOption which sets the maximum message size the
 // client can receive.
 //
-// Deprecated: use WithDefaultCallOptions(MaxCallRecvMsgSize(s)) instead.
+// Deprecated: use WithDefaultCallOptions(MaxCallRecvMsgSize(s)) instead.  Will
+// be supported throughout 1.x.
 func WithMaxMsgSize(s int) DialOption {
 	return WithDefaultCallOptions(MaxCallRecvMsgSize(s))
 }
@@ -168,7 +161,8 @@ func WithDefaultCallOptions(cos ...CallOption) DialOption {
 // WithCodec returns a DialOption which sets a codec for message marshaling and
 // unmarshaling.
 //
-// Deprecated: use WithDefaultCallOptions(ForceCodec(_)) instead.
+// Deprecated: use WithDefaultCallOptions(ForceCodec(_)) instead.  Will be
+// supported throughout 1.x.
 func WithCodec(c Codec) DialOption {
 	return WithDefaultCallOptions(CallCustomCodec(c))
 }
@@ -177,7 +171,7 @@ func WithCodec(c Codec) DialOption {
 // message compression. It has lower priority than the compressor set by the
 // UseCompressor CallOption.
 //
-// Deprecated: use UseCompressor instead.
+// Deprecated: use UseCompressor instead.  Will be supported throughout 1.x.
 func WithCompressor(cp Compressor) DialOption {
 	return newFuncDialOption(func(o *dialOptions) {
 		o.cp = cp
@@ -192,7 +186,8 @@ func WithCompressor(cp Compressor) DialOption {
 // message.  If no compressor is registered for the encoding, an Unimplemented
 // status error will be returned.
 //
-// Deprecated: use encoding.RegisterCompressor instead.
+// Deprecated: use encoding.RegisterCompressor instead.  Will be supported
+// throughout 1.x.
 func WithDecompressor(dc Decompressor) DialOption {
 	return newFuncDialOption(func(o *dialOptions) {
 		o.dc = dc
@@ -203,7 +198,7 @@ func WithDecompressor(dc Decompressor) DialOption {
 // Name resolver will be ignored if this DialOption is specified.
 //
 // Deprecated: use the new balancer APIs in balancer package and
-// WithBalancerName.
+// WithBalancerName.  Will be removed in a future 1.x release.
 func WithBalancer(b Balancer) DialOption {
 	return newFuncDialOption(func(o *dialOptions) {
 		o.balancerBuilder = &balancerWrapperBuilder{
@@ -219,7 +214,8 @@ func WithBalancer(b Balancer) DialOption {
 // The balancer cannot be overridden by balancer option specified by service
 // config.
 //
-// This is an EXPERIMENTAL API.
+// Deprecated: use WithDefaultServiceConfig and WithDisableServiceConfig
+// instead.  Will be removed in a future 1.x release.
 func WithBalancerName(balancerName string) DialOption {
 	builder := balancer.Get(balancerName)
 	if builder == nil {
@@ -240,9 +236,10 @@ func withResolverBuilder(b resolver.Builder) DialOption {
 // WithServiceConfig returns a DialOption which has a channel to read the
 // service configuration.
 //
-// Deprecated: service config should be received through name resolver, as
-// specified here.
-// https://github.com/grpc/grpc/blob/master/doc/service_config.md
+// Deprecated: service config should be received through name resolver or via
+// WithDefaultServiceConfig, as specified at
+// https://github.com/grpc/grpc/blob/master/doc/service_config.md.  Will be
+// removed in a future 1.x release.
 func WithServiceConfig(c <-chan ServiceConfig) DialOption {
 	return newFuncDialOption(func(o *dialOptions) {
 		o.scChan = c
@@ -325,7 +322,8 @@ func WithCredentialsBundle(b credentials.Bundle) DialOption {
 // WithTimeout returns a DialOption that configures a timeout for dialing a
 // ClientConn initially. This is valid if and only if WithBlock() is present.
 //
-// Deprecated: use DialContext and context.WithTimeout instead.
+// Deprecated: use DialContext and context.WithTimeout instead.  Will be
+// supported throughout 1.x.
 func WithTimeout(d time.Duration) DialOption {
 	return newFuncDialOption(func(o *dialOptions) {
 		o.timeout = d
@@ -352,7 +350,8 @@ func init() {
 // is returned by f, gRPC checks the error's Temporary() method to decide if it
 // should try to reconnect to the network address.
 //
-// Deprecated: use WithContextDialer instead
+// Deprecated: use WithContextDialer instead.  Will be supported throughout
+// 1.x.
 func WithDialer(f func(string, time.Duration) (net.Conn, error)) DialOption {
 	return WithContextDialer(
 		func(ctx context.Context, addr string) (net.Conn, error) {
@@ -414,6 +413,17 @@ func WithUnaryInterceptor(f UnaryClientInterceptor) DialOption {
 	})
 }

+// WithChainUnaryInterceptor returns a DialOption that specifies the chained
+// interceptor for unary RPCs. The first interceptor will be the outer most,
+// while the last interceptor will be the inner most wrapper around the real call.
+// All interceptors added by this method will be chained, and the interceptor
+// defined by WithUnaryInterceptor will always be prepended to the chain.
+func WithChainUnaryInterceptor(interceptors ...UnaryClientInterceptor) DialOption {
+	return newFuncDialOption(func(o *dialOptions) {
+		o.chainUnaryInts = append(o.chainUnaryInts, interceptors...)
+	})
+}
+
 // WithStreamInterceptor returns a DialOption that specifies the interceptor for
 // streaming RPCs.
 func WithStreamInterceptor(f StreamClientInterceptor) DialOption {
@@ -422,6 +432,17 @@ func WithStreamInterceptor(f StreamClientInterceptor) DialOption {
 	})
 }

+// WithChainStreamInterceptor returns a DialOption that specifies the chained
+// interceptor for unary RPCs. The first interceptor will be the outer most,
+// while the last interceptor will be the inner most wrapper around the real call.
+// All interceptors added by this method will be chained, and the interceptor
+// defined by WithStreamInterceptor will always be prepended to the chain.
+func WithChainStreamInterceptor(interceptors ...StreamClientInterceptor) DialOption {
+	return newFuncDialOption(func(o *dialOptions) {
+		o.chainStreamInts = append(o.chainStreamInts, interceptors...)
+	})
+}
+
 // WithAuthority returns a DialOption that specifies the value to be used as the
 // :authority pseudo-header. This value only works with WithInsecure and has no
 // effect if TransportCredentials are present.
@@ -440,12 +461,12 @@ func WithChannelzParentID(id int64) DialOption {
 	})
 }

-// WithDisableServiceConfig returns a DialOption that causes grpc to ignore any
+// WithDisableServiceConfig returns a DialOption that causes gRPC to ignore any
 // service config provided by the resolver and provides a hint to the resolver
 // to not fetch service configs.
 //
-// Note that, this dial option only disables service config from resolver. If
-// default service config is provided, grpc will use the default service config.
+// Note that this dial option only disables service config from resolver. If
+// default service config is provided, gRPC will use the default service config.
 func WithDisableServiceConfig() DialOption {
 	return newFuncDialOption(func(o *dialOptions) {
 		o.disableServiceConfig = true
@@ -454,8 +475,10 @@ func WithDisableServiceConfig() DialOption {

 // WithDefaultServiceConfig returns a DialOption that configures the default
 // service config, which will be used in cases where:
-// 1. WithDisableServiceConfig is called.
-// 2. Resolver does not return service config or if the resolver gets and invalid config.
+//
+// 1. WithDisableServiceConfig is also used.
+// 2. Resolver does not return a service config or if the resolver returns an
+//    invalid service config.
 //
 // This API is EXPERIMENTAL.
 func WithDefaultServiceConfig(s string) DialOption {
@@ -511,7 +534,6 @@ func withHealthCheckFunc(f internal.HealthChecker) DialOption {
 func defaultDialOptions() dialOptions {
 	return dialOptions{
 		disableRetry:    !envconfig.Retry,
-		reqHandshake:    envconfig.RequireHandshake,
 		healthCheckFunc: internal.HealthCheckFunc,
 		copts: transport.ConnectOptions{
 			WriteBufferSize: defaultWriteBufSize,
@@ -7,13 +7,13 @@ require (
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/mock v1.1.1
 	github.com/golang/protobuf v1.2.0
+	github.com/google/go-cmp v0.2.0
 	golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3
 	golang.org/x/net v0.0.0-20190311183353-d8887717615a
 	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
-	golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f // indirect
 	golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a
-	golang.org/x/tools v0.0.0-20190311212946-11955173bddd
+	golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135
 	google.golang.org/appengine v1.1.0 // indirect
 	google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8
-	honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099
+	honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc
 )
@@ -26,6 +26,7 @@ import (

 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
 	healthpb "google.golang.org/grpc/health/grpc_health_v1"
 	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/backoff"
@@ -51,7 +52,11 @@ func init() {
 	internal.HealthCheckFunc = clientHealthCheck
 }

-func clientHealthCheck(ctx context.Context, newStream func() (interface{}, error), reportHealth func(bool), service string) error {
+const healthCheckMethod = "/grpc.health.v1.Health/Watch"
+
+// This function implements the protocol defined at:
+// https://github.com/grpc/grpc/blob/master/doc/health-checking.md
+func clientHealthCheck(ctx context.Context, newStream func(string) (interface{}, error), setConnectivityState func(connectivity.State), service string) error {
 	tryCnt := 0

 retryConnection:
@@ -65,7 +70,8 @@ retryConnection:
 		if ctx.Err() != nil {
 			return nil
 		}
-		rawS, err := newStream()
+		setConnectivityState(connectivity.Connecting)
+		rawS, err := newStream(healthCheckMethod)
 		if err != nil {
 			continue retryConnection
 		}
@@ -73,7 +79,7 @@ retryConnection:
 		s, ok := rawS.(grpc.ClientStream)
 		// Ideally, this should never happen. But if it happens, the server is marked as healthy for LBing purposes.
 		if !ok {
-			reportHealth(true)
+			setConnectivityState(connectivity.Ready)
 			return fmt.Errorf("newStream returned %v (type %T); want grpc.ClientStream", rawS, rawS)
 		}

@@ -89,19 +95,23 @@ retryConnection:

 			// Reports healthy for the LBing purposes if health check is not implemented in the server.
 			if status.Code(err) == codes.Unimplemented {
-				reportHealth(true)
+				setConnectivityState(connectivity.Ready)
 				return err
 			}

 			// Reports unhealthy if server's Watch method gives an error other than UNIMPLEMENTED.
 			if err != nil {
-				reportHealth(false)
+				setConnectivityState(connectivity.TransientFailure)
 				continue retryConnection
 			}

 			// As a message has been received, removes the need for backoff for the next retry by reseting the try count.
 			tryCnt = 0
-			reportHealth(resp.Status == healthpb.HealthCheckResponse_SERVING)
+			if resp.Status == healthpb.HealthCheckResponse_SERVING {
+				setConnectivityState(connectivity.Ready)
+			} else {
+				setConnectivityState(connectivity.TransientFailure)
+			}
 		}
 	}
 }
@@ -24,6 +24,7 @@
 package channelz

 import (
+	"fmt"
 	"sort"
 	"sync"
 	"sync/atomic"
@@ -95,9 +96,14 @@ func (d *dbWrapper) get() *channelMap {

 // NewChannelzStorage initializes channelz data storage and id generator.
 //
+// This function returns a cleanup function to wait for all channelz state to be reset by the
+// grpc goroutines when those entities get closed. By using this cleanup function, we make sure tests
+// don't mess up each other, i.e. lingering goroutine from previous test doing entity removal happen
+// to remove some entity just register by the new test, since the id space is the same.
+//
 // Note: This function is exported for testing purpose only. User should not call
 // it in most cases.
-func NewChannelzStorage() {
+func NewChannelzStorage() (cleanup func() error) {
 	db.set(&channelMap{
 		topLevelChannels: make(map[int64]struct{}),
 		channels:         make(map[int64]*channel),
@@ -107,6 +113,28 @@ func NewChannelzStorage() {
 		subChannels:      make(map[int64]*subChannel),
 	})
 	idGen.reset()
+	return func() error {
+		var err error
+		cm := db.get()
+		if cm == nil {
+			return nil
+		}
+		for i := 0; i < 1000; i++ {
+			cm.mu.Lock()
+			if len(cm.topLevelChannels) == 0 && len(cm.servers) == 0 && len(cm.channels) == 0 && len(cm.subChannels) == 0 && len(cm.listenSockets) == 0 && len(cm.normalSockets) == 0 {
+				cm.mu.Unlock()
+				// all things stored in the channelz map have been cleared.
+				return nil
+			}
+			cm.mu.Unlock()
+			time.Sleep(10 * time.Millisecond)
+		}
+
+		cm.mu.Lock()
+		err = fmt.Errorf("after 10s the channelz map has not been cleaned up yet, topchannels: %d, servers: %d, channels: %d, subchannels: %d, listen sockets: %d, normal sockets: %d", len(cm.topLevelChannels), len(cm.servers), len(cm.channels), len(cm.subChannels), len(cm.listenSockets), len(cm.normalSockets))
+		cm.mu.Unlock()
+		return err
+	}
 }

 // GetTopChannels returns a slice of top channel's ChannelMetric, along with a
@@ -25,40 +25,11 @@ import (
 )

 const (
-	prefix              = "GRPC_GO_"
-	retryStr            = prefix + "RETRY"
-	requireHandshakeStr = prefix + "REQUIRE_HANDSHAKE"
-)
-
-// RequireHandshakeSetting describes the settings for handshaking.
-type RequireHandshakeSetting int
-
-const (
-	// RequireHandshakeOn indicates to wait for handshake before considering a
-	// connection ready/successful.
-	RequireHandshakeOn RequireHandshakeSetting = iota
-	// RequireHandshakeOff indicates to not wait for handshake before
-	// considering a connection ready/successful.
-	RequireHandshakeOff
+	prefix   = "GRPC_GO_"
+	retryStr = prefix + "RETRY"
 )

 var (
 	// Retry is set if retry is explicitly enabled via "GRPC_GO_RETRY=on".
 	Retry = strings.EqualFold(os.Getenv(retryStr), "on")
-	// RequireHandshake is set based upon the GRPC_GO_REQUIRE_HANDSHAKE
-	// environment variable.
-	//
-	// Will be removed after the 1.18 release.
-	RequireHandshake = RequireHandshakeOn
 )
-
-func init() {
-	switch strings.ToLower(os.Getenv(requireHandshakeStr)) {
-	case "on":
-		fallthrough
-	default:
-		RequireHandshake = RequireHandshakeOn
-	case "off":
-		RequireHandshake = RequireHandshakeOff
-	}
-}
@@ -23,6 +23,8 @@ package internal
 import (
 	"context"
 	"time"
+
+	"google.golang.org/grpc/connectivity"
 )

 var (
@@ -37,10 +39,25 @@ var (
 	// KeepaliveMinPingTime is the minimum ping interval.  This must be 10s by
 	// default, but tests may wish to set it lower for convenience.
 	KeepaliveMinPingTime = 10 * time.Second
+	// ParseServiceConfig is a function to parse JSON service configs into
+	// opaque data structures.
+	ParseServiceConfig func(sc string) (interface{}, error)
+	// StatusRawProto is exported by status/status.go. This func returns a
+	// pointer to the wrapped Status proto for a given status.Status without a
+	// call to proto.Clone(). The returned Status proto should not be mutated by
+	// the caller.
+	StatusRawProto interface{} // func (*status.Status) *spb.Status
 )

 // HealthChecker defines the signature of the client-side LB channel health checking function.
-type HealthChecker func(ctx context.Context, newStream func() (interface{}, error), reportHealth func(bool), serviceName string) error
+//
+// The implementation is expected to create a health checking RPC stream by
+// calling newStream(), watch for the health status of serviceName, and report
+// it's health back by calling setConnectivityState().
+//
+// The health checking protocol is defined at:
+// https://github.com/grpc/grpc/blob/master/doc/health-checking.md
+type HealthChecker func(ctx context.Context, newStream func(string) (interface{}, error), setConnectivityState func(connectivity.State), serviceName string) error

 const (
 	// CredsBundleModeFallback switches GoogleDefaultCreds to fallback mode.
@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"runtime"
 	"sync"
+	"sync/atomic"

 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/hpack"
@@ -84,12 +85,24 @@ func (il *itemList) isEmpty() bool {
 // the control buffer of transport. They represent different aspects of
 // control tasks, e.g., flow control, settings, streaming resetting, etc.

+// maxQueuedTransportResponseFrames is the most queued "transport response"
+// frames we will buffer before preventing new reads from occurring on the
+// transport.  These are control frames sent in response to client requests,
+// such as RST_STREAM due to bad headers or settings acks.
+const maxQueuedTransportResponseFrames = 50
+
+type cbItem interface {
+	isTransportResponseFrame() bool
+}
+
 // registerStream is used to register an incoming stream with loopy writer.
 type registerStream struct {
 	streamID uint32
 	wq       *writeQuota
 }

+func (*registerStream) isTransportResponseFrame() bool { return false }
+
 // headerFrame is also used to register stream on the client-side.
 type headerFrame struct {
 	streamID   uint32
@@ -102,6 +115,10 @@ type headerFrame struct {
 	onOrphaned func(error)    // Valid on client-side
 }

+func (h *headerFrame) isTransportResponseFrame() bool {
+	return h.cleanup != nil && h.cleanup.rst // Results in a RST_STREAM
+}
+
 type cleanupStream struct {
 	streamID uint32
 	rst      bool
@@ -109,6 +126,8 @@ type cleanupStream struct {
 	onWrite  func()
 }

+func (c *cleanupStream) isTransportResponseFrame() bool { return c.rst } // Results in a RST_STREAM
+
 type dataFrame struct {
 	streamID  uint32
 	endStream bool
@@ -119,27 +138,41 @@ type dataFrame struct {
 	onEachWrite func()
 }

+func (*dataFrame) isTransportResponseFrame() bool { return false }
+
 type incomingWindowUpdate struct {
 	streamID  uint32
 	increment uint32
 }

+func (*incomingWindowUpdate) isTransportResponseFrame() bool { return false }
+
 type outgoingWindowUpdate struct {
 	streamID  uint32
 	increment uint32
 }

+func (*outgoingWindowUpdate) isTransportResponseFrame() bool {
+	return false // window updates are throttled by thresholds
+}
+
 type incomingSettings struct {
 	ss []http2.Setting
 }

+func (*incomingSettings) isTransportResponseFrame() bool { return true } // Results in a settings ACK
+
 type outgoingSettings struct {
 	ss []http2.Setting
 }

+func (*outgoingSettings) isTransportResponseFrame() bool { return false }
+
 type incomingGoAway struct {
 }

+func (*incomingGoAway) isTransportResponseFrame() bool { return false }
+
 type goAway struct {
 	code      http2.ErrCode
 	debugData []byte
@@ -147,15 +180,21 @@ type goAway struct {
 	closeConn bool
 }

+func (*goAway) isTransportResponseFrame() bool { return false }
+
 type ping struct {
 	ack  bool
 	data [8]byte
 }

+func (*ping) isTransportResponseFrame() bool { return true }
+
 type outFlowControlSizeRequest struct {
 	resp chan uint32
 }

+func (*outFlowControlSizeRequest) isTransportResponseFrame() bool { return false }
+
 type outStreamState int

 const (
@@ -238,6 +277,14 @@ type controlBuffer struct {
 	consumerWaiting bool
 	list            *itemList
 	err             error
+
+	// transportResponseFrames counts the number of queued items that represent
+	// the response of an action initiated by the peer.  trfChan is created
+	// when transportResponseFrames >= maxQueuedTransportResponseFrames and is
+	// closed and nilled when transportResponseFrames drops below the
+	// threshold.  Both fields are protected by mu.
+	transportResponseFrames int
+	trfChan                 atomic.Value // *chan struct{}
 }

 func newControlBuffer(done <-chan struct{}) *controlBuffer {
@@ -248,12 +295,24 @@ func newControlBuffer(done <-chan struct{}) *controlBuffer {
 	}
 }

-func (c *controlBuffer) put(it interface{}) error {
+// throttle blocks if there are too many incomingSettings/cleanupStreams in the
+// controlbuf.
+func (c *controlBuffer) throttle() {
+	ch, _ := c.trfChan.Load().(*chan struct{})
+	if ch != nil {
+		select {
+		case <-*ch:
+		case <-c.done:
+		}
+	}
+}
+
+func (c *controlBuffer) put(it cbItem) error {
 	_, err := c.executeAndPut(nil, it)
 	return err
 }

-func (c *controlBuffer) executeAndPut(f func(it interface{}) bool, it interface{}) (bool, error) {
+func (c *controlBuffer) executeAndPut(f func(it interface{}) bool, it cbItem) (bool, error) {
 	var wakeUp bool
 	c.mu.Lock()
 	if c.err != nil {
@@ -271,6 +330,15 @@ func (c *controlBuffer) executeAndPut(f func(it interface{}) bool, it interface{
 		c.consumerWaiting = false
 	}
 	c.list.enqueue(it)
+	if it.isTransportResponseFrame() {
+		c.transportResponseFrames++
+		if c.transportResponseFrames == maxQueuedTransportResponseFrames {
+			// We are adding the frame that puts us over the threshold; create
+			// a throttling channel.
+			ch := make(chan struct{})
+			c.trfChan.Store(&ch)
+		}
+	}
 	c.mu.Unlock()
 	if wakeUp {
 		select {
@@ -304,7 +372,17 @@ func (c *controlBuffer) get(block bool) (interface{}, error) {
 			return nil, c.err
 		}
 		if !c.list.isEmpty() {
-			h := c.list.dequeue()
+			h := c.list.dequeue().(cbItem)
+			if h.isTransportResponseFrame() {
+				if c.transportResponseFrames == maxQueuedTransportResponseFrames {
+					// We are removing the frame that put us over the
+					// threshold; close and clear the throttling channel.
+					ch := c.trfChan.Load().(*chan struct{})
+					close(*ch)
+					c.trfChan.Store((*chan struct{})(nil))
+				}
+				c.transportResponseFrames--
+			}
 			c.mu.Unlock()
 			return h, nil
 		}
@@ -149,6 +149,7 @@ func (f *inFlow) maybeAdjust(n uint32) uint32 {
 		n = uint32(math.MaxInt32)
 	}
 	f.mu.Lock()
+	defer f.mu.Unlock()
 	// estSenderQuota is the receiver's view of the maximum number of bytes the sender
 	// can send without a window update.
 	estSenderQuota := int32(f.limit - (f.pendingData + f.pendingUpdate))
@@ -169,10 +170,8 @@ func (f *inFlow) maybeAdjust(n uint32) uint32 {
 			// is padded; We will fallback on the current available window(at least a 1/4th of the limit).
 			f.delta = n
 		}
-		f.mu.Unlock()
 		return f.delta
 	}
-	f.mu.Unlock()
 	return 0
 }

@@ -24,6 +24,7 @@
 package transport

 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -347,7 +348,7 @@ func (ht *serverHandlerTransport) HandleStreams(startStream func(*Stream), trace
 		ht.stats.HandleRPC(s.ctx, inHeader)
 	}
 	s.trReader = &transportReader{
-		reader:        &recvBufferReader{ctx: s.ctx, ctxDone: s.ctx.Done(), recv: s.buf},
+		reader:        &recvBufferReader{ctx: s.ctx, ctxDone: s.ctx.Done(), recv: s.buf, freeBuffer: func(*bytes.Buffer) {}},
 		windowHandler: func(int) {},
 	}

@@ -361,7 +362,7 @@ func (ht *serverHandlerTransport) HandleStreams(startStream func(*Stream), trace
 		for buf := make([]byte, readSize); ; {
 			n, err := req.Body.Read(buf)
 			if n > 0 {
-				s.buf.put(recvMsg{data: buf[:n:n]})
+				s.buf.put(recvMsg{buffer: bytes.NewBuffer(buf[:n:n])})
 				buf = buf[n:]
 			}
 			if err != nil {
@@ -117,6 +117,8 @@ type http2Client struct {

 	onGoAway func(GoAwayReason)
 	onClose  func()
+
+	bufferPool *bufferPool
 }

 func dial(ctx context.Context, fn func(context.Context, string) (net.Conn, error), addr string) (net.Conn, error) {
@@ -249,6 +251,7 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne
 		onGoAway:              onGoAway,
 		onClose:               onClose,
 		keepaliveEnabled:      keepaliveEnabled,
+		bufferPool:            newBufferPool(),
 	}
 	t.controlBuf = newControlBuffer(t.ctxDone)
 	if opts.InitialWindowSize >= defaultWindowSize {
@@ -367,6 +370,7 @@ func (t *http2Client) newStream(ctx context.Context, callHdr *CallHdr) *Stream {
 			closeStream: func(err error) {
 				t.CloseStream(s, err)
 			},
+			freeBuffer: t.bufferPool.put,
 		},
 		windowHandler: func(n int) {
 			t.updateWindow(s, uint32(n))
@@ -437,6 +441,15 @@ func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr)

 	if md, added, ok := metadata.FromOutgoingContextRaw(ctx); ok {
 		var k string
+		for k, vv := range md {
+			// HTTP doesn't allow you to set pseudoheaders after non pseudoheaders were set.
+			if isReservedHeader(k) {
+				continue
+			}
+			for _, v := range vv {
+				headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)})
+			}
+		}
 		for _, vv := range added {
 			for i, v := range vv {
 				if i%2 == 0 {
@@ -450,15 +463,6 @@ func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr)
 				headerFields = append(headerFields, hpack.HeaderField{Name: strings.ToLower(k), Value: encodeMetadataHeader(k, v)})
 			}
 		}
-		for k, vv := range md {
-			// HTTP doesn't allow you to set pseudoheaders after non pseudoheaders were set.
-			if isReservedHeader(k) {
-				continue
-			}
-			for _, v := range vv {
-				headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)})
-			}
-		}
 	}
 	if md, ok := t.md.(*metadata.MD); ok {
 		for k, vv := range *md {
@@ -489,6 +493,9 @@ func (t *http2Client) createAudience(callHdr *CallHdr) string {
 }

 func (t *http2Client) getTrAuthData(ctx context.Context, audience string) (map[string]string, error) {
+	if len(t.perRPCCreds) == 0 {
+		return nil, nil
+	}
 	authData := map[string]string{}
 	for _, c := range t.perRPCCreds {
 		data, err := c.GetRequestMetadata(ctx, audience)
@@ -509,7 +516,7 @@ func (t *http2Client) getTrAuthData(ctx context.Context, audience string) (map[s
 }

 func (t *http2Client) getCallAuthData(ctx context.Context, audience string, callHdr *CallHdr) (map[string]string, error) {
-	callAuthData := map[string]string{}
+	var callAuthData map[string]string
 	// Check if credentials.PerRPCCredentials were provided via call options.
 	// Note: if these credentials are provided both via dial options and call
 	// options, then both sets of credentials will be applied.
@@ -521,6 +528,7 @@ func (t *http2Client) getCallAuthData(ctx context.Context, audience string, call
 		if err != nil {
 			return nil, status.Errorf(codes.Internal, "transport: %v", err)
 		}
+		callAuthData = make(map[string]string, len(data))
 		for k, v := range data {
 			// Capital header names are illegal in HTTP/2
 			k = strings.ToLower(k)
@@ -549,10 +557,9 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea
 		s.write(recvMsg{err: err})
 		close(s.done)
 		// If headerChan isn't closed, then close it.
-		if atomic.SwapUint32(&s.headerDone, 1) == 0 {
+		if atomic.CompareAndSwapUint32(&s.headerChanClosed, 0, 1) {
 			close(s.headerChan)
 		}
-
 	}
 	hdr := &headerFrame{
 		hf:        headerFields,
@@ -713,7 +720,7 @@ func (t *http2Client) closeStream(s *Stream, err error, rst bool, rstCode http2.
 		s.write(recvMsg{err: err})
 	}
 	// If headerChan isn't closed, then close it.
-	if atomic.SwapUint32(&s.headerDone, 1) == 0 {
+	if atomic.CompareAndSwapUint32(&s.headerChanClosed, 0, 1) {
 		s.noHeaders = true
 		close(s.headerChan)
 	}
@@ -765,6 +772,9 @@ func (t *http2Client) Close() error {
 		t.mu.Unlock()
 		return nil
 	}
+	// Call t.onClose before setting the state to closing to prevent the client
+	// from attempting to create new streams ASAP.
+	t.onClose()
 	t.state = closing
 	streams := t.activeStreams
 	t.activeStreams = nil
@@ -785,7 +795,6 @@ func (t *http2Client) Close() error {
 		}
 		t.statsHandler.HandleConn(t.ctx, connEnd)
 	}
-	t.onClose()
 	return err
 }

@@ -794,21 +803,21 @@ func (t *http2Client) Close() error {
 // stream is closed.  If there are no active streams, the transport is closed
 // immediately.  This does nothing if the transport is already draining or
 // closing.
-func (t *http2Client) GracefulClose() error {
+func (t *http2Client) GracefulClose() {
 	t.mu.Lock()
 	// Make sure we move to draining only from active.
 	if t.state == draining || t.state == closing {
 		t.mu.Unlock()
-		return nil
+		return
 	}
 	t.state = draining
 	active := len(t.activeStreams)
 	t.mu.Unlock()
 	if active == 0 {
-		return t.Close()
+		t.Close()
+		return
 	}
 	t.controlBuf.put(&incomingGoAway{})
-	return nil
 }

 // Write formats the data into HTTP2 data frame(s) and sends it out. The caller
@@ -946,9 +955,10 @@ func (t *http2Client) handleData(f *http2.DataFrame) {
 		// guarantee f.Data() is consumed before the arrival of next frame.
 		// Can this copy be eliminated?
 		if len(f.Data()) > 0 {
-			data := make([]byte, len(f.Data()))
-			copy(data, f.Data())
-			s.write(recvMsg{data: data})
+			buffer := t.bufferPool.get()
+			buffer.Reset()
+			buffer.Write(f.Data())
+			s.write(recvMsg{buffer: buffer})
 		}
 	}
 	// The server has closed the stream without sending trailers.  Record that
@@ -973,9 +983,9 @@ func (t *http2Client) handleRSTStream(f *http2.RSTStreamFrame) {
 		statusCode = codes.Unknown
 	}
 	if statusCode == codes.Canceled {
-		// Our deadline was already exceeded, and that was likely the cause of
-		// this cancelation.  Alter the status code accordingly.
-		if d, ok := s.ctx.Deadline(); ok && d.After(time.Now()) {
+		if d, ok := s.ctx.Deadline(); ok && !d.After(time.Now()) {
+			// Our deadline was already exceeded, and that was likely the cause
+			// of this cancelation.  Alter the status code accordingly.
 			statusCode = codes.DeadlineExceeded
 		}
 	}
@@ -1080,11 +1090,12 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) {
 	default:
 		t.setGoAwayReason(f)
 		close(t.goAway)
-		t.state = draining
 		t.controlBuf.put(&incomingGoAway{})
-
-		// This has to be a new goroutine because we're still using the current goroutine to read in the transport.
+		// Notify the clientconn about the GOAWAY before we set the state to
+		// draining, to allow the client to stop attempting to create streams
+		// before disallowing new streams on this connection.
 		t.onGoAway(t.goAwayReason)
+		t.state = draining
 	}
 	// All streams with IDs greater than the GoAwayId
 	// and smaller than the previous GoAway ID should be killed.
@@ -1142,26 +1153,24 @@ func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) {
 	}
 	endStream := frame.StreamEnded()
 	atomic.StoreUint32(&s.bytesReceived, 1)
-	initialHeader := atomic.SwapUint32(&s.headerDone, 1) == 0
+	initialHeader := atomic.LoadUint32(&s.headerChanClosed) == 0

 	if !initialHeader && !endStream {
-		// As specified by RFC 7540, a HEADERS frame (and associated CONTINUATION frames) can only appear
-		// at the start or end of a stream. Therefore, second HEADERS frame must have EOS bit set.
+		// As specified by gRPC over HTTP2, a HEADERS frame (and associated CONTINUATION frames) can only appear at the start or end of a stream. Therefore, second HEADERS frame must have EOS bit set.
 		st := status.New(codes.Internal, "a HEADERS frame cannot appear in the middle of a stream")
 		t.closeStream(s, st.Err(), true, http2.ErrCodeProtocol, st, nil, false)
 		return
 	}

 	state := &decodeState{}
-	// Initialize isGRPC value to be !initialHeader, since if a gRPC ResponseHeader has been received
-	// which indicates peer speaking gRPC, we are in gRPC mode.
+	// Initialize isGRPC value to be !initialHeader, since if a gRPC Response-Headers has already been received, then it means that the peer is speaking gRPC and we are in gRPC mode.
 	state.data.isGRPC = !initialHeader
 	if err := state.decodeHeader(frame); err != nil {
 		t.closeStream(s, err, true, http2.ErrCodeProtocol, status.Convert(err), nil, endStream)
 		return
 	}

-	var isHeader bool
+	isHeader := false
 	defer func() {
 		if t.statsHandler != nil {
 			if isHeader {
@@ -1180,10 +1189,10 @@ func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) {
 		}
 	}()

-	// If headers haven't been received yet.
-	if initialHeader {
+	// If headerChan hasn't been closed yet
+	if atomic.CompareAndSwapUint32(&s.headerChanClosed, 0, 1) {
 		if !endStream {
-			// Headers frame is ResponseHeader.
+			// HEADERS frame block carries a Response-Headers.
 			isHeader = true
 			// These values can be set without any synchronization because
 			// stream goroutine will read it only after seeing a closed
@@ -1192,14 +1201,17 @@ func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) {
 			if len(state.data.mdata) > 0 {
 				s.header = state.data.mdata
 			}
-			close(s.headerChan)
-			return
+		} else {
+			// HEADERS frame block carries a Trailers-Only.
+			s.noHeaders = true
 		}
-		// Headers frame is Trailers-only.
-		s.noHeaders = true
 		close(s.headerChan)
 	}

+	if !endStream {
+		return
+	}
+
 	// if client received END_STREAM from server while stream was still active, send RST_STREAM
 	rst := s.getState() == streamActive
 	t.closeStream(s, io.EOF, rst, http2.ErrCodeNo, state.status(), state.data.mdata, true)
@@ -1233,6 +1245,7 @@ func (t *http2Client) reader() {

 	// loop to keep reading incoming messages on this transport.
 	for {
+		t.controlBuf.throttle()
 		frame, err := t.framer.fr.ReadFrame()
 		if t.keepaliveEnabled {
 			atomic.CompareAndSwapUint32(&t.activity, 0, 1)
@@ -1320,6 +1333,7 @@ func (t *http2Client) keepalive() {
 					timer.Reset(t.kp.Time)
 					continue
 				}
+				infof("transport: closing client transport due to idleness.")
 				t.Close()
 				return
 			case <-t.ctx.Done():
@@ -35,9 +35,11 @@ import (
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/hpack"

+	spb "google.golang.org/genproto/googleapis/rpc/status"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/channelz"
 	"google.golang.org/grpc/internal/grpcrand"
 	"google.golang.org/grpc/keepalive"
@@ -55,6 +57,9 @@ var (
 	// ErrHeaderListSizeLimitViolation indicates that the header list size is larger
 	// than the limit set by peer.
 	ErrHeaderListSizeLimitViolation = errors.New("transport: trying to send header list size larger than the limit set by peer")
+	// statusRawProto is a function to get to the raw status proto wrapped in a
+	// status.Status without a proto.Clone().
+	statusRawProto = internal.StatusRawProto.(func(*status.Status) *spb.Status)
 )

 // http2Server implements the ServerTransport interface with HTTP2.
@@ -119,6 +124,7 @@ type http2Server struct {
 	// Fields below are for channelz metric collection.
 	channelzID int64 // channelz unique identification number
 	czData     *channelzData
+	bufferPool *bufferPool
 }

 // newHTTP2Server constructs a ServerTransport based on HTTP2. ConnectionError is
@@ -132,7 +138,10 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err
 	}
 	framer := newFramer(conn, writeBufSize, readBufSize, maxHeaderListSize)
 	// Send initial settings as connection preface to client.
-	var isettings []http2.Setting
+	isettings := []http2.Setting{{
+		ID:  http2.SettingMaxFrameSize,
+		Val: http2MaxFrameLen,
+	}}
 	// TODO(zhaoq): Have a better way to signal "no limit" because 0 is
 	// permitted in the HTTP2 spec.
 	maxStreams := config.MaxStreams
@@ -220,6 +229,7 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err
 		kep:               kep,
 		initialWindowSize: iwz,
 		czData:            new(channelzData),
+		bufferPool:        newBufferPool(),
 	}
 	t.controlBuf = newControlBuffer(t.ctxDone)
 	if dynamicWindow {
@@ -405,9 +415,10 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func(
 	s.wq = newWriteQuota(defaultWriteQuota, s.ctxDone)
 	s.trReader = &transportReader{
 		reader: &recvBufferReader{
-			ctx:     s.ctx,
-			ctxDone: s.ctxDone,
-			recv:    s.buf,
+			ctx:        s.ctx,
+			ctxDone:    s.ctxDone,
+			recv:       s.buf,
+			freeBuffer: t.bufferPool.put,
 		},
 		windowHandler: func(n int) {
 			t.updateWindow(s, uint32(n))
@@ -428,6 +439,7 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func(
 func (t *http2Server) HandleStreams(handle func(*Stream), traceCtx func(context.Context, string) context.Context) {
 	defer close(t.readerDone)
 	for {
+		t.controlBuf.throttle()
 		frame, err := t.framer.fr.ReadFrame()
 		atomic.StoreUint32(&t.activity, 1)
 		if err != nil {
@@ -591,9 +603,10 @@ func (t *http2Server) handleData(f *http2.DataFrame) {
 		// guarantee f.Data() is consumed before the arrival of next frame.
 		// Can this copy be eliminated?
 		if len(f.Data()) > 0 {
-			data := make([]byte, len(f.Data()))
-			copy(data, f.Data())
-			s.write(recvMsg{data: data})
+			buffer := t.bufferPool.get()
+			buffer.Reset()
+			buffer.Write(f.Data())
+			s.write(recvMsg{buffer: buffer})
 		}
 	}
 	if f.Header().Flags.Has(http2.FlagDataEndStream) {
@@ -757,6 +770,10 @@ func (t *http2Server) WriteHeader(s *Stream, md metadata.MD) error {
 	return nil
 }

+func (t *http2Server) setResetPingStrikes() {
+	atomic.StoreUint32(&t.resetPingStrikes, 1)
+}
+
 func (t *http2Server) writeHeaderLocked(s *Stream) error {
 	// TODO(mmukhi): Benchmark if the performance gets better if count the metadata and other header fields
 	// first and create a slice of that exact size.
@@ -771,9 +788,7 @@ func (t *http2Server) writeHeaderLocked(s *Stream) error {
 		streamID:  s.id,
 		hf:        headerFields,
 		endStream: false,
-		onWrite: func() {
-			atomic.StoreUint32(&t.resetPingStrikes, 1)
-		},
+		onWrite:   t.setResetPingStrikes,
 	})
 	if !success {
 		if err != nil {
@@ -817,7 +832,7 @@ func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error {
 	headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-status", Value: strconv.Itoa(int(st.Code()))})
 	headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-message", Value: encodeGrpcMessage(st.Message())})

-	if p := st.Proto(); p != nil && len(p.Details) > 0 {
+	if p := statusRawProto(st); p != nil && len(p.Details) > 0 {
 		stBytes, err := proto.Marshal(p)
 		if err != nil {
 			// TODO: return error instead, when callers are able to handle it.
@@ -833,9 +848,7 @@ func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error {
 		streamID:  s.id,
 		hf:        headerFields,
 		endStream: true,
-		onWrite: func() {
-			atomic.StoreUint32(&t.resetPingStrikes, 1)
-		},
+		onWrite:   t.setResetPingStrikes,
 	}
 	s.hdrMu.Unlock()
 	success, err := t.controlBuf.execute(t.checkForHeaderListSize, trailingHeader)
@@ -887,12 +900,10 @@ func (t *http2Server) Write(s *Stream, hdr []byte, data []byte, opts *Options) e
 	hdr = append(hdr, data[:emptyLen]...)
 	data = data[emptyLen:]
 	df := &dataFrame{
-		streamID: s.id,
-		h:        hdr,
-		d:        data,
-		onEachWrite: func() {
-			atomic.StoreUint32(&t.resetPingStrikes, 1)
-		},
+		streamID:    s.id,
+		h:           hdr,
+		d:           data,
+		onEachWrite: t.setResetPingStrikes,
 	}
 	if err := s.wq.get(int32(len(hdr) + len(data))); err != nil {
 		select {
@@ -958,6 +969,7 @@ func (t *http2Server) keepalive() {
 			select {
 			case <-maxAge.C:
 				// Close the connection after grace period.
+				infof("transport: closing server transport due to maximum connection age.")
 				t.Close()
 				// Resetting the timer so that the clean-up doesn't deadlock.
 				maxAge.Reset(infinity)
@@ -971,6 +983,7 @@ func (t *http2Server) keepalive() {
 				continue
 			}
 			if pingSent {
+				infof("transport: closing server transport due to idleness.")
 				t.Close()
 				// Resetting the timer so that the clean-up doesn't deadlock.
 				keepalive.Reset(infinity)
@@ -1019,13 +1032,7 @@ func (t *http2Server) Close() error {
 }

 // deleteStream deletes the stream s from transport's active streams.
-func (t *http2Server) deleteStream(s *Stream, eosReceived bool) (oldState streamState) {
-	oldState = s.swapState(streamDone)
-	if oldState == streamDone {
-		// If the stream was already done, return.
-		return oldState
-	}
-
+func (t *http2Server) deleteStream(s *Stream, eosReceived bool) {
 	// In case stream sending and receiving are invoked in separate
 	// goroutines (e.g., bi-directional streaming), cancel needs to be
 	// called to interrupt the potential blocking on other goroutines.
@@ -1047,15 +1054,13 @@ func (t *http2Server) deleteStream(s *Stream, eosReceived bool) (oldState stream
 			atomic.AddInt64(&t.czData.streamsFailed, 1)
 		}
 	}
-
-	return oldState
 }

 // finishStream closes the stream and puts the trailing headerFrame into controlbuf.
 func (t *http2Server) finishStream(s *Stream, rst bool, rstCode http2.ErrCode, hdr *headerFrame, eosReceived bool) {
-	oldState := t.deleteStream(s, eosReceived)
-	// If the stream is already closed, then don't put trailing header to controlbuf.
+	oldState := s.swapState(streamDone)
 	if oldState == streamDone {
+		// If the stream was already done, return.
 		return
 	}

@@ -1063,14 +1068,18 @@ func (t *http2Server) finishStream(s *Stream, rst bool, rstCode http2.ErrCode, h
 		streamID: s.id,
 		rst:      rst,
 		rstCode:  rstCode,
-		onWrite:  func() {},
+		onWrite: func() {
+			t.deleteStream(s, eosReceived)
+		},
 	}
 	t.controlBuf.put(hdr)
 }

 // closeStream clears the footprint of a stream when the stream is not needed any more.
 func (t *http2Server) closeStream(s *Stream, rst bool, rstCode http2.ErrCode, eosReceived bool) {
+	s.swapState(streamDone)
 	t.deleteStream(s, eosReceived)
+
 	t.controlBuf.put(&cleanupStream{
 		streamID: s.id,
 		rst:      rst,
@@ -667,6 +667,7 @@ func newFramer(conn net.Conn, writeBufferSize, readBufferSize int, maxHeaderList
 		writer: w,
 		fr:     http2.NewFramer(w, r),
 	}
+	f.fr.SetMaxReadFrameSize(http2MaxFrameLen)
 	// Opt-in to Frame reuse API on framer to reduce garbage.
 	// Frames aren't safe to read from after a subsequent call to ReadFrame.
 	f.fr.SetReuseFrames()
@@ -22,6 +22,7 @@
 package transport

 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -39,10 +40,32 @@ import (
 	"google.golang.org/grpc/tap"
 )

+type bufferPool struct {
+	pool sync.Pool
+}
+
+func newBufferPool() *bufferPool {
+	return &bufferPool{
+		pool: sync.Pool{
+			New: func() interface{} {
+				return new(bytes.Buffer)
+			},
+		},
+	}
+}
+
+func (p *bufferPool) get() *bytes.Buffer {
+	return p.pool.Get().(*bytes.Buffer)
+}
+
+func (p *bufferPool) put(b *bytes.Buffer) {
+	p.pool.Put(b)
+}
+
 // recvMsg represents the received msg from the transport. All transport
 // protocol specific info has been removed.
 type recvMsg struct {
-	data []byte
+	buffer *bytes.Buffer
 	// nil: received some data
 	// io.EOF: stream is completed. data is nil.
 	// other non-nil error: transport failure. data is nil.
@@ -117,8 +140,9 @@ type recvBufferReader struct {
 	ctx         context.Context
 	ctxDone     <-chan struct{} // cache of ctx.Done() (for performance).
 	recv        *recvBuffer
-	last        []byte // Stores the remaining data in the previous calls.
+	last        *bytes.Buffer // Stores the remaining data in the previous calls.
 	err         error
+	freeBuffer  func(*bytes.Buffer)
 }

 // Read reads the next len(p) bytes from last. If last is drained, it tries to
@@ -128,10 +152,13 @@ func (r *recvBufferReader) Read(p []byte) (n int, err error) {
 	if r.err != nil {
 		return 0, r.err
 	}
-	if r.last != nil && len(r.last) > 0 {
+	if r.last != nil {
 		// Read remaining data left in last call.
-		copied := copy(p, r.last)
-		r.last = r.last[copied:]
+		copied, _ := r.last.Read(p)
+		if r.last.Len() == 0 {
+			r.freeBuffer(r.last)
+			r.last = nil
+		}
 		return copied, nil
 	}
 	if r.closeStream != nil {
@@ -157,6 +184,19 @@ func (r *recvBufferReader) readClient(p []byte) (n int, err error) {
 	// r.readAdditional acts on that message and returns the necessary error.
 	select {
 	case <-r.ctxDone:
+		// Note that this adds the ctx error to the end of recv buffer, and
+		// reads from the head. This will delay the error until recv buffer is
+		// empty, thus will delay ctx cancellation in Recv().
+		//
+		// It's done this way to fix a race between ctx cancel and trailer. The
+		// race was, stream.Recv() may return ctx error if ctxDone wins the
+		// race, but stream.Trailer() may return a non-nil md because the stream
+		// was not marked as done when trailer is received. This closeStream
+		// call will mark stream as done, thus fix the race.
+		//
+		// TODO: delaying ctx error seems like a unnecessary side effect. What
+		// we really want is to mark the stream as done, and return ctx error
+		// faster.
 		r.closeStream(ContextErr(r.ctx.Err()))
 		m := <-r.recv.get()
 		return r.readAdditional(m, p)
@@ -170,8 +210,13 @@ func (r *recvBufferReader) readAdditional(m recvMsg, p []byte) (n int, err error
 	if m.err != nil {
 		return 0, m.err
 	}
-	copied := copy(p, m.data)
-	r.last = m.data[copied:]
+	copied, _ := m.buffer.Read(p)
+	if m.buffer.Len() == 0 {
+		r.freeBuffer(m.buffer)
+		r.last = nil
+	} else {
+		r.last = m.buffer
+	}
 	return copied, nil
 }

@@ -204,8 +249,8 @@ type Stream struct {
 	// is used to adjust flow control, if needed.
 	requestRead func(int)

-	headerChan chan struct{} // closed to indicate the end of header metadata.
-	headerDone uint32        // set when headerChan is closed. Used to avoid closing headerChan multiple times.
+	headerChan       chan struct{} // closed to indicate the end of header metadata.
+	headerChanClosed uint32        // set when headerChan is closed. Used to avoid closing headerChan multiple times.

 	// hdrMu protects header and trailer metadata on the server-side.
 	hdrMu sync.Mutex
@@ -266,6 +311,14 @@ func (s *Stream) waitOnHeader() error {
 	}
 	select {
 	case <-s.ctx.Done():
+		// We prefer success over failure when reading messages because we delay
+		// context error in stream.Read(). To keep behavior consistent, we also
+		// prefer success here.
+		select {
+		case <-s.headerChan:
+			return nil
+		default:
+		}
 		return ContextErr(s.ctx.Err())
 	case <-s.headerChan:
 		return nil
@@ -578,9 +631,12 @@ type ClientTransport interface {
 	// is called only once.
 	Close() error

-	// GracefulClose starts to tear down the transport. It stops accepting
-	// new RPCs and wait the completion of the pending RPCs.
-	GracefulClose() error
+	// GracefulClose starts to tear down the transport: the transport will stop
+	// accepting new RPCs and NewStream will return error. Once all streams are
+	// finished, the transport will close.
+	//
+	// It does not block.
+	GracefulClose()

 	// Write sends the data for the given stream. A nil stream indicates
 	// the write is to be performed on the transport as a whole.
@@ -17,9 +17,8 @@
 */

 // Package naming defines the naming API and related data structures for gRPC.
-// The interface is EXPERIMENTAL and may be subject to change.
 //
-// Deprecated: please use package resolver.
+// This package is deprecated: please use package resolver instead.
 package naming

 // Operation defines the corresponding operations for a name resolution change.
@@ -120,6 +120,14 @@ func (bp *pickerWrapper) pick(ctx context.Context, failfast bool, opts balancer.
 			bp.mu.Unlock()
 			select {
 			case <-ctx.Done():
+				if connectionErr := bp.connectionError(); connectionErr != nil {
+					switch ctx.Err() {
+					case context.DeadlineExceeded:
+						return nil, nil, status.Errorf(codes.DeadlineExceeded, "latest connection error: %v", connectionErr)
+					case context.Canceled:
+						return nil, nil, status.Errorf(codes.Canceled, "latest connection error: %v", connectionErr)
+					}
+				}
 				return nil, nil, ctx.Err()
 			case <-ch:
 			}
@@ -51,14 +51,18 @@ type pickfirstBalancer struct {

 func (b *pickfirstBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) {
 	if err != nil {
-		grpclog.Infof("pickfirstBalancer: HandleResolvedAddrs called with error %v", err)
+		if grpclog.V(2) {
+			grpclog.Infof("pickfirstBalancer: HandleResolvedAddrs called with error %v", err)
+		}
 		return
 	}
 	if b.sc == nil {
 		b.sc, err = b.cc.NewSubConn(addrs, balancer.NewSubConnOptions{})
 		if err != nil {
 			//TODO(yuxuanli): why not change the cc state to Idle?
-			grpclog.Errorf("pickfirstBalancer: failed to NewSubConn: %v", err)
+			if grpclog.V(2) {
+				grpclog.Errorf("pickfirstBalancer: failed to NewSubConn: %v", err)
+			}
 			return
 		}
 		b.cc.UpdateBalancerState(connectivity.Idle, &picker{sc: b.sc})
@@ -70,9 +74,13 @@ func (b *pickfirstBalancer) HandleResolvedAddrs(addrs []resolver.Address, err er
 }

 func (b *pickfirstBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) {
-	grpclog.Infof("pickfirstBalancer: HandleSubConnStateChange: %p, %v", sc, s)
+	if grpclog.V(2) {
+		grpclog.Infof("pickfirstBalancer: HandleSubConnStateChange: %p, %v", sc, s)
+	}
 	if b.sc != sc {
-		grpclog.Infof("pickfirstBalancer: ignored state change because sc is not recognized")
+		if grpclog.V(2) {
+			grpclog.Infof("pickfirstBalancer: ignored state change because sc is not recognized")
+		}
 		return
 	}
 	if s == connectivity.Shutdown {
@@ -0,0 +1,64 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package grpc
+
+import (
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// PreparedMsg is responsible for creating a Marshalled and Compressed object.
+//
+// This API is EXPERIMENTAL.
+type PreparedMsg struct {
+	// Struct for preparing msg before sending them
+	encodedData []byte
+	hdr         []byte
+	payload     []byte
+}
+
+// Encode marshalls and compresses the message using the codec and compressor for the stream.
+func (p *PreparedMsg) Encode(s Stream, msg interface{}) error {
+	ctx := s.Context()
+	rpcInfo, ok := rpcInfoFromContext(ctx)
+	if !ok {
+		return status.Errorf(codes.Internal, "grpc: unable to get rpcInfo")
+	}
+
+	// check if the context has the relevant information to prepareMsg
+	if rpcInfo.preloaderInfo == nil {
+		return status.Errorf(codes.Internal, "grpc: rpcInfo.preloaderInfo is nil")
+	}
+	if rpcInfo.preloaderInfo.codec == nil {
+		return status.Errorf(codes.Internal, "grpc: rpcInfo.preloaderInfo.codec is nil")
+	}
+
+	// prepare the msg
+	data, err := encode(rpcInfo.preloaderInfo.codec, msg)
+	if err != nil {
+		return err
+	}
+	p.encodedData = data
+	compData, err := compress(data, rpcInfo.preloaderInfo.cp, rpcInfo.preloaderInfo.comp)
+	if err != nil {
+		return err
+	}
+	p.hdr, p.payload = msgHeader(data, compData)
+	return nil
+}
@@ -66,6 +66,9 @@ var (

 var (
 	defaultResolver netResolver = net.DefaultResolver
+	// To prevent excessive re-resolution, we enforce a rate limit on DNS
+	// resolution requests.
+	minDNSResRate = 30 * time.Second
 )

 var customAuthorityDialler = func(authority string) func(ctx context.Context, network, address string) (net.Conn, error) {
@@ -241,7 +244,13 @@ func (d *dnsResolver) watcher() {
 			return
 		case <-d.t.C:
 		case <-d.rn:
+			if !d.t.Stop() {
+				// Before resetting a timer, it should be stopped to prevent racing with
+				// reads on it's channel.
+				<-d.t.C
+			}
 		}
+
 		result, sc := d.lookup()
 		// Next lookup should happen within an interval defined by d.freq. It may be
 		// more often due to exponential retry on empty address list.
@@ -254,6 +263,16 @@ func (d *dnsResolver) watcher() {
 		}
 		d.cc.NewServiceConfig(sc)
 		d.cc.NewAddress(result)
+
+		// Sleep to prevent excessive re-resolutions. Incoming resolution requests
+		// will be queued in d.rn.
+		t := time.NewTimer(minDNSResRate)
+		select {
+		case <-t.C:
+		case <-d.ctx.Done():
+			t.Stop()
+			return
+		}
 	}
 }

@@ -20,6 +20,10 @@
 // All APIs in this package are experimental.
 package resolver

+import (
+	"google.golang.org/grpc/serviceconfig"
+)
+
 var (
 	// m is a map from scheme to resolver builder.
 	m = make(map[string]Builder)
@@ -100,11 +104,12 @@ type BuildOption struct {

 // State contains the current Resolver state relevant to the ClientConn.
 type State struct {
-	Addresses     []Address // Resolved addresses for the target
-	ServiceConfig string    // JSON representation of the service config
+	Addresses []Address // Resolved addresses for the target
+	// ServiceConfig is the parsed service config; obtained from
+	// serviceconfig.Parse.
+	ServiceConfig serviceconfig.Config

 	// TODO: add Err error
-	// TODO: add ParsedServiceConfig interface{}
 }

 // ClientConn contains the callbacks for resolver to notify any updates
@@ -132,6 +137,21 @@ type ClientConn interface {

 // Target represents a target for gRPC, as specified in:
 // https://github.com/grpc/grpc/blob/master/doc/naming.md.
+// It is parsed from the target string that gets passed into Dial or DialContext by the user. And
+// grpc passes it to the resolver and the balancer.
+//
+// If the target follows the naming spec, and the parsed scheme is registered with grpc, we will
+// parse the target string according to the spec. e.g. "dns://some_authority/foo.bar" will be parsed
+// into &Target{Scheme: "dns", Authority: "some_authority", Endpoint: "foo.bar"}
+//
+// If the target does not contain a scheme, we will apply the default scheme, and set the Target to
+// be the full target string. e.g. "foo.bar" will be parsed into
+// &Target{Scheme: resolver.GetDefaultScheme(), Endpoint: "foo.bar"}.
+//
+// If the parsed scheme is not registered (i.e. no corresponding resolver available to resolve the
+// endpoint), we set the Scheme to be the default scheme, and set the Endpoint to be the full target
+// string. e.g. target string "unknown_scheme://authority/endpoint" will be parsed into
+// &Target{Scheme: resolver.GetDefaultScheme(), Endpoint: "unknown_scheme://authority/endpoint"}.
 type Target struct {
 	Scheme    string
 	Authority string
@@ -138,19 +138,22 @@ func (ccr *ccResolverWrapper) NewServiceConfig(sc string) {
 		return
 	}
 	grpclog.Infof("ccResolverWrapper: got new service config: %v", sc)
-	if channelz.IsOn() {
-		ccr.addChannelzTraceEvent(resolver.State{Addresses: ccr.curState.Addresses, ServiceConfig: sc})
+	c, err := parseServiceConfig(sc)
+	if err != nil {
+		return
 	}
-	ccr.curState.ServiceConfig = sc
+	if channelz.IsOn() {
+		ccr.addChannelzTraceEvent(resolver.State{Addresses: ccr.curState.Addresses, ServiceConfig: c})
+	}
+	ccr.curState.ServiceConfig = c
 	ccr.cc.updateResolverState(ccr.curState)
 }

 func (ccr *ccResolverWrapper) addChannelzTraceEvent(s resolver.State) {
-	if s.ServiceConfig == ccr.curState.ServiceConfig && (len(ccr.curState.Addresses) == 0) == (len(s.Addresses) == 0) {
-		return
-	}
 	var updates []string
-	if s.ServiceConfig != ccr.curState.ServiceConfig {
+	oldSC, oldOK := ccr.curState.ServiceConfig.(*ServiceConfig)
+	newSC, newOK := s.ServiceConfig.(*ServiceConfig)
+	if oldOK != newOK || (oldOK && newOK && oldSC.rawJSONString != newSC.rawJSONString) {
 		updates = append(updates, "service config updated")
 	}
 	if len(ccr.curState.Addresses) > 0 && len(s.Addresses) == 0 {
@@ -694,14 +694,34 @@ func recv(p *parser, c baseCodec, s *transport.Stream, dc Decompressor, m interf
 	return nil
 }

+// Information about RPC
 type rpcInfo struct {
-	failfast bool
+	failfast      bool
+	preloaderInfo *compressorInfo
+}
+
+// Information about Preloader
+// Responsible for storing codec, and compressors
+// If stream (s) has  context s.Context which stores rpcInfo that has non nil
+// pointers to codec, and compressors, then we can use preparedMsg for Async message prep
+// and reuse marshalled bytes
+type compressorInfo struct {
+	codec baseCodec
+	cp    Compressor
+	comp  encoding.Compressor
 }

 type rpcInfoContextKey struct{}

-func newContextWithRPCInfo(ctx context.Context, failfast bool) context.Context {
-	return context.WithValue(ctx, rpcInfoContextKey{}, &rpcInfo{failfast: failfast})
+func newContextWithRPCInfo(ctx context.Context, failfast bool, codec baseCodec, cp Compressor, comp encoding.Compressor) context.Context {
+	return context.WithValue(ctx, rpcInfoContextKey{}, &rpcInfo{
+		failfast: failfast,
+		preloaderInfo: &compressorInfo{
+			codec: codec,
+			cp:    cp,
+			comp:  comp,
+		},
+	})
 }

 func rpcInfoFromContext(ctx context.Context) (s *rpcInfo, ok bool) {
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Sebastiaan van StijnandGitHub	eb310fca49	Merge pull request #2373 from tiborvass/19.03-bump-grpc-1.23.1 [19.03 backport] Bump grpc to v1.23.1	2020-03-03 22:59:52 +01:00
Sebastiaan van StijnandTibor Vass	0e40b91921	vendor: bump google.golang.org/grpc v1.23.1 full diff: https://github.com/grpc/grpc-go/compare/v1.23.0...v1.23.1 - grpc/grpc-go#3018 server: set and advertise max frame size of 16KB - grpc/grpc-go#3017 grpclb: fix deadlock in grpclb connection cache - Before the fix, if the timer to remove a SubConn fires at the same time NewSubConn cancels the timer, it caused a mutex leak and deadlock. Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `013151ff78`) Signed-off-by: Tibor Vass <tibor@docker.com>	2020-03-03 21:39:18 +00:00
Tibor Vass	a51e9e6397	vendor: update grpc to v1.23.0 Signed-off-by: Tibor Vass <tibor@docker.com> (cherry picked from commit `70a4eb2a49`) Signed-off-by: Tibor Vass <tibor@docker.com>	2020-03-03 21:39:13 +00:00
Silvin LubeckiandGitHub	3ff19c3755	Merge pull request #2352 from thaJeztah/19.03_backport_add_daemon_redirect [19.03 backport] docs: add redirect for old location of daemon reference	2020-02-26 17:18:59 +01:00
Sebastiaan van Stijn	e94a87225a	docs: add redirect for old location of daemon reference Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `4cd4305b31`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-26 16:35:08 +01:00
Sebastiaan van StijnandGitHub	7c803f7c34	Merge pull request #2344 from thaJeztah/19.03_backport_builder_prune_flags [19.03 backport] fix builder prune flag descriptions	2020-02-26 16:32:45 +01:00
Silvin LubeckiandGitHub	4591b13318	Merge pull request #2342 from thaJeztah/19.03_backport_bump_golang_1.12.17 [19.03 backport] Update Golang 1.12.17	2020-02-19 11:48:11 +01:00
Sebastiaan van Stijn	f9a5c1f152	Fix builder prune -a/--all flag description Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `aad9d2c958`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-18 14:33:42 +01:00
Samuel CochranandSebastiaan van Stijn	829843333b	unused-for is a deprecated synonym See https://github.com/moby/moby/blob/9e7d5ac5ea55dc57185f49476e03d6efa76b3512/builder/builder-next/builder.go#L574 Signed-off-by: Samuel Cochran <sj26@sj26.com> (cherry picked from commit `758d12e651`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-18 14:33:28 +01:00
Sebastiaan van Stijn	f0e74408cf	Update Golang 1.12.17 full diff: https://github.com/golang/go/compare/go1.12.16...go1.12.17 go1.12.17 (released 2020/02/12) includes a fix to the runtime. See the Go 1.12.17 milestone on the issue tracker for details: https://github.com/golang/go/issues?q=milestone%3AGo1.12.17+label%3ACherryPickApproved Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `5e5ef9e0b9`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-17 14:58:54 +01:00
Silvin LubeckiandGitHub	a4bedce165	Merge pull request #2320 from thaJeztah/19.03_docs_backports [19.03 backport] assorted documentation updates	2020-02-10 17:20:36 +01:00
Sebastiaan van Stijn	d4755cfbd5	docs: update deprecated status for --disable-legacy-registry Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `73c9a44d58`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-10 16:55:02 +01:00
Sebastiaan van Stijn	6eef644faf	docs: add status table to "deprecated.md" Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `d4c0de2719`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-10 16:32:37 +01:00
Sebastiaan van Stijn	f47f8e333b	docs: mark deprecation of "docker engine" subcommands Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `e8c2dba697`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-10 16:32:36 +01:00
Sebastiaan van Stijn	64838b6907	docs: mark deprecation of "dab" files and top-level "deploy" subcommand Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `c7aca08497`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-10 16:32:33 +01:00
Sebastiaan van Stijn	c8308ecf6c	docs: sort "deprecated.md" by deprecation release Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `b6875ad690`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-10 16:32:31 +01:00
Sebastiaan van Stijn	793c75015e	docs: update some release-versions in "deprecated.md" Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `672c00e1cc`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-10 16:32:29 +01:00
Sebastiaan van Stijn	776e181628	docs: minor touch-ups in "deprecated.md" Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `c3092d9408`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-10 16:32:27 +01:00
Sebastiaan van Stijn	7fac7eb1af	docs: service create: document os/arch constraints and more examples Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `f26e9a3a61`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-10 16:31:41 +01:00
Sebastiaan van Stijn	27f0612665	docs: service create: document non-matching/exclude constraints Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `b059c93c92`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-10 16:31:39 +01:00
Sebastiaan van Stijn	d5ce81172f	docs: service create: use markdown table for constraints Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `abeb7babdf`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-10 16:31:36 +01:00
Misty Stanley-JonesandSebastiaan van Stijn	3a1d0c67ff	Give an example for --reserve-memory Signed-off-by: Misty Stanley-Jones <misty@docker.com> Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `76852f82ba`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-10 16:30:55 +01:00
Silvin LubeckiandGitHub	e1c31c8910	Merge pull request #2315 from thaJeztah/19.03_backport_carry_855_config_opts [19.03 backport] Add examples for configs	2020-02-10 12:01:37 +01:00
Abreto FUandSebastiaan van Stijn	2d6231ad4d	Remove a useless '\' and enhance a script in an example in the section 'Add entries ... (--add-host)' Signed-off-by: Abreto FU <public@abreto.email> (cherry picked from commit `07436dfe78`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-10 10:41:32 +01:00
Misty Stanley-JonesandSebastiaan van Stijn	be60297514	Add examples for configs Signed-off-by: Misty Stanley-Jones <misty@docker.com> Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `473a9d20cd`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-10 10:38:58 +01:00
Silvin LubeckiandGitHub	528b959ebc	Merge pull request #2311 from thaJeztah/19.03_backport_carry_1889_build_docs_update [19.03 backport] documentation updates	2020-02-06 17:16:35 +01:00
Venkateswara Reddy BukkasamudramandSebastiaan van Stijn	b668f31022	Syntax corrected Below are the changes proposed. - Corrected syntax error. - Updated example commands to maintain consistency. - Provided more clarity. Signed-off-by: Venkateswara Reddy Bukkasamudram <bukkasamudram@outlook.com> Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `74cc062d24`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-06 16:52:11 +01:00
Tonis TiigiandSebastiaan van Stijn	7bfe3d2006	docs: document dockerignore update Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com> Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `4f3bc15817`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-06 16:52:09 +01:00
Tonis TiigiandSebastiaan van Stijn	f6ec0a368e	docs: document build outputs Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com> Signed-off-by: Tibor Vass <tibor@docker.com> Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `f7009ee126`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-06 16:52:07 +01:00
Tonis TiigiandSebastiaan van Stijn	56324e6348	docs: document cache-from Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com> Signed-off-by: Tibor Vass <tibor@docker.com> Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `73cd257d0f`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-06 16:52:04 +01:00
KovahandSebastiaan van Stijn	040920f428	Add notice about port binding and overriding of UFW to docker run reference Signed-off-by: Kovah <mail@kovah.de> Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `a955ed6477`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-02-06 16:52:02 +01:00
Silvin LubeckiandGitHub	5908cdf87c	Merge pull request #2302 from thaJeztah/19.03_bump_golang_1.12.16 [19.03 backport] Update Golang 1.12.16, golang.org/x/crypto (CVE-2020-0601, CVE-2020-7919)	2020-01-30 16:25:51 +01:00
Sebastiaan van Stijn	3179a3b1e5	vendor: update golang.org/x/crypto 69ecbb4d6d5dab05e49161c6e77ea40a030884e1 (CVE-2020-7919) Includes https://github.com/golang/crypto/commit/69ecbb4d6d5dab05e49161c6e77ea40a030884e1 (forward-port of https://github.com/golang/crypto/commit/8b5121be2f68d8fc40bb06467003bdde1040a094), which fixes CVE-2020-7919: - Panic in crypto/x509 certificate parsing and golang.org/x/crypto/cryptobyte On 32-bit architectures, a malformed input to crypto/x509 or the ASN.1 parsing functions of golang.org/x/crypto/cryptobyte can lead to a panic. The malformed certificate can be delivered via a crypto/tls connection to a client, or to a server that accepts client certificates. net/http clients can be made to crash by an HTTPS server, while net/http servers that accept client certificates will recover the panic and are unaffected. Thanks to Project Wycheproof for providing the test cases that led to the discovery of this issue. The issue is CVE-2020-7919 and Go issue golang.org/issue/36837. Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `27d9aa2d9f`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-01-29 22:39:00 +01:00
Sebastiaan van Stijn	4d630fe843	Update Golang 1.12.16 (CVE-2020-0601, CVE-2020-7919) full diff: https://github.com/golang/go/compare/go1.12.15...go1.12.16 go1.12.16 (released 2020/01/28) includes two security fixes. One mitigates the CVE-2020-0601 certificate verification bypass on Windows. The other affects only 32-bit architectures. https://github.com/golang/go/issues?q=milestone%3AGo1.12.16+label%3ACherryPickApproved - X.509 certificate validation bypass on Windows 10 A Windows vulnerability allows attackers to spoof valid certificate chains when the system root store is in use. These releases include a mitigation for Go applications, but it’s strongly recommended that affected users install the Windows security update to protect their system. This issue is CVE-2020-0601 and Go issue golang.org/issue/36834. - Panic in crypto/x509 certificate parsing and golang.org/x/crypto/cryptobyte On 32-bit architectures, a malformed input to crypto/x509 or the ASN.1 parsing functions of golang.org/x/crypto/cryptobyte can lead to a panic. The malformed certificate can be delivered via a crypto/tls connection to a client, or to a server that accepts client certificates. net/http clients can be made to crash by an HTTPS server, while net/http servers that accept client certificates will recover the panic and are unaffected. Thanks to Project Wycheproof for providing the test cases that led to the discovery of this issue. The issue is CVE-2020-7919 and Go issue golang.org/issue/36837. This is also fixed in version v0.0.0-20200124225646-8b5121be2f68 of golang.org/x/crypto/cryptobyte. Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `19fd390c36`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-01-29 22:23:41 +01:00
Silvin LubeckiandGitHub	9b02509a69	Merge pull request #2291 from thaJeztah/19.03_backport_update_flag_description [19.03 backport] Update flag description for docker rm -v	2020-01-28 16:10:01 +01:00
Sebastiaan van Stijn	1d540a50d0	Update flag description for docker rm -v The `-v` option removes anonymous volume only, and keeps named volumes. Signed-off-by: Sebastiaan van Stijn <github@gone.nl> (cherry picked from commit `8f5379b301`) Signed-off-by: Sebastiaan van Stijn <github@gone.nl>	2020-01-27 15:53:37 +01:00