Merge pull request #178 from djdongjin/bump-versions

Add dependabot and bump up golang and dependency versions

.gitattributes

@@ -0,0 +1 @@
+*.go text eol=lf

.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 10
+    groups:
+      golang-x:
+        patterns:
+          - "golang.org/x/*"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 10

.github/workflows/ci.yml

@@ -6,86 +6,117 @@ on:
   pull_request:
     branches: [ main ]
 
-jobs:
+env:
+  GO_VERSION: 1.23.x
 
+permissions:
+  contents: read
+  pull-requests: read
+
+jobs:
+  #
+  # golangci-lint
+  #
+  linters:
+    name: Linters
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          path: src/github.com/containerd/ttrpc
+      - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1
+        with:
+          version: v1.60.3
+          args: --timeout=5m
+          skip-cache: true
+          working-directory: src/github.com/containerd/ttrpc
+
+  #
+  # Project checks
+  #
+  project:
+    name: Project Checks
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
+
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          path: src/github.com/containerd/ttrpc
+          fetch-depth: 25
+      - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - uses: containerd/project-checks@434a07157608eeaa1d5c8d4dd506154204cd9401 # v1.1.0
+        with:
+          working-directory: src/github.com/containerd/ttrpc
+
+  #
+  # Build and Test project
+  #
   build:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-18.04, macos-10.15]
-    name: ${{ matrix.os }}
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        go: [1.22.x, 1.23.x]
+
+    name: ${{ matrix.os }} / ${{ matrix.go }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 5
+    timeout-minutes: 10
     steps:
 
-    - name: Set up Go 1.15
-      uses: actions/setup-go@v2
-      with:
-        go-version: 1.15
-      id: go
-
-    - name: Setup Go binary path
-      shell: bash
-      run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-
-    - name: Check out code
-      uses: actions/checkout@v2
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       with:
         path: src/github.com/containerd/ttrpc
         fetch-depth: 25
-
-    - name: Checkout project
-      uses: actions/checkout@v2
+    - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
-        repository: containerd/project
-        path: src/github.com/containerd/project
-
-    - name: Install dependencies
-      env:
-        GO111MODULE: off
-      run: |
-        go get -u github.com/vbatts/git-validation
-        go get -u github.com/kunalkushwaha/ltag
-
-    - name: Check DCO/whitespace/commit message
-      env:
-        GITHUB_COMMIT_URL: ${{ github.event.pull_request.commits_url }}
-        DCO_VERBOSITY: "-q"
-        DCO_RANGE: ""
-      working-directory: src/github.com/containerd/ttrpc
-      run: |
-        if [ -z "${GITHUB_COMMIT_URL}" ]; then
-          DCO_RANGE=$(jq -r '.before +".."+ .after' ${GITHUB_EVENT_PATH})
-        else
-          DCO_RANGE=$(curl ${GITHUB_COMMIT_URL} | jq -r '.[0].parents[0].sha +".."+ .[-1].sha')
-        fi
-        ../project/script/validate/dco
-
-    - name: Check file headers
-      run: ../project/script/validate/fileheader ../project/
-      working-directory: src/github.com/containerd/ttrpc
+        go-version: ${{ matrix.go }}
 
     - name: Test
       working-directory: src/github.com/containerd/ttrpc
       run: |
-        go test -v -race -coverprofile=coverage.txt -covermode=atomic ./...
+        make test
 
-    - name: Codecov
-      run: bash <(curl -s https://codecov.io/bash)
+    - name: Coverage
       working-directory: src/github.com/containerd/ttrpc
+      run: |
+        make coverage TESTFLAGS_RACE=-race
 
+    - name: Integration Tests
+      working-directory: src/github.com/containerd/ttrpc
+      run: |
+        make integration
+
+  #
+  # Run Protobuild
+  #
   protobuild:
     name: Run Protobuild
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 5
     steps:
 
-    - name: Set up Go 1.17
-      uses: actions/setup-go@v2
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       with:
-        go-version: 1.17
+        path: src/github.com/containerd/ttrpc
+        fetch-depth: 25
+    - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
+      with:
+        go-version: ${{ env.GO_VERSION }}
       id: go
 
     - name: Setup Go binary path
@@ -94,36 +125,18 @@ jobs:
         echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
         echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
 
-    - name: Check out code
-      uses: actions/checkout@v2
-      with:
-        path: src/github.com/containerd/ttrpc
-        fetch-depth: 25
-
-    - name: Install protoc
-      run: |
-        curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.5.0/protoc-3.5.0-linux-x86_64.zip
-        sudo unzip -x protoc-3.5.0-linux-x86_64.zip -d /usr/local
-        sudo chmod -R go+rX /usr/local/include
-        sudo chmod go+x /usr/local/bin/protoc
-
-    - name: Install gogo/protobuf
-      run: |
-        cd $GOPATH/src
-        mkdir -p github.com/gogo
-        cd github.com/gogo
-        git clone --depth 1 --branch v1.3.2 https://github.com/gogo/protobuf
-
-    - name: Build protoc-gen-gogottrpc
+    - name: Install dependencies
       working-directory: src/github.com/containerd/ttrpc
       run: |
-        go build ./cmd/protoc-gen-gogottrpc
+        sudo make install-protobuf
+        make install-protobuild
+
+    - name: Install protoc-gen-go-ttrpc
+      working-directory: src/github.com/containerd/ttrpc
+      run: |
+        make install
 
     - name: Run Protobuild
       working-directory: src/github.com/containerd/ttrpc
       run: |
-        export PATH=$GOPATH/bin:$PWD:$PATH
-        go install github.com/containerd/protobuild@7e5ee24bc1f70e9e289fef15e2631eb3491320bf
-        cd example
-        protobuild
-        git diff --exit-code
+        make check-protos

.gitignore

@@ -1,4 +1,5 @@
 # Binaries for programs and plugins
+/bin/
 *.exe
 *.dll
 *.so
@@ -9,3 +10,4 @@
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+coverage.txt

.golangci.yml

@@ -0,0 +1,52 @@
+linters:
+  enable:
+    - staticcheck
+    - unconvert
+    - gofmt
+    - goimports
+    - revive
+    - ineffassign
+    - govet
+    - unused
+    - misspell
+  disable:
+    - errcheck
+
+linters-settings:
+  revive:
+    ignore-generated-header: true
+    rules:
+      - name: blank-imports
+      - name: context-as-argument
+      - name: context-keys-type
+      - name: dot-imports
+      - name: error-return
+      - name: error-strings
+      - name: error-naming
+      - name: exported
+      - name: if-return
+      - name: increment-decrement
+      - name: var-naming
+        arguments: [["UID", "GID"], []]
+      - name: var-declaration
+      - name: package-comments
+      - name: range
+      - name: receiver-naming
+      - name: time-naming
+      - name: unexported-return
+      - name: indent-error-flow
+      - name: errorf
+      - name: empty-block
+      - name: superfluous-else
+      - name: unused-parameter
+      - name: unreachable-code
+      - name: redefines-builtin-id
+
+issues:
+  include:
+    - EXC0002
+  exclude-dirs:
+    - example
+
+run:
+  timeout: 8m

Makefile

@@ -0,0 +1,180 @@
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+
+# Go command to use for build
+GO ?= go
+INSTALL ?= install
+
+# Root directory of the project (absolute path).
+ROOTDIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+
+WHALE = "🇩"
+ONI = "👹"
+
+# Project binaries.
+COMMANDS=protoc-gen-go-ttrpc protoc-gen-gogottrpc
+
+ifdef BUILDTAGS
+    GO_BUILDTAGS = ${BUILDTAGS}
+endif
+GO_BUILDTAGS ?=
+GO_TAGS=$(if $(GO_BUILDTAGS),-tags "$(strip $(GO_BUILDTAGS))",)
+
+# Project packages.
+PACKAGES=$(shell $(GO) list ${GO_TAGS} ./... | grep -v /example)
+TESTPACKAGES=$(shell $(GO) list ${GO_TAGS} ./... | grep -v /cmd | grep -v /integration | grep -v /example)
+BINPACKAGES=$(addprefix ./cmd/,$(COMMANDS))
+
+#Replaces ":" (*nix), ";" (windows) with newline for easy parsing
+GOPATHS=$(shell echo ${GOPATH} | tr ":" "\n" | tr ";" "\n")
+
+TESTFLAGS_RACE=
+GO_BUILD_FLAGS=
+# See Golang issue re: '-trimpath': https://github.com/golang/go/issues/13809
+GO_GCFLAGS=$(shell				\
+	set -- ${GOPATHS};			\
+	echo "-gcflags=-trimpath=$${1}/src";	\
+	)
+
+BINARIES=$(addprefix bin/,$(COMMANDS))
+
+# Flags passed to `go test`
+TESTFLAGS ?= $(TESTFLAGS_RACE) $(EXTRA_TESTFLAGS)
+TESTFLAGS_PARALLEL ?= 8
+
+# Use this to replace `go test` with, for instance, `gotestsum`
+GOTEST ?= $(GO) test
+
+.PHONY: clean all AUTHORS build binaries test integration generate protos check-protos coverage ci check help install vendor install-protobuf install-protobuild
+.DEFAULT: default
+
+# Forcibly set the default goal to all, in case an include above brought in a rule definition.
+.DEFAULT_GOAL := all
+
+all: binaries
+
+check: proto-fmt ## run all linters
+	@echo "$(WHALE) $@"
+	GOGC=75 golangci-lint run
+
+ci: check binaries check-protos coverage # coverage-integration ## to be used by the CI
+
+AUTHORS: .mailmap .git/HEAD
+	git log --format='%aN <%aE>' | sort -fu > $@
+
+generate: protos
+	@echo "$(WHALE) $@"
+	@PATH="${ROOTDIR}/bin:${PATH}" $(GO) generate -x ${PACKAGES}
+
+protos: bin/protoc-gen-gogottrpc bin/protoc-gen-go-ttrpc ## generate protobuf
+	@echo "$(WHALE) $@"
+	@(PATH="${ROOTDIR}/bin:${PATH}" protobuild --quiet ${PACKAGES})
+
+check-protos: protos ## check if protobufs needs to be generated again
+	@echo "$(WHALE) $@"
+	@test -z "$$(git status --short | grep ".pb.go" | tee /dev/stderr)" || \
+		((git diff | cat) && \
+		(echo "$(ONI) please run 'make protos' when making changes to proto files" && false))
+
+check-api-descriptors: protos ## check that protobuf changes aren't present.
+	@echo "$(WHALE) $@"
+	@test -z "$$(git status --short | grep ".pb.txt" | tee /dev/stderr)" || \
+		((git diff $$(find . -name '*.pb.txt') | cat) && \
+		(echo "$(ONI) please run 'make protos' when making changes to proto files and check-in the generated descriptor file changes" && false))
+
+proto-fmt: ## check format of proto files
+	@echo "$(WHALE) $@"
+	@test -z "$$(find . -name '*.proto' -type f -exec grep -Hn -e "^ " {} \; | tee /dev/stderr)" || \
+		(echo "$(ONI) please indent proto files with tabs only" && false)
+	@test -z "$$(find . -name '*.proto' -type f -exec grep -Hn "Meta meta = " {} \; | grep -v '(gogoproto.nullable) = false' | tee /dev/stderr)" || \
+		(echo "$(ONI) meta fields in proto files must have option (gogoproto.nullable) = false" && false)
+
+build: ## build the go packages
+	@echo "$(WHALE) $@"
+	@$(GO) build ${DEBUG_GO_GCFLAGS} ${GO_GCFLAGS} ${GO_BUILD_FLAGS} ${EXTRA_FLAGS} ${PACKAGES}
+
+test: ## run tests, except integration tests and tests that require root
+	@echo "$(WHALE) $@"
+	@$(GOTEST) ${TESTFLAGS} ${TESTPACKAGES}
+
+integration: ## run integration tests
+	@echo "$(WHALE) $@"
+	@cd "${ROOTDIR}/integration" && $(GOTEST) -v ${TESTFLAGS}  -parallel ${TESTFLAGS_PARALLEL} .
+
+benchmark: ## run benchmarks tests
+	@echo "$(WHALE) $@"
+	@$(GO) test ${TESTFLAGS} -bench . -run Benchmark
+
+FORCE:
+
+define BUILD_BINARY
+@echo "$(WHALE) $@"
+@$(GO) build ${DEBUG_GO_GCFLAGS} ${GO_GCFLAGS} ${GO_BUILD_FLAGS} -o $@ ${GO_TAGS}  ./$<
+endef
+
+# Build a binary from a cmd.
+bin/%: cmd/% FORCE
+	$(call BUILD_BINARY)
+
+binaries: $(BINARIES) ## build binaries
+	@echo "$(WHALE) $@"
+
+clean: ## clean up binaries
+	@echo "$(WHALE) $@"
+	@rm -f $(BINARIES)
+
+install: ## install binaries
+	@echo "$(WHALE) $@ $(BINPACKAGES)"
+	@$(GO) install $(BINPACKAGES)
+
+install-protobuf:
+	@echo "$(WHALE) $@"
+	@script/install-protobuf
+
+install-protobuild:
+	@echo "$(WHALE) $@"
+	@$(GO) install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
+	@$(GO) install github.com/containerd/protobuild@14832ccc41429f5c4f81028e5af08aa233a219cf
+
+coverage: ## generate coverprofiles from the unit tests, except tests that require root
+	@echo "$(WHALE) $@"
+	@rm -f coverage.txt
+	@$(GO) test ${TESTFLAGS} ${TESTPACKAGES} 2> /dev/null
+	@( for pkg in ${PACKAGES}; do \
+		$(GO) test ${TESTFLAGS} \
+			-cover \
+			-coverprofile=profile.out \
+			-covermode=atomic $$pkg || exit; \
+		if [ -f profile.out ]; then \
+			cat profile.out >> coverage.txt; \
+			rm profile.out; \
+		fi; \
+	done )
+
+vendor: ## ensure all the go.mod/go.sum files are up-to-date
+	@echo "$(WHALE) $@"
+	@$(GO) mod tidy
+	@$(GO) mod verify
+
+verify-vendor: ## verify if all the go.mod/go.sum files are up-to-date
+	@echo "$(WHALE) $@"
+	@$(GO) mod tidy
+	@$(GO) mod verify
+	@test -z "$$(git status --short | grep "go.sum" | tee /dev/stderr)" || \
+		((git diff | cat) && \
+		(echo "$(ONI) make sure to checkin changes after go mod tidy" && false))
+
+help: ## this help
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | sort

PROTOCOL.md

@@ -0,0 +1,240 @@
+# Protocol Specification
+
+The ttrpc protocol is client/server protocol to support multiple request streams
+over a single connection with lightweight framing. The client represents the
+process which initiated the underlying connection and the server is the process
+which accepted the connection. The protocol is currently defined as
+asymmetrical, with clients sending requests and servers sending responses. Both
+clients and servers are able to send stream data. The roles are also used in
+determining the stream identifiers, with client initiated streams using odd
+number identifiers and server initiated using even number. The protocol may be
+extended in the future to support server initiated streams, that is not
+supported in the latest version.
+
+## Purpose
+
+The ttrpc protocol is designed to be lightweight and optimized for low latency
+and reliable connections between processes on the same host. The protocol does
+not include features for handling unreliable connections such as handshakes,
+resets, pings, or flow control. The protocol is designed to make low-overhead
+implementations as simple as possible. It is not intended as a suitable
+replacement for HTTP2/3 over the network.
+
+## Message Frame
+
+Each Message Frame consists of a 10-byte message header followed
+by message data. The data length and stream ID are both big-endian
+4-byte unsigned integers. The message type is an unsigned 1-byte
+integer. The flags are also an unsigned 1-byte integer and
+use is defined by the message type.
+
+    +---------------------------------------------------------------+
+    |                       Data Length (32)                        |
+    +---------------------------------------------------------------+
+    |                        Stream ID (32)                         |
+    +---------------+-----------------------------------------------+
+    | Msg Type (8)  |
+    +---------------+
+    |   Flags (8)   |
+    +---------------+-----------------------------------------------+
+    |                           Data (*)                            |
+    +---------------------------------------------------------------+
+
+The Data Length field represents the number of bytes in the Data field. The
+total frame size will always be Data Length + 10 bytes. The maximum data length
+is 4MB and any larger size should be rejected. Due to the maximum data size
+being less than 16MB, the first frame byte should always be zero. This first
+byte should be considered reserved for future use.
+
+The Stream ID must be odd for client initiated streams and even for server
+initiated streams. Server initiated streams are not currently supported.
+
+## Mesage Types
+
+| Message Type | Name     | Description                      |
+|--------------|----------|----------------------------------|
+| 0x01         | Request  | Initiates stream                 |
+| 0x02         | Response | Final stream data and terminates |
+| 0x03         | Data     | Stream data                      |
+
+### Request
+
+The request message is used to initiate stream and send along request data for
+properly routing and handling the stream. The stream may indicate unary without
+any inbound or outbound stream data with only a response is expected on the
+stream. The request may also indicate the stream is still open for more data and
+no response is expected until data is finished. If the remote indicates the
+stream is closed, the request may be considered non-unary but without anymore
+stream data sent. In the case of `remote closed`, the remote still expects to
+receive a response or stream data. For compatibility with non streaming clients,
+a request with empty flags indicates a unary request.
+
+#### Request Flags
+
+| Flag | Name            | Description                                      |
+|------|-----------------|--------------------------------------------------|
+| 0x01 | `remote closed` | Non-unary, but no more data expected from remote |
+| 0x02 | `remote open`   | Non-unary, remote is still sending data          |
+
+### Response
+
+The response message is used to end a stream with data, an empty response, or
+an error. A response message is the only expected message after a unary request.
+A non-unary request does not require a response message if the server is sending
+back stream data. A non-unary stream may return a single response message but no
+other stream data may follow.
+
+#### Response Flags
+
+No response flags are defined at this time, flags should be empty.
+
+### Data
+
+The data message is used to send data on an already initialized stream. Either
+client or server may send data. A data message is not allowed on a unary stream.
+A data message should not be sent after indicating `remote closed` to the peer.
+The last data message on a stream must set the `remote closed` flag.
+
+The `no data` flag is used to indicate that the data message does not include
+any data. This is normally used with the `remote closed` flag to indicate the
+stream is now closed without transmitting any data. Since ttrpc normally
+transmits a single object per message, a zero length data message may be
+interpreted as an empty object. For example, transmitting the number zero as a
+protobuf message ends up with a data length of zero, but the message is still
+considered data and should be processed.
+
+#### Data Flags
+
+| Flag | Name            | Description                       |
+|------|-----------------|-----------------------------------|
+| 0x01 | `remote closed` | No more data expected from remote |
+| 0x04 | `no data`       | This message does not have data   |
+
+## Streaming
+
+All ttrpc requests use streams to transfer data. Unary streams will only have
+two messages sent per stream, a request from a client and a response from the
+server. Non-unary streams, however, may send any numbers of messages from the
+client and the server. This makes stream management more complicated than unary
+streams since both client and server need to track additional state. To keep
+this management as simple as possible, ttrpc minimizes the number of states and
+uses two flags instead of control frames. Each stream has two states while a
+stream is still alive: `local closed` and `remote closed`. Each peer considers
+local and remote from their own perspective and sets flags from the other peer's
+perspective. For example, if a client sends a data frame with the
+`remote closed` flag, that is indicating that the client is now `local closed`
+and the server will be `remote closed`. A unary operation does not need to send
+these flags since each received message always indicates `remote closed`. Once a
+peer is both `local closed` and `remote closed`, the stream is considered
+finished and may be cleaned up.
+
+Due to the asymmetric nature of the current protocol, a client should
+always be in the `local closed` state before `remote closed` and a server should
+always be in the `remote closed` state before `local closed`. This happens
+because the client is always initiating requests and a client always expects a
+final response back from a server to indicate the initiated request has been
+fulfilled. This may mean server sends a final empty response to finish a stream
+even after it has already completed sending data before the client.
+
+### Unary State Diagram
+
+         +--------+                                    +--------+
+         | Client |                                    | Server |
+         +---+----+                                    +----+---+
+             |               +---------+                    |
+      local  >---------------+ Request +--------------------> remote
+      closed |               +---------+                    | closed
+             |                                              |
+             |              +----------+                    |
+    finished <--------------+ Response +--------------------< finished
+             |              +----------+                    |
+             |                                              |
+
+### Non-Unary State Diagrams
+
+RC: `remote closed` flag
+RO: `remote open` flag
+
+         +--------+                                    +--------+
+         | Client |                                    | Server |
+         +---+----+                                    +----+---+
+             |             +--------------+                 |
+             >-------------+ Request [RO] +----------------->
+             |             +--------------+                 |
+             |                                              |
+             |                 +------+                     |
+             >-----------------+ Data +--------------------->
+             |                 +------+                     |
+             |                                              |
+             |               +-----------+                  |
+      local  >---------------+ Data [RC] +------------------> remote
+      closed |               +-----------+                  | closed
+             |                                              |
+             |              +----------+                    |
+    finished <--------------+ Response +--------------------< finished
+             |              +----------+                    |
+             |                                              |
+
+         +--------+                                    +--------+
+         | Client |                                    | Server |
+         +---+----+                                    +----+---+
+             |             +--------------+                 |
+      local  >-------------+ Request [RC] +-----------------> remote
+      closed |             +--------------+                 | closed
+             |                                              |
+             |                 +------+                     |
+             <-----------------+ Data +---------------------<
+             |                 +------+                     |
+             |                                              |
+             |               +-----------+                  |
+    finished <---------------+ Data [RC] +------------------< finished
+             |               +-----------+                  |
+             |                                              |
+
+         +--------+                                    +--------+
+         | Client |                                    | Server |
+         +---+----+                                    +----+---+
+             |             +--------------+                 |
+             >-------------+ Request [RO] +----------------->
+             |             +--------------+                 |
+             |                                              |
+             |                 +------+                     |
+             >-----------------+ Data +--------------------->
+             |                 +------+                     |
+             |                                              |
+             |                 +------+                     |
+             <-----------------+ Data +---------------------<
+             |                 +------+                     |
+             |                                              |
+             |                 +------+                     |
+             >-----------------+ Data +--------------------->
+             |                 +------+                     |
+             |                                              |
+             |               +-----------+                  |
+      local  >---------------+ Data [RC] +------------------> remote
+      closed |               +-----------+                  | closed
+             |                                              |
+             |                 +------+                     |
+             <-----------------+ Data +---------------------<
+             |                 +------+                     |
+             |                                              |
+             |               +-----------+                  |
+    finished <---------------+ Data [RC] +------------------< finished
+             |               +-----------+                  |
+             |                                              |
+
+## RPC
+
+While this protocol is defined primarily to support Remote Procedure Calls, the
+protocol does not define the request and response types beyond the messages
+defined in the protocol. The implementation provides a default protobuf
+definition of request and response which may be used for cross language rpc.
+All implementations should at least define a request type which support
+routing by procedure name and a response type which supports call status.
+
+## Version History
+
+| Version | Features            |
+|---------|---------------------|
+| 1.0     | Unary requests only |
+| 1.2     | Streaming support   |

Protobuild.toml

@@ -0,0 +1,28 @@
+version = "2"
+generators = ["go"]
+
+# Control protoc include paths. Below are usually some good defaults, but feel
+# free to try it without them if it works for your project.
+[includes]
+  # Include paths that will be added before all others. Typically, you want to
+  # treat the root of the project as an include, but this may not be necessary.
+  before = ["."]
+
+  # Paths that will be added untouched to the end of the includes. We use
+  # `/usr/local/include` to pickup the common install location of protobuf.
+  # This is the default.
+  after = ["/usr/local/include"]
+
+# This section maps protobuf imports to Go packages. These will become
+# `-M` directives in the call to the go protobuf generator.
+[packages]
+  "google/protobuf/any.proto" = "github.com/gogo/protobuf/types"
+  "proto/status.proto" = "google.golang.org/genproto/googleapis/rpc/status"
+
+[[overrides]]
+# enable ttrpc and disable fieldpath and grpc for the shim
+prefixes = ["github.com/containerd/ttrpc/integration/streaming"]
+generators = ["go", "go-ttrpc"]
+
+[overrides.parameters.go-ttrpc]
+prefix = "TTRPC"

README.md

@@ -1,7 +1,6 @@
 # ttrpc
 
-[![Build Status](https://github.com/containerd/ttrpc/workflows/CI/badge.svg)](https://github.com/containerd/ttrpc/actions?query=workflow%3ACI)
-[![codecov](https://codecov.io/gh/containerd/ttrpc/branch/main/graph/badge.svg)](https://codecov.io/gh/containerd/ttrpc)
+[![Build Status](https://github.com/containerd/ttrpc/actions/workflows/ci.yml/badge.svg)](https://github.com/containerd/ttrpc/actions/workflows/ci.yml)
 
 GRPC for low-memory environments.
 
@@ -20,13 +19,17 @@ Please note that while this project supports generating either end of the
 protocol, the generated service definitions will be incompatible with regular
 GRPC services, as they do not speak the same protocol.
 
+# Protocol
+
+See the [protocol specification](./PROTOCOL.md).
+
 # Usage
 
 Create a gogo vanity binary (see
 [`cmd/protoc-gen-gogottrpc/main.go`](cmd/protoc-gen-gogottrpc/main.go) for an
 example with the ttrpc plugin enabled.
 
-It's recommended to use [`protobuild`](https://github.com//stevvooe/protobuild)
+It's recommended to use [`protobuild`](https://github.com/containerd/protobuild)
 to build the protobufs for this project, but this will work with protoc
 directly, if required.
 
@@ -37,13 +40,11 @@ directly, if required.
 - The client and server interface are identical whereas in GRPC there is a
   client and server interface that are different.
 - The Go stdlib context package is used instead.
-- No support for streams yet.
 
 # Status
 
 TODO:
 
-- [ ] Document protocol layout
 - [ ] Add testing under concurrent load to ensure
 - [ ] Verify connection error handling
 

channel.go

@@ -38,6 +38,26 @@ type messageType uint8
 const (
 	messageTypeRequest  messageType = 0x1
 	messageTypeResponse messageType = 0x2
+	messageTypeData     messageType = 0x3
+)
+
+func (mt messageType) String() string {
+	switch mt {
+	case messageTypeRequest:
+		return "request"
+	case messageTypeResponse:
+		return "response"
+	case messageTypeData:
+		return "data"
+	default:
+		return "unknown"
+	}
+}
+
+const (
+	flagRemoteClosed uint8 = 0x1
+	flagRemoteOpen   uint8 = 0x2
+	flagNoData       uint8 = 0x4
 )
 
 // messageHeader represents the fixed-length message header of 10 bytes sent
@@ -46,7 +66,7 @@ type messageHeader struct {
 	Length   uint32      // length excluding this header. b[:4]
 	StreamID uint32      // identifies which request stream message is a part of. b[4:8]
 	Type     messageType // message type b[8]
-	Flags    uint8       // reserved          b[9]
+	Flags    uint8       // type specific flags b[9]
 }
 
 func readMessageHeader(p []byte, r io.Reader) (messageHeader, error) {
@@ -111,22 +131,31 @@ func (ch *channel) recv() (messageHeader, []byte, error) {
 		return mh, nil, status.Errorf(codes.ResourceExhausted, "message length %v exceed maximum message size of %v", mh.Length, messageLengthMax)
 	}
 
-	p := ch.getmbuf(int(mh.Length))
-	if _, err := io.ReadFull(ch.br, p); err != nil {
-		return messageHeader{}, nil, fmt.Errorf("failed reading message: %w", err)
+	var p []byte
+	if mh.Length > 0 {
+		p = ch.getmbuf(int(mh.Length))
+		if _, err := io.ReadFull(ch.br, p); err != nil {
+			return messageHeader{}, nil, fmt.Errorf("failed reading message: %w", err)
+		}
 	}
 
 	return mh, p, nil
 }
 
-func (ch *channel) send(streamID uint32, t messageType, p []byte) error {
-	if err := writeMessageHeader(ch.bw, ch.hwbuf[:], messageHeader{Length: uint32(len(p)), StreamID: streamID, Type: t}); err != nil {
+func (ch *channel) send(streamID uint32, t messageType, flags uint8, p []byte) error {
+	if len(p) > messageLengthMax {
+		return OversizedMessageError(len(p))
+	}
+
+	if err := writeMessageHeader(ch.bw, ch.hwbuf[:], messageHeader{Length: uint32(len(p)), StreamID: streamID, Type: t, Flags: flags}); err != nil {
 		return err
 	}
 
-	_, err := ch.bw.Write(p)
-	if err != nil {
-		return err
+	if len(p) > 0 {
+		_, err := ch.bw.Write(p)
+		if err != nil {
+			return err
+		}
 	}
 
 	return ch.bw.Flush()

channel_test.go

@@ -44,7 +44,7 @@ func TestReadWriteMessage(t *testing.T) {
 
 	go func() {
 		for i, msg := range messages {
-			if err := ch.send(uint32(i), 1, msg); err != nil {
+			if err := ch.send(uint32(i), 1, 0, msg); err != nil {
 				errs <- err
 				return
 			}
@@ -89,21 +89,19 @@ func TestReadWriteMessage(t *testing.T) {
 
 func TestMessageOversize(t *testing.T) {
 	var (
-		w, r     = net.Pipe()
-		wch, rch = newChannel(w), newChannel(r)
-		msg      = bytes.Repeat([]byte("a message of massive length"), 512<<10)
-		errs     = make(chan error, 1)
+		w, _ = net.Pipe()
+		wch  = newChannel(w)
+		msg  = bytes.Repeat([]byte("a message of massive length"), 512<<10)
+		errs = make(chan error, 1)
 	)
 
 	go func() {
-		if err := wch.send(1, 1, msg); err != nil {
-			errs <- err
-		}
+		errs <- wch.send(1, 1, 0, msg)
 	}()
 
-	_, _, err := rch.recv()
+	err := <-errs
 	if err == nil {
-		t.Fatalf("error expected reading with small buffer")
+		t.Fatalf("sending oversized message expected to fail")
 	}
 
 	status, ok := status.FromError(err)
@@ -114,12 +112,4 @@ func TestMessageOversize(t *testing.T) {
 	if status.Code() != codes.ResourceExhausted {
 		t.Fatalf("expected grpc status code: %v != %v", status.Code(), codes.ResourceExhausted)
 	}
-
-	select {
-	case err := <-errs:
-		if err != nil {
-			t.Fatal(err)
-		}
-	default:
-	}
 }

client.go

@@ -19,30 +19,30 @@ package ttrpc
 import (
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"net"
-	"os"
 	"strings"
 	"sync"
 	"syscall"
 	"time"
 
-	"github.com/gogo/protobuf/proto"
-	"github.com/sirupsen/logrus"
+	"github.com/containerd/log"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
 )
 
-// ErrClosed is returned by client methods when the underlying connection is
-// closed.
-var ErrClosed = errors.New("ttrpc: closed")
-
 // Client for a ttrpc server
 type Client struct {
 	codec   codec
 	conn    net.Conn
 	channel *channel
-	calls   chan *callRequest
+
+	streamLock   sync.RWMutex
+	streams      map[streamID]*stream
+	nextStreamID streamID
+	sendLock     sync.Mutex
 
 	ctx    context.Context
 	closed func()
@@ -51,8 +51,6 @@ type Client struct {
 	userCloseFunc   func()
 	userCloseWaitCh chan struct{}
 
-	errOnce     sync.Once
-	err         error
 	interceptor UnaryClientInterceptor
 }
 
@@ -73,35 +71,77 @@ func WithUnaryClientInterceptor(i UnaryClientInterceptor) ClientOpts {
 	}
 }
 
+// WithChainUnaryClientInterceptor sets the provided chain of client interceptors
+func WithChainUnaryClientInterceptor(interceptors ...UnaryClientInterceptor) ClientOpts {
+	return func(c *Client) {
+		if len(interceptors) == 0 {
+			return
+		}
+		if c.interceptor != nil {
+			interceptors = append([]UnaryClientInterceptor{c.interceptor}, interceptors...)
+		}
+		c.interceptor = func(
+			ctx context.Context,
+			req *Request,
+			reply *Response,
+			info *UnaryClientInfo,
+			final Invoker,
+		) error {
+			return interceptors[0](ctx, req, reply, info,
+				chainUnaryInterceptors(interceptors[1:], final, info))
+		}
+	}
+}
+
+func chainUnaryInterceptors(interceptors []UnaryClientInterceptor, final Invoker, info *UnaryClientInfo) Invoker {
+	if len(interceptors) == 0 {
+		return final
+	}
+	return func(
+		ctx context.Context,
+		req *Request,
+		reply *Response,
+	) error {
+		return interceptors[0](ctx, req, reply, info,
+			chainUnaryInterceptors(interceptors[1:], final, info))
+	}
+}
+
+// NewClient creates a new ttrpc client using the given connection
 func NewClient(conn net.Conn, opts ...ClientOpts) *Client {
 	ctx, cancel := context.WithCancel(context.Background())
+	channel := newChannel(conn)
 	c := &Client{
 		codec:           codec{},
 		conn:            conn,
-		channel:         newChannel(conn),
-		calls:           make(chan *callRequest),
+		channel:         channel,
+		streams:         make(map[streamID]*stream),
+		nextStreamID:    1,
 		closed:          cancel,
 		ctx:             ctx,
 		userCloseFunc:   func() {},
 		userCloseWaitCh: make(chan struct{}),
-		interceptor:     defaultClientInterceptor,
 	}
 
 	for _, o := range opts {
 		o(c)
 	}
 
+	if c.interceptor == nil {
+		c.interceptor = defaultClientInterceptor
+	}
+
 	go c.run()
 	return c
 }
 
-type callRequest struct {
-	ctx  context.Context
-	req  *Request
-	resp *Response  // response will be written back here
-	errs chan error // error written here on completion
+func (c *Client) send(sid uint32, mt messageType, flags uint8, b []byte) error {
+	c.sendLock.Lock()
+	defer c.sendLock.Unlock()
+	return c.channel.send(sid, mt, flags, b)
 }
 
+// Call makes a unary request and returns with response
 func (c *Client) Call(ctx context.Context, service, method string, req, resp interface{}) error {
 	payload, err := c.codec.Marshal(req)
 	if err != nil {
@@ -113,6 +153,7 @@ func (c *Client) Call(ctx context.Context, service, method string, req, resp int
 			Service: service,
 			Method:  method,
 			Payload: payload,
+			// TODO: metadata from context
 		}
 
 		cresp = &Response{}
@@ -123,7 +164,7 @@ func (c *Client) Call(ctx context.Context, service, method string, req, resp int
 	}
 
 	if dl, ok := ctx.Deadline(); ok {
-		creq.TimeoutNano = dl.Sub(time.Now()).Nanoseconds()
+		creq.TimeoutNano = time.Until(dl).Nanoseconds()
 	}
 
 	info := &UnaryClientInfo{
@@ -143,41 +184,148 @@ func (c *Client) Call(ctx context.Context, service, method string, req, resp int
 	return nil
 }
 
-func (c *Client) dispatch(ctx context.Context, req *Request, resp *Response) error {
-	errs := make(chan error, 1)
-	call := &callRequest{
-		ctx:  ctx,
-		req:  req,
-		resp: resp,
-		errs: errs,
-	}
-
-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	case c.calls <- call:
-	case <-c.ctx.Done():
-		return c.error()
-	}
-
-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	case err := <-errs:
-		return filterCloseErr(err)
-	case <-c.ctx.Done():
-		return c.error()
-	}
+// StreamDesc describes the stream properties, whether the stream has
+// a streaming client, a streaming server, or both
+type StreamDesc struct {
+	StreamingClient bool
+	StreamingServer bool
 }
 
+// ClientStream is used to send or recv messages on the underlying stream
+type ClientStream interface {
+	CloseSend() error
+	SendMsg(m interface{}) error
+	RecvMsg(m interface{}) error
+}
+
+type clientStream struct {
+	ctx          context.Context
+	s            *stream
+	c            *Client
+	desc         *StreamDesc
+	localClosed  bool
+	remoteClosed bool
+}
+
+func (cs *clientStream) CloseSend() error {
+	if !cs.desc.StreamingClient {
+		return fmt.Errorf("%w: cannot close non-streaming client", ErrProtocol)
+	}
+	if cs.localClosed {
+		return ErrStreamClosed
+	}
+	err := cs.s.send(messageTypeData, flagRemoteClosed|flagNoData, nil)
+	if err != nil {
+		return filterCloseErr(err)
+	}
+	cs.localClosed = true
+	return nil
+}
+
+func (cs *clientStream) SendMsg(m interface{}) error {
+	if !cs.desc.StreamingClient {
+		return fmt.Errorf("%w: cannot send data from non-streaming client", ErrProtocol)
+	}
+	if cs.localClosed {
+		return ErrStreamClosed
+	}
+
+	var (
+		payload []byte
+		err     error
+	)
+	if m != nil {
+		payload, err = cs.c.codec.Marshal(m)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = cs.s.send(messageTypeData, 0, payload)
+	if err != nil {
+		return filterCloseErr(err)
+	}
+
+	return nil
+}
+
+func (cs *clientStream) RecvMsg(m interface{}) error {
+	if cs.remoteClosed {
+		return io.EOF
+	}
+
+	var msg *streamMessage
+	select {
+	case <-cs.ctx.Done():
+		return cs.ctx.Err()
+	case <-cs.s.recvClose:
+		// If recv has a pending message, process that first
+		select {
+		case msg = <-cs.s.recv:
+		default:
+			return cs.s.recvErr
+		}
+	case msg = <-cs.s.recv:
+	}
+
+	if msg.header.Type == messageTypeResponse {
+		resp := &Response{}
+		err := proto.Unmarshal(msg.payload[:msg.header.Length], resp)
+		// return the payload buffer for reuse
+		cs.c.channel.putmbuf(msg.payload)
+		if err != nil {
+			return err
+		}
+
+		if err := cs.c.codec.Unmarshal(resp.Payload, m); err != nil {
+			return err
+		}
+
+		if resp.Status != nil && resp.Status.Code != int32(codes.OK) {
+			return status.ErrorProto(resp.Status)
+		}
+
+		cs.c.deleteStream(cs.s)
+		cs.remoteClosed = true
+
+		return nil
+	} else if msg.header.Type == messageTypeData {
+		if !cs.desc.StreamingServer {
+			cs.c.deleteStream(cs.s)
+			cs.remoteClosed = true
+			return fmt.Errorf("received data from non-streaming server: %w", ErrProtocol)
+		}
+		if msg.header.Flags&flagRemoteClosed == flagRemoteClosed {
+			cs.c.deleteStream(cs.s)
+			cs.remoteClosed = true
+
+			if msg.header.Flags&flagNoData == flagNoData {
+				return io.EOF
+			}
+		}
+
+		err := cs.c.codec.Unmarshal(msg.payload[:msg.header.Length], m)
+		cs.c.channel.putmbuf(msg.payload)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+
+	return fmt.Errorf("unexpected %q message received: %w", msg.header.Type, ErrProtocol)
+}
+
+// Close closes the ttrpc connection and underlying connection
 func (c *Client) Close() error {
 	c.closeOnce.Do(func() {
 		c.closed()
+
+		c.conn.Close()
 	})
 	return nil
 }
 
-// UserOnCloseWait is used to blocks untils the user's on-close callback
+// UserOnCloseWait is used to block until the user's on-close callback
 // finishes.
 func (c *Client) UserOnCloseWait(ctx context.Context) error {
 	select {
@@ -188,194 +336,124 @@ func (c *Client) UserOnCloseWait(ctx context.Context) error {
 	}
 }
 
-type message struct {
-	messageHeader
-	p   []byte
-	err error
-}
-
-// callMap provides access to a map of active calls, guarded by a mutex.
-type callMap struct {
-	m           sync.Mutex
-	activeCalls map[uint32]*callRequest
-	closeErr    error
-}
-
-// newCallMap returns a new callMap with an empty set of active calls.
-func newCallMap() *callMap {
-	return &callMap{
-		activeCalls: make(map[uint32]*callRequest),
-	}
-}
-
-// set adds a call entry to the map with the given streamID key.
-func (cm *callMap) set(streamID uint32, cr *callRequest) error {
-	cm.m.Lock()
-	defer cm.m.Unlock()
-	if cm.closeErr != nil {
-		return cm.closeErr
-	}
-	cm.activeCalls[streamID] = cr
-	return nil
-}
-
-// get looks up the call entry for the given streamID key, then removes it
-// from the map and returns it.
-func (cm *callMap) get(streamID uint32) (cr *callRequest, ok bool, err error) {
-	cm.m.Lock()
-	defer cm.m.Unlock()
-	if cm.closeErr != nil {
-		return nil, false, cm.closeErr
-	}
-	cr, ok = cm.activeCalls[streamID]
-	if ok {
-		delete(cm.activeCalls, streamID)
-	}
-	return
-}
-
-// abort sends the given error to each active call, and clears the map.
-// Once abort has been called, any subsequent calls to the callMap will return the error passed to abort.
-func (cm *callMap) abort(err error) error {
-	cm.m.Lock()
-	defer cm.m.Unlock()
-	if cm.closeErr != nil {
-		return cm.closeErr
-	}
-	for streamID, call := range cm.activeCalls {
-		call.errs <- err
-		delete(cm.activeCalls, streamID)
-	}
-	cm.closeErr = err
-	return nil
-}
-
 func (c *Client) run() {
-	var (
-		waiters      = newCallMap()
-		receiverDone = make(chan struct{})
-	)
+	err := c.receiveLoop()
+	c.Close()
+	c.cleanupStreams(err)
 
-	// Sender goroutine
-	// Receives calls from dispatch, adds them to the set of active calls, and sends them
-	// to the server.
-	go func() {
-		var streamID uint32 = 1
-		for {
-			select {
-			case <-c.ctx.Done():
-				return
-			case call := <-c.calls:
-				id := streamID
-				streamID += 2 // enforce odd client initiated request ids
-				if err := waiters.set(id, call); err != nil {
-					call.errs <- err // errs is buffered so should not block.
-					continue
-				}
-				if err := c.send(id, messageTypeRequest, call.req); err != nil {
-					call.errs <- err // errs is buffered so should not block.
-					waiters.get(id)  // remove from waiters set
-				}
-			}
-		}
-	}()
-
-	// Receiver goroutine
-	// Receives responses from the server, looks up the call info in the set of active calls,
-	// and notifies the caller of the response.
-	go func() {
-		defer close(receiverDone)
-		for {
-			select {
-			case <-c.ctx.Done():
-				c.setError(c.ctx.Err())
-				return
-			default:
-				mh, p, err := c.channel.recv()
-				if err != nil {
-					_, ok := status.FromError(err)
-					if !ok {
-						// treat all errors that are not an rpc status as terminal.
-						// all others poison the connection.
-						c.setError(filterCloseErr(err))
-						return
-					}
-				}
-				msg := &message{
-					messageHeader: mh,
-					p:             p[:mh.Length],
-					err:           err,
-				}
-				call, ok, err := waiters.get(mh.StreamID)
-				if err != nil {
-					logrus.Errorf("ttrpc: failed to look up active call: %s", err)
-					continue
-				}
-				if !ok {
-					logrus.Errorf("ttrpc: received message for unknown channel %v", mh.StreamID)
-					continue
-				}
-				call.errs <- c.recv(call.resp, msg)
-			}
-		}
-	}()
-
-	defer func() {
-		c.conn.Close()
-		c.userCloseFunc()
-		close(c.userCloseWaitCh)
-	}()
+	c.userCloseFunc()
+	close(c.userCloseWaitCh)
+}
 
+func (c *Client) receiveLoop() error {
 	for {
 		select {
-		case <-receiverDone:
-			// The receiver has exited.
-			// don't return out, let the close of the context trigger the abort of waiters
-			c.Close()
 		case <-c.ctx.Done():
-			// Abort all active calls. This will also prevent any new calls from being added
-			// to waiters.
-			waiters.abort(c.error())
-			return
+			return ErrClosed
+		default:
+			var (
+				msg = &streamMessage{}
+				err error
+			)
+
+			msg.header, msg.payload, err = c.channel.recv()
+			if err != nil {
+				_, ok := status.FromError(err)
+				if !ok {
+					// treat all errors that are not an rpc status as terminal.
+					// all others poison the connection.
+					return filterCloseErr(err)
+				}
+			}
+			sid := streamID(msg.header.StreamID)
+			s := c.getStream(sid)
+			if s == nil {
+				log.G(c.ctx).WithField("stream", sid).Error("ttrpc: received message on inactive stream")
+				continue
+			}
+
+			if err != nil {
+				s.closeWithError(err)
+			} else {
+				if err := s.receive(c.ctx, msg); err != nil {
+					log.G(c.ctx).WithFields(log.Fields{"error": err, "stream": sid}).Error("ttrpc: failed to handle message")
+				}
+			}
 		}
 	}
 }
 
-func (c *Client) error() error {
-	c.errOnce.Do(func() {
-		if c.err == nil {
-			c.err = ErrClosed
+// createStream creates a new stream and registers it with the client
+// Introduce stream types for multiple or single response
+func (c *Client) createStream(flags uint8, b []byte) (*stream, error) {
+	// sendLock must be held across both allocation of the stream ID and sending it across the wire.
+	// This ensures that new stream IDs sent on the wire are always increasing, which is a
+	// requirement of the TTRPC protocol.
+	// This use of sendLock could be split into another mutex that covers stream creation + first send,
+	// and just use sendLock to guard writing to the wire, but for now it seems simpler to have fewer mutexes.
+	c.sendLock.Lock()
+	defer c.sendLock.Unlock()
+
+	// Check if closed since lock acquired to prevent adding
+	// anything after cleanup completes
+	select {
+	case <-c.ctx.Done():
+		return nil, ErrClosed
+	default:
+	}
+
+	var s *stream
+	if err := func() error {
+		// In the future this could be replaced with a sync.Map instead of streamLock+map.
+		c.streamLock.Lock()
+		defer c.streamLock.Unlock()
+
+		// Check if closed since lock acquired to prevent adding
+		// anything after cleanup completes
+		select {
+		case <-c.ctx.Done():
+			return ErrClosed
+		default:
 		}
-	})
-	return c.err
-}
 
-func (c *Client) setError(err error) {
-	c.errOnce.Do(func() {
-		c.err = err
-	})
-}
+		s = newStream(c.nextStreamID, c)
+		c.streams[s.id] = s
+		c.nextStreamID = c.nextStreamID + 2
 
-func (c *Client) send(streamID uint32, mtype messageType, msg interface{}) error {
-	p, err := c.codec.Marshal(msg)
-	if err != nil {
-		return err
+		return nil
+	}(); err != nil {
+		return nil, err
 	}
 
-	return c.channel.send(streamID, mtype, p)
+	if err := c.channel.send(uint32(s.id), messageTypeRequest, flags, b); err != nil {
+		return s, filterCloseErr(err)
+	}
+
+	return s, nil
 }
 
-func (c *Client) recv(resp *Response, msg *message) error {
-	if msg.err != nil {
-		return msg.err
-	}
+func (c *Client) deleteStream(s *stream) {
+	c.streamLock.Lock()
+	delete(c.streams, s.id)
+	c.streamLock.Unlock()
+	s.closeWithError(nil)
+}
 
-	if msg.Type != messageTypeResponse {
-		return errors.New("unknown message type received")
-	}
+func (c *Client) getStream(sid streamID) *stream {
+	c.streamLock.RLock()
+	s := c.streams[sid]
+	c.streamLock.RUnlock()
+	return s
+}
 
-	defer c.channel.putmbuf(msg.p)
-	return proto.Unmarshal(msg.p, resp)
+func (c *Client) cleanupStreams(err error) {
+	c.streamLock.Lock()
+	defer c.streamLock.Unlock()
+
+	for sid, s := range c.streams {
+		s.closeWithError(err)
+		delete(c.streams, sid)
+	}
 }
 
 // filterCloseErr rewrites EOF and EPIPE errors to ErrClosed. Use when
@@ -388,6 +466,8 @@ func filterCloseErr(err error) error {
 		return nil
 	case err == io.EOF:
 		return ErrClosed
+	case errors.Is(err, io.ErrClosedPipe):
+		return ErrClosed
 	case errors.Is(err, io.EOF):
 		return ErrClosed
 	case strings.Contains(err.Error(), "use of closed network connection"):
@@ -395,11 +475,9 @@ func filterCloseErr(err error) error {
 	default:
 		// if we have an epipe on a write or econnreset on a read , we cast to errclosed
 		var oerr *net.OpError
-		if errors.As(err, &oerr) && (oerr.Op == "write" || oerr.Op == "read") {
-			serr, sok := oerr.Err.(*os.SyscallError)
-			if sok && ((serr.Err == syscall.EPIPE && oerr.Op == "write") ||
-				(serr.Err == syscall.ECONNRESET && oerr.Op == "read")) {
-
+		if errors.As(err, &oerr) {
+			if (oerr.Op == "write" && errors.Is(err, syscall.EPIPE)) ||
+				(oerr.Op == "read" && errors.Is(err, syscall.ECONNRESET)) {
 				return ErrClosed
 			}
 		}
@@ -407,3 +485,86 @@ func filterCloseErr(err error) error {
 
 	return err
 }
+
+// NewStream creates a new stream with the given stream descriptor to the
+// specified service and method. If not a streaming client, the request object
+// may be provided.
+func (c *Client) NewStream(ctx context.Context, desc *StreamDesc, service, method string, req interface{}) (ClientStream, error) {
+	var payload []byte
+	if req != nil {
+		var err error
+		payload, err = c.codec.Marshal(req)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	request := &Request{
+		Service: service,
+		Method:  method,
+		Payload: payload,
+		// TODO: metadata from context
+	}
+	p, err := c.codec.Marshal(request)
+	if err != nil {
+		return nil, err
+	}
+
+	var flags uint8
+	if desc.StreamingClient {
+		flags = flagRemoteOpen
+	} else {
+		flags = flagRemoteClosed
+	}
+	s, err := c.createStream(flags, p)
+	if err != nil {
+		return nil, err
+	}
+
+	return &clientStream{
+		ctx:  ctx,
+		s:    s,
+		c:    c,
+		desc: desc,
+	}, nil
+}
+
+func (c *Client) dispatch(ctx context.Context, req *Request, resp *Response) error {
+	p, err := c.codec.Marshal(req)
+	if err != nil {
+		return err
+	}
+
+	s, err := c.createStream(0, p)
+	if err != nil {
+		return err
+	}
+	defer c.deleteStream(s)
+
+	var msg *streamMessage
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-c.ctx.Done():
+		return ErrClosed
+	case <-s.recvClose:
+		// If recv has a pending message, process that first
+		select {
+		case msg = <-s.recv:
+		default:
+			return s.recvErr
+		}
+	case msg = <-s.recv:
+	}
+
+	if msg.header.Type == messageTypeResponse {
+		err = proto.Unmarshal(msg.payload[:msg.header.Length], resp)
+	} else {
+		err = fmt.Errorf("unexpected %q message received: %w", msg.header.Type, ErrProtocol)
+	}
+
+	// return the payload buffer for reuse
+	c.channel.putmbuf(msg.payload)
+
+	return err
+}

client_test.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"testing"
 	"time"
+
+	"github.com/containerd/ttrpc/internal"
 )
 
 func TestUserOnCloseWait(t *testing.T) {
@@ -46,7 +48,7 @@ func TestUserOnCloseWait(t *testing.T) {
 			}),
 		)
 
-		tp      testPayload
+		tp      internal.TestPayload
 		tclient = newTestingClient(client)
 	)
 
@@ -62,7 +64,7 @@ func TestUserOnCloseWait(t *testing.T) {
 		t.Fatalf("expected error %v, but got %v", context.DeadlineExceeded, err)
 	}
 
-	_ = <-dataCh
+	<-dataCh
 
 	if err := client.UserOnCloseWait(ctx); err != nil {
 		t.Fatalf("expected error nil , but got %v", err)

cmd/protoc-gen-go-ttrpc/generator.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"fmt"
 	"strings"
 
 	"google.golang.org/protobuf/compiler/protogen"
@@ -29,10 +30,19 @@ type generator struct {
 	out *protogen.GeneratedFile
 
 	ident struct {
-		context string
-		server  string
-		client  string
-		method  string
+		context     string
+		server      string
+		client      string
+		method      string
+		stream      string
+		serviceDesc string
+		streamDesc  string
+
+		streamServerIdent protogen.GoIdent
+		streamClientIdent protogen.GoIdent
+
+		streamServer string
+		streamClient string
 	}
 }
 
@@ -54,10 +64,38 @@ func newGenerator(out *protogen.GeneratedFile) *generator {
 		GoImportPath: "github.com/containerd/ttrpc",
 		GoName:       "Method",
 	})
+	gen.ident.stream = out.QualifiedGoIdent(protogen.GoIdent{
+		GoImportPath: "github.com/containerd/ttrpc",
+		GoName:       "Stream",
+	})
+	gen.ident.serviceDesc = out.QualifiedGoIdent(protogen.GoIdent{
+		GoImportPath: "github.com/containerd/ttrpc",
+		GoName:       "ServiceDesc",
+	})
+	gen.ident.streamDesc = out.QualifiedGoIdent(protogen.GoIdent{
+		GoImportPath: "github.com/containerd/ttrpc",
+		GoName:       "StreamDesc",
+	})
+
+	gen.ident.streamServerIdent = protogen.GoIdent{
+		GoImportPath: "github.com/containerd/ttrpc",
+		GoName:       "StreamServer",
+	}
+	gen.ident.streamClientIdent = protogen.GoIdent{
+		GoImportPath: "github.com/containerd/ttrpc",
+		GoName:       "ClientStream",
+	}
+	gen.ident.streamServer = out.QualifiedGoIdent(gen.ident.streamServerIdent)
+	gen.ident.streamClient = out.QualifiedGoIdent(gen.ident.streamClientIdent)
 	return &gen
 }
 
-func generate(plugin *protogen.Plugin, input *protogen.File) error {
+func generate(plugin *protogen.Plugin, input *protogen.File, servicePrefix string) error {
+	if len(input.Services) == 0 {
+		// Only generate a Go file if the file has some services.
+		return nil
+	}
+
 	file := plugin.NewGeneratedFile(input.GeneratedFilenamePrefix+"_ttrpc.pb.go", input.GoImportPath)
 	file.P("// Code generated by protoc-gen-go-ttrpc. DO NOT EDIT.")
 	file.P("// source: ", input.Desc.Path())
@@ -65,6 +103,7 @@ func generate(plugin *protogen.Plugin, input *protogen.File) error {
 
 	gen := newGenerator(file)
 	for _, service := range input.Services {
+		service.GoName = servicePrefix + service.GoName
 		gen.genService(service)
 	}
 	return nil
@@ -74,52 +113,277 @@ func (gen *generator) genService(service *protogen.Service) {
 	fullName := service.Desc.FullName()
 	p := gen.out
 
+	var methods []*protogen.Method
+	var streams []*protogen.Method
+
 	serviceName := service.GoName + "Service"
 	p.P("type ", serviceName, " interface{")
 	for _, method := range service.Methods {
-		p.P(method.GoName,
-			"(ctx ", gen.ident.context, ",",
-			"req *", method.Input.GoIdent, ")",
-			"(*", method.Output.GoIdent, ", error)")
+		var sendArgs, retArgs string
+		if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() {
+			streams = append(streams, method)
+			sendArgs = fmt.Sprintf("%s_%sServer", service.GoName, method.GoName)
+			if !method.Desc.IsStreamingClient() {
+				sendArgs = fmt.Sprintf("*%s, %s", p.QualifiedGoIdent(method.Input.GoIdent), sendArgs)
+			}
+			if method.Desc.IsStreamingServer() {
+				retArgs = "error"
+			} else {
+				retArgs = fmt.Sprintf("(*%s, error)", p.QualifiedGoIdent(method.Output.GoIdent))
+			}
+		} else {
+			methods = append(methods, method)
+			sendArgs = fmt.Sprintf("*%s", p.QualifiedGoIdent(method.Input.GoIdent))
+			retArgs = fmt.Sprintf("(*%s, error)", p.QualifiedGoIdent(method.Output.GoIdent))
+		}
+		p.P(method.GoName, "(", gen.ident.context, ", ", sendArgs, ") ", retArgs)
 	}
 	p.P("}")
+	p.P()
+
+	for _, method := range streams {
+		structName := strings.ToLower(service.GoName) + method.GoName + "Server"
+
+		p.P("type ", service.GoName, "_", method.GoName, "Server interface {")
+		if method.Desc.IsStreamingServer() {
+			p.P("Send(*", method.Output.GoIdent, ") error")
+		}
+		if method.Desc.IsStreamingClient() {
+			p.P("Recv() (*", method.Input.GoIdent, ", error)")
+
+		}
+		p.P(gen.ident.streamServer)
+		p.P("}")
+		p.P()
+
+		p.P("type ", structName, " struct {")
+		p.P(gen.ident.streamServer)
+		p.P("}")
+		p.P()
+
+		if method.Desc.IsStreamingServer() {
+			p.P("func (x *", structName, ") Send(m *", method.Output.GoIdent, ") error {")
+			p.P("return x.StreamServer.SendMsg(m)")
+			p.P("}")
+			p.P()
+		}
+
+		if method.Desc.IsStreamingClient() {
+			p.P("func (x *", structName, ") Recv() (*", method.Input.GoIdent, ", error) {")
+			p.P("m := new(", method.Input.GoIdent, ")")
+			p.P("if err := x.StreamServer.RecvMsg(m); err != nil {")
+			p.P("return nil, err")
+			p.P("}")
+			p.P("return m, nil")
+			p.P("}")
+			p.P()
+		}
+	}
 
 	// registration method
 	p.P("func Register", serviceName, "(srv *", gen.ident.server, ", svc ", serviceName, "){")
-	p.P(`srv.Register("`, fullName, `", map[string]`, gen.ident.method, "{")
-	for _, method := range service.Methods {
-		p.P(`"`, method.GoName, `": func(ctx `, gen.ident.context, ", unmarshal func(interface{}) error)(interface{}, error){")
-		p.P("var req ", method.Input.GoIdent)
-		p.P("if err := unmarshal(&req); err != nil {")
-		p.P("return nil, err")
-		p.P("}")
-		p.P("return svc.", method.GoName, "(ctx, &req)")
+	p.P(`srv.RegisterService("`, fullName, `", &`, gen.ident.serviceDesc, "{")
+	if len(methods) > 0 {
+		p.P(`Methods: map[string]`, gen.ident.method, "{")
+		for _, method := range methods {
+			p.P(`"`, method.GoName, `": func(ctx `, gen.ident.context, ", unmarshal func(interface{}) error)(interface{}, error){")
+			p.P("var req ", method.Input.GoIdent)
+			p.P("if err := unmarshal(&req); err != nil {")
+			p.P("return nil, err")
+			p.P("}")
+			p.P("return svc.", method.GoName, "(ctx, &req)")
+			p.P("},")
+		}
+		p.P("},")
+	}
+	if len(streams) > 0 {
+		p.P(`Streams: map[string]`, gen.ident.stream, "{")
+		for _, method := range streams {
+			p.P(`"`, method.GoName, `": {`)
+			p.P(`Handler: func(ctx `, gen.ident.context, ", stream ", gen.ident.streamServer, ") (interface{}, error) {")
+
+			structName := strings.ToLower(service.GoName) + method.GoName + "Server"
+			var sendArg string
+			if !method.Desc.IsStreamingClient() {
+				sendArg = "m, "
+				p.P("m := new(", method.Input.GoIdent, ")")
+				p.P("if err := stream.RecvMsg(m); err != nil {")
+				p.P("return nil, err")
+				p.P("}")
+			}
+			if method.Desc.IsStreamingServer() {
+				p.P("return nil, svc.", method.GoName, "(ctx, ", sendArg, "&", structName, "{stream})")
+			} else {
+				p.P("return svc.", method.GoName, "(ctx, ", sendArg, "&", structName, "{stream})")
+
+			}
+			p.P("},")
+			if method.Desc.IsStreamingClient() {
+				p.P("StreamingClient: true,")
+			} else {
+				p.P("StreamingClient: false,")
+			}
+			if method.Desc.IsStreamingServer() {
+				p.P("StreamingServer: true,")
+			} else {
+				p.P("StreamingServer: false,")
+			}
+			p.P("},")
+		}
 		p.P("},")
 	}
 	p.P("})")
 	p.P("}")
+	p.P()
 
 	clientType := service.GoName + "Client"
-	clientStructType := strings.ToLower(clientType[:1]) + clientType[1:]
+
+	// For consistency with ttrpc 1.0 without streaming, just use
+	// the service name if no streams are defined
+	clientInterface := serviceName
+	if len(streams) > 0 {
+		clientInterface = clientType
+		// Stream client interfaces are different than the server interface
+		p.P("type ", clientInterface, " interface{")
+		for _, method := range service.Methods {
+			if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() {
+				streams = append(streams, method)
+				var sendArg string
+				if !method.Desc.IsStreamingClient() {
+					sendArg = fmt.Sprintf("*%s, ", p.QualifiedGoIdent(method.Input.GoIdent))
+				}
+				p.P(method.GoName,
+					"(", gen.ident.context, ", ", sendArg,
+					") (", service.GoName, "_", method.GoName, "Client, error)")
+			} else {
+				methods = append(methods, method)
+				p.P(method.GoName,
+					"(", gen.ident.context, ", ",
+					"*", method.Input.GoIdent, ")",
+					"(*", method.Output.GoIdent, ", error)")
+			}
+		}
+		p.P("}")
+		p.P()
+	}
+
+	clientStructType := strings.ToLower(service.GoName) + "Client"
 	p.P("type ", clientStructType, " struct{")
 	p.P("client *", gen.ident.client)
 	p.P("}")
-	p.P("func New", clientType, "(client *", gen.ident.client, ")", serviceName, "{")
+	p.P("func New", clientType, "(client *", gen.ident.client, ")", clientInterface, "{")
 	p.P("return &", clientStructType, "{")
 	p.P("client:client,")
 	p.P("}")
 	p.P("}")
+	p.P()
 
 	for _, method := range service.Methods {
-		p.P("func (c *", clientStructType, ")", method.GoName, "(",
-			"ctx ", gen.ident.context, ",",
-			"req *", method.Input.GoIdent, ")",
-			"(*", method.Output.GoIdent, ", error){")
-		p.P("var resp ", method.Output.GoIdent)
-		p.P(`if err := c.client.Call(ctx, "`, fullName, `", "`, method.Desc.Name(), `", req, &resp); err != nil {`)
-		p.P("return nil, err")
-		p.P("}")
-		p.P("return &resp, nil")
-		p.P("}")
+		var sendArg string
+		if !method.Desc.IsStreamingClient() {
+			sendArg = ", req *" + gen.out.QualifiedGoIdent(method.Input.GoIdent)
+		}
+
+		intName := service.GoName + "_" + method.GoName + "Client"
+		var retArg string
+		if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() {
+			retArg = intName
+		} else {
+			retArg = "*" + gen.out.QualifiedGoIdent(method.Output.GoIdent)
+		}
+
+		p.P("func (c *", clientStructType, ") ", method.GoName,
+			"(ctx ", gen.ident.context, "", sendArg, ") ",
+			"(", retArg, ", error) {")
+
+		if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() {
+			var streamingClient, streamingServer, req string
+			if method.Desc.IsStreamingClient() {
+				streamingClient = "true"
+				req = "nil"
+			} else {
+				streamingClient = "false"
+				req = "req"
+			}
+			if method.Desc.IsStreamingServer() {
+				streamingServer = "true"
+			} else {
+				streamingServer = "false"
+			}
+			p.P("stream, err := c.client.NewStream(ctx, &", gen.ident.streamDesc, "{")
+			p.P("StreamingClient: ", streamingClient, ",")
+			p.P("StreamingServer: ", streamingServer, ",")
+			p.P("}, ", `"`+fullName+`", `, `"`+method.GoName+`", `, req, `)`)
+			p.P("if err != nil {")
+			p.P("return nil, err")
+			p.P("}")
+
+			structName := strings.ToLower(service.GoName) + method.GoName + "Client"
+
+			p.P("x := &", structName, "{stream}")
+
+			p.P("return x, nil")
+			p.P("}")
+			p.P()
+
+			// Create interface
+			p.P("type ", intName, " interface {")
+			if method.Desc.IsStreamingClient() {
+				p.P("Send(*", method.Input.GoIdent, ") error")
+			}
+			if method.Desc.IsStreamingServer() {
+				p.P("Recv() (*", method.Output.GoIdent, ", error)")
+			} else {
+				p.P("CloseAndRecv() (*", method.Output.GoIdent, ", error)")
+			}
+
+			p.P(gen.ident.streamClient)
+			p.P("}")
+			p.P()
+
+			// Create struct
+			p.P("type ", structName, " struct {")
+			p.P(gen.ident.streamClient)
+			p.P("}")
+			p.P()
+
+			if method.Desc.IsStreamingClient() {
+				p.P("func (x *", structName, ") Send(m *", method.Input.GoIdent, ") error {")
+				p.P("return x.", gen.ident.streamClientIdent.GoName, ".SendMsg(m)")
+				p.P("}")
+				p.P()
+			}
+
+			if method.Desc.IsStreamingServer() {
+				p.P("func (x *", structName, ") Recv() (*", method.Output.GoIdent, ", error) {")
+				p.P("m := new(", method.Output.GoIdent, ")")
+				p.P("if err := x.ClientStream.RecvMsg(m); err != nil {")
+				p.P("return nil, err")
+				p.P("}")
+				p.P("return m, nil")
+				p.P("}")
+				p.P()
+			} else {
+				p.P("func (x *", structName, ") CloseAndRecv() (*", method.Output.GoIdent, ", error) {")
+				p.P("if err := x.ClientStream.CloseSend(); err != nil {")
+				p.P("return nil, err")
+				p.P("}")
+				p.P("m := new(", method.Output.GoIdent, ")")
+				p.P("if err := x.ClientStream.RecvMsg(m); err != nil {")
+				p.P("return nil, err")
+				p.P("}")
+				p.P("return m, nil")
+				p.P("}")
+				p.P()
+			}
+		} else {
+			p.P("var resp ", method.Output.GoIdent)
+			p.P(`if err := c.client.Call(ctx, "`, fullName, `", "`, method.Desc.Name(), `", req, &resp); err != nil {`)
+			p.P("return nil, err")
+			p.P("}")
+			p.P("return &resp, nil")
+			p.P("}")
+			p.P()
+		}
 	}
 }

cmd/protoc-gen-go-ttrpc/main.go

@@ -18,15 +18,25 @@ package main
 
 import (
 	"google.golang.org/protobuf/compiler/protogen"
+	"google.golang.org/protobuf/types/pluginpb"
 )
 
 func main() {
-	protogen.Options{}.Run(func(gen *protogen.Plugin) error {
+	var servicePrefix string
+	protogen.Options{
+		ParamFunc: func(name, value string) error {
+			if name == "prefix" {
+				servicePrefix = value
+			}
+			return nil
+		},
+	}.Run(func(gen *protogen.Plugin) error {
+		gen.SupportedFeatures = uint64(pluginpb.CodeGeneratorResponse_FEATURE_PROTO3_OPTIONAL)
 		for _, f := range gen.Files {
 			if !f.Generate {
 				continue
 			}
-			if err := generate(gen, f); err != nil {
+			if err := generate(gen, f, servicePrefix); err != nil {
 				return err
 			}
 		}

codec.go

@@ -19,7 +19,7 @@ package ttrpc
 import (
 	"fmt"
 
-	"github.com/gogo/protobuf/proto"
+	"google.golang.org/protobuf/proto"
 )
 
 type codec struct{}

config.go

@@ -16,7 +16,10 @@
 
 package ttrpc
 
-import "errors"
+import (
+	"context"
+	"errors"
+)
 
 type serverConfig struct {
 	handshaker  Handshaker
@@ -44,9 +47,40 @@ func WithServerHandshaker(handshaker Handshaker) ServerOpt {
 func WithUnaryServerInterceptor(i UnaryServerInterceptor) ServerOpt {
 	return func(c *serverConfig) error {
 		if c.interceptor != nil {
-			return errors.New("only one interceptor allowed per server")
+			return errors.New("only one unchained interceptor allowed per server")
 		}
 		c.interceptor = i
 		return nil
 	}
 }
+
+// WithChainUnaryServerInterceptor sets the provided chain of server interceptors
+func WithChainUnaryServerInterceptor(interceptors ...UnaryServerInterceptor) ServerOpt {
+	return func(c *serverConfig) error {
+		if len(interceptors) == 0 {
+			return nil
+		}
+		if c.interceptor != nil {
+			interceptors = append([]UnaryServerInterceptor{c.interceptor}, interceptors...)
+		}
+		c.interceptor = func(
+			ctx context.Context,
+			unmarshal Unmarshaler,
+			info *UnaryServerInfo,
+			method Method) (interface{}, error) {
+			return interceptors[0](ctx, unmarshal, info,
+				chainUnaryServerInterceptors(info, method, interceptors[1:]))
+		}
+		return nil
+	}
+}
+
+func chainUnaryServerInterceptors(info *UnaryServerInfo, method Method, interceptors []UnaryServerInterceptor) Method {
+	if len(interceptors) == 0 {
+		return method
+	}
+	return func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) {
+		return interceptors[0](ctx, unmarshal, info,
+			chainUnaryServerInterceptors(info, method, interceptors[1:]))
+	}
+}

doc.go

@@ -0,0 +1,23 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+package ttrpc defines and implements a low level simple transfer protocol
+optimized for low latency and reliable connections between processes on the same
+host. The protocol uses simple framing for sending requests, responses, and data
+using multiple streams.
+*/
+package ttrpc

errors.go

@@ -0,0 +1,80 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ttrpc
+
+import (
+	"errors"
+
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+var (
+	// ErrProtocol is a general error in the handling the protocol.
+	ErrProtocol = errors.New("protocol error")
+
+	// ErrClosed is returned by client methods when the underlying connection is
+	// closed.
+	ErrClosed = errors.New("ttrpc: closed")
+
+	// ErrServerClosed is returned when the Server has closed its connection.
+	ErrServerClosed = errors.New("ttrpc: server closed")
+
+	// ErrStreamClosed is when the streaming connection is closed.
+	ErrStreamClosed = errors.New("ttrpc: stream closed")
+)
+
+// OversizedMessageErr is used to indicate refusal to send an oversized message.
+// It wraps a ResourceExhausted grpc Status together with the offending message
+// length.
+type OversizedMessageErr struct {
+	messageLength int
+	err           error
+}
+
+// OversizedMessageError returns an OversizedMessageErr error for the given message
+// length if it exceeds the allowed maximum. Otherwise a nil error is returned.
+func OversizedMessageError(messageLength int) error {
+	if messageLength <= messageLengthMax {
+		return nil
+	}
+
+	return &OversizedMessageErr{
+		messageLength: messageLength,
+		err:           status.Errorf(codes.ResourceExhausted, "message length %v exceed maximum message size of %v", messageLength, messageLengthMax),
+	}
+}
+
+// Error returns the error message for the corresponding grpc Status for the error.
+func (e *OversizedMessageErr) Error() string {
+	return e.err.Error()
+}
+
+// Unwrap returns the corresponding error with our grpc status code.
+func (e *OversizedMessageErr) Unwrap() error {
+	return e.err
+}
+
+// RejectedLength retrieves the rejected message length which triggered the error.
+func (e *OversizedMessageErr) RejectedLength() int {
+	return e.messageLength
+}
+
+// MaximumLength retrieves the maximum allowed message length that triggered the error.
+func (*OversizedMessageErr) MaximumLength() int {
+	return messageLengthMax
+}

example/Protobuild.toml

@@ -1,6 +1,5 @@
-version = "unstable"
-generator = "gogottrpc"
-plugins = ["ttrpc"]
+version = "2"
+generators = ["go", "go-ttrpc"]
 
 # Control protoc include paths. Below are usually some good defaults, but feel
 # free to try it without them if it works for your project.
@@ -9,11 +8,6 @@ plugins = ["ttrpc"]
   # treat the root of the project as an include, but this may not be necessary.
   # before = ["./protobuf"]
 
-  # Paths that should be treated as include roots in relation to the vendor
-  # directory. These will be calculated with the vendor directory nearest the
-  # target package.
-  packages = ["github.com/gogo/protobuf"]
-
   # Paths that will be added untouched to the end of the includes. We use
   # `/usr/local/include` to pickup the common install location of protobuf.
   # This is the default.
@@ -22,11 +16,4 @@ plugins = ["ttrpc"]
 # This section maps protobuf imports to Go packages. These will become
 # `-M` directives in the call to the go protobuf generator.
 [packages]
-  "gogoproto/gogo.proto" = "github.com/gogo/protobuf/gogoproto"
-  "google/protobuf/any.proto" = "github.com/gogo/protobuf/types"
-  "google/protobuf/empty.proto" = "github.com/gogo/protobuf/types"
-  "google/protobuf/descriptor.proto" = "github.com/gogo/protobuf/protoc-gen-gogo/descriptor"
-  "google/protobuf/field_mask.proto" = "github.com/gogo/protobuf/types"
-  "google/protobuf/timestamp.proto" = "github.com/gogo/protobuf/types"
-  "google/protobuf/duration.proto" = "github.com/gogo/protobuf/types"
   "google/rpc/status.proto" = "github.com/containerd/containerd/protobuf/google/rpc"

example/cmd/handshaker_other.go

@@ -1,3 +1,4 @@
+//go:build !linux
 // +build !linux
 
 /*

example/cmd/main.go

@@ -26,7 +26,7 @@ import (
 
 	ttrpc "github.com/containerd/ttrpc"
 	"github.com/containerd/ttrpc/example"
-	"github.com/gogo/protobuf/types"
+	"google.golang.org/protobuf/types/known/emptypb"
 )
 
 const socket = "example-ttrpc-server"
@@ -130,6 +130,6 @@ func (s *exampleServer) Method1(ctx context.Context, r *example.Method1Request)
 	}, nil
 }
 
-func (s *exampleServer) Method2(ctx context.Context, r *example.Method1Request) (*types.Empty, error) {
-	return &types.Empty{}, nil
+func (s *exampleServer) Method2(ctx context.Context, r *example.Method1Request) (*emptypb.Empty, error) {
+	return &emptypb.Empty{}, nil
 }

example/example.proto

@@ -3,7 +3,6 @@ syntax = "proto3";
 package ttrpc.example.v1;
 
 import "google/protobuf/empty.proto";
-import "gogoproto/gogo.proto";
 
 option go_package = "github.com/containerd/ttrpc/example;example";
 

example/example_ttrpc.pb.go

@@ -0,0 +1,57 @@
+// Code generated by protoc-gen-go-ttrpc. DO NOT EDIT.
+// source: github.com/containerd/ttrpc/example/example.proto
+package example
+
+import (
+	context "context"
+	ttrpc "github.com/containerd/ttrpc"
+	empty "github.com/golang/protobuf/ptypes/empty"
+)
+
+type ExampleService interface {
+	Method1(ctx context.Context, req *Method1Request) (*Method1Response, error)
+	Method2(ctx context.Context, req *Method1Request) (*empty.Empty, error)
+}
+
+func RegisterExampleService(srv *ttrpc.Server, svc ExampleService) {
+	srv.Register("ttrpc.example.v1.Example", map[string]ttrpc.Method{
+		"Method1": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) {
+			var req Method1Request
+			if err := unmarshal(&req); err != nil {
+				return nil, err
+			}
+			return svc.Method1(ctx, &req)
+		},
+		"Method2": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) {
+			var req Method1Request
+			if err := unmarshal(&req); err != nil {
+				return nil, err
+			}
+			return svc.Method2(ctx, &req)
+		},
+	})
+}
+
+type exampleClient struct {
+	client *ttrpc.Client
+}
+
+func NewExampleClient(client *ttrpc.Client) ExampleService {
+	return &exampleClient{
+		client: client,
+	}
+}
+func (c *exampleClient) Method1(ctx context.Context, req *Method1Request) (*Method1Response, error) {
+	var resp Method1Response
+	if err := c.client.Call(ctx, "ttrpc.example.v1.Example", "Method1", req, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+func (c *exampleClient) Method2(ctx context.Context, req *Method1Request) (*empty.Empty, error) {
+	var resp empty.Empty
+	if err := c.client.Call(ctx, "ttrpc.example.v1.Example", "Method2", req, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}

go.mod

@@ -1,13 +1,16 @@
 module github.com/containerd/ttrpc
 
-go 1.13
+go 1.22
 
 require (
+	github.com/containerd/log v0.1.0
 	github.com/gogo/protobuf v1.3.2
+	github.com/golang/protobuf v1.5.4
 	github.com/prometheus/procfs v0.6.0
-	github.com/sirupsen/logrus v1.8.1
-	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
-	google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63
-	google.golang.org/grpc v1.27.1
-	google.golang.org/protobuf v1.27.1
+	golang.org/x/sys v0.26.0
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38
+	google.golang.org/grpc v1.69.2
+	google.golang.org/protobuf v1.36.0
 )
+
+require github.com/sirupsen/logrus v1.9.3 // indirect

go.sum

@@ -1,99 +1,69 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
+github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
-github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63 h1:YzfoEYWbODU5Fbt37+h7X16BWQbad7Q4S6gclTKFXM8=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=
-google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
-google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
+google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU=
+google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4=
+google.golang.org/protobuf v1.36.0 h1:mjIs9gYtt56AzC4ZaffQuh88TZurBGhIJMBZGSxNerQ=
+google.golang.org/protobuf v1.36.0/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

handshake.go

@@ -45,6 +45,6 @@ func (fn handshakerFunc) Handshake(ctx context.Context, conn net.Conn) (net.Conn
 	return fn(ctx, conn)
 }
 
-func noopHandshake(ctx context.Context, conn net.Conn) (net.Conn, interface{}, error) {
+func noopHandshake(_ context.Context, conn net.Conn) (net.Conn, interface{}, error) {
 	return conn, nil, nil
 }

integration/streaming/doc.go

@@ -0,0 +1,17 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package streaming

integration/streaming/test.pb.go

@@ -0,0 +1,363 @@
+//
+//Copyright The containerd Authors.
+//
+//Licensed under the Apache License, Version 2.0 (the "License");
+//you may not use this file except in compliance with the License.
+//You may obtain a copy of the License at
+//
+//http://www.apache.org/licenses/LICENSE-2.0
+//
+//Unless required by applicable law or agreed to in writing, software
+//distributed under the License is distributed on an "AS IS" BASIS,
+//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//See the License for the specific language governing permissions and
+//limitations under the License.
+
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.28.1
+// 	protoc        v3.20.1
+// source: github.com/containerd/ttrpc/integration/streaming/test.proto
+
+package streaming
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	emptypb "google.golang.org/protobuf/types/known/emptypb"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type EchoPayload struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Seq uint32 `protobuf:"varint,1,opt,name=seq,proto3" json:"seq,omitempty"`
+	Msg string `protobuf:"bytes,2,opt,name=msg,proto3" json:"msg,omitempty"`
+}
+
+func (x *EchoPayload) Reset() {
+	*x = EchoPayload{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_github_com_containerd_ttrpc_integration_streaming_test_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *EchoPayload) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*EchoPayload) ProtoMessage() {}
+
+func (x *EchoPayload) ProtoReflect() protoreflect.Message {
+	mi := &file_github_com_containerd_ttrpc_integration_streaming_test_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use EchoPayload.ProtoReflect.Descriptor instead.
+func (*EchoPayload) Descriptor() ([]byte, []int) {
+	return file_github_com_containerd_ttrpc_integration_streaming_test_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *EchoPayload) GetSeq() uint32 {
+	if x != nil {
+		return x.Seq
+	}
+	return 0
+}
+
+func (x *EchoPayload) GetMsg() string {
+	if x != nil {
+		return x.Msg
+	}
+	return ""
+}
+
+type Part struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Add int32 `protobuf:"varint,1,opt,name=add,proto3" json:"add,omitempty"`
+}
+
+func (x *Part) Reset() {
+	*x = Part{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_github_com_containerd_ttrpc_integration_streaming_test_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Part) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Part) ProtoMessage() {}
+
+func (x *Part) ProtoReflect() protoreflect.Message {
+	mi := &file_github_com_containerd_ttrpc_integration_streaming_test_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Part.ProtoReflect.Descriptor instead.
+func (*Part) Descriptor() ([]byte, []int) {
+	return file_github_com_containerd_ttrpc_integration_streaming_test_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *Part) GetAdd() int32 {
+	if x != nil {
+		return x.Add
+	}
+	return 0
+}
+
+type Sum struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Sum int32 `protobuf:"varint,1,opt,name=sum,proto3" json:"sum,omitempty"`
+	Num int32 `protobuf:"varint,2,opt,name=num,proto3" json:"num,omitempty"`
+}
+
+func (x *Sum) Reset() {
+	*x = Sum{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_github_com_containerd_ttrpc_integration_streaming_test_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Sum) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Sum) ProtoMessage() {}
+
+func (x *Sum) ProtoReflect() protoreflect.Message {
+	mi := &file_github_com_containerd_ttrpc_integration_streaming_test_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Sum.ProtoReflect.Descriptor instead.
+func (*Sum) Descriptor() ([]byte, []int) {
+	return file_github_com_containerd_ttrpc_integration_streaming_test_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *Sum) GetSum() int32 {
+	if x != nil {
+		return x.Sum
+	}
+	return 0
+}
+
+func (x *Sum) GetNum() int32 {
+	if x != nil {
+		return x.Num
+	}
+	return 0
+}
+
+var File_github_com_containerd_ttrpc_integration_streaming_test_proto protoreflect.FileDescriptor
+
+var file_github_com_containerd_ttrpc_integration_streaming_test_proto_rawDesc = []byte{
+	0x0a, 0x3c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e,
+	0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x74, 0x74, 0x72, 0x70, 0x63, 0x2f, 0x69, 0x6e,
+	0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d,
+	0x69, 0x6e, 0x67, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1b,
+	0x74, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f,
+	0x6e, 0x2e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x1a, 0x1b, 0x67, 0x6f, 0x6f,
+	0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70,
+	0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x31, 0x0a, 0x0b, 0x45, 0x63, 0x68, 0x6f,
+	0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x65, 0x71, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x73, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x73, 0x67,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6d, 0x73, 0x67, 0x22, 0x18, 0x0a, 0x04, 0x50,
+	0x61, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x64, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05,
+	0x52, 0x03, 0x61, 0x64, 0x64, 0x22, 0x29, 0x0a, 0x03, 0x53, 0x75, 0x6d, 0x12, 0x10, 0x0a, 0x03,
+	0x73, 0x75, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x73, 0x75, 0x6d, 0x12, 0x10,
+	0x0a, 0x03, 0x6e, 0x75, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x6e, 0x75, 0x6d,
+	0x32, 0xfa, 0x04, 0x0a, 0x09, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x12, 0x5a,
+	0x0a, 0x04, 0x45, 0x63, 0x68, 0x6f, 0x12, 0x28, 0x2e, 0x74, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x69,
+	0x6e, 0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x73, 0x74, 0x72, 0x65, 0x61,
+	0x6d, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64,
+	0x1a, 0x28, 0x2e, 0x74, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x67, 0x72, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x2e, 0x45,
+	0x63, 0x68, 0x6f, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x64, 0x0a, 0x0a, 0x45, 0x63,
+	0x68, 0x6f, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x28, 0x2e, 0x74, 0x74, 0x72, 0x70, 0x63,
+	0x2e, 0x69, 0x6e, 0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x73, 0x74, 0x72,
+	0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x50, 0x61, 0x79, 0x6c, 0x6f,
+	0x61, 0x64, 0x1a, 0x28, 0x2e, 0x74, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x67,
+	0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67,
+	0x2e, 0x45, 0x63, 0x68, 0x6f, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x28, 0x01, 0x30, 0x01,
+	0x12, 0x52, 0x0a, 0x09, 0x53, 0x75, 0x6d, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x21, 0x2e,
+	0x74, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f,
+	0x6e, 0x2e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x72, 0x74,
+	0x1a, 0x20, 0x2e, 0x74, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x67, 0x72, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x2e, 0x53,
+	0x75, 0x6d, 0x28, 0x01, 0x12, 0x55, 0x0a, 0x0c, 0x44, 0x69, 0x76, 0x69, 0x64, 0x65, 0x53, 0x74,
+	0x72, 0x65, 0x61, 0x6d, 0x12, 0x20, 0x2e, 0x74, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x69, 0x6e, 0x74,
+	0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69,
+	0x6e, 0x67, 0x2e, 0x53, 0x75, 0x6d, 0x1a, 0x21, 0x2e, 0x74, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x69,
+	0x6e, 0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x73, 0x74, 0x72, 0x65, 0x61,
+	0x6d, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x30, 0x01, 0x12, 0x4e, 0x0a, 0x08, 0x45,
+	0x63, 0x68, 0x6f, 0x4e, 0x75, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x74, 0x74, 0x72, 0x70, 0x63, 0x2e,
+	0x69, 0x6e, 0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x73, 0x74, 0x72, 0x65,
+	0x61, 0x6d, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61,
+	0x64, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x28, 0x01, 0x12, 0x56, 0x0a, 0x0e, 0x45,
+	0x63, 0x68, 0x6f, 0x4e, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x28, 0x2e,
+	0x74, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f,
+	0x6e, 0x2e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x63, 0x68, 0x6f,
+	0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x28,
+	0x01, 0x30, 0x01, 0x12, 0x58, 0x0a, 0x12, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x50, 0x61, 0x79, 0x6c,
+	0x6f, 0x61, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
+	0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74,
+	0x79, 0x1a, 0x28, 0x2e, 0x74, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x67, 0x72,
+	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x2e,
+	0x45, 0x63, 0x68, 0x6f, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x30, 0x01, 0x42, 0x3d, 0x5a,
+	0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74,
+	0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x74, 0x74, 0x72, 0x70, 0x63, 0x2f, 0x69, 0x6e, 0x74,
+	0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69,
+	0x6e, 0x67, 0x3b, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_github_com_containerd_ttrpc_integration_streaming_test_proto_rawDescOnce sync.Once
+	file_github_com_containerd_ttrpc_integration_streaming_test_proto_rawDescData = file_github_com_containerd_ttrpc_integration_streaming_test_proto_rawDesc
+)
+
+func file_github_com_containerd_ttrpc_integration_streaming_test_proto_rawDescGZIP() []byte {
+	file_github_com_containerd_ttrpc_integration_streaming_test_proto_rawDescOnce.Do(func() {
+		file_github_com_containerd_ttrpc_integration_streaming_test_proto_rawDescData = protoimpl.X.CompressGZIP(file_github_com_containerd_ttrpc_integration_streaming_test_proto_rawDescData)
+	})
+	return file_github_com_containerd_ttrpc_integration_streaming_test_proto_rawDescData
+}
+
+var file_github_com_containerd_ttrpc_integration_streaming_test_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
+var file_github_com_containerd_ttrpc_integration_streaming_test_proto_goTypes = []interface{}{
+	(*EchoPayload)(nil),   // 0: ttrpc.integration.streaming.EchoPayload
+	(*Part)(nil),          // 1: ttrpc.integration.streaming.Part
+	(*Sum)(nil),           // 2: ttrpc.integration.streaming.Sum
+	(*emptypb.Empty)(nil), // 3: google.protobuf.Empty
+}
+var file_github_com_containerd_ttrpc_integration_streaming_test_proto_depIdxs = []int32{
+	0, // 0: ttrpc.integration.streaming.Streaming.Echo:input_type -> ttrpc.integration.streaming.EchoPayload
+	0, // 1: ttrpc.integration.streaming.Streaming.EchoStream:input_type -> ttrpc.integration.streaming.EchoPayload
+	1, // 2: ttrpc.integration.streaming.Streaming.SumStream:input_type -> ttrpc.integration.streaming.Part
+	2, // 3: ttrpc.integration.streaming.Streaming.DivideStream:input_type -> ttrpc.integration.streaming.Sum
+	0, // 4: ttrpc.integration.streaming.Streaming.EchoNull:input_type -> ttrpc.integration.streaming.EchoPayload
+	0, // 5: ttrpc.integration.streaming.Streaming.EchoNullStream:input_type -> ttrpc.integration.streaming.EchoPayload
+	3, // 6: ttrpc.integration.streaming.Streaming.EmptyPayloadStream:input_type -> google.protobuf.Empty
+	0, // 7: ttrpc.integration.streaming.Streaming.Echo:output_type -> ttrpc.integration.streaming.EchoPayload
+	0, // 8: ttrpc.integration.streaming.Streaming.EchoStream:output_type -> ttrpc.integration.streaming.EchoPayload
+	2, // 9: ttrpc.integration.streaming.Streaming.SumStream:output_type -> ttrpc.integration.streaming.Sum
+	1, // 10: ttrpc.integration.streaming.Streaming.DivideStream:output_type -> ttrpc.integration.streaming.Part
+	3, // 11: ttrpc.integration.streaming.Streaming.EchoNull:output_type -> google.protobuf.Empty
+	3, // 12: ttrpc.integration.streaming.Streaming.EchoNullStream:output_type -> google.protobuf.Empty
+	0, // 13: ttrpc.integration.streaming.Streaming.EmptyPayloadStream:output_type -> ttrpc.integration.streaming.EchoPayload
+	7, // [7:14] is the sub-list for method output_type
+	0, // [0:7] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_github_com_containerd_ttrpc_integration_streaming_test_proto_init() }
+func file_github_com_containerd_ttrpc_integration_streaming_test_proto_init() {
+	if File_github_com_containerd_ttrpc_integration_streaming_test_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_github_com_containerd_ttrpc_integration_streaming_test_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*EchoPayload); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_github_com_containerd_ttrpc_integration_streaming_test_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Part); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_github_com_containerd_ttrpc_integration_streaming_test_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Sum); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_github_com_containerd_ttrpc_integration_streaming_test_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   3,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_github_com_containerd_ttrpc_integration_streaming_test_proto_goTypes,
+		DependencyIndexes: file_github_com_containerd_ttrpc_integration_streaming_test_proto_depIdxs,
+		MessageInfos:      file_github_com_containerd_ttrpc_integration_streaming_test_proto_msgTypes,
+	}.Build()
+	File_github_com_containerd_ttrpc_integration_streaming_test_proto = out.File
+	file_github_com_containerd_ttrpc_integration_streaming_test_proto_rawDesc = nil
+	file_github_com_containerd_ttrpc_integration_streaming_test_proto_goTypes = nil
+	file_github_com_containerd_ttrpc_integration_streaming_test_proto_depIdxs = nil
+}

integration/streaming/test.proto

@@ -0,0 +1,52 @@
+/*
+	Copyright The containerd Authors.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+		http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+syntax = "proto3";
+
+package ttrpc.integration.streaming;
+
+import "google/protobuf/empty.proto";
+
+option go_package = "github.com/containerd/ttrpc/integration/streaming;streaming";
+
+// Shim service is launched for each container and is responsible for owning the IO
+// for the container and its additional processes.  The shim is also the parent of
+// each container and allows reattaching to the IO and receiving the exit status
+// for the container processes.
+
+service Streaming {
+	rpc Echo(EchoPayload) returns (EchoPayload);
+	rpc EchoStream(stream EchoPayload) returns (stream EchoPayload);
+	rpc SumStream(stream Part) returns (Sum);
+	rpc DivideStream(Sum) returns (stream Part);
+	rpc EchoNull(stream EchoPayload) returns (google.protobuf.Empty);
+	rpc EchoNullStream(stream EchoPayload) returns (stream google.protobuf.Empty);
+	rpc EmptyPayloadStream(google.protobuf.Empty) returns (stream EchoPayload);
+}
+
+message EchoPayload {
+	uint32 seq = 1;
+	string msg = 2;
+}
+
+message Part {
+	int32 add = 1;
+}
+
+message Sum {
+	int32 sum = 1;
+	int32 num = 2;
+}

integration/streaming/test_ttrpc.pb.go

@@ -0,0 +1,417 @@
+// Code generated by protoc-gen-go-ttrpc. DO NOT EDIT.
+// source: github.com/containerd/ttrpc/integration/streaming/test.proto
+package streaming
+
+import (
+	context "context"
+	ttrpc "github.com/containerd/ttrpc"
+	emptypb "google.golang.org/protobuf/types/known/emptypb"
+)
+
+type TTRPCStreamingService interface {
+	Echo(context.Context, *EchoPayload) (*EchoPayload, error)
+	EchoStream(context.Context, TTRPCStreaming_EchoStreamServer) error
+	SumStream(context.Context, TTRPCStreaming_SumStreamServer) (*Sum, error)
+	DivideStream(context.Context, *Sum, TTRPCStreaming_DivideStreamServer) error
+	EchoNull(context.Context, TTRPCStreaming_EchoNullServer) (*emptypb.Empty, error)
+	EchoNullStream(context.Context, TTRPCStreaming_EchoNullStreamServer) error
+	EmptyPayloadStream(context.Context, *emptypb.Empty, TTRPCStreaming_EmptyPayloadStreamServer) error
+}
+
+type TTRPCStreaming_EchoStreamServer interface {
+	Send(*EchoPayload) error
+	Recv() (*EchoPayload, error)
+	ttrpc.StreamServer
+}
+
+type ttrpcstreamingEchoStreamServer struct {
+	ttrpc.StreamServer
+}
+
+func (x *ttrpcstreamingEchoStreamServer) Send(m *EchoPayload) error {
+	return x.StreamServer.SendMsg(m)
+}
+
+func (x *ttrpcstreamingEchoStreamServer) Recv() (*EchoPayload, error) {
+	m := new(EchoPayload)
+	if err := x.StreamServer.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+type TTRPCStreaming_SumStreamServer interface {
+	Recv() (*Part, error)
+	ttrpc.StreamServer
+}
+
+type ttrpcstreamingSumStreamServer struct {
+	ttrpc.StreamServer
+}
+
+func (x *ttrpcstreamingSumStreamServer) Recv() (*Part, error) {
+	m := new(Part)
+	if err := x.StreamServer.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+type TTRPCStreaming_DivideStreamServer interface {
+	Send(*Part) error
+	ttrpc.StreamServer
+}
+
+type ttrpcstreamingDivideStreamServer struct {
+	ttrpc.StreamServer
+}
+
+func (x *ttrpcstreamingDivideStreamServer) Send(m *Part) error {
+	return x.StreamServer.SendMsg(m)
+}
+
+type TTRPCStreaming_EchoNullServer interface {
+	Recv() (*EchoPayload, error)
+	ttrpc.StreamServer
+}
+
+type ttrpcstreamingEchoNullServer struct {
+	ttrpc.StreamServer
+}
+
+func (x *ttrpcstreamingEchoNullServer) Recv() (*EchoPayload, error) {
+	m := new(EchoPayload)
+	if err := x.StreamServer.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+type TTRPCStreaming_EchoNullStreamServer interface {
+	Send(*emptypb.Empty) error
+	Recv() (*EchoPayload, error)
+	ttrpc.StreamServer
+}
+
+type ttrpcstreamingEchoNullStreamServer struct {
+	ttrpc.StreamServer
+}
+
+func (x *ttrpcstreamingEchoNullStreamServer) Send(m *emptypb.Empty) error {
+	return x.StreamServer.SendMsg(m)
+}
+
+func (x *ttrpcstreamingEchoNullStreamServer) Recv() (*EchoPayload, error) {
+	m := new(EchoPayload)
+	if err := x.StreamServer.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+type TTRPCStreaming_EmptyPayloadStreamServer interface {
+	Send(*EchoPayload) error
+	ttrpc.StreamServer
+}
+
+type ttrpcstreamingEmptyPayloadStreamServer struct {
+	ttrpc.StreamServer
+}
+
+func (x *ttrpcstreamingEmptyPayloadStreamServer) Send(m *EchoPayload) error {
+	return x.StreamServer.SendMsg(m)
+}
+
+func RegisterTTRPCStreamingService(srv *ttrpc.Server, svc TTRPCStreamingService) {
+	srv.RegisterService("ttrpc.integration.streaming.Streaming", &ttrpc.ServiceDesc{
+		Methods: map[string]ttrpc.Method{
+			"Echo": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) {
+				var req EchoPayload
+				if err := unmarshal(&req); err != nil {
+					return nil, err
+				}
+				return svc.Echo(ctx, &req)
+			},
+		},
+		Streams: map[string]ttrpc.Stream{
+			"EchoStream": {
+				Handler: func(ctx context.Context, stream ttrpc.StreamServer) (interface{}, error) {
+					return nil, svc.EchoStream(ctx, &ttrpcstreamingEchoStreamServer{stream})
+				},
+				StreamingClient: true,
+				StreamingServer: true,
+			},
+			"SumStream": {
+				Handler: func(ctx context.Context, stream ttrpc.StreamServer) (interface{}, error) {
+					return svc.SumStream(ctx, &ttrpcstreamingSumStreamServer{stream})
+				},
+				StreamingClient: true,
+				StreamingServer: false,
+			},
+			"DivideStream": {
+				Handler: func(ctx context.Context, stream ttrpc.StreamServer) (interface{}, error) {
+					m := new(Sum)
+					if err := stream.RecvMsg(m); err != nil {
+						return nil, err
+					}
+					return nil, svc.DivideStream(ctx, m, &ttrpcstreamingDivideStreamServer{stream})
+				},
+				StreamingClient: false,
+				StreamingServer: true,
+			},
+			"EchoNull": {
+				Handler: func(ctx context.Context, stream ttrpc.StreamServer) (interface{}, error) {
+					return svc.EchoNull(ctx, &ttrpcstreamingEchoNullServer{stream})
+				},
+				StreamingClient: true,
+				StreamingServer: false,
+			},
+			"EchoNullStream": {
+				Handler: func(ctx context.Context, stream ttrpc.StreamServer) (interface{}, error) {
+					return nil, svc.EchoNullStream(ctx, &ttrpcstreamingEchoNullStreamServer{stream})
+				},
+				StreamingClient: true,
+				StreamingServer: true,
+			},
+			"EmptyPayloadStream": {
+				Handler: func(ctx context.Context, stream ttrpc.StreamServer) (interface{}, error) {
+					m := new(emptypb.Empty)
+					if err := stream.RecvMsg(m); err != nil {
+						return nil, err
+					}
+					return nil, svc.EmptyPayloadStream(ctx, m, &ttrpcstreamingEmptyPayloadStreamServer{stream})
+				},
+				StreamingClient: false,
+				StreamingServer: true,
+			},
+		},
+	})
+}
+
+type TTRPCStreamingClient interface {
+	Echo(context.Context, *EchoPayload) (*EchoPayload, error)
+	EchoStream(context.Context) (TTRPCStreaming_EchoStreamClient, error)
+	SumStream(context.Context) (TTRPCStreaming_SumStreamClient, error)
+	DivideStream(context.Context, *Sum) (TTRPCStreaming_DivideStreamClient, error)
+	EchoNull(context.Context) (TTRPCStreaming_EchoNullClient, error)
+	EchoNullStream(context.Context) (TTRPCStreaming_EchoNullStreamClient, error)
+	EmptyPayloadStream(context.Context, *emptypb.Empty) (TTRPCStreaming_EmptyPayloadStreamClient, error)
+}
+
+type ttrpcstreamingClient struct {
+	client *ttrpc.Client
+}
+
+func NewTTRPCStreamingClient(client *ttrpc.Client) TTRPCStreamingClient {
+	return &ttrpcstreamingClient{
+		client: client,
+	}
+}
+
+func (c *ttrpcstreamingClient) Echo(ctx context.Context, req *EchoPayload) (*EchoPayload, error) {
+	var resp EchoPayload
+	if err := c.client.Call(ctx, "ttrpc.integration.streaming.Streaming", "Echo", req, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+func (c *ttrpcstreamingClient) EchoStream(ctx context.Context) (TTRPCStreaming_EchoStreamClient, error) {
+	stream, err := c.client.NewStream(ctx, &ttrpc.StreamDesc{
+		StreamingClient: true,
+		StreamingServer: true,
+	}, "ttrpc.integration.streaming.Streaming", "EchoStream", nil)
+	if err != nil {
+		return nil, err
+	}
+	x := &ttrpcstreamingEchoStreamClient{stream}
+	return x, nil
+}
+
+type TTRPCStreaming_EchoStreamClient interface {
+	Send(*EchoPayload) error
+	Recv() (*EchoPayload, error)
+	ttrpc.ClientStream
+}
+
+type ttrpcstreamingEchoStreamClient struct {
+	ttrpc.ClientStream
+}
+
+func (x *ttrpcstreamingEchoStreamClient) Send(m *EchoPayload) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *ttrpcstreamingEchoStreamClient) Recv() (*EchoPayload, error) {
+	m := new(EchoPayload)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *ttrpcstreamingClient) SumStream(ctx context.Context) (TTRPCStreaming_SumStreamClient, error) {
+	stream, err := c.client.NewStream(ctx, &ttrpc.StreamDesc{
+		StreamingClient: true,
+		StreamingServer: false,
+	}, "ttrpc.integration.streaming.Streaming", "SumStream", nil)
+	if err != nil {
+		return nil, err
+	}
+	x := &ttrpcstreamingSumStreamClient{stream}
+	return x, nil
+}
+
+type TTRPCStreaming_SumStreamClient interface {
+	Send(*Part) error
+	CloseAndRecv() (*Sum, error)
+	ttrpc.ClientStream
+}
+
+type ttrpcstreamingSumStreamClient struct {
+	ttrpc.ClientStream
+}
+
+func (x *ttrpcstreamingSumStreamClient) Send(m *Part) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *ttrpcstreamingSumStreamClient) CloseAndRecv() (*Sum, error) {
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	m := new(Sum)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *ttrpcstreamingClient) DivideStream(ctx context.Context, req *Sum) (TTRPCStreaming_DivideStreamClient, error) {
+	stream, err := c.client.NewStream(ctx, &ttrpc.StreamDesc{
+		StreamingClient: false,
+		StreamingServer: true,
+	}, "ttrpc.integration.streaming.Streaming", "DivideStream", req)
+	if err != nil {
+		return nil, err
+	}
+	x := &ttrpcstreamingDivideStreamClient{stream}
+	return x, nil
+}
+
+type TTRPCStreaming_DivideStreamClient interface {
+	Recv() (*Part, error)
+	ttrpc.ClientStream
+}
+
+type ttrpcstreamingDivideStreamClient struct {
+	ttrpc.ClientStream
+}
+
+func (x *ttrpcstreamingDivideStreamClient) Recv() (*Part, error) {
+	m := new(Part)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *ttrpcstreamingClient) EchoNull(ctx context.Context) (TTRPCStreaming_EchoNullClient, error) {
+	stream, err := c.client.NewStream(ctx, &ttrpc.StreamDesc{
+		StreamingClient: true,
+		StreamingServer: false,
+	}, "ttrpc.integration.streaming.Streaming", "EchoNull", nil)
+	if err != nil {
+		return nil, err
+	}
+	x := &ttrpcstreamingEchoNullClient{stream}
+	return x, nil
+}
+
+type TTRPCStreaming_EchoNullClient interface {
+	Send(*EchoPayload) error
+	CloseAndRecv() (*emptypb.Empty, error)
+	ttrpc.ClientStream
+}
+
+type ttrpcstreamingEchoNullClient struct {
+	ttrpc.ClientStream
+}
+
+func (x *ttrpcstreamingEchoNullClient) Send(m *EchoPayload) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *ttrpcstreamingEchoNullClient) CloseAndRecv() (*emptypb.Empty, error) {
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	m := new(emptypb.Empty)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *ttrpcstreamingClient) EchoNullStream(ctx context.Context) (TTRPCStreaming_EchoNullStreamClient, error) {
+	stream, err := c.client.NewStream(ctx, &ttrpc.StreamDesc{
+		StreamingClient: true,
+		StreamingServer: true,
+	}, "ttrpc.integration.streaming.Streaming", "EchoNullStream", nil)
+	if err != nil {
+		return nil, err
+	}
+	x := &ttrpcstreamingEchoNullStreamClient{stream}
+	return x, nil
+}
+
+type TTRPCStreaming_EchoNullStreamClient interface {
+	Send(*EchoPayload) error
+	Recv() (*emptypb.Empty, error)
+	ttrpc.ClientStream
+}
+
+type ttrpcstreamingEchoNullStreamClient struct {
+	ttrpc.ClientStream
+}
+
+func (x *ttrpcstreamingEchoNullStreamClient) Send(m *EchoPayload) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *ttrpcstreamingEchoNullStreamClient) Recv() (*emptypb.Empty, error) {
+	m := new(emptypb.Empty)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *ttrpcstreamingClient) EmptyPayloadStream(ctx context.Context, req *emptypb.Empty) (TTRPCStreaming_EmptyPayloadStreamClient, error) {
+	stream, err := c.client.NewStream(ctx, &ttrpc.StreamDesc{
+		StreamingClient: false,
+		StreamingServer: true,
+	}, "ttrpc.integration.streaming.Streaming", "EmptyPayloadStream", req)
+	if err != nil {
+		return nil, err
+	}
+	x := &ttrpcstreamingEmptyPayloadStreamClient{stream}
+	return x, nil
+}
+
+type TTRPCStreaming_EmptyPayloadStreamClient interface {
+	Recv() (*EchoPayload, error)
+	ttrpc.ClientStream
+}
+
+type ttrpcstreamingEmptyPayloadStreamClient struct {
+	ttrpc.ClientStream
+}
+
+func (x *ttrpcstreamingEmptyPayloadStreamClient) Recv() (*EchoPayload, error) {
+	m := new(EchoPayload)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}

integration/streaming_test.go

@@ -0,0 +1,462 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package integration
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"math/rand"
+	"net"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/containerd/ttrpc"
+	"github.com/containerd/ttrpc/integration/streaming"
+	"github.com/golang/protobuf/ptypes/empty"
+	"google.golang.org/protobuf/types/known/emptypb"
+)
+
+func runService(ctx context.Context, t testing.TB, service streaming.TTRPCStreamingService) (streaming.TTRPCStreamingClient, func()) {
+	server, err := ttrpc.NewServer()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	streaming.RegisterTTRPCStreamingService(server, service)
+
+	addr := t.Name() + ".sock"
+	if err := os.RemoveAll(addr); err != nil {
+		t.Fatal(err)
+	}
+	listener, err := net.Listen("unix", addr)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer func() {
+		if t.Failed() {
+			cancel()
+			server.Close()
+		}
+	}()
+
+	go func() {
+		err := server.Serve(ctx, listener)
+		if err != nil && !errors.Is(err, ttrpc.ErrServerClosed) {
+			t.Error(err)
+		}
+	}()
+
+	conn, err := net.Dial("unix", addr)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	client := ttrpc.NewClient(conn)
+	return streaming.NewTTRPCStreamingClient(client), func() {
+		client.Close()
+		server.Close()
+		conn.Close()
+		cancel()
+	}
+}
+
+type testStreamingService struct {
+	t testing.TB
+}
+
+func (tss *testStreamingService) Echo(_ context.Context, e *streaming.EchoPayload) (*streaming.EchoPayload, error) {
+	e.Seq++
+	return e, nil
+}
+
+func (tss *testStreamingService) EchoStream(_ context.Context, es streaming.TTRPCStreaming_EchoStreamServer) error {
+	for {
+		var e streaming.EchoPayload
+		if err := es.RecvMsg(&e); err != nil {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+		e.Seq++
+		if err := es.SendMsg(&e); err != nil {
+			return err
+		}
+
+	}
+}
+
+func (tss *testStreamingService) SumStream(_ context.Context, ss streaming.TTRPCStreaming_SumStreamServer) (*streaming.Sum, error) {
+	var sum streaming.Sum
+	for {
+		var part streaming.Part
+		if err := ss.RecvMsg(&part); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+		sum.Sum = sum.Sum + part.Add
+		sum.Num++
+	}
+
+	return &sum, nil
+}
+
+func (tss *testStreamingService) DivideStream(_ context.Context, sum *streaming.Sum, ss streaming.TTRPCStreaming_DivideStreamServer) error {
+	parts := divideSum(sum)
+	for _, part := range parts {
+		if err := ss.Send(part); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+func (tss *testStreamingService) EchoNull(_ context.Context, es streaming.TTRPCStreaming_EchoNullServer) (*empty.Empty, error) {
+	msg := "non-empty empty"
+	for seq := uint32(0); ; seq++ {
+		var e streaming.EchoPayload
+		if err := es.RecvMsg(&e); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+		if e.Seq != seq {
+			return nil, fmt.Errorf("unexpected sequence %d, expected %d", e.Seq, seq)
+		}
+		if e.Msg != msg {
+			return nil, fmt.Errorf("unexpected message %q, expected %q", e.Msg, msg)
+		}
+	}
+
+	return &empty.Empty{}, nil
+}
+
+func (tss *testStreamingService) EchoNullStream(_ context.Context, es streaming.TTRPCStreaming_EchoNullStreamServer) error {
+	msg := "non-empty empty"
+	empty := &empty.Empty{}
+	var wg sync.WaitGroup
+	var sendErr error
+	var errOnce sync.Once
+	for seq := uint32(0); ; seq++ {
+		var e streaming.EchoPayload
+		if err := es.RecvMsg(&e); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return err
+		}
+		if e.Seq != seq {
+			return fmt.Errorf("unexpected sequence %d, expected %d", e.Seq, seq)
+		}
+		if e.Msg != msg {
+			return fmt.Errorf("unexpected message %q, expected %q", e.Msg, msg)
+		}
+
+		for i := 0; i < 10; i++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				if err := es.SendMsg(empty); err != nil {
+					errOnce.Do(func() {
+						sendErr = err
+					})
+				}
+			}()
+		}
+	}
+	wg.Wait()
+
+	return sendErr
+}
+
+func (tss *testStreamingService) EmptyPayloadStream(_ context.Context, _ *emptypb.Empty, streamer streaming.TTRPCStreaming_EmptyPayloadStreamServer) error {
+	if err := streamer.Send(&streaming.EchoPayload{Seq: 1}); err != nil {
+		return err
+	}
+
+	return streamer.Send(&streaming.EchoPayload{Seq: 2})
+}
+
+func TestStreamingService(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	client, cleanup := runService(ctx, t, &testStreamingService{t})
+	defer cleanup()
+
+	t.Run("Echo", echoTest(ctx, client))
+	t.Run("EchoStream", echoStreamTest(ctx, client))
+	t.Run("SumStream", sumStreamTest(ctx, client))
+	t.Run("DivideStream", divideStreamTest(ctx, client))
+	t.Run("EchoNull", echoNullTest(ctx, client))
+	t.Run("EchoNullStream", echoNullStreamTest(ctx, client))
+	t.Run("EmptyPayloadStream", emptyPayloadStream(ctx, client))
+}
+
+func echoTest(ctx context.Context, client streaming.TTRPCStreamingClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		echo1 := &streaming.EchoPayload{
+			Seq: 1,
+			Msg: "Echo Me",
+		}
+		resp, err := client.Echo(ctx, echo1)
+		if err != nil {
+			t.Fatal(err)
+		}
+		assertNextEcho(t, echo1, resp)
+	}
+
+}
+
+func echoStreamTest(ctx context.Context, client streaming.TTRPCStreamingClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		stream, err := client.EchoStream(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for i := 0; i < 100; i = i + 2 {
+			echoi := &streaming.EchoPayload{
+				Seq: uint32(i),
+				Msg: fmt.Sprintf("%d: Echo in a stream", i),
+			}
+			if err := stream.Send(echoi); err != nil {
+				t.Fatal(err)
+			}
+
+			resp, err := stream.Recv()
+			if err != nil {
+				t.Fatal(err)
+			}
+			assertNextEcho(t, echoi, resp)
+		}
+
+		if err := stream.CloseSend(); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := stream.Recv(); err != io.EOF {
+			t.Fatalf("Expected io.EOF, got %v", err)
+		}
+	}
+}
+
+func sumStreamTest(ctx context.Context, client streaming.TTRPCStreamingClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		stream, err := client.SumStream(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var sum streaming.Sum
+		if err := stream.Send(&streaming.Part{}); err != nil {
+			t.Fatal(err)
+		}
+		sum.Num++
+		for i := -99; i <= 100; i++ {
+			addi := &streaming.Part{
+				Add: int32(i),
+			}
+			if err := stream.Send(addi); err != nil {
+				t.Fatal(err)
+			}
+			sum.Sum = sum.Sum + int32(i)
+			sum.Num++
+		}
+		if err := stream.Send(&streaming.Part{}); err != nil {
+			t.Fatal(err)
+		}
+		sum.Num++
+
+		ssum, err := stream.CloseAndRecv()
+		if err != nil {
+			t.Fatal(err)
+		}
+		assertSum(t, ssum, &sum)
+	}
+}
+
+func divideStreamTest(ctx context.Context, client streaming.TTRPCStreamingClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		expected := &streaming.Sum{
+			Sum: 392,
+			Num: 30,
+		}
+
+		stream, err := client.DivideStream(ctx, expected)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		var actual streaming.Sum
+		for {
+			part, err := stream.Recv()
+			if err != nil {
+				if err == io.EOF {
+					break
+				}
+				t.Fatal(err)
+			}
+			actual.Sum = actual.Sum + part.Add
+			actual.Num++
+		}
+		assertSum(t, &actual, expected)
+	}
+}
+func echoNullTest(ctx context.Context, client streaming.TTRPCStreamingClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		stream, err := client.EchoNull(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for i := 0; i < 100; i++ {
+			echoi := &streaming.EchoPayload{
+				Seq: uint32(i),
+				Msg: "non-empty empty",
+			}
+			if err := stream.Send(echoi); err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		if _, err := stream.CloseAndRecv(); err != nil {
+			t.Fatal(err)
+		}
+
+	}
+}
+func echoNullStreamTest(ctx context.Context, client streaming.TTRPCStreamingClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		stream, err := client.EchoNullStream(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var c int
+		wait := make(chan error)
+		go func() {
+			defer close(wait)
+			for {
+				_, err := stream.Recv()
+				if err != nil {
+					if err != io.EOF {
+						wait <- err
+					}
+					return
+				}
+				c++
+			}
+
+		}()
+
+		for i := 0; i < 100; i++ {
+			echoi := &streaming.EchoPayload{
+				Seq: uint32(i),
+				Msg: "non-empty empty",
+			}
+			if err := stream.Send(echoi); err != nil {
+				t.Fatal(err)
+			}
+
+		}
+
+		if err := stream.CloseSend(); err != nil {
+			t.Fatal(err)
+		}
+
+		select {
+
+		case err := <-wait:
+			if err != nil {
+				t.Fatal(err)
+			}
+		case <-time.After(time.Second * 10):
+			t.Fatal("did not receive EOF within 10 seconds")
+		}
+
+	}
+}
+
+func emptyPayloadStream(ctx context.Context, client streaming.TTRPCStreamingClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		defer cancel()
+
+		stream, err := client.EmptyPayloadStream(ctx, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		for i := uint32(1); i < 3; i++ {
+			first, err := stream.Recv()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if first.Seq != i {
+				t.Fatalf("unexpected seq: %d != %d", first.Seq, i)
+			}
+		}
+
+		if _, err := stream.Recv(); err != io.EOF {
+			t.Fatalf("Expected io.EOF, got %v", err)
+		}
+	}
+}
+
+func assertNextEcho(t testing.TB, a, b *streaming.EchoPayload) {
+	t.Helper()
+	if a.Msg != b.Msg {
+		t.Fatalf("Mismatched messages: %q != %q", a.Msg, b.Msg)
+	}
+	if b.Seq != a.Seq+1 {
+		t.Fatalf("Wrong sequence ID: got %d, expected %d", b.Seq, a.Seq+1)
+	}
+}
+
+func assertSum(t testing.TB, a, b *streaming.Sum) {
+	t.Helper()
+	if a.Sum != b.Sum {
+		t.Fatalf("Wrong sum %d, expected %d", a.Sum, b.Sum)
+	}
+	if a.Num != b.Num {
+		t.Fatalf("Wrong num %d, expected %d", a.Num, b.Num)
+	}
+}
+
+func divideSum(sum *streaming.Sum) []*streaming.Part {
+	r := rand.New(rand.NewSource(14))
+	var total int32
+	parts := make([]*streaming.Part, sum.Num)
+	for i := int32(1); i < sum.Num-2; i++ {
+		add := r.Int31()%1000 - 500
+		parts[i] = &streaming.Part{
+			Add: add,
+		}
+		total = total + add
+	}
+	parts[0] = &streaming.Part{}
+	parts[sum.Num-2] = &streaming.Part{
+		Add: sum.Sum - total,
+	}
+	parts[sum.Num-1] = &streaming.Part{}
+	return parts
+}

interceptor.go

@@ -28,6 +28,13 @@ type UnaryClientInfo struct {
 	FullMethod string
 }
 
+// StreamServerInfo provides information about the server request
+type StreamServerInfo struct {
+	FullMethod      string
+	StreamingClient bool
+	StreamingServer bool
+}
+
 // Unmarshaler contains the server request data and allows it to be unmarshaled
 // into a concrete type
 type Unmarshaler func(interface{}) error
@@ -41,10 +48,18 @@ type UnaryServerInterceptor func(context.Context, Unmarshaler, *UnaryServerInfo,
 // UnaryClientInterceptor specifies the interceptor function for client request/response
 type UnaryClientInterceptor func(context.Context, *Request, *Response, *UnaryClientInfo, Invoker) error
 
-func defaultServerInterceptor(ctx context.Context, unmarshal Unmarshaler, info *UnaryServerInfo, method Method) (interface{}, error) {
+func defaultServerInterceptor(ctx context.Context, unmarshal Unmarshaler, _ *UnaryServerInfo, method Method) (interface{}, error) {
 	return method(ctx, unmarshal)
 }
 
 func defaultClientInterceptor(ctx context.Context, req *Request, resp *Response, _ *UnaryClientInfo, invoker Invoker) error {
 	return invoker(ctx, req, resp)
 }
+
+type StreamServerInterceptor func(context.Context, StreamServer, *StreamServerInfo, StreamHandler) (interface{}, error)
+
+func defaultStreamServerInterceptor(ctx context.Context, ss StreamServer, _ *StreamServerInfo, stream StreamHandler) (interface{}, error) {
+	return stream(ctx, ss)
+}
+
+type StreamClientInterceptor func(context.Context)

interceptor_test.go

@@ -0,0 +1,245 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ttrpc
+
+import (
+	"context"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/containerd/ttrpc/internal"
+)
+
+func TestUnaryClientInterceptor(t *testing.T) {
+	var (
+		intercepted = false
+		interceptor = func(ctx context.Context, req *Request, reply *Response, _ *UnaryClientInfo, i Invoker) error {
+			intercepted = true
+			return i(ctx, req, reply)
+		}
+
+		ctx             = context.Background()
+		server          = mustServer(t)(NewServer())
+		testImpl        = &testingServer{}
+		addr, listener  = newTestListener(t)
+		client, cleanup = newTestClient(t, addr, WithUnaryClientInterceptor(interceptor))
+		message         = strings.Repeat("a", 16)
+		reply           = strings.Repeat(message, 2)
+	)
+
+	defer listener.Close()
+	defer cleanup()
+
+	registerTestingService(server, testImpl)
+
+	go server.Serve(ctx, listener)
+	defer server.Shutdown(ctx)
+
+	request := &internal.TestPayload{
+		Foo: message,
+	}
+	response := &internal.TestPayload{}
+
+	if err := client.Call(ctx, serviceName, "Test", request, response); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if !intercepted {
+		t.Fatalf("ttrpc client call not intercepted")
+	}
+
+	if response.Foo != reply {
+		t.Fatalf("unexpected test service reply: %q != %q", response.Foo, reply)
+	}
+}
+
+func TestChainUnaryClientInterceptor(t *testing.T) {
+	var (
+		orderIdx  = 0
+		recorded  = []string{}
+		intercept = func(idx int, tag string) UnaryClientInterceptor {
+			return func(ctx context.Context, req *Request, reply *Response, _ *UnaryClientInfo, i Invoker) error {
+				if idx != orderIdx {
+					t.Fatalf("unexpected interceptor invocation order (%d != %d)", orderIdx, idx)
+				}
+				recorded = append(recorded, tag)
+				orderIdx++
+				return i(ctx, req, reply)
+			}
+		}
+
+		ctx             = context.Background()
+		server          = mustServer(t)(NewServer())
+		testImpl        = &testingServer{}
+		addr, listener  = newTestListener(t)
+		client, cleanup = newTestClient(t, addr,
+			WithChainUnaryClientInterceptor(),
+			WithChainUnaryClientInterceptor(
+				intercept(0, "seen it"),
+				intercept(1, "been"),
+				intercept(2, "there"),
+				intercept(3, "done"),
+				intercept(4, "that"),
+			),
+		)
+		expected = []string{
+			"seen it",
+			"been",
+			"there",
+			"done",
+			"that",
+		}
+		message = strings.Repeat("a", 16)
+		reply   = strings.Repeat(message, 2)
+	)
+
+	defer listener.Close()
+	defer cleanup()
+
+	registerTestingService(server, testImpl)
+
+	go server.Serve(ctx, listener)
+	defer server.Shutdown(ctx)
+
+	request := &internal.TestPayload{
+		Foo: message,
+	}
+	response := &internal.TestPayload{}
+	if err := client.Call(ctx, serviceName, "Test", request, response); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if !reflect.DeepEqual(recorded, expected) {
+		t.Fatalf("unexpected ttrpc chained client unary interceptor order (%s != %s)",
+			strings.Join(recorded, " "), strings.Join(expected, " "))
+	}
+
+	if response.Foo != reply {
+		t.Fatalf("unexpected test service reply: %q != %q", response.Foo, reply)
+	}
+}
+
+func TestUnaryServerInterceptor(t *testing.T) {
+	var (
+		intercepted = false
+		interceptor = func(ctx context.Context, unmarshal Unmarshaler, _ *UnaryServerInfo, method Method) (interface{}, error) {
+			intercepted = true
+			return method(ctx, unmarshal)
+		}
+
+		ctx             = context.Background()
+		server          = mustServer(t)(NewServer(WithUnaryServerInterceptor(interceptor)))
+		testImpl        = &testingServer{}
+		addr, listener  = newTestListener(t)
+		client, cleanup = newTestClient(t, addr)
+		message         = strings.Repeat("a", 16)
+		reply           = strings.Repeat(message, 2)
+	)
+
+	defer listener.Close()
+	defer cleanup()
+
+	registerTestingService(server, testImpl)
+
+	go server.Serve(ctx, listener)
+	defer server.Shutdown(ctx)
+
+	request := &internal.TestPayload{
+		Foo: message,
+	}
+	response := &internal.TestPayload{}
+	if err := client.Call(ctx, serviceName, "Test", request, response); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if !intercepted {
+		t.Fatalf("ttrpc server call not intercepted")
+	}
+
+	if response.Foo != reply {
+		t.Fatalf("unexpected test service reply: %q != %q", response.Foo, reply)
+	}
+}
+
+func TestChainUnaryServerInterceptor(t *testing.T) {
+	var (
+		orderIdx  = 0
+		recorded  = []string{}
+		intercept = func(idx int, tag string) UnaryServerInterceptor {
+			return func(ctx context.Context, unmarshal Unmarshaler, _ *UnaryServerInfo, method Method) (interface{}, error) {
+				if orderIdx != idx {
+					t.Fatalf("unexpected interceptor invocation order (%d != %d)", orderIdx, idx)
+				}
+				recorded = append(recorded, tag)
+				orderIdx++
+				return method(ctx, unmarshal)
+			}
+		}
+
+		ctx    = context.Background()
+		server = mustServer(t)(NewServer(
+			WithUnaryServerInterceptor(
+				intercept(0, "seen it"),
+			),
+			WithChainUnaryServerInterceptor(
+				intercept(1, "been"),
+				intercept(2, "there"),
+				intercept(3, "done"),
+				intercept(4, "that"),
+			),
+		))
+		expected = []string{
+			"seen it",
+			"been",
+			"there",
+			"done",
+			"that",
+		}
+		testImpl        = &testingServer{}
+		addr, listener  = newTestListener(t)
+		client, cleanup = newTestClient(t, addr)
+		message         = strings.Repeat("a", 16)
+		reply           = strings.Repeat(message, 2)
+	)
+
+	defer listener.Close()
+	defer cleanup()
+
+	registerTestingService(server, testImpl)
+
+	go server.Serve(ctx, listener)
+	defer server.Shutdown(ctx)
+
+	request := &internal.TestPayload{
+		Foo: message,
+	}
+	response := &internal.TestPayload{}
+
+	if err := client.Call(ctx, serviceName, "Test", request, response); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if !reflect.DeepEqual(recorded, expected) {
+		t.Fatalf("unexpected ttrpc chained server unary interceptor order (%s != %s)",
+			strings.Join(recorded, " "), strings.Join(expected, " "))
+	}
+
+	if response.Foo != reply {
+		t.Fatalf("unexpected test service reply: %q != %q", response.Foo, reply)
+	}
+}

internal/test.pb.go

@@ -0,0 +1,235 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.28.1
+// 	protoc        v3.20.1
+// source: github.com/containerd/ttrpc/test.proto
+
+package internal
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type TestPayload struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Foo      string `protobuf:"bytes,1,opt,name=foo,proto3" json:"foo,omitempty"`
+	Deadline int64  `protobuf:"varint,2,opt,name=deadline,proto3" json:"deadline,omitempty"`
+	Metadata string `protobuf:"bytes,3,opt,name=metadata,proto3" json:"metadata,omitempty"`
+}
+
+func (x *TestPayload) Reset() {
+	*x = TestPayload{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_github_com_containerd_ttrpc_test_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *TestPayload) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*TestPayload) ProtoMessage() {}
+
+func (x *TestPayload) ProtoReflect() protoreflect.Message {
+	mi := &file_github_com_containerd_ttrpc_test_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use TestPayload.ProtoReflect.Descriptor instead.
+func (*TestPayload) Descriptor() ([]byte, []int) {
+	return file_github_com_containerd_ttrpc_test_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *TestPayload) GetFoo() string {
+	if x != nil {
+		return x.Foo
+	}
+	return ""
+}
+
+func (x *TestPayload) GetDeadline() int64 {
+	if x != nil {
+		return x.Deadline
+	}
+	return 0
+}
+
+func (x *TestPayload) GetMetadata() string {
+	if x != nil {
+		return x.Metadata
+	}
+	return ""
+}
+
+type EchoPayload struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Seq int64  `protobuf:"varint,1,opt,name=seq,proto3" json:"seq,omitempty"`
+	Msg string `protobuf:"bytes,2,opt,name=msg,proto3" json:"msg,omitempty"`
+}
+
+func (x *EchoPayload) Reset() {
+	*x = EchoPayload{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_github_com_containerd_ttrpc_test_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *EchoPayload) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*EchoPayload) ProtoMessage() {}
+
+func (x *EchoPayload) ProtoReflect() protoreflect.Message {
+	mi := &file_github_com_containerd_ttrpc_test_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use EchoPayload.ProtoReflect.Descriptor instead.
+func (*EchoPayload) Descriptor() ([]byte, []int) {
+	return file_github_com_containerd_ttrpc_test_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *EchoPayload) GetSeq() int64 {
+	if x != nil {
+		return x.Seq
+	}
+	return 0
+}
+
+func (x *EchoPayload) GetMsg() string {
+	if x != nil {
+		return x.Msg
+	}
+	return ""
+}
+
+var File_github_com_containerd_ttrpc_test_proto protoreflect.FileDescriptor
+
+var file_github_com_containerd_ttrpc_test_proto_rawDesc = []byte{
+	0x0a, 0x26, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e,
+	0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x74, 0x74, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65,
+	0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x74, 0x74, 0x72, 0x70, 0x63, 0x22,
+	0x57, 0x0a, 0x0b, 0x54, 0x65, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x10,
+	0x0a, 0x03, 0x66, 0x6f, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x66, 0x6f, 0x6f,
+	0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x02, 0x20, 0x01,
+	0x28, 0x03, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x1a, 0x0a, 0x08,
+	0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08,
+	0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0x31, 0x0a, 0x0b, 0x45, 0x63, 0x68, 0x6f,
+	0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x65, 0x71, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x73, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x73, 0x67,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6d, 0x73, 0x67, 0x42, 0x26, 0x5a, 0x24, 0x67,
+	0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69,
+	0x6e, 0x65, 0x72, 0x64, 0x2f, 0x74, 0x74, 0x72, 0x70, 0x63, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72,
+	0x6e, 0x61, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_github_com_containerd_ttrpc_test_proto_rawDescOnce sync.Once
+	file_github_com_containerd_ttrpc_test_proto_rawDescData = file_github_com_containerd_ttrpc_test_proto_rawDesc
+)
+
+func file_github_com_containerd_ttrpc_test_proto_rawDescGZIP() []byte {
+	file_github_com_containerd_ttrpc_test_proto_rawDescOnce.Do(func() {
+		file_github_com_containerd_ttrpc_test_proto_rawDescData = protoimpl.X.CompressGZIP(file_github_com_containerd_ttrpc_test_proto_rawDescData)
+	})
+	return file_github_com_containerd_ttrpc_test_proto_rawDescData
+}
+
+var file_github_com_containerd_ttrpc_test_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_github_com_containerd_ttrpc_test_proto_goTypes = []interface{}{
+	(*TestPayload)(nil), // 0: ttrpc.TestPayload
+	(*EchoPayload)(nil), // 1: ttrpc.EchoPayload
+}
+var file_github_com_containerd_ttrpc_test_proto_depIdxs = []int32{
+	0, // [0:0] is the sub-list for method output_type
+	0, // [0:0] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_github_com_containerd_ttrpc_test_proto_init() }
+func file_github_com_containerd_ttrpc_test_proto_init() {
+	if File_github_com_containerd_ttrpc_test_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_github_com_containerd_ttrpc_test_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*TestPayload); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_github_com_containerd_ttrpc_test_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*EchoPayload); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_github_com_containerd_ttrpc_test_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   2,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_github_com_containerd_ttrpc_test_proto_goTypes,
+		DependencyIndexes: file_github_com_containerd_ttrpc_test_proto_depIdxs,
+		MessageInfos:      file_github_com_containerd_ttrpc_test_proto_msgTypes,
+	}.Build()
+	File_github_com_containerd_ttrpc_test_proto = out.File
+	file_github_com_containerd_ttrpc_test_proto_rawDesc = nil
+	file_github_com_containerd_ttrpc_test_proto_goTypes = nil
+	file_github_com_containerd_ttrpc_test_proto_depIdxs = nil
+}

metadata.go

@@ -62,6 +62,34 @@ func (m MD) Append(key string, values ...string) {
 	}
 }
 
+// Clone returns a copy of MD or nil if it's nil.
+// It's copied from golang's `http.Header.Clone` implementation:
+// https://cs.opensource.google/go/go/+/refs/tags/go1.23.4:src/net/http/header.go;l=94
+func (m MD) Clone() MD {
+	if m == nil {
+		return nil
+	}
+
+	// Find total number of values.
+	nv := 0
+	for _, vv := range m {
+		nv += len(vv)
+	}
+	sv := make([]string, nv) // shared backing array for headers' values
+	m2 := make(MD, len(m))
+	for k, vv := range m {
+		if vv == nil {
+			// Preserve nil values.
+			m2[k] = nil
+			continue
+		}
+		n := copy(sv, vv)
+		m2[k] = sv[:n:n]
+		sv = sv[n:]
+	}
+	return m2
+}
+
 func (m MD) setRequest(r *Request) {
 	for k, values := range m {
 		for _, v := range values {

metadata_test.go

@@ -18,6 +18,8 @@ package ttrpc
 
 import (
 	"context"
+	"fmt"
+	"sync"
 	"testing"
 )
 
@@ -106,3 +108,100 @@ func TestMetadataContext(t *testing.T) {
 		t.Errorf("invalid metadata value: %q", bar)
 	}
 }
+
+func TestMetadataClone(t *testing.T) {
+	var metadata MD
+	m2 := metadata.Clone()
+	if m2 != nil {
+		t.Error("MD.Clone() on nil metadata should return nil")
+	}
+
+	metadata = MD{"nil": nil, "foo": {"bar"}, "baz": {"qux", "quxx"}}
+	m2 = metadata.Clone()
+
+	if len(metadata) != len(m2) {
+		t.Errorf("unexpected number of keys: %d, expected: %d", len(m2), len(metadata))
+	}
+
+	for k, v := range metadata {
+		v2, ok := m2[k]
+		if !ok {
+			t.Errorf("key not found: %s", k)
+		}
+		if v == nil && v2 == nil {
+			continue
+		}
+		if v == nil || v2 == nil {
+			t.Errorf("unexpected nil value: %v, expected: %v", v2, v)
+		}
+		if len(v) != len(v2) {
+			t.Errorf("unexpected number of values: %d, expected: %d", len(v2), len(v))
+		}
+		for i := range v {
+			if v[i] != v2[i] {
+				t.Errorf("unexpected value: %s, expected: %s", v2[i], v[i])
+			}
+		}
+	}
+}
+
+func TestMetadataCloneConcurrent(t *testing.T) {
+	metadata := make(MD)
+	metadata.Set("foo", "bar")
+
+	var wg sync.WaitGroup
+	for i := 0; i < 20; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			m2 := metadata.Clone()
+			m2.Set("foo", "baz")
+		}()
+	}
+	wg.Wait()
+	// Concurrent modification should clone the metadata first to avoid panic
+	// due to concurrent map writes.
+	if val, ok := metadata.Get("foo"); !ok {
+		t.Error("metadata not found")
+	} else if val[0] != "bar" {
+		t.Errorf("invalid metadata value: %q", val[0])
+	}
+}
+
+func simpleClone(src MD) MD {
+	md := MD{}
+	for k, v := range src {
+		md[k] = append(md[k], v...)
+	}
+	return md
+}
+
+func BenchmarkMetadataClone(b *testing.B) {
+	for _, sz := range []int{5, 10, 20, 50} {
+		b.Run(fmt.Sprintf("size=%d", sz), func(b *testing.B) {
+			metadata := make(MD)
+			for i := 0; i < sz; i++ {
+				metadata.Set("foo"+fmt.Sprint(i), "bar"+fmt.Sprint(i))
+			}
+
+			for i := 0; i < b.N; i++ {
+				_ = metadata.Clone()
+			}
+		})
+	}
+}
+
+func BenchmarkSimpleMetadataClone(b *testing.B) {
+	for _, sz := range []int{5, 10, 20, 50} {
+		b.Run(fmt.Sprintf("size=%d", sz), func(b *testing.B) {
+			metadata := make(MD)
+			for i := 0; i < sz; i++ {
+				metadata.Set("foo"+fmt.Sprint(i), "bar"+fmt.Sprint(i))
+			}
+
+			for i := 0; i < b.N; i++ {
+				_ = simpleClone(metadata)
+			}
+		})
+	}
+}

proto/status.proto

@@ -0,0 +1,5 @@
+syntax = "proto3";
+
+message Status {
+	int32 code = 1;
+}

request.pb.go

@@ -0,0 +1,396 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.28.1
+// 	protoc        v3.20.1
+// source: github.com/containerd/ttrpc/request.proto
+
+package ttrpc
+
+import (
+	status "google.golang.org/genproto/googleapis/rpc/status"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type Request struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Service     string      `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"`
+	Method      string      `protobuf:"bytes,2,opt,name=method,proto3" json:"method,omitempty"`
+	Payload     []byte      `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"`
+	TimeoutNano int64       `protobuf:"varint,4,opt,name=timeout_nano,json=timeoutNano,proto3" json:"timeout_nano,omitempty"`
+	Metadata    []*KeyValue `protobuf:"bytes,5,rep,name=metadata,proto3" json:"metadata,omitempty"`
+}
+
+func (x *Request) Reset() {
+	*x = Request{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_github_com_containerd_ttrpc_request_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Request) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Request) ProtoMessage() {}
+
+func (x *Request) ProtoReflect() protoreflect.Message {
+	mi := &file_github_com_containerd_ttrpc_request_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Request.ProtoReflect.Descriptor instead.
+func (*Request) Descriptor() ([]byte, []int) {
+	return file_github_com_containerd_ttrpc_request_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *Request) GetService() string {
+	if x != nil {
+		return x.Service
+	}
+	return ""
+}
+
+func (x *Request) GetMethod() string {
+	if x != nil {
+		return x.Method
+	}
+	return ""
+}
+
+func (x *Request) GetPayload() []byte {
+	if x != nil {
+		return x.Payload
+	}
+	return nil
+}
+
+func (x *Request) GetTimeoutNano() int64 {
+	if x != nil {
+		return x.TimeoutNano
+	}
+	return 0
+}
+
+func (x *Request) GetMetadata() []*KeyValue {
+	if x != nil {
+		return x.Metadata
+	}
+	return nil
+}
+
+type Response struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Status  *status.Status `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"`
+	Payload []byte         `protobuf:"bytes,2,opt,name=payload,proto3" json:"payload,omitempty"`
+}
+
+func (x *Response) Reset() {
+	*x = Response{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_github_com_containerd_ttrpc_request_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Response) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Response) ProtoMessage() {}
+
+func (x *Response) ProtoReflect() protoreflect.Message {
+	mi := &file_github_com_containerd_ttrpc_request_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Response.ProtoReflect.Descriptor instead.
+func (*Response) Descriptor() ([]byte, []int) {
+	return file_github_com_containerd_ttrpc_request_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *Response) GetStatus() *status.Status {
+	if x != nil {
+		return x.Status
+	}
+	return nil
+}
+
+func (x *Response) GetPayload() []byte {
+	if x != nil {
+		return x.Payload
+	}
+	return nil
+}
+
+type StringList struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	List []string `protobuf:"bytes,1,rep,name=list,proto3" json:"list,omitempty"`
+}
+
+func (x *StringList) Reset() {
+	*x = StringList{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_github_com_containerd_ttrpc_request_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *StringList) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*StringList) ProtoMessage() {}
+
+func (x *StringList) ProtoReflect() protoreflect.Message {
+	mi := &file_github_com_containerd_ttrpc_request_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use StringList.ProtoReflect.Descriptor instead.
+func (*StringList) Descriptor() ([]byte, []int) {
+	return file_github_com_containerd_ttrpc_request_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *StringList) GetList() []string {
+	if x != nil {
+		return x.List
+	}
+	return nil
+}
+
+type KeyValue struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Key   string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
+	Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
+}
+
+func (x *KeyValue) Reset() {
+	*x = KeyValue{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_github_com_containerd_ttrpc_request_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *KeyValue) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*KeyValue) ProtoMessage() {}
+
+func (x *KeyValue) ProtoReflect() protoreflect.Message {
+	mi := &file_github_com_containerd_ttrpc_request_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use KeyValue.ProtoReflect.Descriptor instead.
+func (*KeyValue) Descriptor() ([]byte, []int) {
+	return file_github_com_containerd_ttrpc_request_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *KeyValue) GetKey() string {
+	if x != nil {
+		return x.Key
+	}
+	return ""
+}
+
+func (x *KeyValue) GetValue() string {
+	if x != nil {
+		return x.Value
+	}
+	return ""
+}
+
+var File_github_com_containerd_ttrpc_request_proto protoreflect.FileDescriptor
+
+var file_github_com_containerd_ttrpc_request_proto_rawDesc = []byte{
+	0x0a, 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e,
+	0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x74, 0x74, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x74, 0x74, 0x72,
+	0x70, 0x63, 0x1a, 0x12, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73,
+	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xa5, 0x01, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x16, 0x0a, 0x06,
+	0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6d, 0x65,
+	0x74, 0x68, 0x6f, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18,
+	0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x21,
+	0x0a, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x04,
+	0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4e, 0x61, 0x6e,
+	0x6f, 0x12, 0x2b, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20,
+	0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x4b, 0x65, 0x79, 0x56,
+	0x61, 0x6c, 0x75, 0x65, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0x45,
+	0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x06, 0x73, 0x74,
+	0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x07, 0x2e, 0x53, 0x74, 0x61,
+	0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x70,
+	0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61,
+	0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x20, 0x0a, 0x0a, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x4c,
+	0x69, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28,
+	0x09, 0x52, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x22, 0x32, 0x0a, 0x08, 0x4b, 0x65, 0x79, 0x56, 0x61,
+	0x6c, 0x75, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x1d, 0x5a, 0x1b, 0x67,
+	0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69,
+	0x6e, 0x65, 0x72, 0x64, 0x2f, 0x74, 0x74, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
+	0x6f, 0x33,
+}
+
+var (
+	file_github_com_containerd_ttrpc_request_proto_rawDescOnce sync.Once
+	file_github_com_containerd_ttrpc_request_proto_rawDescData = file_github_com_containerd_ttrpc_request_proto_rawDesc
+)
+
+func file_github_com_containerd_ttrpc_request_proto_rawDescGZIP() []byte {
+	file_github_com_containerd_ttrpc_request_proto_rawDescOnce.Do(func() {
+		file_github_com_containerd_ttrpc_request_proto_rawDescData = protoimpl.X.CompressGZIP(file_github_com_containerd_ttrpc_request_proto_rawDescData)
+	})
+	return file_github_com_containerd_ttrpc_request_proto_rawDescData
+}
+
+var file_github_com_containerd_ttrpc_request_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
+var file_github_com_containerd_ttrpc_request_proto_goTypes = []interface{}{
+	(*Request)(nil),       // 0: ttrpc.Request
+	(*Response)(nil),      // 1: ttrpc.Response
+	(*StringList)(nil),    // 2: ttrpc.StringList
+	(*KeyValue)(nil),      // 3: ttrpc.KeyValue
+	(*status.Status)(nil), // 4: Status
+}
+var file_github_com_containerd_ttrpc_request_proto_depIdxs = []int32{
+	3, // 0: ttrpc.Request.metadata:type_name -> ttrpc.KeyValue
+	4, // 1: ttrpc.Response.status:type_name -> Status
+	2, // [2:2] is the sub-list for method output_type
+	2, // [2:2] is the sub-list for method input_type
+	2, // [2:2] is the sub-list for extension type_name
+	2, // [2:2] is the sub-list for extension extendee
+	0, // [0:2] is the sub-list for field type_name
+}
+
+func init() { file_github_com_containerd_ttrpc_request_proto_init() }
+func file_github_com_containerd_ttrpc_request_proto_init() {
+	if File_github_com_containerd_ttrpc_request_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_github_com_containerd_ttrpc_request_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Request); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_github_com_containerd_ttrpc_request_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Response); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_github_com_containerd_ttrpc_request_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*StringList); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_github_com_containerd_ttrpc_request_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*KeyValue); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_github_com_containerd_ttrpc_request_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   4,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_github_com_containerd_ttrpc_request_proto_goTypes,
+		DependencyIndexes: file_github_com_containerd_ttrpc_request_proto_depIdxs,
+		MessageInfos:      file_github_com_containerd_ttrpc_request_proto_msgTypes,
+	}.Build()
+	File_github_com_containerd_ttrpc_request_proto = out.File
+	file_github_com_containerd_ttrpc_request_proto_rawDesc = nil
+	file_github_com_containerd_ttrpc_request_proto_goTypes = nil
+	file_github_com_containerd_ttrpc_request_proto_depIdxs = nil
+}

request.proto

@@ -0,0 +1,29 @@
+syntax = "proto3";
+
+package ttrpc;
+
+import "proto/status.proto";
+
+option go_package = "github.com/containerd/ttrpc";
+
+message Request {
+	string service = 1;
+	string method = 2;
+	bytes payload = 3;
+	int64 timeout_nano = 4;
+	repeated KeyValue metadata = 5;
+}
+
+message Response {
+	Status status = 1;
+	bytes payload = 2;
+}
+
+message StringList {
+	repeated string list = 1;
+}
+
+message KeyValue {
+	string key = 1;
+	string value = 2;
+}

script/install-protobuf

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+
+#
+# Downloads and installs protobuf
+#
+set -eu -o pipefail
+
+PROTOBUF_VERSION=3.20.1
+GOARCH=$(go env GOARCH)
+GOOS=$(go env GOOS)
+PROTOBUF_DIR=$(mktemp -d)
+
+case $GOARCH in
+
+arm64)
+	wget -O "$PROTOBUF_DIR/protobuf" "https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protoc-$PROTOBUF_VERSION-linux-aarch64.zip"
+	unzip "$PROTOBUF_DIR/protobuf" -d /usr/local
+	;;
+
+amd64|386)
+	if [ "$GOOS" = windows ]; then
+		wget -O "$PROTOBUF_DIR/protobuf" "https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protoc-$PROTOBUF_VERSION-win32.zip"
+	elif [ "$GOOS" = linux ]; then
+		wget -O "$PROTOBUF_DIR/protobuf" "https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protoc-$PROTOBUF_VERSION-linux-x86_64.zip"
+	fi
+	unzip "$PROTOBUF_DIR/protobuf" -d /usr/local
+	;;
+
+ppc64le)
+	wget -O "$PROTOBUF_DIR/protobuf" "https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protoc-$PROTOBUF_VERSION-linux-ppcle_64.zip"
+	unzip "$PROTOBUF_DIR/protobuf" -d /usr/local
+	;;
+*)
+	wget -O "$PROTOBUF_DIR/protobuf" "https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-cpp-$PROTOBUF_VERSION.zip"
+	unzip "$PROTOBUF_DIR/protobuf" -d /usr/src/protobuf
+	cd "/usr/src/protobuf/protobuf-$PROTOBUF_VERSION"
+	./autogen.sh
+	./configure --disable-shared
+	make
+	make check
+	make install
+	ldconfig
+	;;
+esac
+rm -rf "$PROTOBUF_DIR"

server.go

@@ -24,17 +24,14 @@ import (
 	"net"
 	"sync"
 	"sync/atomic"
+	"syscall"
 	"time"
 
-	"github.com/sirupsen/logrus"
+	"github.com/containerd/log"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )
 
-var (
-	ErrServerClosed = errors.New("ttrpc: server closed")
-)
-
 type Server struct {
 	config   *serverConfig
 	services *serviceSet
@@ -66,14 +63,29 @@ func NewServer(opts ...ServerOpt) (*Server, error) {
 	}, nil
 }
 
+// Register registers a map of methods to method handlers
+// TODO: Remove in 2.0, does not support streams
 func (s *Server) Register(name string, methods map[string]Method) {
-	s.services.register(name, methods)
+	s.services.register(name, &ServiceDesc{Methods: methods})
+}
+
+func (s *Server) RegisterService(name string, desc *ServiceDesc) {
+	s.services.register(name, desc)
 }
 
 func (s *Server) Serve(ctx context.Context, l net.Listener) error {
-	s.addListener(l)
+	s.mu.Lock()
+	s.addListenerLocked(l)
 	defer s.closeListener(l)
 
+	select {
+	case <-s.done:
+		s.mu.Unlock()
+		return ErrServerClosed
+	default:
+	}
+	s.mu.Unlock()
+
 	var (
 		backoff    time.Duration
 		handshaker = s.config.handshaker
@@ -106,7 +118,7 @@ func (s *Server) Serve(ctx context.Context, l net.Listener) error {
 				}
 
 				sleep := time.Duration(rand.Int63n(int64(backoff)))
-				logrus.WithError(err).Errorf("ttrpc: failed accept; backoff %v", sleep)
+				log.G(ctx).WithError(err).Errorf("ttrpc: failed accept; backoff %v", sleep)
 				time.Sleep(sleep)
 				continue
 			}
@@ -118,12 +130,18 @@ func (s *Server) Serve(ctx context.Context, l net.Listener) error {
 
 		approved, handshake, err := handshaker.Handshake(ctx, conn)
 		if err != nil {
-			logrus.WithError(err).Errorf("ttrpc: refusing connection after handshake")
+			log.G(ctx).WithError(err).Error("ttrpc: refusing connection after handshake")
+			conn.Close()
+			continue
+		}
+
+		sc, err := s.newConn(approved, handshake)
+		if err != nil {
+			log.G(ctx).WithError(err).Error("ttrpc: create connection failed")
 			conn.Close()
 			continue
 		}
 
-		sc := s.newConn(approved, handshake)
 		go sc.run(ctx)
 	}
 }
@@ -142,15 +160,20 @@ func (s *Server) Shutdown(ctx context.Context) error {
 	ticker := time.NewTicker(200 * time.Millisecond)
 	defer ticker.Stop()
 	for {
-		if s.closeIdleConns() {
-			return lnerr
+		s.closeIdleConns()
+
+		if s.countConnection() == 0 {
+			break
 		}
+
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
 		case <-ticker.C:
 		}
 	}
+
+	return lnerr
 }
 
 // Close the server without waiting for active connections.
@@ -174,9 +197,7 @@ func (s *Server) Close() error {
 	return err
 }
 
-func (s *Server) addListener(l net.Listener) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
+func (s *Server) addListenerLocked(l net.Listener) {
 	s.listeners[l] = struct{}{}
 }
 
@@ -202,11 +223,18 @@ func (s *Server) closeListeners() error {
 	return err
 }
 
-func (s *Server) addConnection(c *serverConn) {
+func (s *Server) addConnection(c *serverConn) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
+	select {
+	case <-s.done:
+		return ErrServerClosed
+	default:
+	}
+
 	s.connections[c] = struct{}{}
+	return nil
 }
 
 func (s *Server) delConnection(c *serverConn) {
@@ -223,20 +251,17 @@ func (s *Server) countConnection() int {
 	return len(s.connections)
 }
 
-func (s *Server) closeIdleConns() bool {
+func (s *Server) closeIdleConns() {
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	quiescent := true
+
 	for c := range s.connections {
-		st, ok := c.getState()
-		if !ok || st != connStateIdle {
-			quiescent = false
+		if st, ok := c.getState(); !ok || st == connStateActive {
 			continue
 		}
 		c.close()
 		delete(s.connections, c)
 	}
-	return quiescent
 }
 
 type connState int
@@ -260,7 +285,7 @@ func (cs connState) String() string {
 	}
 }
 
-func (s *Server) newConn(conn net.Conn, handshake interface{}) *serverConn {
+func (s *Server) newConn(conn net.Conn, handshake interface{}) (*serverConn, error) {
 	c := &serverConn{
 		server:    s,
 		conn:      conn,
@@ -268,8 +293,11 @@ func (s *Server) newConn(conn net.Conn, handshake interface{}) *serverConn {
 		shutdown:  make(chan struct{}),
 	}
 	c.setState(connStateIdle)
-	s.addConnection(c)
-	return c
+	if err := s.addConnection(c); err != nil {
+		c.close()
+		return nil, err
+	}
+	return c, nil
 }
 
 type serverConn struct {
@@ -301,27 +329,25 @@ func (c *serverConn) close() error {
 
 func (c *serverConn) run(sctx context.Context) {
 	type (
-		request struct {
-			id  uint32
-			req *Request
-		}
-
 		response struct {
-			id   uint32
-			resp *Response
+			id          uint32
+			status      *status.Status
+			data        []byte
+			closeStream bool
+			streaming   bool
 		}
 	)
 
 	var (
-		ch          = newChannel(c.conn)
-		ctx, cancel = context.WithCancel(sctx)
-		active      int
-		state       connState = connStateIdle
-		responses             = make(chan response)
-		requests              = make(chan request)
-		recvErr               = make(chan error, 1)
-		shutdown              = c.shutdown
-		done                  = make(chan struct{})
+		ch                     = newChannel(c.conn)
+		ctx, cancel            = context.WithCancel(sctx)
+		state        connState = connStateIdle
+		responses              = make(chan response)
+		recvErr                = make(chan error, 1)
+		done                   = make(chan struct{})
+		streams                = sync.Map{}
+		active       int32
+		lastStreamID uint32
 	)
 
 	defer c.conn.Close()
@@ -329,27 +355,26 @@ func (c *serverConn) run(sctx context.Context) {
 	defer close(done)
 	defer c.server.delConnection(c)
 
+	sendStatus := func(id uint32, st *status.Status) bool {
+		select {
+		case responses <- response{
+			// even though we've had an invalid stream id, we send it
+			// back on the same stream id so the client knows which
+			// stream id was bad.
+			id:          id,
+			status:      st,
+			closeStream: true,
+		}:
+			return true
+		case <-c.shutdown:
+			return false
+		case <-done:
+			return false
+		}
+	}
+
 	go func(recvErr chan error) {
 		defer close(recvErr)
-		sendImmediate := func(id uint32, st *status.Status) bool {
-			select {
-			case responses <- response{
-				// even though we've had an invalid stream id, we send it
-				// back on the same stream id so the client knows which
-				// stream id was bad.
-				id: id,
-				resp: &Response{
-					Status: st.Proto(),
-				},
-			}:
-				return true
-			case <-c.shutdown:
-				return false
-			case <-done:
-				return false
-			}
-		}
-
 		for {
 			select {
 			case <-c.shutdown:
@@ -369,112 +394,173 @@ func (c *serverConn) run(sctx context.Context) {
 
 				// in this case, we send an error for that particular message
 				// when the status is defined.
-				if !sendImmediate(mh.StreamID, status) {
+				if !sendStatus(mh.StreamID, status) {
 					return
 				}
 
 				continue
 			}
 
-			if mh.Type != messageTypeRequest {
-				// we must ignore this for future compat.
-				continue
-			}
-
-			var req Request
-			if err := c.server.codec.Unmarshal(p, &req); err != nil {
-				ch.putmbuf(p)
-				if !sendImmediate(mh.StreamID, status.Newf(codes.InvalidArgument, "unmarshal request error: %v", err)) {
-					return
-				}
-				continue
-			}
-			ch.putmbuf(p)
-
 			if mh.StreamID%2 != 1 {
 				// enforce odd client initiated identifiers.
-				if !sendImmediate(mh.StreamID, status.Newf(codes.InvalidArgument, "StreamID must be odd for client initiated streams")) {
+				if !sendStatus(mh.StreamID, status.Newf(codes.InvalidArgument, "StreamID must be odd for client initiated streams")) {
 					return
 				}
 				continue
 			}
 
-			// Forward the request to the main loop. We don't wait on s.done
-			// because we have already accepted the client request.
-			select {
-			case requests <- request{
-				id:  mh.StreamID,
-				req: &req,
-			}:
-			case <-done:
-				return
+			if mh.Type == messageTypeData {
+				i, ok := streams.Load(mh.StreamID)
+				if !ok {
+					if !sendStatus(mh.StreamID, status.Newf(codes.InvalidArgument, "StreamID is no longer active")) {
+						return
+					}
+				}
+				sh := i.(*streamHandler)
+				if mh.Flags&flagNoData != flagNoData {
+					unmarshal := func(obj interface{}) error {
+						err := protoUnmarshal(p, obj)
+						ch.putmbuf(p)
+						return err
+					}
+
+					if err := sh.data(unmarshal); err != nil {
+						if !sendStatus(mh.StreamID, status.Newf(codes.InvalidArgument, "data handling error: %v", err)) {
+							return
+						}
+					}
+				}
+
+				if mh.Flags&flagRemoteClosed == flagRemoteClosed {
+					sh.closeSend()
+					if len(p) > 0 {
+						if !sendStatus(mh.StreamID, status.Newf(codes.InvalidArgument, "data close message cannot include data")) {
+							return
+						}
+					}
+				}
+			} else if mh.Type == messageTypeRequest {
+				if mh.StreamID <= lastStreamID {
+					// enforce odd client initiated identifiers.
+					if !sendStatus(mh.StreamID, status.Newf(codes.InvalidArgument, "StreamID cannot be re-used and must increment")) {
+						return
+					}
+					continue
+
+				}
+				lastStreamID = mh.StreamID
+
+				// TODO: Make request type configurable
+				// Unmarshaller which takes in a byte array and returns an interface?
+				var req Request
+				if err := c.server.codec.Unmarshal(p, &req); err != nil {
+					ch.putmbuf(p)
+					if !sendStatus(mh.StreamID, status.Newf(codes.InvalidArgument, "unmarshal request error: %v", err)) {
+						return
+					}
+					continue
+				}
+				ch.putmbuf(p)
+
+				id := mh.StreamID
+				respond := func(status *status.Status, data []byte, streaming, closeStream bool) error {
+					select {
+					case responses <- response{
+						id:          id,
+						status:      status,
+						data:        data,
+						closeStream: closeStream,
+						streaming:   streaming,
+					}:
+					case <-done:
+						return ErrClosed
+					}
+					return nil
+				}
+				sh, err := c.server.services.handle(ctx, &req, respond)
+				if err != nil {
+					status, _ := status.FromError(err)
+					if !sendStatus(mh.StreamID, status) {
+						return
+					}
+					continue
+				}
+
+				streams.Store(id, sh)
+				atomic.AddInt32(&active, 1)
 			}
+			// TODO: else we must ignore this for future compat. log this?
 		}
 	}(recvErr)
 
 	for {
-		newstate := state
-		switch {
-		case active > 0:
+		var (
+			newstate connState
+			shutdown chan struct{}
+		)
+
+		activeN := atomic.LoadInt32(&active)
+		if activeN > 0 {
 			newstate = connStateActive
 			shutdown = nil
-		case active == 0:
+		} else {
 			newstate = connStateIdle
 			shutdown = c.shutdown // only enable this branch in idle mode
 		}
-
 		if newstate != state {
 			c.setState(newstate)
 			state = newstate
 		}
 
 		select {
-		case request := <-requests:
-			active++
-			go func(id uint32) {
-				ctx, cancel := getRequestContext(ctx, request.req)
-				defer cancel()
-
-				p, status := c.server.services.call(ctx, request.req.Service, request.req.Method, request.req.Payload)
-				resp := &Response{
-					Status:  status.Proto(),
-					Payload: p,
-				}
-
-				select {
-				case responses <- response{
-					id:   id,
-					resp: resp,
-				}:
-				case <-done:
-				}
-			}(request.id)
 		case response := <-responses:
-			p, err := c.server.codec.Marshal(response.resp)
-			if err != nil {
-				logrus.WithError(err).Error("failed marshaling response")
-				return
+			if !response.streaming || response.status.Code() != codes.OK {
+				p, err := c.server.codec.Marshal(&Response{
+					Status:  response.status.Proto(),
+					Payload: response.data,
+				})
+				if err != nil {
+					log.G(ctx).WithError(err).Error("failed marshaling response")
+					return
+				}
+
+				if err := ch.send(response.id, messageTypeResponse, 0, p); err != nil {
+					log.G(ctx).WithError(err).Error("failed sending message on channel")
+					return
+				}
+			} else {
+				var flags uint8
+				if response.closeStream {
+					flags = flagRemoteClosed
+				}
+				if response.data == nil {
+					flags = flags | flagNoData
+				}
+				if err := ch.send(response.id, messageTypeData, flags, response.data); err != nil {
+					log.G(ctx).WithError(err).Error("failed sending message on channel")
+					return
+				}
 			}
 
-			if err := ch.send(response.id, messageTypeResponse, p); err != nil {
-				logrus.WithError(err).Error("failed sending message on channel")
-				return
+			if response.closeStream {
+				// The ttrpc protocol currently does not support the case where
+				// the server is localClosed but not remoteClosed. Once the server
+				// is closing, the whole stream may be considered finished
+				streams.Delete(response.id)
+				atomic.AddInt32(&active, -1)
 			}
-
-			active--
 		case err := <-recvErr:
 			// TODO(stevvooe): Not wildly clear what we should do in this
 			// branch. Basically, it means that we are no longer receiving
 			// requests due to a terminal error.
 			recvErr = nil // connection is now "closing"
-			if err == io.EOF || err == io.ErrUnexpectedEOF {
+			if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, syscall.ECONNRESET) {
 				// The client went away and we should stop processing
 				// requests, so that the client connection is closed
 				return
 			}
-			if err != nil {
-				logrus.WithError(err).Error("error receiving message")
-			}
+			log.G(ctx).WithError(err).Error("error receiving message")
+			// else, initiate shutdown
 		case <-shutdown:
 			return
 		}

server_linux_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/containerd/ttrpc/internal"
 	"github.com/prometheus/procfs"
 )
 
@@ -41,7 +42,7 @@ func TestUnixSocketHandshake(t *testing.T) {
 
 	registerTestingService(server, &testingServer{})
 
-	var tp testPayload
+	var tp internal.TestPayload
 	// server shutdown, but we still make a call.
 	if err := client.Call(ctx, serviceName, "Test", &tp, &tp); err != nil {
 		t.Fatalf("unexpected error making call: %v", err)
@@ -70,7 +71,7 @@ func BenchmarkRoundTripUnixSocketCreds(b *testing.B) {
 	go server.Serve(ctx, listener)
 	defer server.Shutdown(ctx)
 
-	var tp testPayload
+	var tp internal.TestPayload
 	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
@@ -96,7 +97,7 @@ func TestServerEOF(t *testing.T) {
 
 	registerTestingService(server, &testingServer{})
 
-	tp := &testPayload{}
+	tp := &internal.TestPayload{}
 	// do a regular call
 	if err := client.Call(ctx, serviceName, "Test", tp, tp); err != nil {
 		t.Fatalf("unexpected error during test call: %v", err)

server_test.go

@@ -17,20 +17,22 @@
 package ttrpc
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
 	"net"
-	"reflect"
 	"runtime"
 	"strings"
 	"sync"
+	"syscall"
 	"testing"
 	"time"
 
-	"github.com/gogo/protobuf/proto"
+	"github.com/containerd/ttrpc/internal"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
 )
 
 const serviceName = "testService"
@@ -40,7 +42,7 @@ const serviceName = "testService"
 // Typically, this is generated. We define it here to ensure that that package
 // primitive has what is required for generated code.
 type testingService interface {
-	Test(ctx context.Context, req *testPayload) (*testPayload, error)
+	Test(ctx context.Context, req *internal.TestPayload) (*internal.TestPayload, error)
 }
 
 type testingClient struct {
@@ -53,26 +55,16 @@ func newTestingClient(client *Client) *testingClient {
 	}
 }
 
-func (tc *testingClient) Test(ctx context.Context, req *testPayload) (*testPayload, error) {
-	var tp testPayload
+func (tc *testingClient) Test(ctx context.Context, req *internal.TestPayload) (*internal.TestPayload, error) {
+	var tp internal.TestPayload
 	return &tp, tc.client.Call(ctx, serviceName, "Test", req, &tp)
 }
 
-type testPayload struct {
-	Foo      string `protobuf:"bytes,1,opt,name=foo,proto3"`
-	Deadline int64  `protobuf:"varint,2,opt,name=deadline,proto3"`
-	Metadata string `protobuf:"bytes,3,opt,name=metadata,proto3"`
-}
-
-func (r *testPayload) Reset()         { *r = testPayload{} }
-func (r *testPayload) String() string { return fmt.Sprintf("%+#v", r) }
-func (r *testPayload) ProtoMessage()  {}
-
 // testingServer is what would be implemented by the user of this package.
 type testingServer struct{}
 
-func (s *testingServer) Test(ctx context.Context, req *testPayload) (*testPayload, error) {
-	tp := &testPayload{Foo: strings.Repeat(req.Foo, 2)}
+func (s *testingServer) Test(ctx context.Context, req *internal.TestPayload) (*internal.TestPayload, error) {
+	tp := &internal.TestPayload{Foo: strings.Repeat(req.Foo, 2)}
 	if dl, ok := ctx.Deadline(); ok {
 		tp.Deadline = dl.UnixNano()
 	}
@@ -90,7 +82,7 @@ func (s *testingServer) Test(ctx context.Context, req *testPayload) (*testPayloa
 func registerTestingService(srv *Server, svc testingService) {
 	srv.Register(serviceName, map[string]Method{
 		"Test": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) {
-			var req testPayload
+			var req internal.TestPayload
 			if err := unmarshal(&req); err != nil {
 				return nil, err
 			}
@@ -99,10 +91,16 @@ func registerTestingService(srv *Server, svc testingService) {
 	})
 }
 
-func init() {
-	proto.RegisterType((*testPayload)(nil), "testPayload")
-	proto.RegisterType((*Request)(nil), "Request")
-	proto.RegisterType((*Response)(nil), "Response")
+func protoEqual(a, b proto.Message) (bool, error) {
+	ma, err := proto.Marshal(a)
+	if err != nil {
+		return false, err
+	}
+	mb, err := proto.Marshal(b)
+	if err != nil {
+		return false, err
+	}
+	return bytes.Equal(ma, mb), nil
 }
 
 func TestServer(t *testing.T) {
@@ -123,16 +121,27 @@ func TestServer(t *testing.T) {
 	go server.Serve(ctx, listener)
 	defer server.Shutdown(ctx)
 
-	const calls = 2
-	results := make(chan callResult, 2)
-	go roundTrip(ctx, t, tclient, "bar", results)
-	go roundTrip(ctx, t, tclient, "baz", results)
+	testCases := []string{"bar", "baz"}
+	results := make(chan callResult, len(testCases))
+	for _, tc := range testCases {
+		go func(expected string) {
+			results <- roundTrip(ctx, tclient, expected)
+		}(tc)
+	}
 
-	for i := 0; i < calls; i++ {
+	for i := 0; i < len(testCases); {
 		result := <-results
-		if !reflect.DeepEqual(result.received, result.expected) {
+		if result.err != nil {
+			t.Fatalf("(%s): %v", result.name, result.err)
+		}
+		equal, err := protoEqual(result.received, result.expected)
+		if err != nil {
+			t.Fatalf("failed to compare %s and %s: %s", result.received, result.expected, err)
+		}
+		if !equal {
 			t.Fatalf("unexpected response: %+#v != %+#v", result.received, result.expected)
 		}
+		i++
 	}
 }
 
@@ -150,7 +159,7 @@ func TestServerUnimplemented(t *testing.T) {
 		errs <- server.Serve(ctx, listener)
 	}()
 
-	var tp testPayload
+	var tp internal.TestPayload
 	if err := client.Call(ctx, "Not", "Found", &tp, &tp); err == nil {
 		t.Fatalf("expected error from non-existent service call")
 	} else if status, ok := status.FromError(err); !ok {
@@ -192,35 +201,33 @@ func TestServerListenerClosed(t *testing.T) {
 func TestServerShutdown(t *testing.T) {
 	const ncalls = 5
 	var (
-		ctx                      = context.Background()
-		server                   = mustServer(t)(NewServer())
-		addr, listener           = newTestListener(t)
-		shutdownStarted          = make(chan struct{})
-		shutdownFinished         = make(chan struct{})
-		handlersStarted          = make(chan struct{})
-		handlersStartedCloseOnce sync.Once
-		proceed                  = make(chan struct{})
-		serveErrs                = make(chan error, 1)
-		callwg                   sync.WaitGroup
-		callErrs                 = make(chan error, ncalls)
-		shutdownErrs             = make(chan error, 1)
-		client, cleanup          = newTestClient(t, addr)
-		_, cleanup2              = newTestClient(t, addr) // secondary connection
+		ctx              = context.Background()
+		server           = mustServer(t)(NewServer())
+		addr, listener   = newTestListener(t)
+		shutdownStarted  = make(chan struct{})
+		shutdownFinished = make(chan struct{})
+		handlersStarted  sync.WaitGroup
+		proceed          = make(chan struct{})
+		serveErrs        = make(chan error, 1)
+		callErrs         = make(chan error, ncalls)
+		shutdownErrs     = make(chan error, 1)
+		client, cleanup  = newTestClient(t, addr)
+		_, cleanup2      = newTestClient(t, addr) // secondary connection
 	)
 	defer cleanup()
 	defer cleanup2()
 
 	// register a service that takes until we tell it to stop
 	server.Register(serviceName, map[string]Method{
-		"Test": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) {
-			var req testPayload
+		"Test": func(_ context.Context, unmarshal func(interface{}) error) (interface{}, error) {
+			var req internal.TestPayload
 			if err := unmarshal(&req); err != nil {
 				return nil, err
 			}
 
-			handlersStartedCloseOnce.Do(func() { close(handlersStarted) })
+			handlersStarted.Done()
 			<-proceed
-			return &testPayload{Foo: "waited"}, nil
+			return &internal.TestPayload{Foo: "waited"}, nil
 		},
 	})
 
@@ -229,20 +236,18 @@ func TestServerShutdown(t *testing.T) {
 	}()
 
 	// send a series of requests that will get blocked
-	for i := 0; i < 5; i++ {
-		callwg.Add(1)
+	for i := 0; i < ncalls; i++ {
+		handlersStarted.Add(1)
 		go func(i int) {
-			callwg.Done()
-			tp := testPayload{Foo: "half" + fmt.Sprint(i)}
+			tp := internal.TestPayload{Foo: "half" + fmt.Sprint(i)}
 			callErrs <- client.Call(ctx, serviceName, "Test", &tp, &tp)
 		}(i)
 	}
 
-	<-handlersStarted
+	handlersStarted.Wait()
 	go func() {
 		close(shutdownStarted)
 		shutdownErrs <- server.Shutdown(ctx)
-		// server.Close()
 		close(shutdownFinished)
 	}()
 
@@ -293,6 +298,36 @@ func TestServerClose(t *testing.T) {
 	checkServerShutdown(t, server)
 }
 
+func TestImmediateServerShutdown(t *testing.T) {
+	var (
+		ctx            = context.Background()
+		server         = mustServer(t)(NewServer())
+		addr, listener = newTestListener(t)
+		errs           = make(chan error, 1)
+		_, cleanup     = newTestClient(t, addr)
+	)
+	defer cleanup()
+	defer listener.Close()
+	go func() {
+		time.Sleep(1 * time.Millisecond)
+		errs <- server.Serve(ctx, listener)
+	}()
+
+	registerTestingService(server, &testingServer{})
+
+	if err := server.Shutdown(ctx); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case err := <-errs:
+		if err != ErrServerClosed {
+			t.Fatal(err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("retreiving error from server.Shutdown() timed out")
+	}
+}
+
 func TestOversizeCall(t *testing.T) {
 	var (
 		ctx             = context.Background()
@@ -309,11 +344,11 @@ func TestOversizeCall(t *testing.T) {
 
 	registerTestingService(server, &testingServer{})
 
-	tp := &testPayload{
+	tp := &internal.TestPayload{
 		Foo: strings.Repeat("a", 1+messageLengthMax),
 	}
 	if err := client.Call(ctx, serviceName, "Test", tp, tp); err == nil {
-		t.Fatalf("expected error from non-existent service call")
+		t.Fatalf("expected error from oversized message")
 	} else if status, ok := status.FromError(err); !ok {
 		t.Fatalf("expected status present in error: %v", err)
 	} else if status.Code() != codes.ResourceExhausted {
@@ -344,7 +379,7 @@ func TestClientEOF(t *testing.T) {
 
 	registerTestingService(server, &testingServer{})
 
-	tp := &testPayload{}
+	tp := &internal.TestPayload{}
 	// do a regular call
 	if err := client.Call(ctx, serviceName, "Test", tp, tp); err != nil {
 		t.Fatalf("unexpected error: %v", err)
@@ -358,10 +393,17 @@ func TestClientEOF(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	client.UserOnCloseWait(ctx)
+
 	// server shutdown, but we still make a call.
 	if err := client.Call(ctx, serviceName, "Test", tp, tp); err == nil {
 		t.Fatalf("expected error when calling against shutdown server")
 	} else if !errors.Is(err, ErrClosed) {
+		var errno syscall.Errno
+		if errors.As(err, &errno) {
+			t.Logf("errno=%d", errno)
+		}
+
 		t.Fatalf("expected to have a cause of ErrClosed, got %v", err)
 	}
 }
@@ -373,7 +415,7 @@ func TestServerRequestTimeout(t *testing.T) {
 		addr, listener  = newTestListener(t)
 		testImpl        = &testingServer{}
 		client, cleanup = newTestClient(t, addr)
-		result          testPayload
+		result          internal.TestPayload
 	)
 	defer cancel()
 	defer cleanup()
@@ -384,13 +426,13 @@ func TestServerRequestTimeout(t *testing.T) {
 	go server.Serve(ctx, listener)
 	defer server.Shutdown(ctx)
 
-	if err := client.Call(ctx, serviceName, "Test", &testPayload{}, &result); err != nil {
+	if err := client.Call(ctx, serviceName, "Test", &internal.TestPayload{}, &result); err != nil {
 		t.Fatalf("unexpected error making call: %v", err)
 	}
 
 	dl, _ := ctx.Deadline()
 	if result.Deadline != dl.UnixNano() {
-		t.Fatalf("expected deadline %v, actual: %v", dl, result.Deadline)
+		t.Fatalf("expected deadline %v, actual: %v", dl, time.Unix(0, result.Deadline))
 	}
 }
 
@@ -410,7 +452,7 @@ func TestServerConnectionsLeak(t *testing.T) {
 
 	registerTestingService(server, &testingServer{})
 
-	tp := &testPayload{}
+	tp := &internal.TestPayload{}
 	// do a regular call
 	if err := client.Call(ctx, serviceName, "Test", tp, tp); err != nil {
 		t.Fatalf("unexpected error during test call: %v", err)
@@ -459,7 +501,7 @@ func BenchmarkRoundTrip(b *testing.B) {
 	go server.Serve(ctx, listener)
 	defer server.Shutdown(ctx)
 
-	var tp testPayload
+	var tp internal.TestPayload
 	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
@@ -473,39 +515,55 @@ func checkServerShutdown(t *testing.T, server *Server) {
 	t.Helper()
 	server.mu.Lock()
 	defer server.mu.Unlock()
+
 	if len(server.listeners) > 0 {
-		t.Fatalf("expected listeners to be empty: %v", server.listeners)
+		t.Errorf("expected listeners to be empty: %v", server.listeners)
+	}
+	for listener := range server.listeners {
+		t.Logf("listener addr=%s", listener.Addr())
 	}
 
 	if len(server.connections) > 0 {
-		t.Fatalf("expected connections to be empty: %v", server.connections)
+		t.Errorf("expected connections to be empty: %v", server.connections)
+	}
+	for conn := range server.connections {
+		state, ok := conn.getState()
+		if !ok {
+			t.Errorf("failed to get state from %v", conn)
+		}
+		t.Logf("conn state=%s", state)
 	}
 }
 
 type callResult struct {
-	input    *testPayload
-	expected *testPayload
-	received *testPayload
+	name     string
+	err      error
+	input    *internal.TestPayload
+	expected *internal.TestPayload
+	received *internal.TestPayload
 }
 
-func roundTrip(ctx context.Context, t *testing.T, client *testingClient, value string, results chan callResult) {
-	t.Helper()
+func roundTrip(ctx context.Context, client *testingClient, name string) callResult {
 	var (
-		tp = &testPayload{
-			Foo: "bar",
+		tp = &internal.TestPayload{
+			Foo: name,
 		}
 	)
 
-	ctx = WithMetadata(ctx, MD{"foo": []string{"bar"}})
+	ctx = WithMetadata(ctx, MD{"foo": []string{name}})
 
 	resp, err := client.Test(ctx, tp)
 	if err != nil {
-		t.Fatal(err)
+		return callResult{
+			name: name,
+			err:  err,
+		}
 	}
 
-	results <- callResult{
+	return callResult{
+		name:     name,
 		input:    tp,
-		expected: &testPayload{Foo: strings.Repeat(tp.Foo, 2), Metadata: "bar"},
+		expected: &internal.TestPayload{Foo: strings.Repeat(tp.Foo, 2), Metadata: name},
 		received: resp,
 	}
 }

services.go

@@ -25,43 +25,62 @@ import (
 	"path"
 	"unsafe"
 
-	"github.com/gogo/protobuf/proto"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
 )
 
 type Method func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error)
 
+type StreamHandler func(context.Context, StreamServer) (interface{}, error)
+
+type Stream struct {
+	Handler         StreamHandler
+	StreamingClient bool
+	StreamingServer bool
+}
+
 type ServiceDesc struct {
 	Methods map[string]Method
-
-	// TODO(stevvooe): Add stream support.
+	Streams map[string]Stream
 }
 
 type serviceSet struct {
-	services    map[string]ServiceDesc
-	interceptor UnaryServerInterceptor
+	services          map[string]*ServiceDesc
+	unaryInterceptor  UnaryServerInterceptor
+	streamInterceptor StreamServerInterceptor
 }
 
 func newServiceSet(interceptor UnaryServerInterceptor) *serviceSet {
 	return &serviceSet{
-		services:    make(map[string]ServiceDesc),
-		interceptor: interceptor,
+		services:          make(map[string]*ServiceDesc),
+		unaryInterceptor:  interceptor,
+		streamInterceptor: defaultStreamServerInterceptor,
 	}
 }
 
-func (s *serviceSet) register(name string, methods map[string]Method) {
+func (s *serviceSet) register(name string, desc *ServiceDesc) {
 	if _, ok := s.services[name]; ok {
 		panic(fmt.Errorf("duplicate service %v registered", name))
 	}
 
-	s.services[name] = ServiceDesc{
-		Methods: methods,
-	}
+	s.services[name] = desc
 }
 
-func (s *serviceSet) call(ctx context.Context, serviceName, methodName string, p []byte) ([]byte, *status.Status) {
-	p, err := s.dispatch(ctx, serviceName, methodName, p)
+func (s *serviceSet) unaryCall(ctx context.Context, method Method, info *UnaryServerInfo, data []byte) (p []byte, st *status.Status) {
+	unmarshal := func(obj interface{}) error {
+		return protoUnmarshal(data, obj)
+	}
+
+	resp, err := s.unaryInterceptor(ctx, unmarshal, info, method)
+	if err == nil {
+		if isNil(resp) {
+			err = errors.New("ttrpc: marshal called with nil")
+		} else {
+			p, err = protoMarshal(resp)
+		}
+	}
+
 	st, ok := status.FromError(err)
 	if !ok {
 		st = status.New(convertCode(err), err.Error())
@@ -70,38 +89,146 @@ func (s *serviceSet) call(ctx context.Context, serviceName, methodName string, p
 	return p, st
 }
 
-func (s *serviceSet) dispatch(ctx context.Context, serviceName, methodName string, p []byte) ([]byte, error) {
-	method, err := s.resolve(serviceName, methodName)
-	if err != nil {
-		return nil, err
+func (s *serviceSet) streamCall(ctx context.Context, stream StreamHandler, info *StreamServerInfo, ss StreamServer) (p []byte, st *status.Status) {
+	resp, err := s.streamInterceptor(ctx, ss, info, stream)
+	if err == nil {
+		p, err = protoMarshal(resp)
+	}
+	st, ok := status.FromError(err)
+	if !ok {
+		st = status.New(convertCode(err), err.Error())
+	}
+	return
+}
+
+func (s *serviceSet) handle(ctx context.Context, req *Request, respond func(*status.Status, []byte, bool, bool) error) (*streamHandler, error) {
+	srv, ok := s.services[req.Service]
+	if !ok {
+		return nil, status.Errorf(codes.Unimplemented, "service %v", req.Service)
 	}
 
-	unmarshal := func(obj interface{}) error {
-		switch v := obj.(type) {
-		case proto.Message:
-			if err := proto.Unmarshal(p, v); err != nil {
-				return status.Errorf(codes.Internal, "ttrpc: error unmarshalling payload: %v", err.Error())
+	if method, ok := srv.Methods[req.Method]; ok {
+		go func() {
+			ctx, cancel := getRequestContext(ctx, req)
+			defer cancel()
+
+			info := &UnaryServerInfo{
+				FullMethod: fullPath(req.Service, req.Method),
 			}
-		default:
-			return status.Errorf(codes.Internal, "ttrpc: error unsupported request type: %T", v)
+			p, st := s.unaryCall(ctx, method, info, req.Payload)
+
+			respond(st, p, false, true)
+		}()
+		return nil, nil
+	}
+	if stream, ok := srv.Streams[req.Method]; ok {
+		ctx, cancel := getRequestContext(ctx, req)
+		info := &StreamServerInfo{
+			FullMethod:      fullPath(req.Service, req.Method),
+			StreamingClient: stream.StreamingClient,
+			StreamingServer: stream.StreamingServer,
 		}
+		sh := &streamHandler{
+			ctx:     ctx,
+			respond: respond,
+			recv:    make(chan Unmarshaler, 5),
+			info:    info,
+		}
+		go func() {
+			defer cancel()
+			p, st := s.streamCall(ctx, stream.Handler, info, sh)
+			respond(st, p, stream.StreamingServer, true)
+		}()
+
+		// Empty proto messages serialized to 0 payloads,
+		// so signatures like: rpc Stream(google.protobuf.Empty) returns (stream Data);
+		// don't get invoked here, which causes hang on client side.
+		// See https://github.com/containerd/ttrpc/issues/126
+		if req.Payload != nil || !info.StreamingClient {
+			unmarshal := func(obj interface{}) error {
+				return protoUnmarshal(req.Payload, obj)
+			}
+			if err := sh.data(unmarshal); err != nil {
+				return nil, err
+			}
+		}
+
+		return sh, nil
+	}
+	return nil, status.Errorf(codes.Unimplemented, "method %v", req.Method)
+}
+
+type streamHandler struct {
+	ctx     context.Context
+	respond func(*status.Status, []byte, bool, bool) error
+	recv    chan Unmarshaler
+	info    *StreamServerInfo
+
+	remoteClosed bool
+	localClosed  bool
+}
+
+func (s *streamHandler) closeSend() {
+	if !s.remoteClosed {
+		s.remoteClosed = true
+		close(s.recv)
+	}
+}
+
+func (s *streamHandler) data(unmarshal Unmarshaler) error {
+	if s.remoteClosed {
+		return ErrStreamClosed
+	}
+	select {
+	case s.recv <- unmarshal:
 		return nil
+	case <-s.ctx.Done():
+		return s.ctx.Err()
 	}
+}
 
-	info := &UnaryServerInfo{
-		FullMethod: fullPath(serviceName, methodName),
+func (s *streamHandler) SendMsg(m interface{}) error {
+	if s.localClosed {
+		return ErrStreamClosed
 	}
-
-	resp, err := s.interceptor(ctx, unmarshal, info, method)
+	p, err := protoMarshal(m)
 	if err != nil {
-		return nil, err
+		return err
+	}
+	return s.respond(nil, p, true, false)
+}
+
+func (s *streamHandler) RecvMsg(m interface{}) error {
+	select {
+	case unmarshal, ok := <-s.recv:
+		if !ok {
+			return io.EOF
+		}
+		return unmarshal(m)
+	case <-s.ctx.Done():
+		return s.ctx.Err()
+
+	}
+}
+
+func protoUnmarshal(p []byte, obj interface{}) error {
+	switch v := obj.(type) {
+	case proto.Message:
+		if err := proto.Unmarshal(p, v); err != nil {
+			return status.Errorf(codes.Internal, "ttrpc: error unmarshalling payload: %v", err.Error())
+		}
+	default:
+		return status.Errorf(codes.Internal, "ttrpc: error unsupported request type: %T", v)
+	}
+	return nil
+}
+
+func protoMarshal(obj interface{}) ([]byte, error) {
+	if obj == nil {
+		return nil, nil
 	}
 
-	if isNil(resp) {
-		return nil, errors.New("ttrpc: marshal called with nil")
-	}
-
-	switch v := resp.(type) {
+	switch v := obj.(type) {
 	case proto.Message:
 		r, err := proto.Marshal(v)
 		if err != nil {
@@ -114,20 +241,6 @@ func (s *serviceSet) dispatch(ctx context.Context, serviceName, methodName strin
 	}
 }
 
-func (s *serviceSet) resolve(service, method string) (Method, error) {
-	srv, ok := s.services[service]
-	if !ok {
-		return nil, status.Errorf(codes.Unimplemented, "service %v", service)
-	}
-
-	mthd, ok := srv.Methods[method]
-	if !ok {
-		return nil, status.Errorf(codes.Unimplemented, "method %v", method)
-	}
-
-	return mthd, nil
-}
-
 // convertCode maps stdlib go errors into grpc space.
 //
 // This is ripped from the grpc-go code base.

stream.go

@@ -0,0 +1,84 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ttrpc
+
+import (
+	"context"
+	"sync"
+)
+
+type streamID uint32
+
+type streamMessage struct {
+	header  messageHeader
+	payload []byte
+}
+
+type stream struct {
+	id     streamID
+	sender sender
+	recv   chan *streamMessage
+
+	closeOnce sync.Once
+	recvErr   error
+	recvClose chan struct{}
+}
+
+func newStream(id streamID, send sender) *stream {
+	return &stream{
+		id:        id,
+		sender:    send,
+		recv:      make(chan *streamMessage, 1),
+		recvClose: make(chan struct{}),
+	}
+}
+
+func (s *stream) closeWithError(err error) error {
+	s.closeOnce.Do(func() {
+		if err != nil {
+			s.recvErr = err
+		} else {
+			s.recvErr = ErrClosed
+		}
+		close(s.recvClose)
+	})
+	return nil
+}
+
+func (s *stream) send(mt messageType, flags uint8, b []byte) error {
+	return s.sender.send(uint32(s.id), mt, flags, b)
+}
+
+func (s *stream) receive(ctx context.Context, msg *streamMessage) error {
+	select {
+	case <-s.recvClose:
+		return s.recvErr
+	default:
+	}
+	select {
+	case <-s.recvClose:
+		return s.recvErr
+	case s.recv <- msg:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+type sender interface {
+	send(uint32, messageType, uint8, []byte) error
+}

stream_server.go

@@ -0,0 +1,22 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ttrpc
+
+type StreamServer interface {
+	SendMsg(m interface{}) error
+	RecvMsg(m interface{}) error
+}

stream_test.go

@@ -0,0 +1,118 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ttrpc
+
+import (
+	"context"
+	"io"
+	"testing"
+
+	"github.com/containerd/ttrpc/internal"
+)
+
+func TestStreamClient(t *testing.T) {
+	var (
+		ctx             = context.Background()
+		server          = mustServer(t)(NewServer())
+		addr, listener  = newTestListener(t)
+		client, cleanup = newTestClient(t, addr)
+		serviceName     = "streamService"
+	)
+
+	defer listener.Close()
+	defer cleanup()
+
+	desc := &ServiceDesc{
+		Methods: map[string]Method{
+			"Echo": func(_ context.Context, unmarshal func(interface{}) error) (interface{}, error) {
+				var req internal.EchoPayload
+				if err := unmarshal(&req); err != nil {
+					return nil, err
+				}
+				req.Seq++
+				return &req, nil
+			},
+		},
+		Streams: map[string]Stream{
+			"EchoStream": {
+				Handler: func(_ context.Context, ss StreamServer) (interface{}, error) {
+					for {
+						var req internal.EchoPayload
+						if err := ss.RecvMsg(&req); err != nil {
+							if err == io.EOF {
+								err = nil
+							}
+							return nil, err
+						}
+						req.Seq++
+						if err := ss.SendMsg(&req); err != nil {
+							return nil, err
+						}
+					}
+
+				},
+				StreamingClient: true,
+				StreamingServer: true,
+			},
+		},
+	}
+	server.RegisterService(serviceName, desc)
+
+	go server.Serve(ctx, listener)
+	defer server.Shutdown(ctx)
+
+	//func (c *Client) NewStream(ctx context.Context, desc *StreamDesc, service, method string) (ClientStream, error) {
+	var req, resp internal.EchoPayload
+	if err := client.Call(ctx, serviceName, "Echo", &req, &resp); err != nil {
+		t.Fatal(err)
+	}
+
+	stream, err := client.NewStream(ctx, &StreamDesc{true, true}, serviceName, "EchoStream", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := 1; i <= 100; i++ {
+		req := internal.EchoPayload{
+			Seq: int64(i),
+			Msg: "should be returned",
+		}
+		if err := stream.SendMsg(&req); err != nil {
+			t.Fatalf("%d: %v", i, err)
+		}
+		var resp internal.EchoPayload
+		if err := stream.RecvMsg(&resp); err != nil {
+			t.Fatalf("%d: %v", i, err)
+		}
+		if resp.Seq != int64(i)+1 {
+			t.Fatalf("%d: unexpected sequence value: %d, expected %d", i, resp.Seq, i+1)
+		}
+		if resp.Msg != req.Msg {
+			t.Fatalf("%d: unexpected message: %q, expected %q", i, resp.Msg, req.Msg)
+		}
+	}
+	if err := stream.CloseSend(); err != nil {
+		t.Fatal(err)
+	}
+
+	err = stream.RecvMsg(&resp)
+	if err == nil {
+		t.Fatal("expected io.EOF after close send")
+	}
+	if err != io.EOF {
+		t.Fatalf("expected io.EOF after close send, got %v", err)
+	}
+}

test.proto

@@ -0,0 +1,16 @@
+syntax = "proto3";
+
+package ttrpc;
+
+option go_package = "github.com/containerd/ttrpc/internal";
+
+message TestPayload {
+	string foo = 1;
+	int64 deadline = 2;
+	string metadata = 3;
+}
+
+message EchoPayload {
+	int64 seq = 1;
+	string msg = 2;
+}

types.go

@@ -1,63 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package ttrpc
-
-import (
-	"fmt"
-
-	spb "google.golang.org/genproto/googleapis/rpc/status"
-)
-
-type Request struct {
-	Service     string      `protobuf:"bytes,1,opt,name=service,proto3"`
-	Method      string      `protobuf:"bytes,2,opt,name=method,proto3"`
-	Payload     []byte      `protobuf:"bytes,3,opt,name=payload,proto3"`
-	TimeoutNano int64       `protobuf:"varint,4,opt,name=timeout_nano,proto3"`
-	Metadata    []*KeyValue `protobuf:"bytes,5,rep,name=metadata,proto3"`
-}
-
-func (r *Request) Reset()         { *r = Request{} }
-func (r *Request) String() string { return fmt.Sprintf("%+#v", r) }
-func (r *Request) ProtoMessage()  {}
-
-type Response struct {
-	Status  *spb.Status `protobuf:"bytes,1,opt,name=status,proto3"`
-	Payload []byte      `protobuf:"bytes,2,opt,name=payload,proto3"`
-}
-
-func (r *Response) Reset()         { *r = Response{} }
-func (r *Response) String() string { return fmt.Sprintf("%+#v", r) }
-func (r *Response) ProtoMessage()  {}
-
-type StringList struct {
-	List []string `protobuf:"bytes,1,rep,name=list,proto3"`
-}
-
-func (r *StringList) Reset()         { *r = StringList{} }
-func (r *StringList) String() string { return fmt.Sprintf("%+#v", r) }
-func (r *StringList) ProtoMessage()  {}
-
-func makeStringList(item ...string) StringList { return StringList{List: item} }
-
-type KeyValue struct {
-	Key   string `protobuf:"bytes,1,opt,name=key,proto3"`
-	Value string `protobuf:"bytes,2,opt,name=value,proto3"`
-}
-
-func (m *KeyValue) Reset()         { *m = KeyValue{} }
-func (*KeyValue) ProtoMessage()    {}
-func (m *KeyValue) String() string { return fmt.Sprintf("%+#v", m) }

unixcreds_linux.go

@@ -29,7 +29,7 @@ import (
 
 type UnixCredentialsFunc func(*unix.Ucred) error
 
-func (fn UnixCredentialsFunc) Handshake(ctx context.Context, conn net.Conn) (net.Conn, interface{}, error) {
+func (fn UnixCredentialsFunc) Handshake(_ context.Context, conn net.Conn) (net.Conn, interface{}, error) {
 	uc, err := requireUnixSocket(conn)
 	if err != nil {
 		return nil, nil, fmt.Errorf("ttrpc.UnixCredentialsFunc: require unix socket: %w", err)
@@ -50,7 +50,7 @@ func (fn UnixCredentialsFunc) Handshake(ctx context.Context, conn net.Conn) (net
 	}
 
 	if ucredErr != nil {
-		return nil, nil, fmt.Errorf("ttrpc.UnixCredentialsFunc: failed to retrieve socket peer credentials: %w", err)
+		return nil, nil, fmt.Errorf("ttrpc.UnixCredentialsFunc: failed to retrieve socket peer credentials: %w", ucredErr)
 	}
 
 	if err := fn(ucred); err != nil {
@@ -88,10 +88,6 @@ func UnixSocketRequireSameUser() UnixCredentialsFunc {
 	return UnixSocketRequireUidGid(euid, egid)
 }
 
-func requireRoot(ucred *unix.Ucred) error {
-	return requireUidGid(ucred, 0, 0)
-}
-
 func requireUidGid(ucred *unix.Ucred, uid, gid int) error {
 	if (uid != -1 && uint32(uid) != ucred.Uid) || (gid != -1 && uint32(gid) != ucred.Gid) {
 		return fmt.Errorf("ttrpc: invalid credentials: %v", syscall.EPERM)