kubernetes
/
kubectl
mirror of https://github.com/kubernetes/kubectl.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Configure fixture processes with ports to listen on 

- APIServer & Etcd get configured, from the outside, on which ports to
  listen on
- Configuration, the subjects under test might be interested in, is
  exposed by Fixtures.Config

Hint: Before we start any process, we get a random port and check if
that random port is acutally free to bind to. As it takes some time
until we actually start anything, we might run into cases, where another
process binds to that port while we are starting up. Even if we do the
port checking closer to actually binding, we still have the same issue.
For now, however, we take that risk - if we run into problems with that,
we are open to refactor that.
This commit is contained in:
Hannes Hörl 2017-12-05 12:22:11 +00:00 committed by Gareth Smith
parent 88e83f4286
commit e8c6a13d49
6 changed files with 121 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
pkg/framework/test/apiserver.go
View File

@ -2,11 +2,11 @@ package test
import (
"fmt"
"io"
"net/url"
"os/exec"
"time"
"io"
"github.com/onsi/gomega"
"github.com/onsi/gomega/gbytes"
"github.com/onsi/gomega/gexec"
@ -19,6 +19,7 @@ type APIServer struct {
EtcdURL string
ProcessStarter simpleSessionStarter
CertDirManager certDirManager
APIServerURL string
session SimpleSession
stdOut *gbytes.Buffer
stdErr *gbytes.Buffer
@ -31,7 +32,8 @@ type certDirManager interface {
//go:generate counterfeiter . certDirManager
func NewAPIServer(pathToAPIServer, etcdURL string) *APIServer {
// NewAPIServer creates a new APIServer Fixture Process
func NewAPIServer(pathToAPIServer, apiServerURL, etcdURL string) *APIServer {
starter := func(command *exec.Cmd, out, err io.Writer) (SimpleSession, error) {
return gexec.Start(command, out, err)
}
@ -41,6 +43,7 @@ func NewAPIServer(pathToAPIServer, etcdURL string) *APIServer {
EtcdURL: etcdURL,
ProcessStarter: starter,
CertDirManager: NewTempDirManager(),
APIServerURL: apiServerURL,
}
return apiserver
@ -56,6 +59,11 @@ func (s *APIServer) Start() error {
return err
}
url, err := url.Parse(s.APIServerURL)
if err != nil {
return err
}
args := []string{
"--authorization-mode=Node,RBAC",
"--runtime-config=admissionregistration.k8s.io/v1alpha1",
@ -63,14 +71,14 @@ func (s *APIServer) Start() error {
"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,SecurityContextDeny,DefaultStorageClass,DefaultTolerationSeconds,GenericAdmissionWebhook,ResourceQuota",
"--admission-control-config-file=",
"--bind-address=0.0.0.0",
"--insecure-bind-address=127.0.0.1",
"--insecure-port=8080",
"--storage-backend=etcd3",
fmt.Sprintf("--etcd-servers=%s", s.EtcdURL),
fmt.Sprintf("--cert-dir=%s", certDir),
fmt.Sprintf("--insecure-port=%s", url.Port()),
fmt.Sprintf("--insecure-bind-address=%s", url.Hostname()),
}
detectedStart := s.stdErr.Detect("Serving insecurely on 127.0.0.1:8080")
detectedStart := s.stdErr.Detect(fmt.Sprintf("Serving insecurely on %s", url.Host))
timedOut := time.After(20 * time.Second)
command := exec.Command(s.Path, args...)

3
pkg/framework/test/democli/integration/integration_suite_test.go
View File

@ -31,7 +31,8 @@ var _ = BeforeSuite(func() {
assetsDir, ok := os.LookupEnv("KUBE_ASSETS_DIR")
Expect(ok).To(BeTrue(), "KUBE_ASSETS_DIR should point to a directory containing etcd and apiserver binaries")
fixtures = test.NewFixtures(filepath.Join(assetsDir, "etcd"), filepath.Join(assetsDir, "kube-apiserver"))
fixtures, err = test.NewFixtures(filepath.Join(assetsDir, "etcd"), filepath.Join(assetsDir, "kube-apiserver"))
Expect(err).NotTo(HaveOccurred())
err = fixtures.Start()
Expect(err).NotTo(HaveOccurred())
})

16
pkg/framework/test/etcd.go
View File

@ -6,6 +6,8 @@ import (
"os/exec"
"time"
"net/url"
"github.com/onsi/gomega"
"github.com/onsi/gomega/gbytes"
"github.com/onsi/gomega/gexec"
@ -15,6 +17,7 @@ import (
type Etcd struct {
Path string
EtcdURL string
EtcdPeerURL string
ProcessStarter simpleSessionStarter
DataDirManager dataDirManager
session SimpleSession
@ -42,7 +45,7 @@ type SimpleSession interface {
type simpleSessionStarter func(command *exec.Cmd, out, err io.Writer) (SimpleSession, error)
// NewEtcd constructs an Etcd Fixture Process
func NewEtcd(pathToEtcd string, etcdURL string) *Etcd {
func NewEtcd(pathToEtcd, etcdURL, etcdPeerURL string) *Etcd {
starter := func(command *exec.Cmd, out, err io.Writer) (SimpleSession, error) {
return gexec.Start(command, out, err)
}
@ -50,6 +53,7 @@ func NewEtcd(pathToEtcd string, etcdURL string) *Etcd {
etcd := &Etcd{
Path: pathToEtcd,
EtcdURL: etcdURL,
EtcdPeerURL: etcdPeerURL,
ProcessStarter: starter,
DataDirManager: NewTempDirManager(),
}
@ -73,11 +77,19 @@ func (e *Etcd) Start() error {
e.EtcdURL,
"--listen-client-urls",
e.EtcdURL,
"--listen-peer-urls",
e.EtcdPeerURL,
"--data-dir",
dataDir,
}
detectedStart := e.stdErr.Detect("serving insecure client requests on 127.0.0.1:2379")
url, err := url.Parse(e.EtcdURL)
if err != nil {
return err
}
detectedStart := e.stdErr.Detect(fmt.Sprintf(
"serving insecure client requests on %s", url.Host))
timedOut := time.After(20 * time.Second)
command := exec.Command(e.Path, args...)

56
pkg/framework/test/fixtures.go
View File

@ -1,11 +1,23 @@
package test
import (
"fmt"
"net"
)
// Fixtures is a struct that knows how to start all your test fixtures.
//
// Right now, that means Etcd and your APIServer. This is likely to increase in future.
type Fixtures struct {
Etcd FixtureProcess
APIServer FixtureProcess
Config FixturesConfig
}
// FixturesConfig is a datastructure that exposes configuration that should be used by clients to talk
// to the fixture processes.
type FixturesConfig struct {
APIServerURL string
}
// FixtureProcess knows how to start and stop a Fixture processes.
@ -19,13 +31,32 @@ type FixtureProcess interface {
//go:generate counterfeiter . FixtureProcess
// NewFixtures will give you a Fixtures struct that's properly wired together.
func NewFixtures(pathToEtcd, pathToAPIServer string) *Fixtures {
etcdURL := "http://127.0.0.1:2379"
return &Fixtures{
Etcd: NewEtcd(pathToEtcd, etcdURL),
APIServer: NewAPIServer(pathToAPIServer, etcdURL),
func NewFixtures(pathToEtcd, pathToAPIServer string) (*Fixtures, error) {
urls := map[string]string{
"etcdClients": "",
"etcdPeers": "",
"apiServerClients": "",
}
host := "127.0.0.1"
for name := range urls {
port, err := getFreePort(host)
if err != nil {
return nil, err
}
urls[name] = fmt.Sprintf("http://%s:%d", host, port)
}
fixtures := &Fixtures{
Etcd: NewEtcd(pathToEtcd, urls["etcdClients"], urls["etcdPeers"]),
APIServer: NewAPIServer(pathToAPIServer, urls["apiServerClients"], urls["etcdClients"]),
}
fixtures.Config = FixturesConfig{
APIServerURL: urls["apiServerClients"],
}
return fixtures, nil
}
// Start will start all your fixtures. To stop them, call Stop().
@ -58,3 +89,16 @@ func (f *Fixtures) Stop() error {
f.Etcd.Stop()
return nil
}
func getFreePort(host string) (int, error) {
addr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:0", host))
if err != nil {
return 0, err
}
l, err := net.ListenTCP("tcp", addr)
if err != nil {
return 0, err
}
defer l.Close()
return l.Addr().(*net.TCPAddr).Port, nil
}

7
pkg/framework/test/fixtures_test.go
View File

@ -12,7 +12,8 @@ import (
var _ = Describe("Fixtures", func() {
It("can construct a properly wired Fixtures struct", func() {
f := NewFixtures("path to etcd", "path to apiserver")
f, err := NewFixtures("path to etcd", "path to apiserver")
Expect(err).NotTo(HaveOccurred())
Expect(f.Etcd.(*Etcd).Path).To(Equal("path to etcd"))
Expect(f.APIServer.(*APIServer).Path).To(Equal("path to apiserver"))
})
@ -38,11 +39,11 @@ var _ = Describe("Fixtures", func() {
By("starting Etcd")
Expect(fakeEtcdProcess.StartCallCount()).To(Equal(1),
"the EtcdStartStopper should be called exactly once")
"the Etcd process should be started exactly once")
By("starting APIServer")
Expect(fakeAPIServerProcess.StartCallCount()).To(Equal(1),
"the APIServerStartStopper should be called exactly once")
"the APIServer process should be started exactly once")
})
Context("when starting etcd fails", func() {

47
pkg/framework/test/integration/integration_test.go
View File

@ -1,10 +1,13 @@
package integration_test
import (
"fmt"
"net"
"time"
"net/url"
"fmt"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"k8s.io/kubectl/pkg/framework/test"
@ -12,22 +15,45 @@ import (
var _ = Describe("The Testing Framework", func() {
It("Successfully manages the fixtures lifecycle", func() {
fixtures := test.NewFixtures(defaultPathToEtcd, defaultPathToApiserver)
fixtures, err := test.NewFixtures(defaultPathToEtcd, defaultPathToApiserver)
Expect(err).NotTo(HaveOccurred())
err := fixtures.Start()
By("Starting all the fixture processes")
err = fixtures.Start()
Expect(err).NotTo(HaveOccurred(), "Expected fixtures to start successfully")
isEtcdListening := isSomethingListeningOnPort(2379)
isAPIServerListening := isSomethingListeningOnPort(8080)
var etcdURL, etcdPeerURL, apiServerURL *url.URL
etcd := fixtures.Etcd.(*test.Etcd)
apiServer := fixtures.APIServer.(*test.APIServer)
Expect(isEtcdListening()).To(BeTrue(), "Expected Etcd to listen on 2379")
etcdURL, err = url.Parse(etcd.EtcdURL)
Expect(err).NotTo(HaveOccurred())
etcdPeerURL, err = url.Parse(etcd.EtcdPeerURL)
Expect(err).NotTo(HaveOccurred())
apiServerURL, err = url.Parse(apiServer.APIServerURL)
Expect(err).NotTo(HaveOccurred())
Expect(isAPIServerListening()).To(BeTrue(), "Expected APIServer to listen on 8080")
isEtcdListening := isSomethingListeningOnPort(etcdURL.Host)
isEtcdPeerListening := isSomethingListeningOnPort(etcdPeerURL.Host)
isAPIServerListening := isSomethingListeningOnPort(apiServerURL.Host)
By("Ensuring Etcd is listening")
Expect(isEtcdListening()).To(BeTrue(),
fmt.Sprintf("Expected Etcd to listen on %s", etcdURL.Host))
Expect(isEtcdPeerListening()).To(BeTrue(),
fmt.Sprintf("Expected Etcd to listen for peers on %s", etcdPeerURL.Host))
By("Ensuring APIServer is listening")
Expect(isAPIServerListening()).To(BeTrue(),
fmt.Sprintf("Expected APIServer to listen on %s", apiServerURL.Host))
By("Stopping all the fixture processes")
err = fixtures.Stop()
Expect(err).NotTo(HaveOccurred(), "Expected fixtures to stop successfully")
By("Ensuring Etcd is not listening anymore")
Expect(isEtcdListening()).To(BeFalse(), "Expected Etcd not to listen anymore")
Expect(isEtcdPeerListening()).To(BeFalse(), "Expected Etcd not to listen for peers anymore")
By("Ensuring APIServer is not listening anymore")
Expect(isAPIServerListening()).To(BeFalse(), "Expected APIServer not to listen anymore")
@ -35,7 +61,8 @@ var _ = Describe("The Testing Framework", func() {
Measure("It should be fast to bring up and tear down the fixtures", func(b Benchmarker) {
b.Time("lifecycle", func() {
fixtures := test.NewFixtures(defaultPathToEtcd, defaultPathToApiserver)
fixtures, err := test.NewFixtures(defaultPathToEtcd, defaultPathToApiserver)
Expect(err).NotTo(HaveOccurred())
fixtures.Start()
fixtures.Stop()
@ -45,9 +72,9 @@ var _ = Describe("The Testing Framework", func() {
type portChecker func() bool
func isSomethingListeningOnPort(port int) portChecker {
func isSomethingListeningOnPort(hostAndPort string) portChecker {
return func() bool {
conn, err := net.DialTimeout("tcp", net.JoinHostPort("", fmt.Sprintf("%d", port)), 1*time.Second)
conn, err := net.DialTimeout("tcp", hostAndPort, 1*time.Second)
if err != nil {
return false