Merge pull request #1467 from dgageot/master

Support Static IPs and Preemptible instances on Google Compute Engine
This commit is contained in:
Evan Hazlett 2015-07-31 05:45:06 -07:00
commit 12f7ac57ae
10 changed files with 7297 additions and 1825 deletions

8
Godeps/Godeps.json generated
View File

@ -152,13 +152,17 @@
"ImportPath": "golang.org/x/crypto/ssh",
"Rev": "1fbbd62cfec66bd39d91e97749579579d4d3037e"
},
{
"ImportPath": "golang.org/x/net/context",
"Rev": "d9558e5c97f85372afee28cf2b6059d7d3818919"
},
{
"ImportPath": "google.golang.org/api/compute/v1",
"Rev": "aa91ac681e18e52b1a0dfe29b9d8354e88c0dcf5"
"Rev": "a09229c13c2f13bbdedf7b31b506cad4c83ef3bf"
},
{
"ImportPath": "google.golang.org/api/googleapi",
"Rev": "aa91ac681e18e52b1a0dfe29b9d8354e88c0dcf5"
"Rev": "a09229c13c2f13bbdedf7b31b506cad4c83ef3bf"
}
]
}

View File

@ -0,0 +1,447 @@
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package context defines the Context type, which carries deadlines,
// cancelation signals, and other request-scoped values across API boundaries
// and between processes.
//
// Incoming requests to a server should create a Context, and outgoing calls to
// servers should accept a Context. The chain of function calls between must
// propagate the Context, optionally replacing it with a modified copy created
// using WithDeadline, WithTimeout, WithCancel, or WithValue.
//
// Programs that use Contexts should follow these rules to keep interfaces
// consistent across packages and enable static analysis tools to check context
// propagation:
//
// Do not store Contexts inside a struct type; instead, pass a Context
// explicitly to each function that needs it. The Context should be the first
// parameter, typically named ctx:
//
// func DoSomething(ctx context.Context, arg Arg) error {
// // ... use ctx ...
// }
//
// Do not pass a nil Context, even if a function permits it. Pass context.TODO
// if you are unsure about which Context to use.
//
// Use context Values only for request-scoped data that transits processes and
// APIs, not for passing optional parameters to functions.
//
// The same Context may be passed to functions running in different goroutines;
// Contexts are safe for simultaneous use by multiple goroutines.
//
// See http://blog.golang.org/context for example code for a server that uses
// Contexts.
package context
import (
"errors"
"fmt"
"sync"
"time"
)
// A Context carries a deadline, a cancelation signal, and other values across
// API boundaries.
//
// Context's methods may be called by multiple goroutines simultaneously.
type Context interface {
// Deadline returns the time when work done on behalf of this context
// should be canceled. Deadline returns ok==false when no deadline is
// set. Successive calls to Deadline return the same results.
Deadline() (deadline time.Time, ok bool)
// Done returns a channel that's closed when work done on behalf of this
// context should be canceled. Done may return nil if this context can
// never be canceled. Successive calls to Done return the same value.
//
// WithCancel arranges for Done to be closed when cancel is called;
// WithDeadline arranges for Done to be closed when the deadline
// expires; WithTimeout arranges for Done to be closed when the timeout
// elapses.
//
// Done is provided for use in select statements:
//
// // Stream generates values with DoSomething and sends them to out
// // until DoSomething returns an error or ctx.Done is closed.
// func Stream(ctx context.Context, out <-chan Value) error {
// for {
// v, err := DoSomething(ctx)
// if err != nil {
// return err
// }
// select {
// case <-ctx.Done():
// return ctx.Err()
// case out <- v:
// }
// }
// }
//
// See http://blog.golang.org/pipelines for more examples of how to use
// a Done channel for cancelation.
Done() <-chan struct{}
// Err returns a non-nil error value after Done is closed. Err returns
// Canceled if the context was canceled or DeadlineExceeded if the
// context's deadline passed. No other values for Err are defined.
// After Done is closed, successive calls to Err return the same value.
Err() error
// Value returns the value associated with this context for key, or nil
// if no value is associated with key. Successive calls to Value with
// the same key returns the same result.
//
// Use context values only for request-scoped data that transits
// processes and API boundaries, not for passing optional parameters to
// functions.
//
// A key identifies a specific value in a Context. Functions that wish
// to store values in Context typically allocate a key in a global
// variable then use that key as the argument to context.WithValue and
// Context.Value. A key can be any type that supports equality;
// packages should define keys as an unexported type to avoid
// collisions.
//
// Packages that define a Context key should provide type-safe accessors
// for the values stores using that key:
//
// // Package user defines a User type that's stored in Contexts.
// package user
//
// import "golang.org/x/net/context"
//
// // User is the type of value stored in the Contexts.
// type User struct {...}
//
// // key is an unexported type for keys defined in this package.
// // This prevents collisions with keys defined in other packages.
// type key int
//
// // userKey is the key for user.User values in Contexts. It is
// // unexported; clients use user.NewContext and user.FromContext
// // instead of using this key directly.
// var userKey key = 0
//
// // NewContext returns a new Context that carries value u.
// func NewContext(ctx context.Context, u *User) context.Context {
// return context.WithValue(ctx, userKey, u)
// }
//
// // FromContext returns the User value stored in ctx, if any.
// func FromContext(ctx context.Context) (*User, bool) {
// u, ok := ctx.Value(userKey).(*User)
// return u, ok
// }
Value(key interface{}) interface{}
}
// Canceled is the error returned by Context.Err when the context is canceled.
var Canceled = errors.New("context canceled")
// DeadlineExceeded is the error returned by Context.Err when the context's
// deadline passes.
var DeadlineExceeded = errors.New("context deadline exceeded")
// An emptyCtx is never canceled, has no values, and has no deadline. It is not
// struct{}, since vars of this type must have distinct addresses.
type emptyCtx int
func (*emptyCtx) Deadline() (deadline time.Time, ok bool) {
return
}
func (*emptyCtx) Done() <-chan struct{} {
return nil
}
func (*emptyCtx) Err() error {
return nil
}
func (*emptyCtx) Value(key interface{}) interface{} {
return nil
}
func (e *emptyCtx) String() string {
switch e {
case background:
return "context.Background"
case todo:
return "context.TODO"
}
return "unknown empty Context"
}
var (
background = new(emptyCtx)
todo = new(emptyCtx)
)
// Background returns a non-nil, empty Context. It is never canceled, has no
// values, and has no deadline. It is typically used by the main function,
// initialization, and tests, and as the top-level Context for incoming
// requests.
func Background() Context {
return background
}
// TODO returns a non-nil, empty Context. Code should use context.TODO when
// it's unclear which Context to use or it's is not yet available (because the
// surrounding function has not yet been extended to accept a Context
// parameter). TODO is recognized by static analysis tools that determine
// whether Contexts are propagated correctly in a program.
func TODO() Context {
return todo
}
// A CancelFunc tells an operation to abandon its work.
// A CancelFunc does not wait for the work to stop.
// After the first call, subsequent calls to a CancelFunc do nothing.
type CancelFunc func()
// WithCancel returns a copy of parent with a new Done channel. The returned
// context's Done channel is closed when the returned cancel function is called
// or when the parent context's Done channel is closed, whichever happens first.
//
// Canceling this context releases resources associated with it, so code should
// call cancel as soon as the operations running in this Context complete.
func WithCancel(parent Context) (ctx Context, cancel CancelFunc) {
c := newCancelCtx(parent)
propagateCancel(parent, &c)
return &c, func() { c.cancel(true, Canceled) }
}
// newCancelCtx returns an initialized cancelCtx.
func newCancelCtx(parent Context) cancelCtx {
return cancelCtx{
Context: parent,
done: make(chan struct{}),
}
}
// propagateCancel arranges for child to be canceled when parent is.
func propagateCancel(parent Context, child canceler) {
if parent.Done() == nil {
return // parent is never canceled
}
if p, ok := parentCancelCtx(parent); ok {
p.mu.Lock()
if p.err != nil {
// parent has already been canceled
child.cancel(false, p.err)
} else {
if p.children == nil {
p.children = make(map[canceler]bool)
}
p.children[child] = true
}
p.mu.Unlock()
} else {
go func() {
select {
case <-parent.Done():
child.cancel(false, parent.Err())
case <-child.Done():
}
}()
}
}
// parentCancelCtx follows a chain of parent references until it finds a
// *cancelCtx. This function understands how each of the concrete types in this
// package represents its parent.
func parentCancelCtx(parent Context) (*cancelCtx, bool) {
for {
switch c := parent.(type) {
case *cancelCtx:
return c, true
case *timerCtx:
return &c.cancelCtx, true
case *valueCtx:
parent = c.Context
default:
return nil, false
}
}
}
// removeChild removes a context from its parent.
func removeChild(parent Context, child canceler) {
p, ok := parentCancelCtx(parent)
if !ok {
return
}
p.mu.Lock()
if p.children != nil {
delete(p.children, child)
}
p.mu.Unlock()
}
// A canceler is a context type that can be canceled directly. The
// implementations are *cancelCtx and *timerCtx.
type canceler interface {
cancel(removeFromParent bool, err error)
Done() <-chan struct{}
}
// A cancelCtx can be canceled. When canceled, it also cancels any children
// that implement canceler.
type cancelCtx struct {
Context
done chan struct{} // closed by the first cancel call.
mu sync.Mutex
children map[canceler]bool // set to nil by the first cancel call
err error // set to non-nil by the first cancel call
}
func (c *cancelCtx) Done() <-chan struct{} {
return c.done
}
func (c *cancelCtx) Err() error {
c.mu.Lock()
defer c.mu.Unlock()
return c.err
}
func (c *cancelCtx) String() string {
return fmt.Sprintf("%v.WithCancel", c.Context)
}
// cancel closes c.done, cancels each of c's children, and, if
// removeFromParent is true, removes c from its parent's children.
func (c *cancelCtx) cancel(removeFromParent bool, err error) {
if err == nil {
panic("context: internal error: missing cancel error")
}
c.mu.Lock()
if c.err != nil {
c.mu.Unlock()
return // already canceled
}
c.err = err
close(c.done)
for child := range c.children {
// NOTE: acquiring the child's lock while holding parent's lock.
child.cancel(false, err)
}
c.children = nil
c.mu.Unlock()
if removeFromParent {
removeChild(c.Context, c)
}
}
// WithDeadline returns a copy of the parent context with the deadline adjusted
// to be no later than d. If the parent's deadline is already earlier than d,
// WithDeadline(parent, d) is semantically equivalent to parent. The returned
// context's Done channel is closed when the deadline expires, when the returned
// cancel function is called, or when the parent context's Done channel is
// closed, whichever happens first.
//
// Canceling this context releases resources associated with it, so code should
// call cancel as soon as the operations running in this Context complete.
func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) {
if cur, ok := parent.Deadline(); ok && cur.Before(deadline) {
// The current deadline is already sooner than the new one.
return WithCancel(parent)
}
c := &timerCtx{
cancelCtx: newCancelCtx(parent),
deadline: deadline,
}
propagateCancel(parent, c)
d := deadline.Sub(time.Now())
if d <= 0 {
c.cancel(true, DeadlineExceeded) // deadline has already passed
return c, func() { c.cancel(true, Canceled) }
}
c.mu.Lock()
defer c.mu.Unlock()
if c.err == nil {
c.timer = time.AfterFunc(d, func() {
c.cancel(true, DeadlineExceeded)
})
}
return c, func() { c.cancel(true, Canceled) }
}
// A timerCtx carries a timer and a deadline. It embeds a cancelCtx to
// implement Done and Err. It implements cancel by stopping its timer then
// delegating to cancelCtx.cancel.
type timerCtx struct {
cancelCtx
timer *time.Timer // Under cancelCtx.mu.
deadline time.Time
}
func (c *timerCtx) Deadline() (deadline time.Time, ok bool) {
return c.deadline, true
}
func (c *timerCtx) String() string {
return fmt.Sprintf("%v.WithDeadline(%s [%s])", c.cancelCtx.Context, c.deadline, c.deadline.Sub(time.Now()))
}
func (c *timerCtx) cancel(removeFromParent bool, err error) {
c.cancelCtx.cancel(false, err)
if removeFromParent {
// Remove this timerCtx from its parent cancelCtx's children.
removeChild(c.cancelCtx.Context, c)
}
c.mu.Lock()
if c.timer != nil {
c.timer.Stop()
c.timer = nil
}
c.mu.Unlock()
}
// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)).
//
// Canceling this context releases resources associated with it, so code should
// call cancel as soon as the operations running in this Context complete:
//
// func slowOperationWithTimeout(ctx context.Context) (Result, error) {
// ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
// defer cancel() // releases resources if slowOperation completes before timeout elapses
// return slowOperation(ctx)
// }
func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) {
return WithDeadline(parent, time.Now().Add(timeout))
}
// WithValue returns a copy of parent in which the value associated with key is
// val.
//
// Use context Values only for request-scoped data that transits processes and
// APIs, not for passing optional parameters to functions.
func WithValue(parent Context, key interface{}, val interface{}) Context {
return &valueCtx{parent, key, val}
}
// A valueCtx carries a key-value pair. It implements Value for that key and
// delegates all other calls to the embedded Context.
type valueCtx struct {
Context
key, val interface{}
}
func (c *valueCtx) String() string {
return fmt.Sprintf("%v.WithValue(%#v, %#v)", c.Context, c.key, c.val)
}
func (c *valueCtx) Value(key interface{}) interface{} {
if c.key == key {
return c.val
}
return c.Context.Value(key)
}

View File

@ -0,0 +1,575 @@
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package context
import (
"fmt"
"math/rand"
"runtime"
"strings"
"sync"
"testing"
"time"
)
// otherContext is a Context that's not one of the types defined in context.go.
// This lets us test code paths that differ based on the underlying type of the
// Context.
type otherContext struct {
Context
}
func TestBackground(t *testing.T) {
c := Background()
if c == nil {
t.Fatalf("Background returned nil")
}
select {
case x := <-c.Done():
t.Errorf("<-c.Done() == %v want nothing (it should block)", x)
default:
}
if got, want := fmt.Sprint(c), "context.Background"; got != want {
t.Errorf("Background().String() = %q want %q", got, want)
}
}
func TestTODO(t *testing.T) {
c := TODO()
if c == nil {
t.Fatalf("TODO returned nil")
}
select {
case x := <-c.Done():
t.Errorf("<-c.Done() == %v want nothing (it should block)", x)
default:
}
if got, want := fmt.Sprint(c), "context.TODO"; got != want {
t.Errorf("TODO().String() = %q want %q", got, want)
}
}
func TestWithCancel(t *testing.T) {
c1, cancel := WithCancel(Background())
if got, want := fmt.Sprint(c1), "context.Background.WithCancel"; got != want {
t.Errorf("c1.String() = %q want %q", got, want)
}
o := otherContext{c1}
c2, _ := WithCancel(o)
contexts := []Context{c1, o, c2}
for i, c := range contexts {
if d := c.Done(); d == nil {
t.Errorf("c[%d].Done() == %v want non-nil", i, d)
}
if e := c.Err(); e != nil {
t.Errorf("c[%d].Err() == %v want nil", i, e)
}
select {
case x := <-c.Done():
t.Errorf("<-c.Done() == %v want nothing (it should block)", x)
default:
}
}
cancel()
time.Sleep(100 * time.Millisecond) // let cancelation propagate
for i, c := range contexts {
select {
case <-c.Done():
default:
t.Errorf("<-c[%d].Done() blocked, but shouldn't have", i)
}
if e := c.Err(); e != Canceled {
t.Errorf("c[%d].Err() == %v want %v", i, e, Canceled)
}
}
}
func TestParentFinishesChild(t *testing.T) {
// Context tree:
// parent -> cancelChild
// parent -> valueChild -> timerChild
parent, cancel := WithCancel(Background())
cancelChild, stop := WithCancel(parent)
defer stop()
valueChild := WithValue(parent, "key", "value")
timerChild, stop := WithTimeout(valueChild, 10000*time.Hour)
defer stop()
select {
case x := <-parent.Done():
t.Errorf("<-parent.Done() == %v want nothing (it should block)", x)
case x := <-cancelChild.Done():
t.Errorf("<-cancelChild.Done() == %v want nothing (it should block)", x)
case x := <-timerChild.Done():
t.Errorf("<-timerChild.Done() == %v want nothing (it should block)", x)
case x := <-valueChild.Done():
t.Errorf("<-valueChild.Done() == %v want nothing (it should block)", x)
default:
}
// The parent's children should contain the two cancelable children.
pc := parent.(*cancelCtx)
cc := cancelChild.(*cancelCtx)
tc := timerChild.(*timerCtx)
pc.mu.Lock()
if len(pc.children) != 2 || !pc.children[cc] || !pc.children[tc] {
t.Errorf("bad linkage: pc.children = %v, want %v and %v",
pc.children, cc, tc)
}
pc.mu.Unlock()
if p, ok := parentCancelCtx(cc.Context); !ok || p != pc {
t.Errorf("bad linkage: parentCancelCtx(cancelChild.Context) = %v, %v want %v, true", p, ok, pc)
}
if p, ok := parentCancelCtx(tc.Context); !ok || p != pc {
t.Errorf("bad linkage: parentCancelCtx(timerChild.Context) = %v, %v want %v, true", p, ok, pc)
}
cancel()
pc.mu.Lock()
if len(pc.children) != 0 {
t.Errorf("pc.cancel didn't clear pc.children = %v", pc.children)
}
pc.mu.Unlock()
// parent and children should all be finished.
check := func(ctx Context, name string) {
select {
case <-ctx.Done():
default:
t.Errorf("<-%s.Done() blocked, but shouldn't have", name)
}
if e := ctx.Err(); e != Canceled {
t.Errorf("%s.Err() == %v want %v", name, e, Canceled)
}
}
check(parent, "parent")
check(cancelChild, "cancelChild")
check(valueChild, "valueChild")
check(timerChild, "timerChild")
// WithCancel should return a canceled context on a canceled parent.
precanceledChild := WithValue(parent, "key", "value")
select {
case <-precanceledChild.Done():
default:
t.Errorf("<-precanceledChild.Done() blocked, but shouldn't have")
}
if e := precanceledChild.Err(); e != Canceled {
t.Errorf("precanceledChild.Err() == %v want %v", e, Canceled)
}
}
func TestChildFinishesFirst(t *testing.T) {
cancelable, stop := WithCancel(Background())
defer stop()
for _, parent := range []Context{Background(), cancelable} {
child, cancel := WithCancel(parent)
select {
case x := <-parent.Done():
t.Errorf("<-parent.Done() == %v want nothing (it should block)", x)
case x := <-child.Done():
t.Errorf("<-child.Done() == %v want nothing (it should block)", x)
default:
}
cc := child.(*cancelCtx)
pc, pcok := parent.(*cancelCtx) // pcok == false when parent == Background()
if p, ok := parentCancelCtx(cc.Context); ok != pcok || (ok && pc != p) {
t.Errorf("bad linkage: parentCancelCtx(cc.Context) = %v, %v want %v, %v", p, ok, pc, pcok)
}
if pcok {
pc.mu.Lock()
if len(pc.children) != 1 || !pc.children[cc] {
t.Errorf("bad linkage: pc.children = %v, cc = %v", pc.children, cc)
}
pc.mu.Unlock()
}
cancel()
if pcok {
pc.mu.Lock()
if len(pc.children) != 0 {
t.Errorf("child's cancel didn't remove self from pc.children = %v", pc.children)
}
pc.mu.Unlock()
}
// child should be finished.
select {
case <-child.Done():
default:
t.Errorf("<-child.Done() blocked, but shouldn't have")
}
if e := child.Err(); e != Canceled {
t.Errorf("child.Err() == %v want %v", e, Canceled)
}
// parent should not be finished.
select {
case x := <-parent.Done():
t.Errorf("<-parent.Done() == %v want nothing (it should block)", x)
default:
}
if e := parent.Err(); e != nil {
t.Errorf("parent.Err() == %v want nil", e)
}
}
}
func testDeadline(c Context, wait time.Duration, t *testing.T) {
select {
case <-time.After(wait):
t.Fatalf("context should have timed out")
case <-c.Done():
}
if e := c.Err(); e != DeadlineExceeded {
t.Errorf("c.Err() == %v want %v", e, DeadlineExceeded)
}
}
func TestDeadline(t *testing.T) {
c, _ := WithDeadline(Background(), time.Now().Add(100*time.Millisecond))
if got, prefix := fmt.Sprint(c), "context.Background.WithDeadline("; !strings.HasPrefix(got, prefix) {
t.Errorf("c.String() = %q want prefix %q", got, prefix)
}
testDeadline(c, 200*time.Millisecond, t)
c, _ = WithDeadline(Background(), time.Now().Add(100*time.Millisecond))
o := otherContext{c}
testDeadline(o, 200*time.Millisecond, t)
c, _ = WithDeadline(Background(), time.Now().Add(100*time.Millisecond))
o = otherContext{c}
c, _ = WithDeadline(o, time.Now().Add(300*time.Millisecond))
testDeadline(c, 200*time.Millisecond, t)
}
func TestTimeout(t *testing.T) {
c, _ := WithTimeout(Background(), 100*time.Millisecond)
if got, prefix := fmt.Sprint(c), "context.Background.WithDeadline("; !strings.HasPrefix(got, prefix) {
t.Errorf("c.String() = %q want prefix %q", got, prefix)
}
testDeadline(c, 200*time.Millisecond, t)
c, _ = WithTimeout(Background(), 100*time.Millisecond)
o := otherContext{c}
testDeadline(o, 200*time.Millisecond, t)
c, _ = WithTimeout(Background(), 100*time.Millisecond)
o = otherContext{c}
c, _ = WithTimeout(o, 300*time.Millisecond)
testDeadline(c, 200*time.Millisecond, t)
}
func TestCanceledTimeout(t *testing.T) {
c, _ := WithTimeout(Background(), 200*time.Millisecond)
o := otherContext{c}
c, cancel := WithTimeout(o, 400*time.Millisecond)
cancel()
time.Sleep(100 * time.Millisecond) // let cancelation propagate
select {
case <-c.Done():
default:
t.Errorf("<-c.Done() blocked, but shouldn't have")
}
if e := c.Err(); e != Canceled {
t.Errorf("c.Err() == %v want %v", e, Canceled)
}
}
type key1 int
type key2 int
var k1 = key1(1)
var k2 = key2(1) // same int as k1, different type
var k3 = key2(3) // same type as k2, different int
func TestValues(t *testing.T) {
check := func(c Context, nm, v1, v2, v3 string) {
if v, ok := c.Value(k1).(string); ok == (len(v1) == 0) || v != v1 {
t.Errorf(`%s.Value(k1).(string) = %q, %t want %q, %t`, nm, v, ok, v1, len(v1) != 0)
}
if v, ok := c.Value(k2).(string); ok == (len(v2) == 0) || v != v2 {
t.Errorf(`%s.Value(k2).(string) = %q, %t want %q, %t`, nm, v, ok, v2, len(v2) != 0)
}
if v, ok := c.Value(k3).(string); ok == (len(v3) == 0) || v != v3 {
t.Errorf(`%s.Value(k3).(string) = %q, %t want %q, %t`, nm, v, ok, v3, len(v3) != 0)
}
}
c0 := Background()
check(c0, "c0", "", "", "")
c1 := WithValue(Background(), k1, "c1k1")
check(c1, "c1", "c1k1", "", "")
if got, want := fmt.Sprint(c1), `context.Background.WithValue(1, "c1k1")`; got != want {
t.Errorf("c.String() = %q want %q", got, want)
}
c2 := WithValue(c1, k2, "c2k2")
check(c2, "c2", "c1k1", "c2k2", "")
c3 := WithValue(c2, k3, "c3k3")
check(c3, "c2", "c1k1", "c2k2", "c3k3")
c4 := WithValue(c3, k1, nil)
check(c4, "c4", "", "c2k2", "c3k3")
o0 := otherContext{Background()}
check(o0, "o0", "", "", "")
o1 := otherContext{WithValue(Background(), k1, "c1k1")}
check(o1, "o1", "c1k1", "", "")
o2 := WithValue(o1, k2, "o2k2")
check(o2, "o2", "c1k1", "o2k2", "")
o3 := otherContext{c4}
check(o3, "o3", "", "c2k2", "c3k3")
o4 := WithValue(o3, k3, nil)
check(o4, "o4", "", "c2k2", "")
}
func TestAllocs(t *testing.T) {
bg := Background()
for _, test := range []struct {
desc string
f func()
limit float64
gccgoLimit float64
}{
{
desc: "Background()",
f: func() { Background() },
limit: 0,
gccgoLimit: 0,
},
{
desc: fmt.Sprintf("WithValue(bg, %v, nil)", k1),
f: func() {
c := WithValue(bg, k1, nil)
c.Value(k1)
},
limit: 3,
gccgoLimit: 3,
},
{
desc: "WithTimeout(bg, 15*time.Millisecond)",
f: func() {
c, _ := WithTimeout(bg, 15*time.Millisecond)
<-c.Done()
},
limit: 8,
gccgoLimit: 13,
},
{
desc: "WithCancel(bg)",
f: func() {
c, cancel := WithCancel(bg)
cancel()
<-c.Done()
},
limit: 5,
gccgoLimit: 8,
},
{
desc: "WithTimeout(bg, 100*time.Millisecond)",
f: func() {
c, cancel := WithTimeout(bg, 100*time.Millisecond)
cancel()
<-c.Done()
},
limit: 8,
gccgoLimit: 25,
},
} {
limit := test.limit
if runtime.Compiler == "gccgo" {
// gccgo does not yet do escape analysis.
// TOOD(iant): Remove this when gccgo does do escape analysis.
limit = test.gccgoLimit
}
if n := testing.AllocsPerRun(100, test.f); n > limit {
t.Errorf("%s allocs = %f want %d", test.desc, n, int(limit))
}
}
}
func TestSimultaneousCancels(t *testing.T) {
root, cancel := WithCancel(Background())
m := map[Context]CancelFunc{root: cancel}
q := []Context{root}
// Create a tree of contexts.
for len(q) != 0 && len(m) < 100 {
parent := q[0]
q = q[1:]
for i := 0; i < 4; i++ {
ctx, cancel := WithCancel(parent)
m[ctx] = cancel
q = append(q, ctx)
}
}
// Start all the cancels in a random order.
var wg sync.WaitGroup
wg.Add(len(m))
for _, cancel := range m {
go func(cancel CancelFunc) {
cancel()
wg.Done()
}(cancel)
}
// Wait on all the contexts in a random order.
for ctx := range m {
select {
case <-ctx.Done():
case <-time.After(1 * time.Second):
buf := make([]byte, 10<<10)
n := runtime.Stack(buf, true)
t.Fatalf("timed out waiting for <-ctx.Done(); stacks:\n%s", buf[:n])
}
}
// Wait for all the cancel functions to return.
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
select {
case <-done:
case <-time.After(1 * time.Second):
buf := make([]byte, 10<<10)
n := runtime.Stack(buf, true)
t.Fatalf("timed out waiting for cancel functions; stacks:\n%s", buf[:n])
}
}
func TestInterlockedCancels(t *testing.T) {
parent, cancelParent := WithCancel(Background())
child, cancelChild := WithCancel(parent)
go func() {
parent.Done()
cancelChild()
}()
cancelParent()
select {
case <-child.Done():
case <-time.After(1 * time.Second):
buf := make([]byte, 10<<10)
n := runtime.Stack(buf, true)
t.Fatalf("timed out waiting for child.Done(); stacks:\n%s", buf[:n])
}
}
func TestLayersCancel(t *testing.T) {
testLayers(t, time.Now().UnixNano(), false)
}
func TestLayersTimeout(t *testing.T) {
testLayers(t, time.Now().UnixNano(), true)
}
func testLayers(t *testing.T, seed int64, testTimeout bool) {
rand.Seed(seed)
errorf := func(format string, a ...interface{}) {
t.Errorf(fmt.Sprintf("seed=%d: %s", seed, format), a...)
}
const (
timeout = 200 * time.Millisecond
minLayers = 30
)
type value int
var (
vals []*value
cancels []CancelFunc
numTimers int
ctx = Background()
)
for i := 0; i < minLayers || numTimers == 0 || len(cancels) == 0 || len(vals) == 0; i++ {
switch rand.Intn(3) {
case 0:
v := new(value)
ctx = WithValue(ctx, v, v)
vals = append(vals, v)
case 1:
var cancel CancelFunc
ctx, cancel = WithCancel(ctx)
cancels = append(cancels, cancel)
case 2:
var cancel CancelFunc
ctx, cancel = WithTimeout(ctx, timeout)
cancels = append(cancels, cancel)
numTimers++
}
}
checkValues := func(when string) {
for _, key := range vals {
if val := ctx.Value(key).(*value); key != val {
errorf("%s: ctx.Value(%p) = %p want %p", when, key, val, key)
}
}
}
select {
case <-ctx.Done():
errorf("ctx should not be canceled yet")
default:
}
if s, prefix := fmt.Sprint(ctx), "context.Background."; !strings.HasPrefix(s, prefix) {
t.Errorf("ctx.String() = %q want prefix %q", s, prefix)
}
t.Log(ctx)
checkValues("before cancel")
if testTimeout {
select {
case <-ctx.Done():
case <-time.After(timeout + timeout/10):
errorf("ctx should have timed out")
}
checkValues("after timeout")
} else {
cancel := cancels[rand.Intn(len(cancels))]
cancel()
select {
case <-ctx.Done():
default:
errorf("ctx should be canceled")
}
checkValues("after cancel")
}
}
func TestCancelRemoves(t *testing.T) {
checkChildren := func(when string, ctx Context, want int) {
if got := len(ctx.(*cancelCtx).children); got != want {
t.Errorf("%s: context has %d children, want %d", when, got, want)
}
}
ctx, _ := WithCancel(Background())
checkChildren("after creation", ctx, 0)
_, cancel := WithCancel(ctx)
checkChildren("with WithCancel child ", ctx, 1)
cancel()
checkChildren("after cancelling WithCancel child", ctx, 0)
ctx, _ = WithCancel(Background())
checkChildren("after creation", ctx, 0)
_, cancel = WithTimeout(ctx, 60*time.Minute)
checkChildren("with WithTimeout child ", ctx, 1)
cancel()
checkChildren("after cancelling WithTimeout child", ctx, 0)
}

View File

@ -0,0 +1,26 @@
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package context_test
import (
"fmt"
"time"
"golang.org/x/net/context"
)
func ExampleWithTimeout() {
// Pass a context with a timeout to tell a blocking function that it
// should abandon its work after the timeout elapses.
ctx, _ := context.WithTimeout(context.Background(), 100*time.Millisecond)
select {
case <-time.After(200 * time.Millisecond):
fmt.Println("overslept")
case <-ctx.Done():
fmt.Println(ctx.Err()) // prints "context deadline exceeded"
}
// Output:
// context deadline exceeded
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -9,6 +9,7 @@ package googleapi
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
@ -16,9 +17,13 @@ import (
"net/http"
"net/textproto"
"net/url"
"os"
"regexp"
"strconv"
"strings"
"sync"
"time"
"golang.org/x/net/context"
"google.golang.org/api/googleapi/internal/uritemplates"
)
@ -30,7 +35,25 @@ type ContentTyper interface {
ContentType() string
}
const Version = "0.5"
// A SizeReaderAt is a ReaderAt with a Size method.
// An io.SectionReader implements SizeReaderAt.
type SizeReaderAt interface {
io.ReaderAt
Size() int64
}
const (
Version = "0.5"
// statusResumeIncomplete is the code returned by the Google uploader when the transfer is not yet complete.
statusResumeIncomplete = 308
// UserAgent is the header string used to identify this package.
UserAgent = "google-api-go-client/" + Version
// uploadPause determines the delay between failed upload attempts
uploadPause = 1 * time.Second
)
// Error contains an error response from the server.
type Error struct {
@ -130,14 +153,41 @@ func getMediaType(media io.Reader) (io.Reader, string) {
return media, typer.ContentType()
}
pr, pw := io.Pipe()
typ := "application/octet-stream"
buf, err := ioutil.ReadAll(io.LimitReader(media, 512))
if err != nil {
pw.CloseWithError(fmt.Errorf("error reading media: %v", err))
return pr, typ
}
typ = http.DetectContentType(buf)
mr := io.MultiReader(bytes.NewReader(buf), media)
go func() {
_, err = io.Copy(pw, mr)
if err != nil {
pw.CloseWithError(fmt.Errorf("error reading media: %v", err))
return
}
pw.Close()
}()
return pr, typ
}
// DetectMediaType detects and returns the content type of the provided media.
// If the type can not be determined, "application/octet-stream" is returned.
func DetectMediaType(media io.ReaderAt) string {
if typer, ok := media.(ContentTyper); ok {
return typer.ContentType()
}
typ := "application/octet-stream"
buf := make([]byte, 1024)
n, err := media.Read(buf)
n, err := media.ReadAt(buf, 0)
buf = buf[:n]
if err == nil {
if err == nil || err == io.EOF {
typ = http.DetectContentType(buf)
}
return io.MultiReader(bytes.NewBuffer(buf), media), typ
return typ
}
type Lengther interface {
@ -145,7 +195,7 @@ type Lengther interface {
}
// endingWithErrorReader from r until it returns an error. If the
// final error from r is os.EOF and e is non-nil, e is used instead.
// final error from r is io.EOF and e is non-nil, e is used instead.
type endingWithErrorReader struct {
r io.Reader
e error
@ -159,38 +209,6 @@ func (er endingWithErrorReader) Read(p []byte) (n int, err error) {
return
}
func getReaderSize(r io.Reader) (io.Reader, int64) {
// Ideal case, the reader knows its own size.
if lr, ok := r.(Lengther); ok {
return r, int64(lr.Len())
}
// But maybe it's a seeker and we can seek to the end to find its size.
if s, ok := r.(io.Seeker); ok {
pos0, err := s.Seek(0, os.SEEK_CUR)
if err == nil {
posend, err := s.Seek(0, os.SEEK_END)
if err == nil {
_, err = s.Seek(pos0, os.SEEK_SET)
if err == nil {
return r, posend - pos0
} else {
// We moved it forward but can't restore it.
// Seems unlikely, but can't really restore now.
return endingWithErrorReader{strings.NewReader(""), err}, posend - pos0
}
}
}
}
// Otherwise we have to make a copy to calculate how big the reader is.
buf := new(bytes.Buffer)
// TODO(bradfitz): put a cap on this copy? spill to disk after
// a certain point?
_, err := io.Copy(buf, r)
return endingWithErrorReader{buf, err}, int64(buf.Len())
}
func typeHeader(contentType string) textproto.MIMEHeader {
h := make(textproto.MIMEHeader)
h.Set("Content-Type", contentType)
@ -219,60 +237,185 @@ func (w countingWriter) Write(p []byte) (int, error) {
// to the "multipart/related" content type, with random boundary.
//
// The return value is the content-length of the entire multpart body.
func ConditionallyIncludeMedia(media io.Reader, bodyp *io.Reader, ctypep *string) (totalContentLength int64, ok bool) {
func ConditionallyIncludeMedia(media io.Reader, bodyp *io.Reader, ctypep *string) (cancel func(), ok bool) {
if media == nil {
return
}
// Get the media type and size. The type check might return a
// different reader instance, so do the size check first,
// which looks at the specific type of the io.Reader.
// Get the media type, which might return a different reader instance.
var mediaType string
if typer, ok := media.(ContentTyper); ok {
mediaType = typer.ContentType()
}
media, mediaSize := getReaderSize(media)
if mediaType == "" {
media, mediaType = getMediaType(media)
}
body, bodyType := *bodyp, *ctypep
body, bodySize := getReaderSize(body)
media, mediaType = getMediaType(media)
// Calculate how big the the multipart will be.
{
totalContentLength = bodySize + mediaSize
mpw := multipart.NewWriter(countingWriter{&totalContentLength})
mpw.CreatePart(typeHeader(bodyType))
mpw.CreatePart(typeHeader(mediaType))
mpw.Close()
}
body, bodyType := *bodyp, *ctypep
pr, pw := io.Pipe()
mpw := multipart.NewWriter(pw)
*bodyp = pr
*ctypep = "multipart/related; boundary=" + mpw.Boundary()
go func() {
defer pw.Close()
defer mpw.Close()
w, err := mpw.CreatePart(typeHeader(bodyType))
if err != nil {
mpw.Close()
pw.CloseWithError(fmt.Errorf("googleapi: body CreatePart failed: %v", err))
return
}
_, err = io.Copy(w, body)
if err != nil {
mpw.Close()
pw.CloseWithError(fmt.Errorf("googleapi: body Copy failed: %v", err))
return
}
w, err = mpw.CreatePart(typeHeader(mediaType))
if err != nil {
mpw.Close()
pw.CloseWithError(fmt.Errorf("googleapi: media CreatePart failed: %v", err))
return
}
_, err = io.Copy(w, media)
if err != nil {
mpw.Close()
pw.CloseWithError(fmt.Errorf("googleapi: media Copy failed: %v", err))
return
}
mpw.Close()
pw.Close()
}()
return totalContentLength, true
cancel = func() { pw.CloseWithError(errAborted) }
return cancel, true
}
var errAborted = errors.New("googleapi: upload aborted")
// ProgressUpdater is a function that is called upon every progress update of a resumable upload.
// This is the only part of a resumable upload (from googleapi) that is usable by the developer.
// The remaining usable pieces of resumable uploads is exposed in each auto-generated API.
type ProgressUpdater func(current, total int64)
// ResumableUpload is used by the generated APIs to provide resumable uploads.
// It is not used by developers directly.
type ResumableUpload struct {
Client *http.Client
// URI is the resumable resource destination provided by the server after specifying "&uploadType=resumable".
URI string
UserAgent string // User-Agent for header of the request
// Media is the object being uploaded.
Media io.ReaderAt
// MediaType defines the media type, e.g. "image/jpeg".
MediaType string
// ContentLength is the full size of the object being uploaded.
ContentLength int64
mu sync.Mutex // guards progress
progress int64 // number of bytes uploaded so far
// Callback is an optional function that will be called upon every progress update.
Callback ProgressUpdater
}
var (
// rangeRE matches the transfer status response from the server. $1 is the last byte index uploaded.
rangeRE = regexp.MustCompile(`^bytes=0\-(\d+)$`)
// chunkSize is the size of the chunks created during a resumable upload and should be a power of two.
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
chunkSize int64 = 1 << 18
)
// Progress returns the number of bytes uploaded at this point.
func (rx *ResumableUpload) Progress() int64 {
rx.mu.Lock()
defer rx.mu.Unlock()
return rx.progress
}
func (rx *ResumableUpload) transferStatus() (int64, *http.Response, error) {
req, _ := http.NewRequest("POST", rx.URI, nil)
req.ContentLength = 0
req.Header.Set("User-Agent", rx.UserAgent)
req.Header.Set("Content-Range", fmt.Sprintf("bytes */%v", rx.ContentLength))
res, err := rx.Client.Do(req)
if err != nil || res.StatusCode != statusResumeIncomplete {
return 0, res, err
}
var start int64
if m := rangeRE.FindStringSubmatch(res.Header.Get("Range")); len(m) == 2 {
start, err = strconv.ParseInt(m[1], 10, 64)
if err != nil {
return 0, nil, fmt.Errorf("unable to parse range size %v", m[1])
}
start += 1 // Start at the next byte
}
return start, res, nil
}
type chunk struct {
body io.Reader
size int64
err error
}
func (rx *ResumableUpload) transferChunks(ctx context.Context) (*http.Response, error) {
start, res, err := rx.transferStatus()
if err != nil || res.StatusCode != statusResumeIncomplete {
return res, err
}
for {
select { // Check for cancellation
case <-ctx.Done():
res.StatusCode = http.StatusRequestTimeout
return res, ctx.Err()
default:
}
reqSize := rx.ContentLength - start
if reqSize > chunkSize {
reqSize = chunkSize
}
r := io.NewSectionReader(rx.Media, start, reqSize)
req, _ := http.NewRequest("POST", rx.URI, r)
req.ContentLength = reqSize
req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, rx.ContentLength))
req.Header.Set("Content-Type", rx.MediaType)
req.Header.Set("User-Agent", rx.UserAgent)
res, err = rx.Client.Do(req)
start += reqSize
if err == nil && (res.StatusCode == statusResumeIncomplete || res.StatusCode == http.StatusOK) {
rx.mu.Lock()
rx.progress = start // keep track of number of bytes sent so far
rx.mu.Unlock()
if rx.Callback != nil {
rx.Callback(start, rx.ContentLength)
}
}
if err != nil || res.StatusCode != statusResumeIncomplete {
break
}
}
return res, err
}
var sleep = time.Sleep // override in unit tests
// Upload starts the process of a resumable upload with a cancellable context.
// It retries indefinitely (with a pause of uploadPause between attempts) until cancelled.
// It is called from the auto-generated API code and is not visible to the user.
// rx is private to the auto-generated API code.
func (rx *ResumableUpload) Upload(ctx context.Context) (*http.Response, error) {
var res *http.Response
var err error
for {
res, err = rx.transferChunks(ctx)
if err != nil || res.StatusCode == http.StatusCreated || res.StatusCode == http.StatusOK {
return res, err
}
select { // Check for cancellation
case <-ctx.Done():
res.StatusCode = http.StatusRequestTimeout
return res, ctx.Err()
default:
}
sleep(uploadPause)
}
return res, err
}
func ResolveRelative(basestr, relstr string) string {

View File

@ -11,9 +11,15 @@ import (
"io/ioutil"
"net/http"
"net/url"
"os"
"reflect"
"regexp"
"strconv"
"strings"
"testing"
"time"
"golang.org/x/net/context"
)
type SetOpaqueTest struct {
@ -359,3 +365,234 @@ func TestConvertVariant(t *testing.T) {
}
}
}
type unexpectedReader struct{}
func (unexpectedReader) Read([]byte) (int, error) {
return 0, fmt.Errorf("unexpected read in test.")
}
var contentRangeRE = regexp.MustCompile(`^bytes (\d+)\-(\d+)/(\d+)$`)
func (t *testTransport) RoundTrip(req *http.Request) (*http.Response, error) {
t.req = req
if rng := req.Header.Get("Content-Range"); rng != "" && !strings.HasPrefix(rng, "bytes */") { // Read the data
m := contentRangeRE.FindStringSubmatch(rng)
if len(m) != 4 {
return nil, fmt.Errorf("unable to parse content range: %v", rng)
}
start, err := strconv.ParseInt(m[1], 10, 64)
if err != nil {
return nil, fmt.Errorf("unable to parse content range: %v", rng)
}
end, err := strconv.ParseInt(m[2], 10, 64)
if err != nil {
return nil, fmt.Errorf("unable to parse content range: %v", rng)
}
totalSize, err := strconv.ParseInt(m[3], 10, 64)
if err != nil {
return nil, fmt.Errorf("unable to parse content range: %v", rng)
}
partialSize := end - start + 1
t.buf, err = ioutil.ReadAll(req.Body)
if err != nil || int64(len(t.buf)) != partialSize {
return nil, fmt.Errorf("unable to read %v bytes from request data, n=%v: %v", partialSize, len(t.buf), err)
}
if totalSize == end+1 {
t.statusCode = 200 // signify completion of transfer
}
}
f := ioutil.NopCloser(unexpectedReader{})
res := &http.Response{
Body: f,
StatusCode: t.statusCode,
Header: http.Header{},
}
if t.rangeVal != "" {
res.Header.Set("Range", t.rangeVal)
}
return res, nil
}
type testTransport struct {
req *http.Request
statusCode int
rangeVal string
want int64
buf []byte
}
var statusTests = []*testTransport{
&testTransport{statusCode: 308, want: 0},
&testTransport{statusCode: 308, rangeVal: "bytes=0-0", want: 1},
&testTransport{statusCode: 308, rangeVal: "bytes=0-42", want: 43},
}
func TestTransferStatus(t *testing.T) {
for _, tr := range statusTests {
rx := &ResumableUpload{
Client: &http.Client{Transport: tr},
}
g, _, err := rx.transferStatus()
if err != nil {
t.Error(err)
}
if g != tr.want {
t.Errorf("transferStatus got %v, want %v", g, tr.want)
}
}
}
func (t *interruptedTransport) RoundTrip(req *http.Request) (*http.Response, error) {
t.req = req
if rng := req.Header.Get("Content-Range"); rng != "" && !strings.HasPrefix(rng, "bytes */") {
t.interruptCount += 1
if t.interruptCount%7 == 0 { // Respond with a "service unavailable" error
res := &http.Response{
StatusCode: http.StatusServiceUnavailable,
Header: http.Header{},
}
t.rangeVal = fmt.Sprintf("bytes=0-%v", len(t.buf)-1) // Set the response for next time
return res, nil
}
m := contentRangeRE.FindStringSubmatch(rng)
if len(m) != 4 {
return nil, fmt.Errorf("unable to parse content range: %v", rng)
}
start, err := strconv.ParseInt(m[1], 10, 64)
if err != nil {
return nil, fmt.Errorf("unable to parse content range: %v", rng)
}
end, err := strconv.ParseInt(m[2], 10, 64)
if err != nil {
return nil, fmt.Errorf("unable to parse content range: %v", rng)
}
totalSize, err := strconv.ParseInt(m[3], 10, 64)
if err != nil {
return nil, fmt.Errorf("unable to parse content range: %v", rng)
}
partialSize := end - start + 1
buf, err := ioutil.ReadAll(req.Body)
if err != nil || int64(len(buf)) != partialSize {
return nil, fmt.Errorf("unable to read %v bytes from request data, n=%v: %v", partialSize, len(buf), err)
}
t.buf = append(t.buf, buf...)
if totalSize == end+1 {
t.statusCode = 200 // signify completion of transfer
}
}
f := ioutil.NopCloser(unexpectedReader{})
res := &http.Response{
Body: f,
StatusCode: t.statusCode,
Header: http.Header{},
}
if t.rangeVal != "" {
res.Header.Set("Range", t.rangeVal)
}
return res, nil
}
type interruptedTransport struct {
req *http.Request
statusCode int
rangeVal string
interruptCount int
buf []byte
progressBuf string
}
func (tr *interruptedTransport) ProgressUpdate(current, total int64) {
tr.progressBuf += fmt.Sprintf("%v, %v\n", current, total)
}
func TestInterruptedTransferChunks(t *testing.T) {
f, err := os.Open("googleapi.go")
if err != nil {
t.Fatalf("unable to open googleapi.go: %v", err)
}
defer f.Close()
slurp, err := ioutil.ReadAll(f)
if err != nil {
t.Fatalf("unable to slurp file: %v", err)
}
st, err := f.Stat()
if err != nil {
t.Fatalf("unable to stat googleapi.go: %v", err)
}
tr := &interruptedTransport{
statusCode: 308,
buf: make([]byte, 0, st.Size()),
}
oldChunkSize := chunkSize
defer func() { chunkSize = oldChunkSize }()
chunkSize = 100 // override to process small chunks for test.
sleep = func(time.Duration) {} // override time.Sleep
rx := &ResumableUpload{
Client: &http.Client{Transport: tr},
Media: f,
MediaType: "text/plain",
ContentLength: st.Size(),
Callback: tr.ProgressUpdate,
}
res, err := rx.Upload(context.Background())
if err != nil || res == nil || res.StatusCode != http.StatusOK {
if res == nil {
t.Errorf("transferChunks not successful, res=nil: %v", err)
} else {
t.Errorf("transferChunks not successful, statusCode=%v: %v", res.StatusCode, err)
}
}
if len(tr.buf) != len(slurp) || bytes.Compare(tr.buf, slurp) != 0 {
t.Errorf("transfered file corrupted:\ngot %s\nwant %s", tr.buf, slurp)
}
w := ""
for i := chunkSize; i <= st.Size(); i += chunkSize {
w += fmt.Sprintf("%v, %v\n", i, st.Size())
}
if st.Size()%chunkSize != 0 {
w += fmt.Sprintf("%v, %v\n", st.Size(), st.Size())
}
if tr.progressBuf != w {
t.Errorf("progress update error, got %v, want %v", tr.progressBuf, w)
}
}
func TestCancelUpload(t *testing.T) {
f, err := os.Open("googleapi.go")
if err != nil {
t.Fatalf("unable to open googleapi.go: %v", err)
}
defer f.Close()
st, err := f.Stat()
if err != nil {
t.Fatalf("unable to stat googleapi.go: %v", err)
}
tr := &interruptedTransport{
statusCode: 308,
buf: make([]byte, 0, st.Size()),
}
oldChunkSize := chunkSize
defer func() { chunkSize = oldChunkSize }()
chunkSize = 100 // override to process small chunks for test.
sleep = func(time.Duration) {} // override time.Sleep
rx := &ResumableUpload{
Client: &http.Client{Transport: tr},
Media: f,
MediaType: "text/plain",
ContentLength: st.Size(),
Callback: tr.ProgressUpdate,
}
ctx, cancelFunc := context.WithCancel(context.Background())
cancelFunc() // stop the upload that hasn't started yet
res, err := rx.Upload(ctx)
if err == nil || res == nil || res.StatusCode != http.StatusRequestTimeout {
if res == nil {
t.Errorf("transferChunks not successful, got res=nil, err=%v, want StatusRequestTimeout", err)
} else {
t.Errorf("transferChunks not successful, got statusCode=%v, err=%v, want StatusRequestTimeout", res.StatusCode, err)
}
}
}

View File

@ -4,6 +4,7 @@ import (
"fmt"
"io/ioutil"
"net/url"
"regexp"
"strings"
"time"
@ -19,6 +20,8 @@ type ComputeUtil struct {
userName string
project string
diskTypeURL string
address string
preemptible bool
service *raw.Service
zoneURL string
authTokenPath string
@ -51,6 +54,8 @@ func newComputeUtil(driver *Driver) (*ComputeUtil, error) {
userName: driver.SSHUser,
project: driver.Project,
diskTypeURL: driver.DiskType,
address: driver.Address,
preemptible: driver.Preemptible,
service: service,
zoneURL: apiURL + driver.Project + "/zones/" + driver.Zone,
globalURL: apiURL + driver.Project + "/global",
@ -84,6 +89,31 @@ func (c *ComputeUtil) deleteDisk() error {
return c.waitForRegionalOp(op.Name)
}
// staticAddress returns the external static IP address.
func (c *ComputeUtil) staticAddress() (string, error) {
// is the address a name?
isName, err := regexp.MatchString("[a-z]([-a-z0-9]*[a-z0-9])?", c.address)
if err != nil {
return "", err
}
if (!isName) {
return c.address, nil
}
// resolve the address by name
externalAddress, err := c.service.Addresses.Get(c.project, c.region(), c.address).Do()
if err != nil {
return "", err
}
return externalAddress.Address, nil
}
func (c *ComputeUtil) region() (string) {
return c.zone[:len(c.zone)-2]
}
func (c *ComputeUtil) firewallRule() (*raw.Firewall, error) {
return c.service.Firewalls.Get(c.project, firewallRule).Do()
}
@ -178,7 +208,20 @@ func (c *ComputeUtil) createInstance(d *Driver) error {
Scopes: strings.Split(d.Scopes, ","),
},
},
Scheduling: &raw.Scheduling{
Preemptible: c.preemptible,
},
}
if c.address != "" {
staticAddress, err := c.staticAddress()
if err != nil {
return err
}
instance.NetworkInterfaces[0].AccessConfigs[0].NatIP = staticAddress
}
disk, err := c.disk()
if disk == nil || err != nil {
instance.Disks[0].InitializeParams = &raw.AttachedDiskInitializeParams{

View File

@ -16,6 +16,8 @@ type Driver struct {
Zone string
MachineType string
DiskType string
Address string
Preemptible bool
Scopes string
DiskSize int
AuthTokenPath string
@ -79,6 +81,16 @@ func GetCreateFlags() []cli.Flag {
Value: "pd-standard",
EnvVar: "GOOGLE_DISK_TYPE",
},
cli.StringFlag{
Name: "google-address",
Usage: "GCE Instance External IP",
EnvVar: "GOOGLE_ADDRESS",
},
cli.BoolFlag{
Name: "google-preemptible",
Usage: "GCE Instance Preemptibility",
EnvVar: "GOOGLE_PREEMPTIBLE",
},
}
}
@ -111,6 +123,8 @@ func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) error {
d.MachineType = flags.String("google-machine-type")
d.DiskSize = flags.Int("google-disk-size")
d.DiskType = flags.String("google-disk-type")
d.Address = flags.String("google-address")
d.Preemptible = flags.Bool("google-preemptible")
d.AuthTokenPath = flags.String("google-auth-token")
d.Project = flags.String("google-project")
d.Scopes = flags.String("google-scopes")