Update `google/safebrowsing` lib to tip of master. (#3006)

This commit updates the `github.com/google/safebrowsing` dependency to
commit f387af, the tip of master at the time of writing.

Unit tests were confirmed to pass per CONTRIBUTING.md:
```
$ go test ./...
ok    github.com/google/safebrowsing  2.500s
?     github.com/google/safebrowsing/cmd/sblookup [no test files]
?     github.com/google/safebrowsing/cmd/sbserver [no test files]
?     github.com/google/safebrowsing/cmd/sbserver/statik  [no test files]
?     github.com/google/safebrowsing/internal/safebrowsing_proto  [no test files]
```

Godeps/Godeps.json

@@ -148,11 +148,11 @@
 		},
 		{
 			"ImportPath": "github.com/google/safebrowsing",
-			"Rev": "a8c029efb52bae66853e05241150ab338e98fbc7"
+			"Rev": "f387afacc9e702b5ed3e90100d3375871e724c08"
 		},
 		{
 			"ImportPath": "github.com/google/safebrowsing/internal/safebrowsing_proto",
-			"Rev": "a8c029efb52bae66853e05241150ab338e98fbc7"
+			"Rev": "f387afacc9e702b5ed3e90100d3375871e724c08"
 		},
 		{
 			"ImportPath": "github.com/grpc-ecosystem/go-grpc-prometheus",

vendor/github.com/google/safebrowsing/database.go

@@ -117,16 +117,16 @@ func (db *database) Init(config *Config, logger *log.Logger) bool {
 		db.setError(err)
 		return false
 	}
-
+	// Validate that the database threat list stored on disk is not too stale.
-	// Validate that the database threat list stored on disk is at least a
+	if db.isStale(dbf.Time) {
-	// superset of the specified configuration.
-	if db.config.now().Sub(dbf.Time) > (db.config.UpdatePeriod + jitter) {
 		db.log.Printf("database loaded is stale")
 		db.ml.Lock()
 		defer db.ml.Unlock()
 		db.setStale()
 		return false
 	}
+	// Validate that the database threat list stored on disk is at least a
+	// superset of the specified configuration.
 	tfuNew := make(threatsForUpdate)
 	for _, td := range db.config.ThreatLists {
 		if row, ok := dbf.Table[td]; ok {
@@ -142,8 +142,9 @@ func (db *database) Init(config *Config, logger *log.Logger) bool {
 	return true
 }
 
-// Status reports the health of the database. If in a faulted state, the db
+// Status reports the health of the database. The database is considered faulted
-// may repair itself on the next Update.
+// if there was an error during update or if the last update has gone stale. If
+// in a faulted state, the db may repair itself on the next Update.
 func (db *database) Status() error {
 	db.ml.RLock()
 	defer db.ml.RUnlock()
@@ -151,7 +152,7 @@ func (db *database) Status() error {
 	if db.err != nil {
 		return db.err
 	}
-	if db.config.now().Sub(db.last) > (db.config.UpdatePeriod + jitter) {
+	if db.isStale(db.last) {
 		db.setStale()
 		return db.err
 	}
@@ -306,6 +307,16 @@ func (db *database) setError(err error) {
 	db.ml.Unlock()
 }
 
+// isStale checks whether the last successful update should be considered stale.
+// Staleness is defined as being older than two of the configured update periods
+// plus jitter.
+func (db *database) isStale(lastUpdate time.Time) bool {
+	if db.config.now().Sub(lastUpdate) > 2*(db.config.UpdatePeriod+jitter) {
+		return true
+	}
+	return false
+}
+
 // setStale sets the error state to a stale message, without clearing
 // the database state.
 //

vendor/github.com/google/safebrowsing/safebrowser.go

@@ -402,33 +402,34 @@ func (sb *SafeBrowser) LookupURLs(urls []string) (threats [][]URLThreat, err err
 		return threats, err
 	}
 
-	// TODO: There are some optimizations to be made here:
+	hashes := make(map[hashPrefix]string)
-	//	1.) We could force a database update if it is in error.
+	hash2idx := make(map[hashPrefix]int)
-	//  However, we must ensure that we perform some form of rate-limiting.
+
-	//	2.) We should batch all of the partial hashes together such that we
+	// Construct the follow-up request being made to the server.
-	//	call api.HashLookup only once.
+	// In the request, we only ask for partial hashes for privacy reasons.
+	req := &pb.FindFullHashesRequest{
+		Client: &pb.ClientInfo{
+			ClientId:      sb.config.ID,
+			ClientVersion: sb.config.Version,
+		},
+		ThreatInfo: &pb.ThreatInfo{},
+	}
+	ttm := make(map[pb.ThreatType]bool)
+	ptm := make(map[pb.PlatformType]bool)
+	tetm := make(map[pb.ThreatEntryType]bool)
 
 	for i, url := range urls {
-		hashes, err := generateHashes(url)
+		urlhashes, err := generateHashes(url)
 		if err != nil {
-			sb.log.Printf("error generating hashes: %v", err)
+			sb.log.Printf("error generating urlhashes: %v", err)
 			atomic.AddInt64(&sb.stats.QueriesFail, int64(len(urls)-i))
 			return threats, err
 		}
 
-		// Construct the follow-up request being made to the server.
+		for fullHash, pattern := range urlhashes {
-		// In the request, we only ask for partial hashes for privacy reasons.
+			hashes[fullHash] = pattern
-		req := &pb.FindFullHashesRequest{
+			hash2idx[fullHash] = i
-			Client: &pb.ClientInfo{
+
-				ClientId:      sb.config.ID,
-				ClientVersion: sb.config.Version,
-			},
-			ThreatInfo: &pb.ThreatInfo{},
-		}
-		ttm := make(map[pb.ThreatType]bool)
-		ptm := make(map[pb.PlatformType]bool)
-		tetm := make(map[pb.ThreatEntryType]bool)
-		for fullHash, pattern := range hashes {
 			// Lookup in database according to threat list.
 			partialHash, unsureThreats := sb.db.Lookup(fullHash)
 			if len(unsureThreats) == 0 {
@@ -451,6 +452,7 @@ func (sb *SafeBrowser) LookupURLs(urls []string) (threats [][]URLThreat, err err
 						})
 					}
 				}
+				atomic.AddInt64(&sb.stats.QueriesByCache, 1)
 			case negativeCacheHit:
 				// This is cached as a non-threat.
 				atomic.AddInt64(&sb.stats.QueriesByCache, 1)
@@ -467,27 +469,23 @@ func (sb *SafeBrowser) LookupURLs(urls []string) (threats [][]URLThreat, err err
 					&pb.ThreatEntry{Hash: []byte(partialHash)})
 			}
 		}
-		for tt := range ttm {
+	}
-			req.ThreatInfo.ThreatTypes = append(req.ThreatInfo.ThreatTypes, tt)
+	for tt := range ttm {
-		}
+		req.ThreatInfo.ThreatTypes = append(req.ThreatInfo.ThreatTypes, tt)
-		for pt := range ptm {
+	}
-			req.ThreatInfo.PlatformTypes = append(req.ThreatInfo.PlatformTypes, pt)
+	for pt := range ptm {
-		}
+		req.ThreatInfo.PlatformTypes = append(req.ThreatInfo.PlatformTypes, pt)
-		for tet := range tetm {
+	}
-			req.ThreatInfo.ThreatEntryTypes = append(req.ThreatInfo.ThreatEntryTypes, tet)
+	for tet := range tetm {
-		}
+		req.ThreatInfo.ThreatEntryTypes = append(req.ThreatInfo.ThreatEntryTypes, tet)
+	}
 
-		// All results are known, so just continue.
+	// Actually query the Safe Browsing API for exact full hash matches.
-		if len(req.ThreatInfo.ThreatEntries) == 0 {
+	if len(req.ThreatInfo.ThreatEntries) != 0 {
-			atomic.AddInt64(&sb.stats.QueriesByCache, 1)
-			continue
-		}
-
-		// Actually query the Safe Browsing API for exact full hash matches.
 		resp, err := sb.api.HashLookup(req)
 		if err != nil {
 			sb.log.Printf("HashLookup failure: %v", err)
-			atomic.AddInt64(&sb.stats.QueriesFail, int64(len(urls)-i))
+			atomic.AddInt64(&sb.stats.QueriesFail, 1)
 			return threats, err
 		}
 
@@ -500,7 +498,9 @@ func (sb *SafeBrowser) LookupURLs(urls []string) (threats [][]URLThreat, err err
 			if !fullHash.IsFull() {
 				continue
 			}
-			if pattern, ok := hashes[fullHash]; ok {
+			pattern, ok := hashes[fullHash]
+			idx, findidx := hash2idx[fullHash]
+			if findidx && ok {
 				td := ThreatDescriptor{
 					ThreatType:      ThreatType(tm.ThreatType),
 					PlatformType:    PlatformType(tm.PlatformType),
@@ -509,7 +509,7 @@ func (sb *SafeBrowser) LookupURLs(urls []string) (threats [][]URLThreat, err err
 				if !sb.lists[td] {
 					continue
 				}
-				threats[i] = append(threats[i], URLThreat{
+				threats[idx] = append(threats[idx], URLThreat{
 					Pattern:          pattern,
 					ThreatDescriptor: td,
 				})