apiserver/pkg/endpoints/discovery/aggregated/handler_test.go

626 lines
23 KiB
Go

/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aggregated_test
import (
"encoding/json"
"math/rand"
"net/http"
"net/http/httptest"
"sort"
"strconv"
"strings"
"sync"
"testing"
fuzz "github.com/google/gofuzz"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/version"
discoveryendpoint "k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
)
var scheme = runtime.NewScheme()
var codecs = runtimeserializer.NewCodecFactory(scheme)
const discoveryPath = "/apis"
func init() {
// Add all builtin types to scheme
apidiscoveryv2beta1.AddToScheme(scheme)
codecs = runtimeserializer.NewCodecFactory(scheme)
}
func fuzzAPIGroups(atLeastNumGroups, maxNumGroups int, seed int64) apidiscoveryv2beta1.APIGroupDiscoveryList {
fuzzer := fuzz.NewWithSeed(seed)
fuzzer.NumElements(atLeastNumGroups, maxNumGroups)
fuzzer.NilChance(0)
fuzzer.Funcs(func(o *apidiscoveryv2beta1.APIGroupDiscovery, c fuzz.Continue) {
c.FuzzNoCustom(o)
// The ResourceManager will just not serve the group if its versions
// list is empty
atLeastOne := apidiscoveryv2beta1.APIVersionDiscovery{}
c.Fuzz(&atLeastOne)
o.Versions = append(o.Versions, atLeastOne)
sort.Slice(o.Versions[:], func(i, j int) bool {
return version.CompareKubeAwareVersionStrings(o.Versions[i].Version, o.Versions[j].Version) > 0
})
o.TypeMeta = metav1.TypeMeta{}
var name string
c.Fuzz(&name)
o.ObjectMeta = metav1.ObjectMeta{
Name: name,
}
})
var apis []apidiscoveryv2beta1.APIGroupDiscovery
fuzzer.Fuzz(&apis)
sort.Slice(apis[:], func(i, j int) bool {
return apis[i].Name < apis[j].Name
})
return apidiscoveryv2beta1.APIGroupDiscoveryList{
TypeMeta: metav1.TypeMeta{
Kind: "APIGroupDiscoveryList",
APIVersion: "apidiscovery.k8s.io/v2beta1",
},
Items: apis,
}
}
func fetchPath(handler http.Handler, acceptPrefix string, path string, etag string) (*http.Response, []byte, *apidiscoveryv2beta1.APIGroupDiscoveryList) {
// Expect json-formatted apis group list
w := httptest.NewRecorder()
req := httptest.NewRequest("GET", discoveryPath, nil)
// Ask for JSON response
req.Header.Set("Accept", acceptPrefix+";g=apidiscovery.k8s.io;v=v2beta1;as=APIGroupDiscoveryList")
if etag != "" {
// Quote provided etag if unquoted
quoted := etag
if !strings.HasPrefix(etag, "\"") {
quoted = strconv.Quote(etag)
}
req.Header.Set("If-None-Match", quoted)
}
handler.ServeHTTP(w, req)
bytes := w.Body.Bytes()
var decoded *apidiscoveryv2beta1.APIGroupDiscoveryList
if len(bytes) > 0 {
decoded = &apidiscoveryv2beta1.APIGroupDiscoveryList{}
runtime.DecodeInto(codecs.UniversalDecoder(), bytes, decoded)
}
return w.Result(), bytes, decoded
}
// Add all builtin APIServices to the manager and check the output
func TestBasicResponse(t *testing.T) {
manager := discoveryendpoint.NewResourceManager()
apis := fuzzAPIGroups(1, 3, 10)
manager.SetGroups(apis.Items)
response, body, decoded := fetchPath(manager, "application/json", discoveryPath, "")
jsonFormatted, err := json.Marshal(&apis)
require.NoError(t, err, "json marshal should always succeed")
assert.Equal(t, http.StatusOK, response.StatusCode, "response should be 200 OK")
assert.Equal(t, "application/json;g=apidiscovery.k8s.io;v=v2beta1;as=APIGroupDiscoveryList", response.Header.Get("Content-Type"), "Content-Type response header should be as requested in Accept header if supported")
assert.NotEmpty(t, response.Header.Get("ETag"), "E-Tag should be set")
assert.NoError(t, err, "decode should always succeed")
assert.EqualValues(t, &apis, decoded, "decoded value should equal input")
assert.Equal(t, string(jsonFormatted)+"\n", string(body), "response should be the api group list")
}
// Test that protobuf is outputted correctly
func TestBasicResponseProtobuf(t *testing.T) {
manager := discoveryendpoint.NewResourceManager()
apis := fuzzAPIGroups(1, 3, 10)
manager.SetGroups(apis.Items)
response, _, decoded := fetchPath(manager, "application/vnd.kubernetes.protobuf", discoveryPath, "")
assert.Equal(t, http.StatusOK, response.StatusCode, "response should be 200 OK")
assert.Equal(t, "application/vnd.kubernetes.protobuf;g=apidiscovery.k8s.io;v=v2beta1;as=APIGroupDiscoveryList", response.Header.Get("Content-Type"), "Content-Type response header should be as requested in Accept header if supported")
assert.NotEmpty(t, response.Header.Get("ETag"), "E-Tag should be set")
assert.EqualValues(t, &apis, decoded, "decoded value should equal input")
}
// Test that an etag associated with the service only depends on the apiresources
// e.g.: Multiple services with the same contents should have the same etag.
func TestEtagConsistent(t *testing.T) {
// Create 2 managers, add a bunch of services to each
manager1 := discoveryendpoint.NewResourceManager()
manager2 := discoveryendpoint.NewResourceManager()
apis := fuzzAPIGroups(1, 3, 11)
manager1.SetGroups(apis.Items)
manager2.SetGroups(apis.Items)
// Make sure etag of each is the same
res1_initial, _, _ := fetchPath(manager1, "application/json", discoveryPath, "")
res2_initial, _, _ := fetchPath(manager2, "application/json", discoveryPath, "")
assert.NotEmpty(t, res1_initial.Header.Get("ETag"), "Etag should be populated")
assert.NotEmpty(t, res2_initial.Header.Get("ETag"), "Etag should be populated")
assert.Equal(t, res1_initial.Header.Get("ETag"), res2_initial.Header.Get("ETag"), "etag should be deterministic")
// Then add one service to only one.
// Make sure etag is changed, but other is the same
apis = fuzzAPIGroups(1, 1, 11)
for _, group := range apis.Items {
for _, version := range group.Versions {
manager1.AddGroupVersion(group.Name, version)
}
}
res1_addedToOne, _, _ := fetchPath(manager1, "application/json", discoveryPath, "")
res2_addedToOne, _, _ := fetchPath(manager2, "application/json", discoveryPath, "")
assert.NotEmpty(t, res1_addedToOne.Header.Get("ETag"), "Etag should be populated")
assert.NotEmpty(t, res2_addedToOne.Header.Get("ETag"), "Etag should be populated")
assert.NotEqual(t, res1_initial.Header.Get("ETag"), res1_addedToOne.Header.Get("ETag"), "ETag should be changed since version was added")
assert.Equal(t, res2_initial.Header.Get("ETag"), res2_addedToOne.Header.Get("ETag"), "ETag should be unchanged since data was unchanged")
// Then add service to other one
// Make sure etag is the same
for _, group := range apis.Items {
for _, version := range group.Versions {
manager2.AddGroupVersion(group.Name, version)
}
}
res1_addedToBoth, _, _ := fetchPath(manager1, "application/json", discoveryPath, "")
res2_addedToBoth, _, _ := fetchPath(manager2, "application/json", discoveryPath, "")
assert.NotEmpty(t, res1_addedToOne.Header.Get("ETag"), "Etag should be populated")
assert.NotEmpty(t, res2_addedToOne.Header.Get("ETag"), "Etag should be populated")
assert.Equal(t, res1_addedToBoth.Header.Get("ETag"), res2_addedToBoth.Header.Get("ETag"), "ETags should be equal since content is equal")
assert.NotEqual(t, res2_initial.Header.Get("ETag"), res2_addedToBoth.Header.Get("ETag"), "ETag should be changed since data was changed")
// Remove the group version from both. Initial E-Tag should be restored
for _, group := range apis.Items {
for _, version := range group.Versions {
manager1.RemoveGroupVersion(metav1.GroupVersion{
Group: group.Name,
Version: version.Version,
})
manager2.RemoveGroupVersion(metav1.GroupVersion{
Group: group.Name,
Version: version.Version,
})
}
}
res1_removeFromBoth, _, _ := fetchPath(manager1, "application/json", discoveryPath, "")
res2_removeFromBoth, _, _ := fetchPath(manager2, "application/json", discoveryPath, "")
assert.NotEmpty(t, res1_addedToOne.Header.Get("ETag"), "Etag should be populated")
assert.NotEmpty(t, res2_addedToOne.Header.Get("ETag"), "Etag should be populated")
assert.Equal(t, res1_removeFromBoth.Header.Get("ETag"), res2_removeFromBoth.Header.Get("ETag"), "ETags should be equal since content is equal")
assert.Equal(t, res1_initial.Header.Get("ETag"), res1_removeFromBoth.Header.Get("ETag"), "ETag should be equal to initial value since added content was removed")
}
// Test that if a request comes in with an If-None-Match header with an incorrect
// E-Tag, that fresh content is returned.
func TestEtagNonMatching(t *testing.T) {
manager := discoveryendpoint.NewResourceManager()
apis := fuzzAPIGroups(1, 3, 12)
manager.SetGroups(apis.Items)
// fetch the document once
initial, _, _ := fetchPath(manager, "application/json", discoveryPath, "")
assert.NotEmpty(t, initial.Header.Get("ETag"), "ETag should be populated")
// Send another request with a wrong e-tag. The same response should
// get sent again
second, _, _ := fetchPath(manager, "application/json", discoveryPath, "wrongetag")
assert.Equal(t, http.StatusOK, initial.StatusCode, "response should be 200 OK")
assert.Equal(t, http.StatusOK, second.StatusCode, "response should be 200 OK")
assert.Equal(t, initial.Header.Get("ETag"), second.Header.Get("ETag"), "ETag of both requests should be equal")
}
// Test that if a request comes in with an If-None-Match header with a correct
// E-Tag, that 304 Not Modified is returned
func TestEtagMatching(t *testing.T) {
manager := discoveryendpoint.NewResourceManager()
apis := fuzzAPIGroups(1, 3, 12)
manager.SetGroups(apis.Items)
// fetch the document once
initial, initialBody, _ := fetchPath(manager, "application/json", discoveryPath, "")
assert.NotEmpty(t, initial.Header.Get("ETag"), "ETag should be populated")
assert.NotEmpty(t, initialBody, "body should not be empty")
// Send another request with a wrong e-tag. The same response should
// get sent again
second, secondBody, _ := fetchPath(manager, "application/json", discoveryPath, initial.Header.Get("ETag"))
assert.Equal(t, http.StatusOK, initial.StatusCode, "initial response should be 200 OK")
assert.Equal(t, http.StatusNotModified, second.StatusCode, "second response should be 304 Not Modified")
assert.Equal(t, initial.Header.Get("ETag"), second.Header.Get("ETag"), "ETag of both requests should be equal")
assert.Empty(t, secondBody, "body should be empty when returning 304 Not Modified")
}
// Test that if a request comes in with an If-None-Match header with an old
// E-Tag, that fresh content is returned
func TestEtagOutdated(t *testing.T) {
manager := discoveryendpoint.NewResourceManager()
apis := fuzzAPIGroups(1, 3, 15)
manager.SetGroups(apis.Items)
// fetch the document once
initial, initialBody, _ := fetchPath(manager, "application/json", discoveryPath, "")
assert.NotEmpty(t, initial.Header.Get("ETag"), "ETag should be populated")
assert.NotEmpty(t, initialBody, "body should not be empty")
// Then add some services so the etag changes
apis = fuzzAPIGroups(1, 3, 14)
for _, group := range apis.Items {
for _, version := range group.Versions {
manager.AddGroupVersion(group.Name, version)
}
}
// Send another request with the old e-tag. Response should not be 304 Not Modified
second, secondBody, _ := fetchPath(manager, "application/json", discoveryPath, initial.Header.Get("ETag"))
assert.Equal(t, http.StatusOK, initial.StatusCode, "initial response should be 200 OK")
assert.Equal(t, http.StatusOK, second.StatusCode, "second response should be 304 Not Modified")
assert.NotEqual(t, initial.Header.Get("ETag"), second.Header.Get("ETag"), "ETag of both requests should be unequal since contents differ")
assert.NotEmpty(t, secondBody, "body should be not empty when returning 304 Not Modified")
}
// Test that an api service can be added or removed
func TestAddRemove(t *testing.T) {
manager := discoveryendpoint.NewResourceManager()
apis := fuzzAPIGroups(1, 3, 15)
for _, group := range apis.Items {
for _, version := range group.Versions {
manager.AddGroupVersion(group.Name, version)
}
}
_, _, initialDocument := fetchPath(manager, "application/json", discoveryPath, "")
for _, group := range apis.Items {
for _, version := range group.Versions {
manager.RemoveGroupVersion(metav1.GroupVersion{
Group: group.Name,
Version: version.Version,
})
}
}
_, _, secondDocument := fetchPath(manager, "application/json", discoveryPath, "")
require.NotNil(t, initialDocument, "initial document should parse")
require.NotNil(t, secondDocument, "second document should parse")
assert.Len(t, initialDocument.Items, len(apis.Items), "initial document should have set number of groups")
assert.Len(t, secondDocument.Items, 0, "second document should have no groups")
}
// Show that updating an existing service replaces and does not add the entry
// and instead replaces it
func TestUpdateService(t *testing.T) {
manager := discoveryendpoint.NewResourceManager()
apis := fuzzAPIGroups(1, 3, 15)
for _, group := range apis.Items {
for _, version := range group.Versions {
manager.AddGroupVersion(group.Name, version)
}
}
_, _, initialDocument := fetchPath(manager, "application/json", discoveryPath, "")
assert.Equal(t, initialDocument, &apis, "should have returned expected document")
b, err := json.Marshal(apis)
if err != nil {
t.Error(err)
}
var newapis apidiscoveryv2beta1.APIGroupDiscoveryList
err = json.Unmarshal(b, &newapis)
if err != nil {
t.Error(err)
}
newapis.Items[0].Versions[0].Resources[0].Resource = "changed a resource name!"
for _, group := range newapis.Items {
for _, version := range group.Versions {
manager.AddGroupVersion(group.Name, version)
}
}
_, _, secondDocument := fetchPath(manager, "application/json", discoveryPath, "")
assert.Equal(t, secondDocument, &newapis, "should have returned expected document")
assert.NotEqual(t, secondDocument, initialDocument, "should have returned expected document")
}
// Show the discovery manager is capable of serving requests to multiple users
// with unchanging data
func TestConcurrentRequests(t *testing.T) {
manager := discoveryendpoint.NewResourceManager()
apis := fuzzAPIGroups(1, 3, 15)
manager.SetGroups(apis.Items)
waitGroup := sync.WaitGroup{}
numReaders := 100
numRequestsPerReader := 100
// Spawn a bunch of readers that will keep sending requests to the server
for i := 0; i < numReaders; i++ {
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
etag := ""
for j := 0; j < numRequestsPerReader; j++ {
usedEtag := etag
if j%2 == 0 {
// Disable use of etag for every second request
usedEtag = ""
}
response, body, document := fetchPath(manager, "application/json", discoveryPath, usedEtag)
if usedEtag != "" {
assert.Equal(t, http.StatusNotModified, response.StatusCode, "response should be Not Modified if etag was used")
assert.Empty(t, body, "body should be empty if etag used")
} else {
assert.Equal(t, http.StatusOK, response.StatusCode, "response should be OK if etag was unused")
assert.Equal(t, &apis, document, "document should be equal")
}
etag = response.Header.Get("ETag")
}
}()
}
waitGroup.Wait()
}
// Show the handler is capable of serving many concurrent readers and many
// concurrent writers without tripping up. Good to run with go '-race' detector
// since there are not many "correctness" checks
func TestAbuse(t *testing.T) {
manager := discoveryendpoint.NewResourceManager()
numReaders := 100
numRequestsPerReader := 1000
numWriters := 10
numWritesPerWriter := 1000
waitGroup := sync.WaitGroup{}
// Spawn a bunch of writers that randomly add groups, remove groups, and
// reset the list of groups
for i := 0; i < numWriters; i++ {
source := rand.NewSource(int64(i))
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
// track list of groups we've added so that we can remove them
// randomly
var addedGroups []metav1.GroupVersion
for j := 0; j < numWritesPerWriter; j++ {
switch source.Int63() % 3 {
case 0:
// Add a fuzzed group
apis := fuzzAPIGroups(1, 2, 15)
for _, group := range apis.Items {
for _, version := range group.Versions {
manager.AddGroupVersion(group.Name, version)
addedGroups = append(addedGroups, metav1.GroupVersion{
Group: group.Name,
Version: version.Version,
})
}
}
case 1:
// Remove a group that we have added
if len(addedGroups) > 0 {
manager.RemoveGroupVersion(addedGroups[0])
addedGroups = addedGroups[1:]
} else {
// Send a request and try to remove a group someone else
// might have added
_, _, document := fetchPath(manager, "application/json", discoveryPath, "")
assert.NotNil(t, document, "manager should always succeed in returning a document")
if len(document.Items) > 0 {
manager.RemoveGroupVersion(metav1.GroupVersion{
Group: document.Items[0].Name,
Version: document.Items[0].Versions[0].Version,
})
}
}
case 2:
manager.SetGroups(nil)
addedGroups = nil
default:
panic("unreachable")
}
}
}()
}
// Spawn a bunch of readers that will keep sending requests to the server
// and making sure the response makes sense
for i := 0; i < numReaders; i++ {
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
etag := ""
for j := 0; j < numRequestsPerReader; j++ {
response, body, document := fetchPath(manager, "application/json", discoveryPath, etag)
if response.StatusCode == http.StatusNotModified {
assert.Equal(t, etag, response.Header.Get("ETag"))
assert.Empty(t, body, "body should be empty if etag used")
assert.Nil(t, document)
} else {
assert.Equal(t, http.StatusOK, response.StatusCode, "response should be OK if etag was unused")
assert.NotNil(t, document)
}
etag = response.Header.Get("ETag")
}
}()
}
waitGroup.Wait()
}
func TestVersionSortingNoPriority(t *testing.T) {
manager := discoveryendpoint.NewResourceManager()
manager.AddGroupVersion("default", apidiscoveryv2beta1.APIVersionDiscovery{
Version: "v1alpha1",
})
manager.AddGroupVersion("default", apidiscoveryv2beta1.APIVersionDiscovery{
Version: "v2beta1",
})
manager.AddGroupVersion("default", apidiscoveryv2beta1.APIVersionDiscovery{
Version: "v1",
})
manager.AddGroupVersion("default", apidiscoveryv2beta1.APIVersionDiscovery{
Version: "v1beta1",
})
manager.AddGroupVersion("default", apidiscoveryv2beta1.APIVersionDiscovery{
Version: "v2",
})
response, _, decoded := fetchPath(manager, "application/json", discoveryPath, "")
assert.Equal(t, http.StatusOK, response.StatusCode, "response should be 200 OK")
versions := decoded.Items[0].Versions
// Ensure that v1 is sorted before v1alpha1
assert.Equal(t, versions[0].Version, "v2")
assert.Equal(t, versions[1].Version, "v1")
assert.Equal(t, versions[2].Version, "v2beta1")
assert.Equal(t, versions[3].Version, "v1beta1")
assert.Equal(t, versions[4].Version, "v1alpha1")
}
func TestVersionSortingWithPriority(t *testing.T) {
manager := discoveryendpoint.NewResourceManager()
manager.AddGroupVersion("default", apidiscoveryv2beta1.APIVersionDiscovery{
Version: "v1",
})
manager.SetGroupVersionPriority(metav1.GroupVersion{Group: "default", Version: "v1"}, 1000, 100)
manager.AddGroupVersion("default", apidiscoveryv2beta1.APIVersionDiscovery{
Version: "v1alpha1",
})
manager.SetGroupVersionPriority(metav1.GroupVersion{Group: "default", Version: "v1alpha1"}, 1000, 200)
response, _, decoded := fetchPath(manager, "application/json", discoveryPath, "")
assert.Equal(t, http.StatusOK, response.StatusCode, "response should be 200 OK")
versions := decoded.Items[0].Versions
// Ensure that reverse alpha sort order can be overridden by setting group version priorities.
assert.Equal(t, versions[0].Version, "v1alpha1")
assert.Equal(t, versions[1].Version, "v1")
}
// if two apiservices declare conflicting priorities for their group priority, take the higher one.
func TestGroupVersionSortingConflictingPriority(t *testing.T) {
manager := discoveryendpoint.NewResourceManager()
manager.AddGroupVersion("default", apidiscoveryv2beta1.APIVersionDiscovery{
Version: "v1",
})
manager.SetGroupVersionPriority(metav1.GroupVersion{Group: "default", Version: "v1"}, 1000, 100)
manager.AddGroupVersion("test", apidiscoveryv2beta1.APIVersionDiscovery{
Version: "v1alpha1",
})
manager.SetGroupVersionPriority(metav1.GroupVersion{Group: "test", Version: "v1alpha1"}, 500, 100)
manager.AddGroupVersion("test", apidiscoveryv2beta1.APIVersionDiscovery{
Version: "v1alpha2",
})
manager.SetGroupVersionPriority(metav1.GroupVersion{Group: "test", Version: "v1alpha1"}, 2000, 100)
response, _, decoded := fetchPath(manager, "application/json", discoveryPath, "")
assert.Equal(t, http.StatusOK, response.StatusCode, "response should be 200 OK")
groups := decoded.Items
// Ensure that reverse alpha sort order can be overridden by setting group version priorities.
assert.Equal(t, groups[0].Name, "test")
assert.Equal(t, groups[1].Name, "default")
}
// Show that the GroupPriorityMinimum is not sticky if a higher group version is removed
// after a lower one is added
func TestStatelessGroupPriorityMinimum(t *testing.T) {
manager := discoveryendpoint.NewResourceManager()
stableGroup := "stable.example.com"
experimentalGroup := "experimental.example.com"
manager.AddGroupVersion(stableGroup, apidiscoveryv2beta1.APIVersionDiscovery{
Version: "v1",
})
manager.SetGroupVersionPriority(metav1.GroupVersion{Group: stableGroup, Version: "v1"}, 1000, 100)
manager.AddGroupVersion(experimentalGroup, apidiscoveryv2beta1.APIVersionDiscovery{
Version: "v1",
})
manager.SetGroupVersionPriority(metav1.GroupVersion{Group: experimentalGroup, Version: "v1"}, 100, 100)
manager.AddGroupVersion(experimentalGroup, apidiscoveryv2beta1.APIVersionDiscovery{
Version: "v1alpha1",
})
manager.SetGroupVersionPriority(metav1.GroupVersion{Group: experimentalGroup, Version: "v1alpha1"}, 10000, 100)
// Expect v1alpha1's group priority to be used and sort it first in the list
response, _, decoded := fetchPath(manager, "application/json", discoveryPath, "")
assert.Equal(t, http.StatusOK, response.StatusCode, "response should be 200 OK")
assert.Equal(t, decoded.Items[0].Name, "experimental.example.com")
assert.Equal(t, decoded.Items[1].Name, "stable.example.com")
// Remove v1alpha1 and expect the new lower priority to take hold
manager.RemoveGroupVersion(metav1.GroupVersion{Group: experimentalGroup, Version: "v1alpha1"})
response, _, decoded = fetchPath(manager, "application/json", discoveryPath, "")
assert.Equal(t, http.StatusOK, response.StatusCode, "response should be 200 OK")
assert.Equal(t, decoded.Items[0].Name, "stable.example.com")
assert.Equal(t, decoded.Items[1].Name, "experimental.example.com")
}