refactor: isolate asset construction out of cloudup

Continuing the node/infra splitting.
This commit is contained in:
justinsb 2024-03-28 07:50:50 -04:00
parent 8c33116d7d
commit e5d5175e08
9 changed files with 215 additions and 176 deletions

View File

@ -34,6 +34,7 @@ import (
"k8s.io/kops/pkg/client/simple/vfsclientset"
"k8s.io/kops/pkg/model"
"k8s.io/kops/pkg/model/resources"
"k8s.io/kops/pkg/nodemodel/wellknownassets"
"k8s.io/kops/pkg/wellknownservices"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup"
@ -213,8 +214,7 @@ func (r *KopsConfigReconciler) buildBootstrapData(ctx context.Context) ([]byte,
nodeUpAssets := make(map[architectures.Architecture]*mirrors.MirroredAsset)
for _, arch := range architectures.GetSupported() {
asset, err := cloudup.NodeUpAsset(assetBuilder, arch)
asset, err := wellknownassets.NodeUpAsset(assetBuilder, arch)
if err != nil {
return nil, err
}

View File

@ -61,7 +61,7 @@ sed -i.bak -e "s@DNS_CONTROLLER_TAG=${KOPS_RELEASE_VERSION}@DNS_CONTROLLER_TAG=$
sed -i.bak -e "s@KOPS_CONTROLLER_TAG=${KOPS_RELEASE_VERSION}@KOPS_CONTROLLER_TAG=${NEW_RELEASE_VERSION}@g" Makefile
sed -i.bak -e "s@KUBE_APISERVER_HEALTHCHECK_TAG=${KOPS_RELEASE_VERSION}@KUBE_APISERVER_HEALTHCHECK_TAG=${NEW_RELEASE_VERSION}@g" Makefile
sed -i.bak -e "s@\"${KOPS_RELEASE_VERSION}\"@\"${NEW_RELEASE_VERSION}\"@g" upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go
sed -i.bak -e "s@${KOPS_RELEASE_VERSION}@${NEW_RELEASE_VERSION}@g" upup/pkg/fi/cloudup/urls_test.go
sed -i.bak -e "s@${KOPS_RELEASE_VERSION}@${NEW_RELEASE_VERSION}@g" pkg/nodemodel/wellknownassets/kopsassets_test.go
git grep -l registry.k8s.io/kops/dns-controller | xargs -I {} sed -i.bak -e "s@dns-controller:${KOPS_RELEASE_VERSION}@dns-controller:${NEW_RELEASE_VERSION}@g" {}
git grep -l "version..v${KOPS_RELEASE_VERSION}" upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/ | xargs -I {} sed -i.bak -e "s@version: v${KOPS_RELEASE_VERSION}@version: v${NEW_RELEASE_VERSION}@g" {}

View File

@ -49,6 +49,7 @@ import (
"k8s.io/kops/pkg/featureflag"
"k8s.io/kops/pkg/model"
"k8s.io/kops/pkg/model/resources"
"k8s.io/kops/pkg/nodemodel/wellknownassets"
"k8s.io/kops/pkg/wellknownservices"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup"
@ -424,8 +425,7 @@ func buildBootstrapData(ctx context.Context, clientset simple.Clientset, cluster
nodeUpAssets := make(map[architectures.Architecture]*mirrors.MirroredAsset)
for _, arch := range architectures.GetSupported() {
asset, err := cloudup.NodeUpAsset(assetBuilder, arch)
asset, err := wellknownassets.NodeUpAsset(assetBuilder, arch)
if err != nil {
return nil, err
}

196
pkg/nodemodel/fileassets.go Normal file
View File

@ -0,0 +1,196 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nodemodel
import (
"fmt"
"net/url"
"path"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/apis/kops/model"
"k8s.io/kops/pkg/apis/kops/util"
"k8s.io/kops/pkg/assets"
"k8s.io/kops/pkg/model/components"
"k8s.io/kops/pkg/nodemodel/wellknownassets"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/util/pkg/architectures"
"k8s.io/kops/util/pkg/hashing"
"k8s.io/kops/util/pkg/mirrors"
)
type FileAssets struct {
// Assets is a list of sources for files (primarily when not using everything containerized)
// Formats:
// raw url: http://... or https://...
// url with hash: <hex>@http://... or <hex>@https://...
Assets map[architectures.Architecture][]*mirrors.MirroredAsset
// NodeUpAssets are the assets for downloading nodeup
NodeUpAssets map[architectures.Architecture]*mirrors.MirroredAsset
Cluster *kops.Cluster
}
// AddFileAssets adds the file assets within the assetBuilder
func (c *FileAssets) AddFileAssets(assetBuilder *assets.AssetBuilder) error {
var baseURL string
if components.IsBaseURL(c.Cluster.Spec.KubernetesVersion) {
baseURL = c.Cluster.Spec.KubernetesVersion
} else {
baseURL = "https://dl.k8s.io/release/v" + c.Cluster.Spec.KubernetesVersion
}
c.Assets = make(map[architectures.Architecture][]*mirrors.MirroredAsset)
c.NodeUpAssets = make(map[architectures.Architecture]*mirrors.MirroredAsset)
for _, arch := range architectures.GetSupported() {
c.Assets[arch] = []*mirrors.MirroredAsset{}
k8sAssetsNames := []string{
fmt.Sprintf("/bin/linux/%s/kubelet", arch),
fmt.Sprintf("/bin/linux/%s/kubectl", arch),
}
if needsMounterAsset(c.Cluster) {
k8sAssetsNames = append(k8sAssetsNames, fmt.Sprintf("/bin/linux/%s/mounter", arch))
}
for _, an := range k8sAssetsNames {
k, err := url.Parse(baseURL)
if err != nil {
return err
}
k.Path = path.Join(k.Path, an)
u, hash, err := assetBuilder.RemapFileAndSHA(k)
if err != nil {
return err
}
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash))
}
kubernetesVersion, _ := util.ParseKubernetesVersion(c.Cluster.Spec.KubernetesVersion)
cloudProvider := c.Cluster.Spec.GetCloudProvider()
if ok := model.UseExternalKubeletCredentialProvider(*kubernetesVersion, cloudProvider); ok {
switch cloudProvider {
case kops.CloudProviderGCE:
binaryLocation := c.Cluster.Spec.CloudProvider.GCE.BinariesLocation
if binaryLocation == nil {
binaryLocation = fi.PtrTo("https://storage.googleapis.com/k8s-staging-cloud-provider-gcp/auth-provider-gcp")
}
// VALID FOR 60 DAYS WE REALLY NEED TO MERGE https://github.com/kubernetes/cloud-provider-gcp/pull/601 and CUT A RELEASE
k, err := url.Parse(fmt.Sprintf("%s/linux-%s/v20231005-providersv0.27.1-65-g8fbe8d27", *binaryLocation, arch))
if err != nil {
return err
}
hashes := map[architectures.Architecture]string{
"amd64": "827d558953d861b81a35c3b599191a73f53c1f63bce42c61e7a3fee21a717a89",
"arm64": "f1617c0ef77f3718e12a3efc6f650375d5b5e96eebdbcbad3e465e89e781bdfa",
}
hash, err := hashing.FromString(hashes[arch])
if err != nil {
return fmt.Errorf("unable to parse auth-provider-gcp binary asset hash %q: %v", hashes[arch], err)
}
u, err := assetBuilder.RemapFileAndSHAValue(k, hashes[arch])
if err != nil {
return err
}
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash))
case kops.CloudProviderAWS:
binaryLocation := c.Cluster.Spec.CloudProvider.AWS.BinariesLocation
if binaryLocation == nil {
binaryLocation = fi.PtrTo("https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1")
}
k, err := url.Parse(fmt.Sprintf("%s/linux/%s/ecr-credential-provider-linux-%s", *binaryLocation, arch, arch))
if err != nil {
return err
}
u, hash, err := assetBuilder.RemapFileAndSHA(k)
if err != nil {
return err
}
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash))
}
}
{
cniAsset, cniAssetHash, err := wellknownassets.FindCNIAssets(c.Cluster, assetBuilder, arch)
if err != nil {
return err
}
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(cniAsset, cniAssetHash))
}
if c.Cluster.Spec.Containerd == nil || !c.Cluster.Spec.Containerd.SkipInstall {
containerdAssetUrl, containerdAssetHash, err := wellknownassets.FindContainerdAsset(c.Cluster, assetBuilder, arch)
if err != nil {
return err
}
if containerdAssetUrl != nil && containerdAssetHash != nil {
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(containerdAssetUrl, containerdAssetHash))
}
runcAssetUrl, runcAssetHash, err := wellknownassets.FindRuncAsset(c.Cluster, assetBuilder, arch)
if err != nil {
return err
}
if runcAssetUrl != nil && runcAssetHash != nil {
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(runcAssetUrl, runcAssetHash))
}
nerdctlAssetUrl, nerdctlAssetHash, err := wellknownassets.FindNerdctlAsset(c.Cluster, assetBuilder, arch)
if err != nil {
return err
}
if nerdctlAssetUrl != nil && nerdctlAssetHash != nil {
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(nerdctlAssetUrl, nerdctlAssetHash))
}
}
crictlAssetUrl, crictlAssetHash, err := wellknownassets.FindCrictlAsset(c.Cluster, assetBuilder, arch)
if err != nil {
return err
}
if crictlAssetUrl != nil && crictlAssetHash != nil {
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(crictlAssetUrl, crictlAssetHash))
}
asset, err := wellknownassets.NodeUpAsset(assetBuilder, arch)
if err != nil {
return err
}
c.NodeUpAssets[arch] = asset
}
return nil
}
// needsMounterAsset checks if we need the mounter program
// This is only needed currently on ContainerOS i.e. GCE, but we don't have a nice way to detect it yet
func needsMounterAsset(c *kops.Cluster) bool {
// TODO: Do real detection of ContainerOS (but this has to work with image names, and maybe even forked images)
switch c.Spec.GetCloudProvider() {
case kops.CloudProviderGCE:
return true
default:
return false
}
}

View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package cloudup
package wellknownassets
import (
"fmt"
@ -47,7 +47,7 @@ const (
ENV_VAR_CNI_ASSET_HASH = "CNI_ASSET_HASH_STRING"
)
func findCNIAssets(c *kopsapi.Cluster, assetBuilder *assets.AssetBuilder, arch architectures.Architecture) (*url.URL, *hashing.Hash, error) {
func FindCNIAssets(c *kopsapi.Cluster, assetBuilder *assets.AssetBuilder, arch architectures.Architecture) (*url.URL, *hashing.Hash, error) {
// Override CNI packages from env vars
cniAssetURL := os.Getenv(ENV_VAR_CNI_ASSET_URL)
cniAssetHash := os.Getenv(ENV_VAR_CNI_ASSET_HASH)

View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package cloudup
package wellknownassets
import (
"testing"
@ -36,7 +36,7 @@ func Test_FindCNIAssetFromEnvironmentVariable(t *testing.T) {
cluster.Spec.KubernetesVersion = "v1.18.0"
assetBuilder := assets.NewAssetBuilder(vfs.Context, cluster.Spec.Assets, cluster.Spec.KubernetesVersion, false)
cniAsset, cniAssetHash, err := findCNIAssets(cluster, assetBuilder, architectures.ArchitectureAmd64)
cniAsset, cniAssetHash, err := FindCNIAssets(cluster, assetBuilder, architectures.ArchitectureAmd64)
if err != nil {
t.Errorf("Unable to parse CNI version %s", err)
}
@ -58,7 +58,7 @@ func Test_FindCNIAssetFromDefaults122(t *testing.T) {
cluster.Spec.KubernetesVersion = "v1.22.0"
assetBuilder := assets.NewAssetBuilder(vfs.Context, cluster.Spec.Assets, cluster.Spec.KubernetesVersion, false)
cniAsset, cniAssetHash, err := findCNIAssets(cluster, assetBuilder, architectures.ArchitectureAmd64)
cniAsset, cniAssetHash, err := FindCNIAssets(cluster, assetBuilder, architectures.ArchitectureAmd64)
if err != nil {
t.Errorf("Unable to parse CNI version %s", err)
}

View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package cloudup
package wellknownassets
import (
"fmt"

View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package cloudup
package wellknownassets
import (
"fmt"

View File

@ -57,6 +57,7 @@ import (
"k8s.io/kops/pkg/model/iam"
"k8s.io/kops/pkg/model/openstackmodel"
"k8s.io/kops/pkg/model/scalewaymodel"
"k8s.io/kops/pkg/nodemodel"
"k8s.io/kops/pkg/nodemodel/wellknownassets"
"k8s.io/kops/pkg/templates"
"k8s.io/kops/pkg/wellknownports"
@ -74,7 +75,6 @@ import (
"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
"k8s.io/kops/upup/pkg/fi/cloudup/terraformWriter"
"k8s.io/kops/util/pkg/architectures"
"k8s.io/kops/util/pkg/hashing"
"k8s.io/kops/util/pkg/mirrors"
"k8s.io/kops/util/pkg/vfs"
)
@ -103,9 +103,6 @@ type ApplyClusterCmd struct {
InstanceGroups []*kops.InstanceGroup
// NodeUpAssets are the assets for downloading nodeup
NodeUpAssets map[architectures.Architecture]*mirrors.MirroredAsset
// TargetName specifies how we are operating e.g. direct to GCE, or AWS, or dry-run, or terraform
TargetName string
@ -115,12 +112,6 @@ type ApplyClusterCmd struct {
// OutDir is a local directory in which we place output, can cache files etc
OutDir string
// Assets is a list of sources for files (primarily when not using everything containerized)
// Formats:
// raw url: http://... or https://...
// url with hash: <hex>@http://... or <hex>@https://...
Assets map[architectures.Architecture][]*mirrors.MirroredAsset
Clientset simple.Clientset
// DryRun is true if this is only a dry run
@ -401,7 +392,8 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error {
}
}
if err := c.addFileAssets(assetBuilder); err != nil {
fileAssets := &nodemodel.FileAssets{Cluster: cluster}
if err := fileAssets.AddFileAssets(assetBuilder); err != nil {
return err
}
@ -521,7 +513,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error {
cloud: cloud,
}
configBuilder, err := NewNodeUpConfigBuilder(cluster, assetBuilder, c.Assets, encryptionConfigSecretHash)
configBuilder, err := NewNodeUpConfigBuilder(cluster, assetBuilder, fileAssets.Assets, encryptionConfigSecretHash)
if err != nil {
return err
}
@ -529,7 +521,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error {
KopsModelContext: modelContext,
Lifecycle: clusterLifecycle,
NodeUpConfigBuilder: configBuilder,
NodeUpAssets: c.NodeUpAssets,
NodeUpAssets: fileAssets.NodeUpAssets,
}
{
@ -1036,143 +1028,6 @@ func (c *ApplyClusterCmd) validateKubernetesVersion() error {
return nil
}
// addFileAssets adds the file assets within the assetBuilder
func (c *ApplyClusterCmd) addFileAssets(assetBuilder *assets.AssetBuilder) error {
var baseURL string
if components.IsBaseURL(c.Cluster.Spec.KubernetesVersion) {
baseURL = c.Cluster.Spec.KubernetesVersion
} else {
baseURL = "https://dl.k8s.io/release/v" + c.Cluster.Spec.KubernetesVersion
}
c.Assets = make(map[architectures.Architecture][]*mirrors.MirroredAsset)
c.NodeUpAssets = make(map[architectures.Architecture]*mirrors.MirroredAsset)
for _, arch := range architectures.GetSupported() {
c.Assets[arch] = []*mirrors.MirroredAsset{}
k8sAssetsNames := []string{
fmt.Sprintf("/bin/linux/%s/kubelet", arch),
fmt.Sprintf("/bin/linux/%s/kubectl", arch),
}
if needsMounterAsset(c.Cluster, c.InstanceGroups) {
k8sAssetsNames = append(k8sAssetsNames, fmt.Sprintf("/bin/linux/%s/mounter", arch))
}
for _, an := range k8sAssetsNames {
k, err := url.Parse(baseURL)
if err != nil {
return err
}
k.Path = path.Join(k.Path, an)
u, hash, err := assetBuilder.RemapFileAndSHA(k)
if err != nil {
return err
}
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash))
}
kubernetesVersion, _ := util.ParseKubernetesVersion(c.Cluster.Spec.KubernetesVersion)
cloudProvider := c.Cluster.Spec.GetCloudProvider()
if ok := apiModel.UseExternalKubeletCredentialProvider(*kubernetesVersion, cloudProvider); ok {
switch cloudProvider {
case kops.CloudProviderGCE:
binaryLocation := c.Cluster.Spec.CloudProvider.GCE.BinariesLocation
if binaryLocation == nil {
binaryLocation = fi.PtrTo("https://storage.googleapis.com/k8s-staging-cloud-provider-gcp/auth-provider-gcp")
}
// VALID FOR 60 DAYS WE REALLY NEED TO MERGE https://github.com/kubernetes/cloud-provider-gcp/pull/601 and CUT A RELEASE
k, err := url.Parse(fmt.Sprintf("%s/linux-%s/v20231005-providersv0.27.1-65-g8fbe8d27", *binaryLocation, arch))
if err != nil {
return err
}
hashes := map[architectures.Architecture]string{
"amd64": "827d558953d861b81a35c3b599191a73f53c1f63bce42c61e7a3fee21a717a89",
"arm64": "f1617c0ef77f3718e12a3efc6f650375d5b5e96eebdbcbad3e465e89e781bdfa",
}
hash, err := hashing.FromString(hashes[arch])
if err != nil {
return fmt.Errorf("unable to parse auth-provider-gcp binary asset hash %q: %v", hashes[arch], err)
}
u, err := assetBuilder.RemapFileAndSHAValue(k, hashes[arch])
if err != nil {
return err
}
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash))
case kops.CloudProviderAWS:
binaryLocation := c.Cluster.Spec.CloudProvider.AWS.BinariesLocation
if binaryLocation == nil {
binaryLocation = fi.PtrTo("https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1")
}
k, err := url.Parse(fmt.Sprintf("%s/linux/%s/ecr-credential-provider-linux-%s", *binaryLocation, arch, arch))
if err != nil {
return err
}
u, hash, err := assetBuilder.RemapFileAndSHA(k)
if err != nil {
return err
}
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash))
}
}
{
cniAsset, cniAssetHash, err := findCNIAssets(c.Cluster, assetBuilder, arch)
if err != nil {
return err
}
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(cniAsset, cniAssetHash))
}
if c.Cluster.Spec.Containerd == nil || !c.Cluster.Spec.Containerd.SkipInstall {
containerdAssetUrl, containerdAssetHash, err := wellknownassets.FindContainerdAsset(c.Cluster, assetBuilder, arch)
if err != nil {
return err
}
if containerdAssetUrl != nil && containerdAssetHash != nil {
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(containerdAssetUrl, containerdAssetHash))
}
runcAssetUrl, runcAssetHash, err := wellknownassets.FindRuncAsset(c.Cluster, assetBuilder, arch)
if err != nil {
return err
}
if runcAssetUrl != nil && runcAssetHash != nil {
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(runcAssetUrl, runcAssetHash))
}
nerdctlAssetUrl, nerdctlAssetHash, err := wellknownassets.FindNerdctlAsset(c.Cluster, assetBuilder, arch)
if err != nil {
return err
}
if nerdctlAssetUrl != nil && nerdctlAssetHash != nil {
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(nerdctlAssetUrl, nerdctlAssetHash))
}
}
crictlAssetUrl, crictlAssetHash, err := wellknownassets.FindCrictlAsset(c.Cluster, assetBuilder, arch)
if err != nil {
return err
}
if crictlAssetUrl != nil && crictlAssetHash != nil {
c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(crictlAssetUrl, crictlAssetHash))
}
asset, err := NodeUpAsset(assetBuilder, arch)
if err != nil {
return err
}
c.NodeUpAssets[arch] = asset
}
return nil
}
// buildPermalink returns a link to our "permalink docs", to further explain an error message
func buildPermalink(key, anchor string) string {
url := "https://github.com/kubernetes/kops/blob/master/permalinks/" + key + ".md"
@ -1190,18 +1045,6 @@ func ChannelForCluster(vfsContext *vfs.VFSContext, c *kops.Cluster) (*kops.Chann
return kops.LoadChannel(vfsContext, channelLocation)
}
// needsMounterAsset checks if we need the mounter program
// This is only needed currently on ContainerOS i.e. GCE, but we don't have a nice way to detect it yet
func needsMounterAsset(c *kops.Cluster, instanceGroups []*kops.InstanceGroup) bool {
// TODO: Do real detection of ContainerOS (but this has to work with image names, and maybe even forked images)
switch c.Spec.GetCloudProvider() {
case kops.CloudProviderGCE:
return true
default:
return false
}
}
type nodeUpConfigBuilder struct {
// Assets is a list of sources for files (primarily when not using everything containerized)
// Formats:
@ -1240,7 +1083,7 @@ func NewNodeUpConfigBuilder(cluster *kops.Cluster, assetBuilder *assets.AssetBui
channelsAsset := map[architectures.Architecture][]*mirrors.MirroredAsset{}
for _, arch := range architectures.GetSupported() {
asset, err := ProtokubeAsset(assetBuilder, arch)
asset, err := wellknownassets.ProtokubeAsset(assetBuilder, arch)
if err != nil {
return nil, err
}
@ -1248,7 +1091,7 @@ func NewNodeUpConfigBuilder(cluster *kops.Cluster, assetBuilder *assets.AssetBui
}
for _, arch := range architectures.GetSupported() {
asset, err := ChannelsAsset(assetBuilder, arch)
asset, err := wellknownassets.ChannelsAsset(assetBuilder, arch)
if err != nil {
return nil, err
}