kubernetes
/
registry.k8s.io
mirror of https://github.com/kubernetes/registry.k8s.io.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor codegen and parse gcp data

This commit is contained in:
Benjamin Elder 2023-02-07 20:38:39 -08:00
parent e20b14aaf4
commit 843d3f2d0c
14 changed files with 4984 additions and 5189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5900
pkg/net/cloudcidrs/internal/ranges2go/zz_generated_rawdata.go → pkg/net/cloudcidrs/internal/ranges2go/data/aws-ip-ranges.json
View File

File diff suppressed because it is too large Load Diff

2305
pkg/net/cloudcidrs/internal/ranges2go/data/gcp-cloud.json Normal file
View File

File diff suppressed because it is too large Load Diff

52
pkg/net/cloudcidrs/internal/ranges2go/gen.go
View File

@ -46,19 +46,51 @@ import (
"net/netip"
)
const AWS = "AWS"
// regionToRanges contains a preparsed map of AWS regions to netip.Prefix
var regionToRanges = map[IPInfo][]netip.Prefix{
`
func generateRangesGo(w io.Writer, rtp regionsToPrefixes) error {
// generate source file
func generateRangesGo(w io.Writer, cloudToRTP map[string]regionsToPrefixes) error {
// generate source file header
if _, err := io.WriteString(w, fileHeader); err != nil {
return err
}
// ensure iteration order is predictable
// ensure iteration order is predictable for reproducible codegen
clouds := make([]string, 0, len(cloudToRTP))
for cloud := range cloudToRTP {
clouds = append(clouds, cloud)
}
sort.Strings(clouds)
// generate constants for each cloud
for _, cloud := range clouds {
if _, err := fmt.Fprintf(w, "// %s cloud\nconst %s = %q\n", cloud, cloud, cloud); err != nil {
return err
}
}
// generate main data variable
if _, err := io.WriteString(w, `
// regionToRanges contains a preparsed map of cloud IPInfo to netip.Prefix
var regionToRanges = map[IPInfo][]netip.Prefix{
`,
); err != nil {
return err
}
for _, cloud := range clouds {
rtp := cloudToRTP[cloud]
if err := genCloud(w, cloud, rtp); err != nil {
return err
}
}
if _, err := io.WriteString(w, "}\n"); err != nil {
return err
}
return nil
}
func genCloud(w io.Writer, cloud string, rtp regionsToPrefixes) error {
// ensure iteration order is predictable for reproducible codegen
regions := make([]string, 0, len(rtp))
for region := range rtp {
regions = append(regions, region)
@ -66,7 +98,7 @@ func generateRangesGo(w io.Writer, rtp regionsToPrefixes) error {
sort.Strings(regions)
for _, region := range regions {
prefixes := rtp[region]
if _, err := fmt.Fprintf(w, "\t{Cloud: AWS, Region: %q}: {\n", region); err != nil {
if _, err := fmt.Fprintf(w, "\t{Cloud: %s, Region: %q}: {\n", cloud, region); err != nil {
return err
}
for _, prefix := range prefixes {
@ -99,9 +131,5 @@ func generateRangesGo(w io.Writer, rtp regionsToPrefixes) error {
return err
}
}
if _, err := io.WriteString(w, "}\n"); err != nil {
return err
}
return nil
}

7
pkg/net/cloudcidrs/internal/ranges2go/gen_test.go
View File

@ -80,7 +80,7 @@ func TestGenerateRangesGo(t *testing.T) {
]
}
`
rtp, err := regionsToPrefixesFromRaw(rawData)
rtp, err := parseAWS(rawData)
if err != nil {
t.Fatalf("unexpected error parsing test data: %v", err)
}
@ -110,9 +110,10 @@ import (
"net/netip"
)
// AWS cloud
const AWS = "AWS"
// regionToRanges contains a preparsed map of AWS regions to netip.Prefix
// regionToRanges contains a preparsed map of cloud IPInfo to netip.Prefix
var regionToRanges = map[IPInfo][]netip.Prefix{
{Cloud: AWS, Region: "ap-northeast-2"}: {
netip.PrefixFrom(netip.AddrFrom4([4]byte{3, 5, 140, 0}), 22),
@ -130,7 +131,7 @@ var regionToRanges = map[IPInfo][]netip.Prefix{
`
// generate and compare
w := &bytes.Buffer{}
if err := generateRangesGo(w, rtp); err != nil {
if err := generateRangesGo(w, map[string]regionsToPrefixes{"AWS": rtp}); err != nil {
t.Fatalf("unexpected error generating: %v", err)
}
result := w.String()

54
pkg/net/cloudcidrs/internal/ranges2go/genrawdata.sh
View File

@ -1,54 +0,0 @@
#!/usr/bin/env bash
# Copyright 2022 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit -o nounset -o pipefail
# cd to self
cd "$(dirname "${BASH_SOURCE[0]}")"
DATA_URL='https://ip-ranges.amazonaws.com/ip-ranges.json'
# emit ip ranges data into a go source file with the string contents
# this data changes infrequently and this simplifies generating the runtime
# data
{
cat <<EOF
/*
Copyright $(date +%Y) The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// File generated by genrawdata.sh DO NOT EDIT
package main
// ipRangesRaw contains the contents of ${DATA_URL}
var ipRangesRaw = \`
EOF
curl "${DATA_URL}"
echo '`'
}>zz_generated_rawdata.go

33
pkg/net/cloudcidrs/internal/ranges2go/main.go
View File

@ -18,16 +18,31 @@ limitations under the License.
// See also genrawdata.sh for downloading the raw data to this binary.
package main
import "os"
import (
"os"
"path/filepath"
)
func main() {
// overridable for make verify
outputPath := os.Getenv("OUT_FILE")
dataDir := os.Getenv("DATA_DIR")
if outputPath == "" {
outputPath = "./zz_generated_range_data.go"
}
if dataDir == "" {
dataDir = "./internal/ranges2go/data"
}
// read in data
awsRaw := mustReadFile(filepath.Join(dataDir, "aws-ip-ranges.json"))
gcpRaw := mustReadFile(filepath.Join(dataDir, "gcp-cloud.json"))
// parse raw AWS IP range data
rtp, err := regionsToPrefixesFromRaw(ipRangesRaw)
awsRTP, err := parseAWS(awsRaw)
if err != nil {
panic(err)
}
// parse GCP IP range data
gcpRTP, err := parseGCP(gcpRaw)
if err != nil {
panic(err)
}
@ -36,7 +51,19 @@ func main() {
if err != nil {
panic(err)
}
if err := generateRangesGo(f, rtp); err != nil {
cloudToRTP := map[string]regionsToPrefixes{
"AWS": awsRTP,
"GCP": gcpRTP,
}
if err := generateRangesGo(f, cloudToRTP); err != nil {
panic(err)
}
}
func mustReadFile(filePath string) string {
contents, err := os.ReadFile(filePath)
if err != nil {
panic(err)
}
return string(contents)
}

46
pkg/net/cloudcidrs/internal/ranges2go/parse.go → pkg/net/cloudcidrs/internal/ranges2go/parse_aws.go
View File

@ -22,14 +22,14 @@ import (
"sort"
)
// regionsToPrefixesFromRaw parses raw AWS IP ranges JSON data
// parseAWS parses raw AWS IP ranges JSON data
// and processes it to a regionsToPrefixes map
func regionsToPrefixesFromRaw(raw string) (regionsToPrefixes, error) {
parsed, err := parseIPRangesJSON([]byte(raw))
func parseAWS(raw string) (regionsToPrefixes, error) {
parsed, err := parseAWSIPRangesJSON([]byte(raw))
if err != nil {
return nil, err
}
return regionsToPrefixesFromData(parsed)
return awsRegionsToPrefixesFromData(parsed)
}
/*
@ -37,20 +37,20 @@ func regionsToPrefixesFromRaw(raw string) (regionsToPrefixes, error) {
https://docs.aws.amazon.com/general/latest/gr/aws-ip-ranges.html
*/
type IPRangesJSON struct {
Prefixes []Prefix `json:"prefixes"`
IPv6Prefixes []IPv6Prefix `json:"ipv6_prefixes"`
type AWSIPRangesJSON struct {
Prefixes []AWSPrefix `json:"prefixes"`
IPv6Prefixes []AWSIPv6Prefix `json:"ipv6_prefixes"`
// syncToken and createDate omitted
}
type Prefix struct {
type AWSPrefix struct {
IPPrefix string `json:"ip_prefix"`
Region string `json:"region"`
Service string `json:"service"`
// network_border_group omitted
}
type IPv6Prefix struct {
type AWSIPv6Prefix struct {
IPv6Prefix string `json:"ipv6_prefix"`
Region string `json:"region"`
Service string `json:"service"`
@ -59,19 +59,16 @@ type IPv6Prefix struct {
// parseIPRangesJSON parse AWS IP ranges JSON data
// https://docs.aws.amazon.com/general/latest/gr/aws-ip-ranges.html
func parseIPRangesJSON(rawJSON []byte) (*IPRangesJSON, error) {
r := &IPRangesJSON{}
func parseAWSIPRangesJSON(rawJSON []byte) (*AWSIPRangesJSON, error) {
r := &AWSIPRangesJSON{}
if err := json.Unmarshal(rawJSON, r); err != nil {
return nil, err
}
return r, nil
}
// regionsToPrefixes is the structure we process the JSON into
type regionsToPrefixes map[string][]netip.Prefix
// regionsToPrefixesFromData processes the raw unmarshalled JSON into regionsToPrefixes map
func regionsToPrefixesFromData(data *IPRangesJSON) (regionsToPrefixes, error) {
// awsRegionsToPrefixesFromData processes the raw unmarshalled JSON into regionsToPrefixes map
func awsRegionsToPrefixesFromData(data *AWSIPRangesJSON) (regionsToPrefixes, error) {
// convert from AWS published structure to a map by region, parse Prefixes
rtp := regionsToPrefixes{}
for _, prefix := range data.Prefixes {
@ -105,20 +102,3 @@ func regionsToPrefixesFromData(data *IPRangesJSON) (regionsToPrefixes, error) {
return rtp, nil
}
func dedupeSortedPrefixes(s []netip.Prefix) []netip.Prefix {
l := len(s)
// nothing to do for <= 1
if l <= 1 {
return s
}
// for 1..len(s) if previous entry does not match, keep current
j := 0
for i := 1; i < l; i++ {
if s[i].String() != s[i-1].String() {
s[j] = s[i]
j++
}
}
return s[0:j]
}

32
pkg/net/cloudcidrs/internal/ranges2go/parse_test.go → pkg/net/cloudcidrs/internal/ranges2go/parse_aws_test.go
View File

@ -21,7 +21,7 @@ import (
"testing"
)
func TestParseIPRangesJSON(t *testing.T) {
func TestAWSParseIPRangesJSON(t *testing.T) {
// parse a snapshot of a valid subsest of data
const testData = `{
"syncToken": "1649878400",
@ -43,15 +43,15 @@ func TestParseIPRangesJSON(t *testing.T) {
}
]
}`
expectedParsed := &IPRangesJSON{
Prefixes: []Prefix{
expectedParsed := &AWSIPRangesJSON{
Prefixes: []AWSPrefix{
{
IPPrefix: "3.5.140.0/22",
Region: "ap-northeast-2",
Service: "AMAZON",
},
},
IPv6Prefixes: []IPv6Prefix{
IPv6Prefixes: []AWSIPv6Prefix{
{
IPv6Prefix: "2a05:d07a:a000::/40",
Region: "eu-south-1",
@ -59,7 +59,7 @@ func TestParseIPRangesJSON(t *testing.T) {
},
},
}
parsed, err := parseIPRangesJSON([]byte(testData))
parsed, err := parseAWSIPRangesJSON([]byte(testData))
if err != nil {
t.Fatalf("unexpected error parsing testdata: %v", err)
}
@ -72,17 +72,17 @@ func TestParseIPRangesJSON(t *testing.T) {
}
// parse some bogus data
_, err = parseIPRangesJSON([]byte(`{"prefixes": false}`))
_, err = parseAWSIPRangesJSON([]byte(`{"prefixes": false}`))
if err == nil {
t.Fatal("expected error parsing garbage data but got none")
}
}
func TestRegionsToPrefixesFromData(t *testing.T) {
func TestAWSRegionsToPrefixesFromData(t *testing.T) {
t.Run("bad IPv4 prefixes", func(t *testing.T) {
t.Parallel()
badV4Prefixes := &IPRangesJSON{
Prefixes: []Prefix{
badV4Prefixes := &AWSIPRangesJSON{
Prefixes: []AWSPrefix{
{
IPPrefix: "asdf;asdf,",
Service: "AMAZON",
@ -90,22 +90,22 @@ func TestRegionsToPrefixesFromData(t *testing.T) {
},
},
}
_, err := regionsToPrefixesFromData(badV4Prefixes)
_, err := awsRegionsToPrefixesFromData(badV4Prefixes)
if err == nil {
t.Fatal("expected error parsing bogus prefix but got none")
}
})
t.Run("bad IPv6 prefixes", func(t *testing.T) {
t.Parallel()
badV6Prefixes := &IPRangesJSON{
Prefixes: []Prefix{
badV6Prefixes := &AWSIPRangesJSON{
Prefixes: []AWSPrefix{
{
IPPrefix: "127.0.0.1/32",
Service: "AMAZON",
Region: "us-east-1",
},
},
IPv6Prefixes: []IPv6Prefix{
IPv6Prefixes: []AWSIPv6Prefix{
{
IPv6Prefix: "asdfasdf----....",
Service: "AMAZON",
@ -113,18 +113,18 @@ func TestRegionsToPrefixesFromData(t *testing.T) {
},
},
}
_, err := regionsToPrefixesFromData(badV6Prefixes)
_, err := awsRegionsToPrefixesFromData(badV6Prefixes)
if err == nil {
t.Fatal("expected error parsing bogus prefix but got none")
}
})
}
func TestRegionsToPrefixesFromRaw(t *testing.T) {
func TestParseAWS(t *testing.T) {
t.Run("unparsable data", func(t *testing.T) {
t.Parallel()
badJSON := `{"prefixes":false}`
_, err := regionsToPrefixesFromRaw(badJSON)
_, err := parseAWS(badJSON)
if err == nil {
t.Fatal("expected error parsing bogus raw JSON but got none")
}

93
pkg/net/cloudcidrs/internal/ranges2go/parse_gcp.go Normal file
View File

@ -0,0 +1,93 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"encoding/json"
"errors"
"net/netip"
"sort"
)
// parseGCP parses raw GCP cloud.json data
// and processes it to a regionsToPrefixes map
func parseGCP(raw string) (regionsToPrefixes, error) {
parsed, err := parseGCPCloudJSON([]byte(raw))
if err != nil {
return nil, err
}
return gcpRegionsToPrefixesFromData(parsed)
}
type GCPCloudJSON struct {
Prefixes []GCPPrefix `json:"prefixes"`
// syncToken and createDate omitted
}
type GCPPrefix struct {
IPv4Prefix string `json:"ipv4Prefix"`
IPv6Prefix string `json:"ipv6Prefix"`
Scope string `json:"scope"`
// service omitted
}
// parseGCPCloudJSON parses GCP cloud.json IP ranges JSON data
func parseGCPCloudJSON(rawJSON []byte) (*GCPCloudJSON, error) {
r := &GCPCloudJSON{}
if err := json.Unmarshal(rawJSON, r); err != nil {
return nil, err
}
return r, nil
}
// gcpRegionsToPrefixesFromData processes the raw unmarshalled JSON into regionsToPrefixes map
func gcpRegionsToPrefixesFromData(data *GCPCloudJSON) (regionsToPrefixes, error) {
// convert from AWS published structure to a map by region, parse Prefixes
rtp := regionsToPrefixes{}
for _, prefix := range data.Prefixes {
region := prefix.Scope
if prefix.IPv4Prefix != "" {
ipPrefix, err := netip.ParsePrefix(prefix.IPv4Prefix)
if err != nil {
return nil, err
}
rtp[region] = append(rtp[region], ipPrefix)
} else if prefix.IPv6Prefix != "" {
ipPrefix, err := netip.ParsePrefix(prefix.IPv6Prefix)
if err != nil {
return nil, err
}
rtp[region] = append(rtp[region], ipPrefix)
} else {
return nil, errors.New("unexpected entry with no ipv4Prefix or ipv6Prefix")
}
}
// flatten
numPrefixes := 0
for region := range rtp {
// this approach allows us to produce consistent generated results
// since the ip ranges will be ordered
sort.Slice(rtp[region], func(i, j int) bool {
return rtp[region][i].String() < rtp[region][j].String()
})
rtp[region] = dedupeSortedPrefixes(rtp[region])
numPrefixes += len(rtp[region])
}
return rtp, nil
}

117
pkg/net/cloudcidrs/internal/ranges2go/parse_gcp_test.go Normal file
View File

@ -0,0 +1,117 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"reflect"
"testing"
)
func TestGCPParseIPRangesJSON(t *testing.T) {
// parse a snapshot of a valid subsest of data
const testData = `{
"syncToken": "1675807451971",
"creationTime": "2023-02-07T14:04:11.9716",
"prefixes": [{
"ipv4Prefix": "34.80.0.0/15",
"service": "Google Cloud",
"scope": "asia-east1"
}, {
"ipv6Prefix": "2600:1900:4180::/44",
"service": "Google Cloud",
"scope": "us-west4"
}]
}
`
expectedParsed := &GCPCloudJSON{
Prefixes: []GCPPrefix{
{
IPv4Prefix: "34.80.0.0/15",
Scope: "asia-east1",
},
{
IPv6Prefix: "2600:1900:4180::/44",
Scope: "us-west4",
},
},
}
parsed, err := parseGCPCloudJSON([]byte(testData))
if err != nil {
t.Fatalf("unexpected error parsing testdata: %v", err)
}
if !reflect.DeepEqual(expectedParsed, parsed) {
t.Error("parsed did not match expected:")
t.Errorf("%#v", expectedParsed)
t.Error("parsed: ")
t.Errorf("%#v", parsed)
t.Fail()
}
// parse some bogus data
_, err = parseAWSIPRangesJSON([]byte(`{"prefixes": false}`))
if err == nil {
t.Fatal("expected error parsing garbage data but got none")
}
}
func TestGCPRegionsToPrefixesFromData(t *testing.T) {
t.Run("bad IPv4 prefixes", func(t *testing.T) {
t.Parallel()
badV4Prefixes := &GCPCloudJSON{
Prefixes: []GCPPrefix{
{
IPv4Prefix: "asdf;asdf,",
Scope: "us-east-1",
},
},
}
_, err := gcpRegionsToPrefixesFromData(badV4Prefixes)
if err == nil {
t.Fatal("expected error parsing bogus prefix but got none")
}
})
t.Run("bad IPv6 prefixes", func(t *testing.T) {
t.Parallel()
badV6Prefixes := &GCPCloudJSON{
Prefixes: []GCPPrefix{
{
IPv4Prefix: "127.0.0.1/32",
Scope: "us-east-1",
},
{
IPv6Prefix: "asdfasdf----....",
Scope: "us-east-1",
},
},
}
_, err := gcpRegionsToPrefixesFromData(badV6Prefixes)
if err == nil {
t.Fatal("expected error parsing bogus prefix but got none")
}
})
}
func TestParseGCP(t *testing.T) {
t.Run("unparsable data", func(t *testing.T) {
t.Parallel()
badJSON := `{"prefixes":false}`
_, err := parseGCP(badJSON)
if err == nil {
t.Fatal("expected error parsing bogus raw JSON but got none")
}
})
}

36
pkg/net/cloudcidrs/internal/ranges2go/prefix_helpers.go Normal file
View File

@ -0,0 +1,36 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import "net/netip"
func dedupeSortedPrefixes(s []netip.Prefix) []netip.Prefix {
l := len(s)
// nothing to do for <= 1
if l <= 1 {
return s
}
// for 1..len(s) if previous entry does not match, keep current
j := 0
for i := 1; i < l; i++ {
if s[i].String() != s[i-1].String() {
s[j] = s[i]
j++
}
}
return s[0:j]
}

24
pkg/net/cloudcidrs/internal/ranges2go/regionstoprefixes.go Normal file
View File

@ -0,0 +1,24 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"net/netip"
)
// regionToPrefixes is the structure we process the JSON into
type regionsToPrefixes map[string][]netip.Prefix

24
pkg/net/cloudcidrs/internal/ranges2go/update-data.sh Executable file
View File

@ -0,0 +1,24 @@
#!/usr/bin/env bash
# Copyright 2022 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit -o nounset -o pipefail
# cd to self
cd "$(dirname "${BASH_SOURCE[0]}")"
# fetch data for each supported cloud
curl -Lo 'data/aws-ip-ranges.json' 'https://ip-ranges.amazonaws.com/ip-ranges.json'
curl -Lo 'data/gcp-cloud.json' 'https://www.gstatic.com/ipranges/cloud.json'

1450
pkg/net/cloudcidrs/zz_generated_range_data.go
View File

File diff suppressed because it is too large Load Diff