From 5c2d09a22b3bd2b8f12f0da477ad87bfabf5e1d0 Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Thu, 18 Jan 2024 10:59:11 +0000 Subject: [PATCH] Update aws regions and add script to automate this --- lib/shared/addon/utils/amazon.js | 52 ++++---- scripts/aws/update-data | 202 +++++++++++++++++++++++++++++++ 2 files changed, 230 insertions(+), 24 deletions(-) create mode 100755 scripts/aws/update-data diff --git a/lib/shared/addon/utils/amazon.js b/lib/shared/addon/utils/amazon.js index b75051b56..57089d2c5 100644 --- a/lib/shared/addon/utils/amazon.js +++ b/lib/shared/addon/utils/amazon.js @@ -629,71 +629,75 @@ export const INSTANCE_TYPES = [ // 2) Managing EC2 Nodes. See https://cloud-images.ubuntu.com/locator/ec2/ (and https://github.com/rancher/machine/blob/master/drivers/amazonec2/region.go#L13) export const REGIONS = [ 'af-south-1', + 'ap-east-1', 'ap-northeast-1', 'ap-northeast-2', + 'ap-northeast-3', + 'ap-south-1', + 'ap-south-2', 'ap-southeast-1', 'ap-southeast-2', 'ap-southeast-3', 'ap-southeast-4', - 'ap-east-1', - 'ap-south-1', - 'ap-south-2', - 'me-south-1', - 'me-central-1', 'ca-central-1', - 'cn-north-1', - 'cn-northwest-1', + 'eu-central-1', + 'eu-central-2', 'eu-north-1', + 'eu-south-1', + 'eu-south-2', 'eu-west-1', 'eu-west-2', 'eu-west-3', - 'eu-central-1', - 'eu-central-2', - 'eu-south-1', - 'eu-south-2', + 'il-central-1', + 'me-central-1', + 'me-south-1', 'sa-east-1', 'us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', - 'us-gov-west-1', 'us-gov-east-1', + 'us-gov-west-1', + 'cn-north-1', + 'cn-northwest-1', ]; // Supports // 1) Managing EKS clusters. See https://docs.aws.amazon.com/general/latest/gr/eks.html export const EKS_REGIONS = [ 'af-south-1', + 'ap-east-1', 'ap-northeast-1', 'ap-northeast-2', 'ap-northeast-3', + 'ap-south-1', + 'ap-south-2', 'ap-southeast-1', 'ap-southeast-2', 'ap-southeast-3', 'ap-southeast-4', - 'ap-east-1', - 'ap-south-1', - 'ap-south-2', - 'me-south-1', - 'me-central-1', 'ca-central-1', - 'cn-north-1', - 'cn-northwest-1', + 'ca-west-1', + 'eu-central-1', + 'eu-central-2', 'eu-north-1', + 'eu-south-1', + 'eu-south-2', 'eu-west-1', 'eu-west-2', 'eu-west-3', - 'eu-central-1', - 'eu-central-2', - 'eu-south-1', - 'eu-south-2', + 'il-central-1', + 'me-central-1', + 'me-south-1', 'sa-east-1', 'us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', - 'us-gov-west-1', 'us-gov-east-1', + 'us-gov-west-1', + 'cn-north-1', + 'cn-northwest-1', ]; // from https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html diff --git a/scripts/aws/update-data b/scripts/aws/update-data new file mode 100755 index 000000000..326f7ca01 --- /dev/null +++ b/scripts/aws/update-data @@ -0,0 +1,202 @@ +#!/usr/bin/env node + +/** + * This script reads the AWS region data from GitHub (the AWS SDK project) and + * checks the regions against the amazon.js file. + * + * It will update this file with the latest regions if there are changes (regions added or removed) + * + */ + +const fs = require('fs'); +const path = require('path'); + +console.log('Updating EC2 and EKS region lists'); +console.log('================================='); +console.log(''); + +const SCRIPT = process.argv[1]; +const DIR = path.resolve(SCRIPT, '../../..'); +const PARTITIONS = ['aws', 'aws-us-gov', 'aws-cn']; +const ENDPOINTS_URL = 'https://raw.githubusercontent.com/boto/botocore/develop/botocore/data/endpoints.json'; +const JS_FILE = path.resolve(DIR, 'lib/shared/addon/utils/amazon.js'); + +const request = require('request'); + +function cleanArrayItem(item) { + item = item.trim(); + + if (item.startsWith('\'')) { + item = item.substr(1); + } + if (item.endsWith(',')) { + item = item.slice(0, -1); + } + if (item.endsWith('\'')) { + item = item.slice(0, -1); + } + return item; +} + +function readExisting(name) { + const data = fs.readFileSync(JS_FILE).toString(); + + const items = []; + + let processing = false; + let done = false; + + data.split(/\r?\n/).forEach(line => { + if (!done) { + if (!processing) { + if (line.includes(`export const ${ name } = [`)) { + processing = true; + } + } else { + if (line === '];') { + processing = false; + done = true; + } else { + items.push(cleanArrayItem(line.trim())); + } + } + } + }); + + return items; +} + +function patchRegions(key, latest) { + const data = fs.readFileSync(JS_FILE).toString(); + + const items = []; + const out = []; + + let processing = false; + let done = false; + + data.split(/\r?\n/).forEach(line => { + if (!done) { + if (!processing) { + if (line.includes(`export const ${ key } = [`)) { + processing = true; + } + out.push(line); + } else { + if (line === '];') { + processing = false; + done = true; + + // We've skipped the existing list - now write the new list + latest.forEach((r) => { + out.push(` '${ r }',`); + }); + + out.push(line); + } else { + items.push(cleanArrayItem(line.trim())); + } + } + } else { + out.push(line); + } + }); + + // Write the file back out; + fs.writeFileSync(JS_FILE, out.join('\n')); +} + +// List regions in a partition +function listRegions(endpoints, partition, svc) { + const p = endpoints.partitions.find((p => p.partition === partition)); + + if (!p) { + console.error(`Can not find partition ${ partition }`); + return; + } + + const regions = []; + + Object.keys(p.services?.[svc]?.endpoints || {}).forEach((r) => { + const v = p.services[svc].endpoints[r]; + + if (!v.deprecated) { + regions.push(r); + // console.log(` ${ r }`); + } + }); + + regions.sort(); + + return regions; +} + +function checkService(endpoints, svc, key) { + const latest = []; + + console.log(`Checking regions for ${ svc }`); + + PARTITIONS.forEach((p) => latest.push(...listRegions(endpoints, p, svc))); + + const existing = readExisting(key); + + const latestMap = {}; + const existingMap = {}; + + latest.forEach((r) => latestMap[r] = true); + existing.forEach((r) => existingMap[r] = true); + + let changes = false; + + latest.forEach((r) => { + if (!existingMap[r]) { + console.log(' + new region ' + r); + changes = true; + } + }); + + existing.forEach((r) => { + if (!latestMap[r]) { + console.log(' - removed region ' + r); + changes = true; + } + }); + + // Patch the changes + + if (changes) { + patchRegions(key, latest); + } else { + console.log(' No region changes'); + } + + console.log(''); + + return changes; +} + +request(ENDPOINTS_URL, function (error, res, body) { + if (error) { + console.error('Could not fetch endpoints data'); + + process.exit(1); + } + + if (res.statusCode !== 200) { + console.error(`Could not fetch endpoints data - status ${ res.statusCode } ${ res.statusMessage }`); + + process.exit(2); + } + + try { + const endpoints = JSON.parse(body); + + checkService(endpoints, 'eks', 'EKS_REGIONS'); + checkService(endpoints, 'ec2', 'REGIONS'); + } catch (e) { + console.error('Error parsing and processing data'); + console.error(e); + + process.exit(3); + } +});