From 92948cbfd760bfed4540751a3bb81aef970ed547 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Wed, 17 Aug 2022 13:35:51 +0200 Subject: [PATCH] lab: cloudfront update with lambda edge function for redirects Signed-off-by: CrazyMax --- .github/workflows/publish.yml | 17 ++ _releaser/Dockerfile | 28 +++- _releaser/aws.go | 199 ++++++++++++++++++++++- _releaser/cloudfront-lambda-redirects.js | 26 +++ docker-bake.hcl | 18 +- 5 files changed, 276 insertions(+), 12 deletions(-) create mode 100644 _releaser/cloudfront-lambda-redirects.js diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 020e0fae1b..a3a423f908 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -21,6 +21,8 @@ jobs: DOCS_URL="https://docs-stage.docker.com" DOCS_S3_BUCKET="docs.docker.com-stage-us-east-1" DOCS_S3_CONFIG="_website-config-docs-stage.json" + DOCS_CLOUDFRONT_ID="E2Q9X128R7SWCF" # FIXME: use correct cloudfront ID + DOCS_LAMBDA_FUNCTION_REDIRECTS="DockerDocsRedirectFunction" # FIXME: make sure this lambda edge function eixsts DOCS_LAMBDA_FUNCTION_CACHE="arn:aws:lambda:us-east-1:710015040892:function:docs-stage-cache-invalidator" DOCS_SLACK_MSG="Successfully promoted docs-stage from master. https://docs-stage.docker.com/" elif [ "${{ github.ref }}" = "refs/heads/published" ]; then @@ -28,6 +30,8 @@ jobs: DOCS_URL="https://docs.docker.com" DOCS_S3_BUCKET="docs.docker.com-us-east-1" DOCS_S3_CONFIG="_website-config-docs.json" + DOCS_CLOUDFRONT_ID="E2Q9X128R7SWCF" # FIXME: use correct cloudfront ID + DOCS_LAMBDA_FUNCTION_REDIRECTS="DockerDocsRedirectFunction" # FIXME: make sure this lambda edge function eixsts DOCS_LAMBDA_FUNCTION_CACHE="arn:aws:lambda:us-east-1:710015040892:function:docs-cache-invalidator" DOCS_SLACK_MSG="Successfully published docs. https://docs.docker.com/" elif [ "${{ github.ref }}" = "refs/heads/lab" ]; then @@ -44,6 +48,8 @@ jobs: echo "DOCS_AWS_REGION=$DOCS_AWS_REGION" >> $GITHUB_ENV echo "DOCS_S3_BUCKET=$DOCS_S3_BUCKET" >> $GITHUB_ENV echo "DOCS_S3_CONFIG=$DOCS_S3_CONFIG" >> $GITHUB_ENV + echo "DOCS_CLOUDFRONT_ID=$DOCS_CLOUDFRONT_ID" >> $GITHUB_ENV + echo "DOCS_LAMBDA_FUNCTION_REDIRECTS=$DOCS_LAMBDA_FUNCTION_REDIRECTS" >> $GITHUB_ENV echo "DOCS_LAMBDA_FUNCTION_CACHE=$DOCS_LAMBDA_FUNCTION_CACHE" >> $GITHUB_ENV echo "DOCS_SLACK_MSG=$DOCS_SLACK_MSG" >> $GITHUB_ENV - @@ -80,6 +86,17 @@ jobs: AWS_REGION: ${{ env.DOCS_AWS_REGION }} AWS_S3_BUCKET: ${{ env.DOCS_S3_BUCKET }} AWS_S3_CONFIG: ${{ env.DOCS_S3_CONFIG }} + - + name: Update Cloudfront config + uses: docker/bake-action@v2 + with: + targets: aws-cloudfront-update + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: us-east-1 # cloudfront and lambda edge functions are only available in us-east-1 region + AWS_CLOUDFRONT_ID: ${{ env.DOCS_CLOUDFRONT_ID }} + AWS_LAMBDA_FUNCTION: ${{ env.DOCS_LAMBDA_FUNCTION_REDIRECTS }} - name: Invalidate docs website cache if: ${{ env.DOCS_LAMBDA_FUNCTION_CACHE != '' }} diff --git a/_releaser/Dockerfile b/_releaser/Dockerfile index 54be23c84f..a88ada60f8 100644 --- a/_releaser/Dockerfile +++ b/_releaser/Dockerfile @@ -8,15 +8,13 @@ FROM golang:${GO_VERSION}-alpine AS base RUN apk add --no-cache jq openssl ENV CGO_ENABLED=0 WORKDIR /src - -FROM base AS vendor -COPY go.mod go.sum *.go ./ +COPY go.mod go.sum ./ RUN --mount=type=cache,target=/go/pkg/mod \ - go mod tidy && go mod download + go mod download -FROM vendor AS releaser -COPY go.mod go.sum *.go ./ -RUN --mount=type=cache,target=/go/pkg/mod \ +FROM base AS releaser +RUN --mount=type=bind,target=. \ + --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ go build -o /out/releaser . @@ -42,6 +40,7 @@ RUN --mount=type=bind,target=. \ --mount=type=bind,from=releaser,source=/out/releaser,target=/usr/bin/releaser \ --mount=type=secret,id=AWS_ACCESS_KEY_ID \ --mount=type=secret,id=AWS_SECRET_ACCESS_KEY \ + --mount=type=secret,id=AWS_SESSION_TOKEN \ releaser aws s3-update-config FROM base AS aws-lambda-invoke @@ -50,4 +49,19 @@ ARG AWS_LAMBDA_FUNCTION RUN --mount=type=bind,from=releaser,source=/out/releaser,target=/usr/bin/releaser \ --mount=type=secret,id=AWS_ACCESS_KEY_ID \ --mount=type=secret,id=AWS_SECRET_ACCESS_KEY \ + --mount=type=secret,id=AWS_SESSION_TOKEN \ releaser aws lambda-invoke + +FROM base AS aws-cloudfront-update +ARG AWS_REGION +ARG AWS_LAMBDA_FUNCTION +ARG AWS_CLOUDFRONT_ID +RUN --mount=type=bind,target=. \ + --mount=type=bind,from=sitedir,target=/site \ + --mount=type=bind,from=releaser,source=/out/releaser,target=/usr/bin/releaser \ + --mount=type=secret,id=AWS_ACCESS_KEY_ID \ + --mount=type=secret,id=AWS_SECRET_ACCESS_KEY \ + --mount=type=secret,id=AWS_SESSION_TOKEN \ + AWS_LAMBDA_FUNCTION_FILE=cloudfront-lambda-redirects.js \ + REDIRECTS_JSON=$(cat /site/redirects.json) \ + releaser aws cloudfront-update diff --git a/_releaser/aws.go b/_releaser/aws.go index 7469b26c5b..529ff0ac2c 100644 --- a/_releaser/aws.go +++ b/_releaser/aws.go @@ -1,21 +1,29 @@ package main import ( + "archive/zip" + "bytes" "encoding/json" "fmt" - "io/ioutil" "log" + "os" + "path" + "text/template" + "time" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudfront" "github.com/aws/aws-sdk-go/service/lambda" "github.com/aws/aws-sdk-go/service/s3" ) type AwsCmd struct { - S3UpdateConfig AwsS3UpdateConfigCmd `kong:"cmd,name=s3-update-config"` - LambdaInvoke AwsLambdaInvokeCmd `kong:"cmd,name=lambda-invoke"` + S3UpdateConfig AwsS3UpdateConfigCmd `kong:"cmd,name=s3-update-config"` + LambdaInvoke AwsLambdaInvokeCmd `kong:"cmd,name=lambda-invoke"` + CloudfrontUpdate AwsCloudfrontUpdateCmd `kong:"cmd,name=cloudfront-update"` } type AwsS3UpdateConfigCmd struct { @@ -25,7 +33,7 @@ type AwsS3UpdateConfigCmd struct { } func (s *AwsS3UpdateConfigCmd) Run() error { - file, err := ioutil.ReadFile(s.S3Config) + file, err := os.ReadFile(s.S3Config) if err != nil { return fmt.Errorf("failed to read s3 config file %s: %w", s.S3Config, err) } @@ -40,6 +48,9 @@ func (s *AwsS3UpdateConfigCmd) Run() error { Credentials: awsCredentials(), Region: aws.String(s.Region), }) + if err != nil { + return fmt.Errorf("failed to create session: %w", err) + } svc := s3.New(sess) @@ -84,6 +95,186 @@ func (s *AwsLambdaInvokeCmd) Run() error { return nil } +type AwsCloudfrontUpdateCmd struct { + Region string `kong:"name='region',env='AWS_REGION'"` + Function string `kong:"name='lambda-function',env='AWS_LAMBDA_FUNCTION'"` + FunctionFile string `kong:"name='lambda-function-file',env='AWS_LAMBDA_FUNCTION_FILE'"` + CloudfrontID string `kong:"name='cloudfront-id',env='AWS_CLOUDFRONT_ID'"` + RedirectsJSON string `kong:"name='redirects-json',env='REDIRECTS_JSON'"` +} + +func (s *AwsCloudfrontUpdateCmd) Run() error { + var err error + ver := time.Now().UTC().Format(time.RFC3339) + + zipdt, err := getLambdaFunctionZip(s.FunctionFile, s.RedirectsJSON) + if err != nil { + return fmt.Errorf("cannot create lambda function zip: %w", err) + } + + svc := lambda.New(session.Must(session.NewSessionWithOptions(session.Options{ + SharedConfigState: session.SharedConfigEnable, + })), &aws.Config{ + Credentials: awsCredentials(), + Region: aws.String(s.Region), + }) + + function, err := svc.GetFunction(&lambda.GetFunctionInput{ + FunctionName: aws.String(s.Function), + }) + if err != nil { + if aerr, ok := err.(awserr.Error); ok && aerr.Code() != lambda.ErrCodeResourceNotFoundException { + return fmt.Errorf("cannot find lambda function %q: %w", s.Function, err) + } + _, err = svc.CreateFunction(&lambda.CreateFunctionInput{ + FunctionName: aws.String(s.Function), + }) + if aerr, ok := err.(awserr.Error); ok && aerr.Code() != lambda.ErrCodeResourceConflictException { + return err + } + } + codeSha256 := *function.Configuration.CodeSha256 + log.Printf("INFO: updating lambda function %q\n", s.Function) + + updateConfig, err := svc.UpdateFunctionCode(&lambda.UpdateFunctionCodeInput{ + FunctionName: aws.String(s.Function), + ZipFile: zipdt, + }) + if err != nil { + return fmt.Errorf("failed to update lambda function code: %s", err) + } + log.Printf("INFO: lambda function updated successfully (%s)\n", *updateConfig.FunctionArn) + + if codeSha256 == *updateConfig.CodeSha256 { + log.Printf("INFO: lambda function code has not changed. skipping publication...") + return nil + } + + log.Printf("INFO: waiting for lambda function to be processed\n") + // the lambda function code image is never ready right away, AWS has to + // process it, so we wait 3 seconds before trying to publish the version. + time.Sleep(3 * time.Second) + + publishConfig, err := svc.PublishVersion(&lambda.PublishVersionInput{ + FunctionName: aws.String(s.Function), + CodeSha256: aws.String(*updateConfig.CodeSha256), + Description: aws.String(ver), + }) + if err != nil { + return fmt.Errorf("failed to publish lambda function version %q for %q: %w", ver, s.Function, err) + } + log.Printf("INFO: lambda function version %q published successfully (%s)\n", ver, *publishConfig.FunctionArn) + + sess, err := session.NewSession(&aws.Config{ + Credentials: awsCredentials(), + Region: aws.String(s.Region)}, + ) + if err != nil { + return fmt.Errorf("failed to create session: %w", err) + } + + cfrt := cloudfront.New(sess) + cfrtDistrib, err := cfrt.GetDistribution(&cloudfront.GetDistributionInput{ + Id: aws.String(s.CloudfrontID), + }) + if err != nil { + return fmt.Errorf("cannot find cloudfront distribution %q: %w", s.CloudfrontID, err) + } + log.Printf("INFO: cloudfront distribution %q loaded\n", *cfrtDistrib.Distribution.Id) + + cfrtDistribConfig, err := cfrt.GetDistributionConfig(&cloudfront.GetDistributionConfigInput{ + Id: aws.String(s.CloudfrontID), + }) + if err != nil { + return fmt.Errorf("cannot load cloudfront distribution config: %w", err) + } + log.Printf("INFO: cloudfront distribution configuration loaded\n") + + distribConfig := cfrtDistribConfig.DistributionConfig + if distribConfig.DefaultCacheBehavior == nil { + log.Printf("INFO: cloudfront distribution default cache behavior not found. skipping...") + return nil + } + + for _, funcAssoc := range distribConfig.DefaultCacheBehavior.LambdaFunctionAssociations.Items { + if *funcAssoc.EventType != cloudfront.EventTypeViewerRequest { + continue + } + log.Printf("INFO: cloudfront distribution viewer request function ARN found: %q\n", *funcAssoc.LambdaFunctionARN) + } + + log.Printf("INFO: updating cloudfront config with viewer request function ARN %q", *publishConfig.FunctionArn) + distribConfig.DefaultCacheBehavior.LambdaFunctionAssociations = &cloudfront.LambdaFunctionAssociations{ + Quantity: aws.Int64(1), + Items: []*cloudfront.LambdaFunctionAssociation{ + { + EventType: aws.String(cloudfront.EventTypeViewerRequest), + IncludeBody: aws.Bool(false), + LambdaFunctionARN: publishConfig.FunctionArn, + }, + }, + } + _, err = cfrt.UpdateDistribution(&cloudfront.UpdateDistributionInput{ + Id: aws.String(s.CloudfrontID), + IfMatch: cfrtDistrib.ETag, + DistributionConfig: distribConfig, + }) + if err != nil { + return err + } + + log.Printf("INFO: cloudfront config updated successfully\n") + return nil +} + +func getLambdaFunctionZip(funcFilename string, redirectsJSON string) ([]byte, error) { + funcdt, err := os.ReadFile(funcFilename) + if err != nil { + return nil, fmt.Errorf("failed to read lambda function file %q: %w", err) + } + + var funcbuf bytes.Buffer + functpl := template.Must(template.New("").Parse(string(funcdt))) + if err = functpl.Execute(&funcbuf, struct { + RedirectsJSON string + }{ + redirectsJSON, + }); err != nil { + return nil, err + } + + tmpdir, err := os.MkdirTemp("", "lambda-zip") + if err != nil { + return nil, err + } + defer os.RemoveAll(tmpdir) + + zipfile, err := os.Create(path.Join(tmpdir, "lambda-function.zip")) + if err != nil { + return nil, err + } + defer zipfile.Close() + + zipwrite := zip.NewWriter(zipfile) + zipindex, err := zipwrite.Create("index.js") + if err != nil { + return nil, err + } + if _, err = zipindex.Write(funcbuf.Bytes()); err != nil { + return nil, err + } + if err = zipwrite.Close(); err != nil { + return nil, err + } + + zipdt, err := os.ReadFile(zipfile.Name()) + if err != nil { + return nil, err + } + + return zipdt, nil +} + func awsCredentials() *credentials.Credentials { return credentials.NewChainCredentials( []credentials.Provider{ diff --git a/_releaser/cloudfront-lambda-redirects.js b/_releaser/cloudfront-lambda-redirects.js new file mode 100644 index 0000000000..9d86e1288c --- /dev/null +++ b/_releaser/cloudfront-lambda-redirects.js @@ -0,0 +1,26 @@ +'use strict'; + +exports.handler = (event, context, callback) => { + //console.log("event", JSON.stringify(event)); + const request = event.Records[0].cf.request; + const redirects = JSON.parse(`{{.RedirectsJSON}}`); + for (let key in redirects) { + if (key !== request.uri) { + continue; + } + //console.log(`redirect: ${key} to ${redirects[key]}`); + const response = { + status: '301', + statusDescription: 'Moved Permanently', + headers: { + location: [{ + key: 'Location', + value: redirects[key], + }], + }, + } + callback(null, response); + return + } + callback(null, request); +}; diff --git a/docker-bake.hcl b/docker-bake.hcl index bd37880d3e..32e9ca305c 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -114,6 +114,9 @@ variable "AWS_S3_BUCKET" { variable "AWS_S3_CONFIG" { default = "_website-config-docs-stage.json" } +variable "AWS_CLOUDFRONT_ID" { + default = "" +} variable "AWS_LAMBDA_FUNCTION" { default = "" } @@ -123,11 +126,13 @@ target "_common-aws" { AWS_REGION = AWS_REGION AWS_S3_BUCKET = AWS_S3_BUCKET AWS_S3_CONFIG = AWS_S3_CONFIG + AWS_CLOUDFRONT_ID = AWS_CLOUDFRONT_ID AWS_LAMBDA_FUNCTION = AWS_LAMBDA_FUNCTION } secret = [ "id=AWS_ACCESS_KEY_ID,env=AWS_ACCESS_KEY_ID", - "id=AWS_SECRET_ACCESS_KEY,env=AWS_SECRET_ACCESS_KEY" + "id=AWS_SECRET_ACCESS_KEY,env=AWS_SECRET_ACCESS_KEY", + "id=AWS_SESSION_TOKEN,env=AWS_SESSION_TOKEN" ] } @@ -146,3 +151,14 @@ target "aws-lambda-invoke" { no-cache-filter = ["aws-lambda-invoke"] output = ["type=cacheonly"] } + +target "aws-cloudfront-update" { + inherits = ["_common-aws"] + context = "_releaser" + target = "aws-cloudfront-update" + contexts = { + sitedir = DOCS_SITE_DIR + } + no-cache-filter = ["aws-cloudfront-update"] + output = ["type=cacheonly"] +}