grpc: Implement a static multiple IP address gRPC resolver (#6270)

- Implement a static resolver for the gPRC dialer under the scheme `static:///` which allows the dialer to resolve a backend from a static list of IPv4/IPv6 addresses passed via the existing JSON config. - Add config key `serverAddresses` to the `GRPCClientConfig` which, when populated, enables static IP resolution of gRPC server backends. - Set `config-next` to use static gRPC backend resolution for all SA clients. - Generate a new SA certificate which adds `10.77.77.77` and `10.88.88.88` to the SANs. Resolves #6255
2022-08-05 10:20:57 -07:00 · 2022-08-05 10:20:57 -07:00 · 576b6777b5
parent b6c4d9bc21
commit 576b6777b5
17 changed files with 292 additions and 61 deletions
--- a/cmd/config.go
+++ b/cmd/config.go
@ -245,8 +245,17 @@ func (d *ConfigDuration) UnmarshalYAML(unmarshal func(interface{}) error) error

 // GRPCClientConfig contains the information needed to talk to the gRPC service
 type GRPCClientConfig struct {
+	// ServerAddress is a single host:port combination that the gRPC client
+	// will, if necessary, resolve via DNS and then connect to. This field
+	// cannot be used in combination with `ServerIPAddresses` field.
 	ServerAddress string
-	Timeout       ConfigDuration
+	// ServerIPAddresses is a list of IPv4/6 addresses, in the format IPv4:port,
+	// [IPv6]:port or :port, that the gRPC client will connect to. Note that the
+	// server's certificate will be validated against these IP addresses, so
+	// they must be present in the SANs of the server certificate. This field
+	// cannot be used in combination with `ServerAddress`.
+	ServerIPAddresses []string
+	Timeout           ConfigDuration
 }

 // GRPCServerConfig contains the information needed to run a gRPC service
--- a/grpc/client.go
+++ b/grpc/client.go
@ -4,6 +4,7 @@ import (
 	"crypto/tls"
 	"errors"
 	"net"
+	"strings"

 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/honeycombio/beeline-go/wrappers/hnygrpc"
@ -26,8 +27,10 @@ func ClientSetup(c *cmd.GRPCClientConfig, tlsConfig *tls.Config, metrics clientM
 	if c == nil {
 		return nil, errors.New("nil gRPC client config provided. JSON config is probably missing a fooService section.")
 	}
-	if c.ServerAddress == "" {
-		return nil, errors.New("ServerAddress must not be empty")
+	if c.ServerIPAddresses != nil && c.ServerAddress != "" {
+		return nil, errors.New(
+			"both 'serverIPAddresses' and 'serverAddress' are set in gRPC client config provided. Only one should be set.",
+		)
 	}
 	if tlsConfig == nil {
 		return nil, errNilTLS
@ -40,17 +43,27 @@ func ClientSetup(c *cmd.GRPCClientConfig, tlsConfig *tls.Config, metrics clientM
 		hnygrpc.UnaryClientInterceptor(),
 	}
 	allInterceptors = append(interceptors, allInterceptors...)
-	host, _, err := net.SplitHostPort(c.ServerAddress)
-	if err != nil {
-		return nil, err
+
+	var target string
+	var hostOverride string
+	if c.ServerAddress != "" {
+		var splitHostPortErr error
+		hostOverride, _, splitHostPortErr = net.SplitHostPort(c.ServerAddress)
+		if splitHostPortErr != nil {
+			return nil, splitHostPortErr
+		}
+		target = "dns:///" + c.ServerAddress
+	} else {
+		target = "static:///" + strings.Join(c.ServerIPAddresses, ",")
 	}
-	creds := bcreds.NewClientCredentials(tlsConfig.RootCAs, tlsConfig.Certificates, host)
+	creds := bcreds.NewClientCredentials(tlsConfig.RootCAs, tlsConfig.Certificates, hostOverride)
 	return grpc.Dial(
-		"dns:///"+c.ServerAddress,
+		target,
 		grpc.WithBalancerName("round_robin"),
 		grpc.WithTransportCredentials(creds),
 		grpc.WithChainUnaryInterceptor(allInterceptors...),
 	)
+
 }

 type registry interface {
--- a/grpc/client_test.go
+++ b/grpc/client_test.go
@ -0,0 +1,43 @@
+package grpc
+
+import (
+	"crypto/tls"
+	"testing"
+
+	"github.com/jmhodges/clock"
+	"github.com/letsencrypt/boulder/cmd"
+	"github.com/letsencrypt/boulder/test"
+	"google.golang.org/grpc"
+	_ "google.golang.org/grpc/health"
+)
+
+func TestClientSetup(t *testing.T) {
+	tests := []struct {
+		name         string
+		cfg          *cmd.GRPCClientConfig
+		expectTarget string
+		wantErr      bool
+	}{
+		{"valid, address provided", &cmd.GRPCClientConfig{ServerAddress: "localhost:8080"}, "dns:///localhost:8080", false},
+		{"valid, implicit localhost with port provided", &cmd.GRPCClientConfig{ServerAddress: ":8080"}, "dns:///:8080", false},
+		{"valid, IPv6 address provided", &cmd.GRPCClientConfig{ServerAddress: "[::1]:8080"}, "dns:///[::1]:8080", false},
+		{"valid, two addresses provided", &cmd.GRPCClientConfig{ServerIPAddresses: []string{"127.0.0.1:8080", "127.0.0.2:8080"}}, "static:///127.0.0.1:8080,127.0.0.2:8080", false},
+		{"valid, two addresses provided, one has an implicit localhost, ", &cmd.GRPCClientConfig{ServerIPAddresses: []string{":8080", "127.0.0.2:8080"}}, "static:///:8080,127.0.0.2:8080", false},
+		{"valid, two addresses provided, one is IPv6, ", &cmd.GRPCClientConfig{ServerIPAddresses: []string{"[::1]:8080", "127.0.0.2:8080"}}, "static:///[::1]:8080,127.0.0.2:8080", false},
+		{"invalid, both address and addresses provided", &cmd.GRPCClientConfig{ServerAddress: "localhost:8080", ServerIPAddresses: []string{"127.0.0.1:8080"}}, "", true},
+		{"invalid, no address or addresses provided", &cmd.GRPCClientConfig{}, "", true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			client, err := ClientSetup(tt.cfg, &tls.Config{}, clientMetrics{}, clock.NewFake(), []grpc.UnaryClientInterceptor{}...)
+			if tt.wantErr {
+				test.AssertError(t, err, "expected error, got nil")
+			} else {
+				test.AssertNotError(t, err, "unexpected error")
+			}
+			if tt.expectTarget != "" {
+				test.AssertEquals(t, client.Target(), tt.expectTarget)
+			}
+		})
+	}
+}
--- a/grpc/resolver.go
+++ b/grpc/resolver.go
@ -0,0 +1,101 @@
+package grpc
+
+import (
+	"fmt"
+	"net"
+	"strings"
+
+	"google.golang.org/grpc/resolver"
+)
+
+// staticBuilder implements the `resolver.Builder` interface.
+type staticBuilder struct{}
+
+// newStaticBuilder creates a `staticBuilder` used to construct static DNS
+// resolvers.
+func newStaticBuilder() resolver.Builder {
+	return &staticBuilder{}
+}
+
+// Build implements the `resolver.Builder` interface and is usually called by
+// the gRPC dialer. It takes a target containing a comma separated list of
+// IPv4/6 addresses and a `resolver.ClientConn` and returns a `staticResolver`
+// which implements the `resolver.Resolver` interface.
+func (sb *staticBuilder) Build(target resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) {
+	var resolverAddrs []resolver.Address
+	for _, address := range strings.Split(target.Endpoint, ",") {
+		parsedAddress, err := parseResolverIPAddress(address)
+		if err != nil {
+			return nil, err
+		}
+		resolverAddrs = append(resolverAddrs, *parsedAddress)
+	}
+	return newStaticResolver(cc, resolverAddrs), nil
+}
+
+// Scheme returns the scheme that `staticBuilder` will be registered for, for
+// example: `static:///`.
+func (sb *staticBuilder) Scheme() string {
+	return "static"
+}
+
+// staticResolver is used to wrap an inner `resolver.ClientConn` and implements
+// the `resolver.Resolver` interface.
+type staticResolver struct {
+	cc resolver.ClientConn
+}
+
+// newStaticResolver takes a `resolver.ClientConn` and a list of
+// `resolver.Addresses`. It updates the state of the `resolver.ClientConn` with
+// the provided addresses and returns a `staticResolver` which wraps the
+// `resolver.ClientConn` and implements the `resolver.Resolver` interface.
+func newStaticResolver(cc resolver.ClientConn, resolverAddrs []resolver.Address) resolver.Resolver {
+	cc.UpdateState(resolver.State{Addresses: resolverAddrs})
+	return &staticResolver{cc: cc}
+}
+
+// ResolveNow is a no-op necessary for `staticResolver` to implement the
+// `resolver.Resolver` interface. This resolver is constructed once by
+// staticBuilder.Build and the state of the inner `resolver.ClientConn` is never
+// updated.
+func (sr *staticResolver) ResolveNow(_ resolver.ResolveNowOptions) {}
+
+// Close is a no-op necessary for `staticResolver` to implement the
+// `resolver.Resolver` interface.
+func (sr *staticResolver) Close() {}
+
+// parseResolverIPAddress takes an IPv4/6 address (ip:port, [ip]:port, or :port)
+// and returns a properly formatted `resolver.Address` object. The `Addr` and
+// `ServerName` fields of the returned `resolver.Address` will both be set to
+// host:port or [host]:port if the host is an IPv6 address.
+func parseResolverIPAddress(addr string) (*resolver.Address, error) {
+	host, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil, fmt.Errorf("splitting host and port for address %q: %w", addr, err)
+	}
+	if port == "" {
+		// If the port field is empty the address ends with colon (e.g.
+		// "[::1]:").
+		return nil, fmt.Errorf("address %q missing port after port-separator colon", addr)
+	}
+	if host == "" {
+		// Address only has a port (i.e ipv4-host:port, [ipv6-host]:port,
+		// host-name:port). Keep consistent with net.Dial(); if the host is
+		// empty (e.g. :80), the local system is assumed.
+		host = "127.0.0.1"
+	}
+	if net.ParseIP(host) == nil {
+		// Host is a DNS name or an IPv6 address without brackets.
+		return nil, fmt.Errorf("address %q is not an IP address", addr)
+	}
+	parsedAddr := net.JoinHostPort(host, port)
+	return &resolver.Address{
+		Addr:       parsedAddr,
+		ServerName: parsedAddr,
+	}, nil
+}
+
+// init registers the `staticBuilder` with the gRPC resolver registry.
+func init() {
+	resolver.Register(newStaticBuilder())
+}
--- a/grpc/resolver_test.go
+++ b/grpc/resolver_test.go
@ -0,0 +1,34 @@
+package grpc
+
+import (
+	"testing"
+
+	"github.com/letsencrypt/boulder/test"
+	"google.golang.org/grpc/resolver"
+)
+
+func Test_parseResolverIPAddress(t *testing.T) {
+	tests := []struct {
+		name         string
+		addr         string
+		expectTarget *resolver.Address
+		wantErr      bool
+	}{
+		{"valid, IPv4 address", "127.0.0.1:1337", &resolver.Address{Addr: "127.0.0.1:1337", ServerName: "127.0.0.1:1337"}, false},
+		{"valid, IPv6 address", "[::1]:1337", &resolver.Address{Addr: "[::1]:1337", ServerName: "[::1]:1337"}, false},
+		{"valid, port only", ":1337", &resolver.Address{Addr: "127.0.0.1:1337", ServerName: "127.0.0.1:1337"}, false},
+		{"invalid, hostname address", "localhost:1337", nil, true},
+		{"invalid, IPv6 address, no brackets", "::1:1337", nil, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseResolverIPAddress(tt.addr)
+			if tt.wantErr {
+				test.AssertError(t, err, "expected error, got nil")
+			} else {
+				test.AssertNotError(t, err, "unexpected error")
+			}
+			test.AssertDeepEquals(t, got, tt.expectTarget)
+		})
+	}
+}
--- a/test/PKI.md
+++ b/test/PKI.md
@ -19,8 +19,8 @@ for you, you may need to create a combined bundle containing
 The gRPC PKI is under test/grpc-creds/. Each Boulder component has two
 hostnames, each resolving to a different IP address in our test environment,
 plus a third hostname that resolves to both IP addresses. Certificates for these
-components contain all three hostnames, and are stored under
-test/grpc-creds/SERVICE.boulder.
+components contain all three hostnames, both test IP addresses, and are stored
+under test/grpc-creds/SERVICE.boulder.

 To issue new certificates in the WFE or gRPC PKI, install
 https://github.com/jsha/minica, cd to the directory containing minica.pem for
--- a/test/config-next/admin-revoker.json
+++ b/test/config-next/admin-revoker.json
@ -14,7 +14,10 @@
      "timeout": "15s"
    },
    "saService": {
-      "serverAddress": "sa.boulder:9095",
+      "serverIPAddresses": [
+        "10.77.77.77:9095",
+        "10.88.88.88:9095"
+      ],
      "timeout": "15s"
    },
    "features": {
--- a/test/config-next/ca-a.json
+++ b/test/config-next/ca-a.json
@ -33,7 +33,10 @@
      ]
    },
    "saService": {
-      "serverAddress": "sa.boulder:9095",
+      "serverIPAddresses": [
+        "10.77.77.77:9095",
+        "10.88.88.88:9095"
+      ],
      "timeout": "15s"
    },
    "issuance": {
--- a/test/config-next/ca-b.json
+++ b/test/config-next/ca-b.json
@ -33,7 +33,10 @@
      ]
    },
    "saService": {
-      "serverAddress": "sa.boulder:9095",
+      "serverIPAddresses": [
+        "10.77.77.77:9095",
+        "10.88.88.88:9095"
+      ],
      "timeout": "15s"
    },
    "issuance": {
--- a/test/config-next/crl-updater.json
+++ b/test/config-next/crl-updater.json
@ -11,7 +11,10 @@
      "timeout": "15s"
    },
    "saService": {
-      "serverAddress": "sa.boulder:9095",
+      "serverIPAddresses": [
+        "10.77.77.77:9095",
+        "10.88.88.88:9095"
+      ],
      "timeout": "15s"
    },
    "issuerCerts": [
--- a/test/config-next/expiration-mailer.json
+++ b/test/config-next/expiration-mailer.json
@ -20,7 +20,10 @@
      "keyFile": "test/grpc-creds/expiration-mailer.boulder/key.pem"
    },
    "saService": {
-      "serverAddress": "sa.boulder:9095",
+      "serverIPAddresses": [
+        "10.77.77.77:9095",
+        "10.88.88.88:9095"
+      ],
      "timeout": "15s"
    },
    "SMTPTrustedRootFile": "test/mail-test-srv/minica.pem",
--- a/test/config-next/orphan-finder.json
+++ b/test/config-next/orphan-finder.json
@ -22,7 +22,10 @@
    "timeout": "15s"
  },
  "saService": {
-    "serverAddress": "sa.boulder:9095",
+    "serverIPAddresses": [
+      "10.77.77.77:9095",
+      "10.88.88.88:9095"
+    ],
    "timeout": "15s"
  }
 }
--- a/test/config-next/ra.json
+++ b/test/config-next/ra.json
@ -37,7 +37,10 @@
      "timeout": "300s"
    },
    "saService": {
-      "serverAddress": "sa.boulder:9095",
+      "serverIPAddresses": [
+        "10.77.77.77:9095",
+        "10.88.88.88:9095"
+      ],
      "timeout": "15s"
    },
    "akamaiPurgerService": {
--- a/test/config-next/wfe2.json
+++ b/test/config-next/wfe2.json
@ -24,7 +24,10 @@
      "timeout": "15s"
    },
    "saService": {
-      "serverAddress": "sa.boulder:9095",
+      "serverIPAddresses": [
+        "10.77.77.77:9095",
+        "10.88.88.88:9095"
+      ],
      "timeout": "15s"
    },
    "accountCache": {
--- a/test/grpc-creds/generate.sh
+++ b/test/grpc-creds/generate.sh
@ -14,6 +14,12 @@ for SERVICE in admin-revoker expiration-mailer ocsp-updater ocsp-responder \
  minica -domains "${SERVICE}.boulder"
 done

+NEEDIPSANS=( "sa" )
 for SERVICE in publisher nonce ra ca sa va ; do
-  minica -domains "${SERVICE}.boulder,${SERVICE}1.boulder,${SERVICE}2.boulder"
+  if [[ "${NEEDIPSANS[@]}" =~ "${SERVICE}" ]]; then
+    minica -domains "${SERVICE}.boulder,${SERVICE}1.boulder,${SERVICE}2.boulder" \
+      -ip-addresses "10.77.77.77,10.88.88.88"
+  else
+    minica -domains "${SERVICE}.boulder,${SERVICE}1.boulder,${SERVICE}2.boulder"
+  fi
 done
--- a/test/grpc-creds/sa.boulder/cert.pem
+++ b/test/grpc-creds/sa.boulder/cert.pem
@ -1,19 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIIDKTCCAhGgAwIBAgIIHaeuruCOx1AwDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE
-AxMVbWluaWNhIHJvb3QgY2EgM2I4YjJjMCAXDTE5MDEzMTIwMDg0MVoYDzIxMDkw
-MTMxMjAwODQxWjAVMRMwEQYDVQQDEwpzYS5ib3VsZGVyMIIBIjANBgkqhkiG9w0B
-AQEFAAOCAQ8AMIIBCgKCAQEAywJTI+hX1RvKcOuD+g6/dy09tRpoO5NzWQZ3Mu/2
-IDmYRFFq7MCPvB2qPWKLNm/5gBrviy0c2IFlpiXypC6YSye8oax0eOQjqLgSDIM/
-MR/rOFBq5m/h9Fb2SVNNcfCfYlh+UXlBKzRMVdl6K3/l5iuxSuAyZsw9TWmHhBOU
-smqYTfqnjtclRu9yPDZmoYbyO0K+7ZIifFewJkBPQhP+YeK1PwSGr4/7V7ee68Ax
-8bPX68MznLjaBhwGXSs/vzoKpJRXxcrRRGxHiXjknynQA8iOnLmeRPy5gs+rHNtf
-wBC84CQ4taebpuUQwbM02HV84BHtPYSclXhb9F3OhIlacwIDAQABo3AwbjAOBgNV
-HQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1Ud
-EwEB/wQCMAAwLwYDVR0RBCgwJoIKc2EuYm91bGRlcoILc2ExLmJvdWxkZXKCC3Nh
-Mi5ib3VsZGVyMA0GCSqGSIb3DQEBCwUAA4IBAQBntsWlFhN5UHYK7ok7nRNaO5e/
-e4gAOajaELXhirRLRmWXifM7ZcwykmM+bT6/nt8Z6B4sy1T3EGpPqnyiBkaXpc8K
-OH3x9RmPXfIvgBjch/fccFE25zv0p+0uwlHlZk1+8MoMG2qJfAPqqTfNcE1iHejz
-GYdxsmydIPH8ux3yalxyOyqk6twbjbTDVZffAkKKHz3IvD6ERGYUzhr8nt02YvUw
-GnloyTmpnmMqX5JJBnt3/K8tMuZnSVQhTmoGRdWuAM6uD6iwr6oKerOEY5uiASh6
-ShNWLF0PPv9O/6XxThd2UJ79/hJn7gMqfYIeFD5Xx9ZD+0oRHqRjiF4+3ci1
+MIIDMzCCAhugAwIBAgIILdXJR+8c3vgwDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE
+AxMVbWluaWNhIHJvb3QgY2EgM2I4YjJjMB4XDTIyMDgwMTIyMjE1MFoXDTI0MDgz
+MTIyMjE1MFowFTETMBEGA1UEAxMKc2EuYm91bGRlcjCCASIwDQYJKoZIhvcNAQEB
+BQADggEPADCCAQoCggEBAKZqmzz0hyp+3O+CBbRQblZli7nOvJbdyfpDOKwno9yY
+/z+dZAY+Xm946i5kgVHTVFfi2ntVBH86DJSnfH/M0N2zo7j2YIHeQXfrLgVG++Am
+5rL0IGGpjf5skkNrynTuwiEoRGVaS3H2xiXx5v1x/Dlu/ZNxkdGw+0ztRks6lQH/
+yhmoWP8Z0QnwHFm+3wiky00mbDkr3JuLjgkOvdTFHFSHRq5TBiG4U1S4fONMBqZS
+TGr8E6yNm5U+pIl6TupwHSYe7BdAhOF0d9titBnsq75rf/sY/f9AKuEF3gBd87Rn
+OtcMXvcdhst/3vGSWqgg2GgyORd4y4cpbO/k/mS/dzsCAwEAAaN8MHowDgYDVR0P
+AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
+Af8EAjAAMDsGA1UdEQQ0MDKCCnNhLmJvdWxkZXKCC3NhMS5ib3VsZGVyggtzYTIu
+Ym91bGRlcocECk1NTYcEClhYWDANBgkqhkiG9w0BAQsFAAOCAQEAh/CrCjLo70Mj
+m3S9QAMuW/qXQkGDdShstf4s+oafCl3/YK5HpyRX1OKFrfbyqLYd/YvUnx4s2nZ
+UlZyak7N/qa01ZL+koejxf0RPEwaZByS/GKokcsaWMmBS1xDi+w8xpq5WIYKgHPO
+QCxK9/c9ehGX9rSzpodWKt9EpLxYlv3kP2/QAvVzOVJkyieZT5R4wP8PHKkSVtEf
+sPlT5YksuJib/5zvOabSY+OpLfhfaDHnO53b0CQx3OI85jNIdivCzg4J08laH+hk
+ouMkG+W2inHDoRqeH8jK0PvwPUWL8O6DSGvMUDR+3yY8XvAI/opOzwiYdtLyDkdc
+q7nLPXzpwA==
 -----END CERTIFICATE-----
--- a/test/grpc-creds/sa.boulder/key.pem
+++ b/test/grpc-creds/sa.boulder/key.pem
@ -1,27 +1,27 @@
 -----BEGIN RSA PRIVATE KEY-----
-MIIEpAIBAAKCAQEAywJTI+hX1RvKcOuD+g6/dy09tRpoO5NzWQZ3Mu/2IDmYRFFq
-7MCPvB2qPWKLNm/5gBrviy0c2IFlpiXypC6YSye8oax0eOQjqLgSDIM/MR/rOFBq
-5m/h9Fb2SVNNcfCfYlh+UXlBKzRMVdl6K3/l5iuxSuAyZsw9TWmHhBOUsmqYTfqn
-jtclRu9yPDZmoYbyO0K+7ZIifFewJkBPQhP+YeK1PwSGr4/7V7ee68Ax8bPX68Mz
-nLjaBhwGXSs/vzoKpJRXxcrRRGxHiXjknynQA8iOnLmeRPy5gs+rHNtfwBC84CQ4
-taebpuUQwbM02HV84BHtPYSclXhb9F3OhIlacwIDAQABAoIBAQCSm4YxW1z7AUvs
-ypkQIFBzn899l6mIxM67jyjMN3GXCiEFzYqbmeDzqLTGCgGhebJEqq8t4pF/Zctj
-9unJWgEP3Y2jcGjY2WFLOkuGj842CugTTjg1XL1geOD6SVeXnn9sRZ5L9nZy3hz2
-rs1E1uPPmuQ5v7mGdZ8b5cdBQebUBU3NfS/vQlJTur/HYRo4i07kD7OMk/0QrwDR
-2pyvuxTOI+4+MeBYt2WL9VK/H9ZYQmFcz3lEN45/4k7CqK6VPWXlSoV7khTbCuSB
-NPHcKvmQ+DfsmBhGkYMFUURF4MRctx7Dw7dKg4N4DOXWLdzQ/P8TjR4tjhvMSkP+
-tqJIGSX5AoGBAP7wNrKVIE0l8AuegVh40hl7/DVEapYk1bAcNZN0NK/ApvloB0Yc
-jfZ9hZRAnaL18pW0jOBPgfYbNq+VsL0rVyprOy+ueA9QfGiDoIqhbUX8bBz/lGhs
-N8CmY0HFhpv/rcNVqOehRjA+bKR/lpgcFs5BjN9IeN5PVBfra4Ay+CyVAoGBAMva
-wAGQp6cbeulPzCSR95GutGlpPrCKTupUF+2zKgdhya8eBjkGi1sn0ej6qjvmx8+Z
-XSO7SvblKdWaWs/1Xyw3SmWVvPRedk69GQgF/GuWZt9xic/hDpyaAAS3vW0sXkLP
-p3QTAfqTDZShVR03Hxl/PR9WghQJPJENlB1UzKDnAoGAVFi/kBg8xqmdoQqBOvdG
-c/4MdMc9CI4JUSoUI8QXxmpUFEJx3aWG5p2i+2jhftAmwZcp4PENS5K3ZiJ9hij3
-vuYZf+4WdOKpNp8OF6/PCo+4aWO6A06Cp6+lOVaT8bsHD5CgwHogUcflhAtelNb+
-SKFxbVJ6Avt2FC/kslaqu30CgYBgkPYlh6WzhlP6E5/rru2sqCq0SHO24z2wyTcw
-lY+SQaNtffaKquv2uW05RQzBJXh/gfHaDE3dmP7xPZZJLr3vzx1B8+W3iMvYTsF9
-yIAjYvLGZB1ZSQ5H5redhICKJ9tbIMz9MkfcsC5duvL7zPHBfUGyB6PE4/8540nH
-fzUT/wKBgQCKpQMInn1Qbm06nCOSII0kwYGiEpqxgODjEb0Ev8KnP28u4wrWnQop
-FCIDJG6UnLCtjh5Y17MCJpOPcf/izMpYgubrXo4JpFvSPp+kcPNUWnggRBZYgmIN
-E9fEvdQXaLt63G0lkYOVGLJomRo6yiR6Cd1UYF6o1eGMxgSsQ46erg==
+MIIEowIBAAKCAQEApmqbPPSHKn7c74IFtFBuVmWLuc68lt3J+kM4rCej3Jj/P51k
+Bj5eb3jqLmSBUdNUV+Lae1UEfzoMlKd8f8zQ3bOjuPZggd5Bd+suBUb74CbmsvQg
+YamN/mySQ2vKdO7CIShEZVpLcfbGJfHm/XH8OW79k3GR0bD7TO1GSzqVAf/KGahY
+/xnRCfAcWb7fCKTLTSZsOSvcm4uOCQ691MUcVIdGrlMGIbhTVLh840wGplJMavwT
+rI2blT6kiXpO6nAdJh7sF0CE4XR322K0Geyrvmt/+xj9/0Aq4QXeAF3ztGc61wxe
+9x2Gy3/e8ZJaqCDYaDI5F3jLhyls7+T+ZL93OwIDAQABAoIBABm0f9QThUlYHTJV
+qCHpBpILz2BdAZ5gFdG1TmyxFst9Snf+DLQ2MAoR6EJQPfVwqieFH/BK+o3YXpcb
+o8xty5ljJRft3oxQ01X9mrcv9rnx6FKeCN5s/UTeal1FqhkTxCKxAJeTMfIhhwvX
+aLGfSLFqZnq+4SI9ryN7xw7Ztqh/JUZhslIAV0e7J74cESkQquhjL7oYHeKv0XCz
+50Sd6QmfyVXVOXOitFT8GMIYH1opAhkeRb5Lq6Xc3HShZQ7fL0OgmlG6GX0ITCrb
+1CO8OhhPqAtoCNFJYCyW0g3SAI55wktTJmCZb3EQZTnR8gOYljj6vIblptw5jNht
+o97qqgkCgYEAwT1Yr0+vBdy/5XFq7HRIiwfk1uyJlCzcXmWfl1wr7vS/Nf2TtEmi
+XvWJx5u0u13yyFmxBpX+IiiprxggrhiYgcsc0PUxhafwzM3BRlh5mcY6fZCStLCP
+qeFq66iA5WLPZIEatTbBHHJUos58yVSBkBZ02KY3guJC7rnIDe1B+xUCgYEA3HcU
+i3j6h2Z7aW2bAK/68Jvky4/1AOuRkcwDpdAk6EvoqeaF1KrXAt7HlhfzG2DX/GOf
+5idxaD1N0BqWZ8lQTrwYiePpvawhFjlOACOcpkaK+n/D6OxPglb0EuypiGbT7fqQ
+VV5YhRdlfkmYUNN+Qn0PjZEwwD0aqY6jKl1G/Q8CgYEAq6Bvxuzv1zYT3ZXZUH+K
+qidL+JP4yHg65o4nzdG8enAdhRs/kA0DUDpCTca0xsDCbzXhLDtUKtq4c99HwNL
+WGPsiQ5s2HerYEsScQcdoV01D9a65e62+jvyObGkIZeb+XjNHsutwjUtRJg+rpX2
+qOG5D765KO8KYXrx2qSx6G0CgYBDaE1pz5WaLKVzOMX50dTCGIg+DUf1/hDGKUC8
+RAXk2MDMoeUtfwa6o+8WX0A/kL65NqS/NDQikQvnoxl1pkHdsPk3v+3JxoVfTCoj
+NMBCvrLjoIkAALDWw0thNecoA3is/L2AvJYIK30mvc3KeL/UCHTm7guf2475ZNlS
+rK511QKBgF1tTR41mMMYyqhBgDmG621S+043LSPwnqmitcme6guVq4vEDZ8j8HP4
+2f+DMtUIEL5Vonu/GuZt3w/FytHZcRWi1CP3c4PwYyHyAAKno0zdZQMvTcxkOJkJ
+8ITPp9511oJsfYaayorxRDZE7UhP1Sb7x/YZmDsB8qjc3tBsatOK
 -----END RSA PRIVATE KEY-----