mirror of https://github.com/knative/pkg.git
Add support for client TLS to pkg/metrics (#1045)
* Document the plan for metrics * Update with issue links * Update with other @mattmoor comments. * Address feedback from @anniefu * Update export address since `localhost` won't work for a DaemonSet. * Add an explicit action item to migrate Prometheus * Fix possible (ambiguous) link vs checkbox. * Fix typo in metrics/README.md * Add support for client TLS to pkg/metrics * Address comments from @vagababov and @anniefu * Finish removing Resource code * Update name to be shorter and not require a double migration. * Add tests for opencensus exporting with TLS
This commit is contained in:
Evan Anderson
GitHub
parent
9b3ea736c6
commit
945b556708
|
|
@ -1401,6 +1401,7 @@
|
|||
"google.golang.org/api/iterator",
|
||||
"google.golang.org/api/option",
|
||||
"google.golang.org/grpc",
|
||||
"google.golang.org/grpc/credentials",
|
||||
"gopkg.in/yaml.v2",
|
||||
"k8s.io/api/admission/v1beta1",
|
||||
"k8s.io/api/admissionregistration/v1beta1",
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ import (
|
|||
"go.uber.org/zap"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
kubeclient "knative.dev/pkg/client/injection/kube/client"
|
||||
secrets "knative.dev/pkg/client/injection/kube/informers/core/v1/secret"
|
||||
"knative.dev/pkg/configmap"
|
||||
"knative.dev/pkg/controller"
|
||||
"knative.dev/pkg/injection"
|
||||
|
|
@ -189,8 +190,9 @@ func MainWithConfig(ctx context.Context, component string, cfg *rest.Config, cto
|
|||
// Watch the observability config map
|
||||
if _, err := kubeclient.Get(ctx).CoreV1().ConfigMaps(system.Namespace()).Get(metrics.ConfigMapName(),
|
||||
metav1.GetOptions{}); err == nil {
|
||||
secretLister := secrets.Get(ctx).Lister()
|
||||
cmw.Watch(metrics.ConfigMapName(),
|
||||
metrics.UpdateExporterFromConfigMap(component, logger),
|
||||
metrics.ConfigMapWatcher(component, secretLister, logger),
|
||||
profilingHandler.UpdateFromConfigMap)
|
||||
} else if !apierrors.IsNotFound(err) {
|
||||
logger.With(zap.Error(err)).Fatalf("Error reading ConfigMap %q", metrics.ConfigMapName())
|
||||
|
|
|
|||
|
|
@ -94,8 +94,8 @@ statistics for a short period of time if not.
|
|||
**This is true today.**
|
||||
[Ensure this on an ongoing basis.](https://github.com/knative/pkg/issues/957)
|
||||
- [ ] Google to implement OpenCensus Agent configuration to match what they are
|
||||
doing for Stackdriver now. (No public issue link because this shoud be in
|
||||
Google's vendor-specific configuration.)
|
||||
doing for Stackdriver now. (No public issue link because this should be
|
||||
in Google's vendor-specific configuration.)
|
||||
- [ ] Document how to configure OpenCensus/OpenTelemetry Agent + Prometheus to
|
||||
achieve the current level of application visibility, and determine a
|
||||
long-term course for how to maintain this as a "bare minimum" supported
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import (
|
|||
|
||||
"go.opencensus.io/stats"
|
||||
"go.uber.org/zap"
|
||||
clientv1 "k8s.io/client-go/listers/core/v1"
|
||||
"knative.dev/pkg/metrics/metricskey"
|
||||
)
|
||||
|
||||
|
|
@ -86,6 +87,10 @@ type metricsConfig struct {
|
|||
// writing the metrics to the stats.RecordWithOptions interface.
|
||||
recorder func(context.Context, []stats.Measurement, ...stats.Options) error
|
||||
|
||||
// secretsLister provides access for fetching Kubernetes Secrets from an
|
||||
// informer cache.
|
||||
secretsLister clientv1.SecretLister
|
||||
|
||||
// ---- OpenCensus specific below ----
|
||||
// collectorAddress is the address of the collector, if not `localhost:55678`
|
||||
collectorAddress string
|
||||
|
|
@ -156,6 +161,10 @@ func (mc *metricsConfig) record(ctx context.Context, mss []stats.Measurement, ro
|
|||
func createMetricsConfig(ops ExporterOptions, logger *zap.SugaredLogger) (*metricsConfig, error) {
|
||||
var mc metricsConfig
|
||||
|
||||
// We don't check if this is `nil` right now, because this is a transition step.
|
||||
// Eventually, this should be a startup check.
|
||||
mc.secretsLister = ops.Secrets
|
||||
|
||||
if ops.Domain == "" {
|
||||
return nil, errors.New("metrics domain cannot be empty")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import (
|
|||
"go.opencensus.io/stats/view"
|
||||
"go.uber.org/zap"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
clientv1 "k8s.io/client-go/listers/core/v1"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -65,17 +66,29 @@ type ExporterOptions struct {
|
|||
// See https://github.com/knative/serving/blob/master/config/config-observability.yaml
|
||||
// for details.
|
||||
ConfigMap map[string]string
|
||||
|
||||
// A lister for Secrets to allow dynamic configuration of outgoing TLS client cert.
|
||||
Secrets clientv1.SecretLister `json:"-"`
|
||||
}
|
||||
|
||||
// UpdateExporterFromConfigMap returns a helper func that can be used to update the exporter
|
||||
// when a config map is updated.
|
||||
// DEPRECATED: Callers should migrate to ConfigMapWatcher.
|
||||
func UpdateExporterFromConfigMap(component string, logger *zap.SugaredLogger) func(configMap *corev1.ConfigMap) {
|
||||
return ConfigMapWatcher(component, nil, logger)
|
||||
}
|
||||
|
||||
// ConfigMapWatcher returns a helper func which updates the exporter configuration based on
|
||||
// values in the supplied ConfigMap. This method captures a corev1.SecretLister which is used
|
||||
// to configure mTLS with the opencensus agent.
|
||||
func ConfigMapWatcher(component string, secrets clientv1.SecretLister, logger *zap.SugaredLogger) func(*corev1.ConfigMap) {
|
||||
domain := Domain()
|
||||
return func(configMap *corev1.ConfigMap) {
|
||||
UpdateExporter(ExporterOptions{
|
||||
Domain: domain,
|
||||
Component: component,
|
||||
ConfigMap: configMap.Data,
|
||||
Secrets: secrets,
|
||||
}, logger)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,9 +14,16 @@ limitations under the License.
|
|||
package metrics
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
|
||||
"contrib.go.opencensus.io/exporter/ocagent"
|
||||
"go.opencensus.io/stats/view"
|
||||
"go.uber.org/zap"
|
||||
"google.golang.org/grpc/credentials"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
clientv1 "k8s.io/client-go/listers/core/v1"
|
||||
"knative.dev/pkg/system"
|
||||
)
|
||||
|
||||
func newOpenCensusExporter(config *metricsConfig, logger *zap.SugaredLogger) (view.Exporter, error) {
|
||||
|
|
@ -24,7 +31,9 @@ func newOpenCensusExporter(config *metricsConfig, logger *zap.SugaredLogger) (vi
|
|||
if config.collectorAddress != "" {
|
||||
opts = append(opts, ocagent.WithAddress(config.collectorAddress))
|
||||
}
|
||||
if !config.requireSecure {
|
||||
if config.requireSecure {
|
||||
opts = append(opts, ocagent.WithTLSCredentials(credentialFetcher(config.component, config.secretsLister, logger)))
|
||||
} else {
|
||||
opts = append(opts, ocagent.WithInsecure())
|
||||
}
|
||||
e, err := ocagent.NewExporter(opts...)
|
||||
|
|
@ -36,3 +45,36 @@ func newOpenCensusExporter(config *metricsConfig, logger *zap.SugaredLogger) (vi
|
|||
view.RegisterExporter(e)
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// credentialFetcher attempts to locate a secret containing TLS credentials
|
||||
// for communicating with the OpenCensus Agent. To do this, it first looks
|
||||
// for a secret named "<component>-opencensus", then for a generic
|
||||
// "opencensus" secret.
|
||||
func credentialFetcher(component string, lister clientv1.SecretLister, logger *zap.SugaredLogger) credentials.TransportCredentials {
|
||||
if lister == nil {
|
||||
logger.Errorf("No secret lister provided for component %q; cannot use requireSecure=true", component)
|
||||
return nil
|
||||
}
|
||||
return credentials.NewTLS(&tls.Config{
|
||||
GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
|
||||
// We ignore the CertificateRequestInfo for now, and hand back a single fixed certificate.
|
||||
// TODO(evankanderson): maybe do something SPIFFE-ier?
|
||||
cert, err := certificateFetcher(component+"-opencensus", lister)
|
||||
if errors.IsNotFound(err) {
|
||||
cert, err = certificateFetcher("opencensus", lister)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Unable to fetch opencensus secret for %q, cannot use requireSecure=true: %+v", component, err)
|
||||
}
|
||||
return &cert, err
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func certificateFetcher(secretName string, lister clientv1.SecretLister) (tls.Certificate, error) {
|
||||
secret, err := lister.Secrets(system.Namespace()).Get(secretName)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, err
|
||||
}
|
||||
return tls.X509KeyPair(secret.Data["client-cert.pem"], secret.Data["client-key.pem"])
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,183 @@
|
|||
/*
|
||||
Copyright 2020 The Knative Authors.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"contrib.go.opencensus.io/exporter/ocagent"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"go.opencensus.io/stats/view"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
clientv1 "k8s.io/client-go/listers/core/v1"
|
||||
logtesting "knative.dev/pkg/logging/testing"
|
||||
)
|
||||
|
||||
func TestOpenCensusConfig(t *testing.T) {
|
||||
cert, err := ioutil.ReadFile(filepath.Join("testdata", "client-cert.pem"))
|
||||
if err != nil {
|
||||
t.Fatalf("Couldn't find testdata/client-cert.pem: %v", err)
|
||||
}
|
||||
key, err := ioutil.ReadFile(filepath.Join("testdata", "client-key.pem"))
|
||||
if err != nil {
|
||||
t.Fatalf("Couldn't find testdata/client-key.pem: %v", err)
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
desc string
|
||||
config metricsConfig
|
||||
tls *tls.Config
|
||||
err error
|
||||
wantFunc func(*testing.T, view.Exporter)
|
||||
}{{
|
||||
desc: "No TLS mostly default",
|
||||
config: metricsConfig{
|
||||
domain: "test",
|
||||
component: "test",
|
||||
backendDestination: OpenCensus,
|
||||
},
|
||||
wantFunc: func(t *testing.T, v view.Exporter) {
|
||||
if v == nil {
|
||||
t.Error("Expected view to be non-nil")
|
||||
}
|
||||
},
|
||||
}, {
|
||||
desc: "With TLS",
|
||||
|
||||
config: metricsConfig{
|
||||
domain: "secure",
|
||||
component: "test",
|
||||
backendDestination: OpenCensus,
|
||||
secretsLister: fakeSecretList(corev1.Secret{
|
||||
Data: map[string][]byte{
|
||||
"client-cert.pem": cert,
|
||||
"client-key.pem": key,
|
||||
},
|
||||
}),
|
||||
requireSecure: true,
|
||||
},
|
||||
tls: &tls.Config{},
|
||||
wantFunc: func(t *testing.T, v view.Exporter) {
|
||||
if v == nil {
|
||||
t.Error("Expected view to be non-nil")
|
||||
}
|
||||
oc, ok := v.(*ocagent.Exporter)
|
||||
if !ok {
|
||||
t.Errorf("Did not get an OpenCensus exporter: %+v", v)
|
||||
}
|
||||
oc.Flush()
|
||||
},
|
||||
}}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.desc, func(t *testing.T) {
|
||||
var server net.Listener
|
||||
var shutdown chan error
|
||||
var err error
|
||||
if c.err == nil {
|
||||
server, shutdown, err = GetServer(c.tls)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to start server: %v", err)
|
||||
}
|
||||
c.config.collectorAddress = server.Addr().String()
|
||||
}
|
||||
|
||||
got, gotErr := newOpenCensusExporter(&c.config, logtesting.TestLogger(t))
|
||||
if c.err != nil {
|
||||
if diff := cmp.Diff(c.err, gotErr); diff != "" {
|
||||
t.Errorf("wrong err (-want +got) = %v", diff)
|
||||
}
|
||||
return
|
||||
}
|
||||
if gotErr != nil {
|
||||
t.Errorf("unexpected err: %v", gotErr)
|
||||
return
|
||||
}
|
||||
if c.wantFunc != nil {
|
||||
c.wantFunc(t, got)
|
||||
}
|
||||
|
||||
t.Logf("Awaiting channel shutdown at %s", server.Addr().String())
|
||||
err = <-shutdown
|
||||
if err != nil {
|
||||
t.Errorf("Error from server: %v", err)
|
||||
}
|
||||
err = server.Close()
|
||||
if err != nil {
|
||||
t.Errorf("Failed to shut down server: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type fakeSecrets struct {
|
||||
secrets []corev1.Secret
|
||||
}
|
||||
|
||||
func fakeSecretList(s ...corev1.Secret) *fakeSecrets {
|
||||
return &fakeSecrets{secrets: s}
|
||||
}
|
||||
|
||||
func (f *fakeSecrets) List(selector labels.Selector) ([]*corev1.Secret, error) {
|
||||
return nil, fmt.Errorf("Not implemented yet!")
|
||||
}
|
||||
|
||||
func (f *fakeSecrets) Secrets(namespace string) clientv1.SecretNamespaceLister {
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *fakeSecrets) Get(name string) (*corev1.Secret, error) {
|
||||
return &f.secrets[0], nil
|
||||
}
|
||||
|
||||
func GetServer(config *tls.Config) (net.Listener, chan error, error) {
|
||||
var server net.Listener
|
||||
var err error
|
||||
if config == nil {
|
||||
server, err = net.Listen("tcp", "localhost:0")
|
||||
} else {
|
||||
if config.Certificates == nil {
|
||||
serverCert, err := tls.LoadX509KeyPair(
|
||||
filepath.Join("testdata", "server-cert.pem"), filepath.Join("testdata", "server-key.pem"))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("Unable to load server cert from testadata: %v", err)
|
||||
}
|
||||
config.Certificates = []tls.Certificate{serverCert}
|
||||
}
|
||||
server, err = tls.Listen("tcp", "localhost:0", config)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("Unable to create listen server: %v", err)
|
||||
}
|
||||
shutdown := make(chan error)
|
||||
go func() {
|
||||
c, err := server.Accept()
|
||||
if err != nil {
|
||||
shutdown <- fmt.Errorf("Failed to accept connection: %v", err)
|
||||
return
|
||||
}
|
||||
err = c.Close()
|
||||
if err != nil {
|
||||
shutdown <- fmt.Errorf("Failed to close server connection: %v", err)
|
||||
return
|
||||
}
|
||||
shutdown <- nil
|
||||
}()
|
||||
return server, shutdown, err
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
# Test files for metrics
|
||||
|
||||
The cert files were generated with:
|
||||
|
||||
```shell
|
||||
openssl req -x509 -nodes -newkey dsa:<(openssl dsaparam 1024) -keyout client-key.pem -out client-cert.pem -days 10000
|
||||
```
|
||||
|
||||
Note that there are some manual prompts later in the process. This seemed simpler than generating the certs in Go.
|
||||
|
||||
To view the cert:
|
||||
```shell
|
||||
openssl x509 -noout -text -in client-cert.pem
|
||||
```
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
-----BEGIN CERTIFICATE-----
|
||||
MIICfDCCAjoCCQDJUTmi/6JoGDALBglghkgBZQMEAwIwIjEgMB4GA1UEAwwXdGVz
|
||||
dC1jbGllbnQuZXhhbXBsZS5jb20wHhcNMjAwMjE0MTg1OTE2WhcNNDcwNzAyMTg1
|
||||
OTE2WjAiMSAwHgYDVQQDDBd0ZXN0LWNsaWVudC5leGFtcGxlLmNvbTCCAbYwggEr
|
||||
BgcqhkjOOAQBMIIBHgKBgQCEMFcdoo8FsDPTY5lupaggF7LrvxEKYOKR+wSug9uk
|
||||
mzreh3KtfJW2YseHUmjqafd367//Ych7gJim8UTEkdjZEfFKK8iRLTRyjNwK0KRb
|
||||
1JqL4qT9wmNpsrZ90KDX3xaJukbOZHumlb5AmJXD1RGSjnSaWqLzuXxmvVVkPnhe
|
||||
AwIVAMj19ML2hIr0O6cqiQg3Zmh/It/VAoGAfLoW+TmD6JAJyxtp7QMVujbP/OY+
|
||||
jV4wwekXBzgZITUWupC0ULtMY+6VLPvS6Up0mnnZxgVFnABpvoCqrc+tG1p7Lnyi
|
||||
2XG9AQ1V9JMRTjerQoDGCS/kMDD7jx37N59tkq6sJ+XYFOVSjfAlRTmW6wV1DpUW
|
||||
m7olKlj6mEL3qI0DgYQAAoGACz1U5FNb5ANwgcc70dvRU0PrBs9HEZd+jaH7yfBV
|
||||
g9Eas6aBAx5yAHK6g3tYvI9dzVUgZZmKgEspNsjusB1cnSBBVa7YxlCKn6MZB523
|
||||
5+8KnKFMtOYMivRM19Dr+bBvvCkwOc37PJREcCrmddN1OWAM1sEinwxWINumktLO
|
||||
QmkwCwYJYIZIAWUDBAMCAy8AMCwCFGjZ2Hc70n9AanALEmAaOCZ5yCJ6AhQXyIXt
|
||||
4m9Fu8NFOK5qJYhxxDOp9Q==
|
||||
-----END CERTIFICATE-----
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
-----BEGIN PRIVATE KEY-----
|
||||
MIIBSgIBADCCASsGByqGSM44BAEwggEeAoGBAIQwVx2ijwWwM9NjmW6lqCAXsuu/
|
||||
EQpg4pH7BK6D26SbOt6Hcq18lbZix4dSaOpp93frv/9hyHuAmKbxRMSR2NkR8Uor
|
||||
yJEtNHKM3ArQpFvUmovipP3CY2mytn3QoNffFom6Rs5ke6aVvkCYlcPVEZKOdJpa
|
||||
ovO5fGa9VWQ+eF4DAhUAyPX0wvaEivQ7pyqJCDdmaH8i39UCgYB8uhb5OYPokAnL
|
||||
G2ntAxW6Ns/85j6NXjDB6RcHOBkhNRa6kLRQu0xj7pUs+9LpSnSaednGBUWcAGm+
|
||||
gKqtz60bWnsufKLZcb0BDVX0kxFON6tCgMYJL+QwMPuPHfs3n22Srqwn5dgU5VKN
|
||||
8CVFOZbrBXUOlRabuiUqWPqYQveojQQWAhRn0IIk7GKtluNxnlqYpCuNazN6gQ==
|
||||
-----END PRIVATE KEY-----
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
-----BEGIN CERTIFICATE-----
|
||||
MIICwDCCAagCCQCenPDbJ3W+6zANBgkqhkiG9w0BAQsFADAiMSAwHgYDVQQDDBd0
|
||||
ZXN0LXNlcnZlci5leGFtcGxlLmNvbTAeFw0yMDAyMTQyMDEyMzBaFw0yMjExMTAy
|
||||
MDEyMzBaMCIxIDAeBgNVBAMMF3Rlc3Qtc2VydmVyLmV4YW1wbGUuY29tMIIBIjAN
|
||||
BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx4OIOJdBv2mxvSI3izpUu/RL5ND5
|
||||
9IQ1hM5UDCnH2UifweWSEePXDnJgqi8563+gzQHMTN/BBNR4HiIbUulq1KDZjYKE
|
||||
7FtT8PCE8XXwTxieyV/uiIlfd0b74NcSvuayKzGwOZ2h44kU/n8hZ5ubwmS1Dlir
|
||||
jCcJtFK3zcqnjqhmjqgj6gZJA/v5ipEnO78+kWU37XvRAGbfaK1/NzyENozg52q7
|
||||
OhFeVci2SEvn0gE0XC0jbgKFRvaZtMEhfwkggaUYv122btHntP5/Y4iNzxllDocD
|
||||
h6GxXqgbnopdrDEscEbObJc6FJtUvHtMF1lnM2SMXWdMCueUvGUFLGxAqwIDAQAB
|
||||
MA0GCSqGSIb3DQEBCwUAA4IBAQBiMAv5lL+jwUgEeaVn7B9Uma3nyW3yAOce6j2/
|
||||
7ADR4KNNomEzrEJuH67FQNQQj8CkNMtjSsHEwhLrDlkE6SKhZn1ikNw36+5DDa0o
|
||||
42Dzv/UrUUY0dJc1KB6Sro4K0k/ivNWHXe+cdCbqbrZk29QXUVAnBiqJ3r3VWOMG
|
||||
6lgO9kV6Dr5M7RnEOtNT7Dfr7/T817vXODZqa830J8XrM6e2znSB2QVTgswQ0HTf
|
||||
K3SObMYeZc76hg/yOUzdinBMRjZ/Op/a9oE5W71YS3aG/t2v8b2Z2gwIBR4Sa+cr
|
||||
y4VcW99RLXlzjtTCKnVvX9sRLFBZa96Rslm0nFWzbz40umaD
|
||||
-----END CERTIFICATE-----
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDHg4g4l0G/abG9
|
||||
IjeLOlS79Evk0Pn0hDWEzlQMKcfZSJ/B5ZIR49cOcmCqLznrf6DNAcxM38EE1Hge
|
||||
IhtS6WrUoNmNgoTsW1Pw8ITxdfBPGJ7JX+6IiV93Rvvg1xK+5rIrMbA5naHjiRT+
|
||||
fyFnm5vCZLUOWKuMJwm0UrfNyqeOqGaOqCPqBkkD+/mKkSc7vz6RZTfte9EAZt9o
|
||||
rX83PIQ2jODnars6EV5VyLZIS+fSATRcLSNuAoVG9pm0wSF/CSCBpRi/XbZu0ee0
|
||||
/n9jiI3PGWUOhwOHobFeqBueil2sMSxwRs5slzoUm1S8e0wXWWczZIxdZ0wK55S8
|
||||
ZQUsbECrAgMBAAECggEAIp6kSI2WjwxcFyGU2cfpZCPj93R7qv41+zGCTAoD76Q2
|
||||
dILNceVDL/KQ63b+aerfkDM7rCs3ZwsnPLNWYnC2ZOb0WSXIwuqmCizyJKP+avsu
|
||||
smq/DVopAp2Cn2Uyj4WgbPZWSekcaksjJXYR6dSKlpS7Bh5ExjEP8gZYdpEvugUY
|
||||
LOnenhrHJ+eGI1nQ5Sb2fuAOXyM6UYPw0H4d8s/2xwe8fzgOUO8v4PAYPnA6M/R/
|
||||
Y/p6qIuRzEuHwqYuTZ7S1g9UG5hW0HaEuMaQZM9UtK2PxfEP3WHPJZ8zjIVgLhIf
|
||||
yjPWo4s9X477NURYX9R2JeoLgea5sYCgqXvsSSvA4QKBgQDpqxjP8NxC0KwtonvS
|
||||
08EtUwGZaPQ9aROVBqa5u4BLyJ+YwBbBt2DlWTSKJ1eDC9olaca1/pRPxysS4N7x
|
||||
CieZkFES3sS0oaLssh+BQ4VsvvkkBdsdcWAcK6Joj7uIjlEfl72hJq8mUeR14gyy
|
||||
G2wE/sZ1m5wopqItGjcuR4qd5wKBgQDalNBaAcbyD1K415sh0uFCujwPoh5ezr4f
|
||||
VNBUL1VaOBztOyAp7szoo/lyfMpq+6tCn1X61DtHyBJkM3fp9IpaQpc9AnHfedx4
|
||||
0iwispzkpGFY0S88HZotB3oxN9kU+Ll3/i7unXqPmbxNEEoYlZ3UJmLOiqKgdccc
|
||||
qv2I+toGnQKBgDT9Vf0h8/E6/TDEHixrVO2AW2Z8xJaAk65B+eE4whltf7PWK9L2
|
||||
UQTxu9ZwoYnYUDoXyLZQ6zVER2JamHQ1B1HtxlTvK9CCrz3aDwbzVviYPkuLAGum
|
||||
4FLDGmt33OFU1NTDRn+bFDEudQ6+mn5xdYeUd1EIXtthHnn37feSxb6VAoGAXLZ+
|
||||
YY8baZTiS5D4NjKSZZFE5ISpSSF8NyHsc6jYFTpz2pQXonGt7IeQyOTxnss86zdW
|
||||
atwWgO32DxZdqJiXDo3sRG6DCn1P7NeI7PbB4aFvwRKJbIBJ4wum4rWDmIefc6wX
|
||||
EBMv5zUYT7+3DhJ4LYJSqrTXIiSS3jAQ9kcgr2kCgYBE8W+Nx4stlLlQ3EpocikZ
|
||||
D2GcFyTE0TfZf82NV9uPkDXZ8vYQQ9NEIn+crY1gC3edknMXJ5v1IUaRacxQPp2C
|
||||
0UewG3HQqgybhq2wvnMrw+dJqo2vuOInDZAC/Zvv+RGXtc3H+v6o875n3TYOro67
|
||||
P4jKdY+Mxl3BLvI6KhJJrA==
|
||||
-----END PRIVATE KEY-----
|
||||
Loading…
Reference in New Issue