Use different ports for instances of the same service (#7246)

Part of #7245.

This just provides a unique port for each instance, and breaks the
service<->port mapping. A subsequent PR will move to listening on the
same IP.

Remove unused `-b` variants of crl-storer and akamai-purger.

The new port scheme is that the first instance of a service is on `93xx`
and the second instance of a service is on `94xx`.

Part of a stacked change with #7243.
This commit is contained in:
Jacob Hoffman-Andrews 2024-01-10 14:32:33 -08:00 committed by GitHub
parent c7a0987971
commit 7b347dd6c3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 83 additions and 95 deletions

View File

@ -45,12 +45,12 @@
},
"remoteVAs": [
{
"serverAddress": "rva1.service.consul:9097",
"serverAddress": "rva1.service.consul:9397",
"timeout": "15s",
"hostOverride": "rva1.boulder"
},
{
"serverAddress": "rva1.service.consul:9098",
"serverAddress": "rva1.service.consul:9498",
"timeout": "15s",
"hostOverride": "rva1.boulder"
}

View File

@ -7,7 +7,7 @@
"keyFile": "test/grpc-creds/crl-storer.boulder/key.pem"
},
"grpc": {
"address": ":9109",
"address": ":9309",
"maxConnectionAge": "30s",
"services": {
"storer.CRLStorer": {

View File

@ -20,7 +20,7 @@
},
"grpc": {
"maxConnectionAge": "30s",
"address": ":9098",
"address": ":9498",
"services": {
"va.VA": {
"clientNames": [

View File

@ -20,7 +20,6 @@
},
"grpc": {
"maxConnectionAge": "30s",
"address": ":9092",
"services": {
"va.VA": {
"clientNames": [
@ -45,12 +44,12 @@
},
"remoteVAs": [
{
"serverAddress": "rva1.service.consul:9097",
"serverAddress": "rva1.service.consul:9397",
"timeout": "15s",
"hostOverride": "rva1.boulder"
},
{
"serverAddress": "rva1.service.consul:9098",
"serverAddress": "rva1.service.consul:9498",
"timeout": "15s",
"hostOverride": "rva1.boulder"
}

View File

@ -1,3 +1,5 @@
# Keep this file in sync with the ports bound in test/startservers.py
client_addr = "0.0.0.0"
bind_addr = "10.55.55.10"
log_level = "ERROR"
@ -27,15 +29,7 @@ services {
id = "akamai-purger-a"
name = "akamai-purger"
address = "10.77.77.77"
port = 9099
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
services {
id = "akamai-purger-b"
name = "akamai-purger"
address = "10.88.88.88"
port = 9099
port = 9399
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -55,7 +49,7 @@ services {
id = "ca-a"
name = "ca"
address = "10.77.77.77"
port = 9093
port = 9393
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -63,7 +57,7 @@ services {
id = "ca-b"
name = "ca"
address = "10.88.88.88"
port = 9093
port = 9493
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -71,7 +65,7 @@ services {
id = "ca1"
name = "ca1"
address = "10.77.77.77"
port = 9093
port = 9393
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -79,7 +73,7 @@ services {
id = "ca2"
name = "ca2"
address = "10.88.88.88"
port = 9093
port = 9493
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -87,15 +81,7 @@ services {
id = "crl-storer-a"
name = "crl-storer"
address = "10.77.77.77"
port = 9109
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
services {
id = "crl-storer-b"
name = "crl-storer"
address = "10.88.88.88"
port = 9109
port = 9309
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -119,7 +105,7 @@ services {
id = "doh-a"
name = "doh"
address = "10.77.77.77"
port = 8443
port = 8343
tags = ["tcp"]
}
@ -135,7 +121,7 @@ services {
id = "nonce-a"
name = "nonce"
address = "10.77.77.77"
port = 9101
port = 9301
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -143,7 +129,7 @@ services {
id = "nonce-b"
name = "nonce"
address = "10.88.88.88"
port = 9101
port = 9401
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -151,7 +137,7 @@ services {
id = "nonce1"
name = "nonce1"
address = "10.77.77.77"
port = 9101
port = 9301
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -159,7 +145,7 @@ services {
id = "nonce2"
name = "nonce2"
address = "10.88.88.88"
port = 9101
port = 9401
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -167,7 +153,7 @@ services {
id = "publisher-a"
name = "publisher"
address = "10.77.77.77"
port = 9091
port = 9391
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -175,7 +161,7 @@ services {
id = "publisher-b"
name = "publisher"
address = "10.88.88.88"
port = 9091
port = 9491
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -183,7 +169,7 @@ services {
id = "publisher1"
name = "publisher1"
address = "10.77.77.77"
port = 9091
port = 9391
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -191,7 +177,7 @@ services {
id = "publisher2"
name = "publisher2"
address = "10.88.88.88"
port = 9091
port = 9491
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -199,7 +185,7 @@ services {
id = "ra-a"
name = "ra"
address = "10.77.77.77"
port = 9094
port = 9394
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -207,7 +193,7 @@ services {
id = "ra-b"
name = "ra"
address = "10.88.88.88"
port = 9094
port = 9494
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -215,7 +201,7 @@ services {
id = "ra1"
name = "ra1"
address = "10.77.77.77"
port = 9094
port = 9394
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -223,7 +209,7 @@ services {
id = "ra2"
name = "ra2"
address = "10.88.88.88"
port = 9094
port = 9494
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -231,7 +217,7 @@ services {
id = "rva1-a"
name = "rva1"
address = "10.77.77.77"
port = 9097
port = 9397
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -239,7 +225,7 @@ services {
id = "rva1-b"
name = "rva1"
address = "10.77.77.77"
port = 9098
port = 9498
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -247,13 +233,13 @@ services {
id = "sa-a"
name = "sa"
address = "10.77.77.77"
port = 9095
port = 9395
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
checks = [
{
id = "sa-a-grpc"
name = "sa-a-grpc"
grpc = "10.77.77.77:9095"
grpc = "10.77.77.77:9395"
grpc_use_tls = true
tls_server_name = "sa.boulder"
tls_skip_verify = false
@ -262,7 +248,7 @@ services {
{
id = "sa-a-grpc-sa"
name = "sa-a-grpc-sa"
grpc = "10.77.77.77:9095/sa.StorageAuthority"
grpc = "10.77.77.77:9395/sa.StorageAuthority"
grpc_use_tls = true
tls_server_name = "sa.boulder"
tls_skip_verify = false
@ -271,7 +257,7 @@ services {
{
id = "sa-a-grpc-saro"
name = "sa-a-grpc-saro"
grpc = "10.77.77.77:9095/sa.StorageAuthorityReadOnly"
grpc = "10.77.77.77:9395/sa.StorageAuthorityReadOnly"
grpc_use_tls = true
tls_server_name = "sa.boulder"
tls_skip_verify = false
@ -284,13 +270,13 @@ services {
id = "sa-b"
name = "sa"
address = "10.88.88.88"
port = 9095
port = 9495
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
checks = [
{
id = "sa-b-grpc"
name = "sa-b-grpc"
grpc = "10.88.88.88:9095"
grpc = "10.88.88.88:9495"
grpc_use_tls = true
tls_server_name = "sa.boulder"
tls_skip_verify = false
@ -299,7 +285,7 @@ services {
{
id = "sa-b-grpc-sa"
name = "sa-b-grpc-sa"
grpc = "10.88.88.88:9095/sa.StorageAuthority"
grpc = "10.88.88.88:9495/sa.StorageAuthority"
grpc_use_tls = true
tls_server_name = "sa.boulder"
tls_skip_verify = false
@ -308,7 +294,7 @@ services {
{
id = "sa-b-grpc-saro"
name = "sa-b-grpc-saro"
grpc = "10.88.88.88:9095/sa.StorageAuthorityReadOnly"
grpc = "10.88.88.88:9495/sa.StorageAuthorityReadOnly"
grpc_use_tls = true
tls_server_name = "sa.boulder"
tls_skip_verify = false
@ -321,7 +307,7 @@ services {
id = "sa1"
name = "sa1"
address = "10.77.77.77"
port = 9095
port = 9395
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -329,7 +315,7 @@ services {
id = "sa2"
name = "sa2"
address = "10.88.88.88"
port = 9095
port = 9495
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -337,7 +323,7 @@ services {
id = "va-a"
name = "va"
address = "10.77.77.77"
port = 9092
port = 9392
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -345,7 +331,7 @@ services {
id = "va-b"
name = "va"
address = "10.88.88.88"
port = 9092
port = 9492
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -353,7 +339,7 @@ services {
id = "va1"
name = "va1"
address = "10.77.77.77"
port = 9092
port = 9392
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -361,7 +347,7 @@ services {
id = "va2"
name = "va2"
address = "10.88.88.88"
port = 9092
port = 9492
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -382,7 +368,9 @@ services {
}
//
// The following services are used for testing the gRPC DNS resolver.
// The following services are used for testing the gRPC DNS resolver in
// test/integration/srv_resolver_test.go and
// test/integration/testdata/srv-resolver-config.json.
//
// CaseOne config will have 2 SRV records. The first will have 0 backends, the
@ -391,7 +379,7 @@ services {
id = "case1a"
name = "case1a"
address = "10.77.77.77"
port = 9101
port = 9301
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
checks = [
{
@ -408,7 +396,7 @@ services {
id = "case1b"
name = "case1b"
address = "10.88.88.88"
port = 9101
port = 9401
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -418,7 +406,7 @@ services {
id = "case2b"
name = "case2b"
address = "10.88.88.88"
port = 9101
port = 9401
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
}
@ -432,7 +420,7 @@ services {
name = "case4a"
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
address = "10.77.77.77"
port = 9101
port = 9301
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
checks = [
{
@ -449,7 +437,7 @@ services {
id = "case4b"
name = "case4b"
address = "10.88.88.88"
port = 9101
port = 9401
tags = ["tcp"] // Required for SRV RR support in gRPC DNS resolution.
checks = [
{

View File

@ -14,33 +14,34 @@ from helpers import waithealth, waitport, config_dir, CONFIG_NEXT
Service = collections.namedtuple('Service', ('name', 'debug_port', 'grpc_addr', 'cmd', 'deps'))
# Keep these ports in sync with consul/config.hcl
SERVICES = (
Service('boulder-remoteva-a',
8011, 'rva1.service.consul:9097',
('./bin/boulder', 'boulder-remoteva', '--config', os.path.join(config_dir, 'va-remote-a.json'), '--addr', ':9097', '--debug-addr', ':8011'),
8011, 'rva1.service.consul:9397',
('./bin/boulder', 'boulder-remoteva', '--config', os.path.join(config_dir, 'va-remote-a.json'), '--addr', ':9397', '--debug-addr', ':8011'),
None),
Service('boulder-remoteva-b',
8012, 'rva1.service.consul:9098',
('./bin/boulder', 'boulder-remoteva', '--config', os.path.join(config_dir, 'va-remote-b.json'), '--addr', ':9098', '--debug-addr', ':8012'),
8012, 'rva1.service.consul:9498',
('./bin/boulder', 'boulder-remoteva', '--config', os.path.join(config_dir, 'va-remote-b.json'), '--addr', ':9498', '--debug-addr', ':8012'),
None),
Service('boulder-sa-1',
8003, 'sa1.service.consul:9095',
('./bin/boulder', 'boulder-sa', '--config', os.path.join(config_dir, 'sa.json'), '--addr', 'sa1.service.consul:9095', '--debug-addr', ':8003'),
8003, 'sa1.service.consul:9395',
('./bin/boulder', 'boulder-sa', '--config', os.path.join(config_dir, 'sa.json'), '--addr', 'sa1.service.consul:9395', '--debug-addr', ':8003'),
None),
Service('boulder-sa-2',
8103, 'sa2.service.consul:9095',
('./bin/boulder', 'boulder-sa', '--config', os.path.join(config_dir, 'sa.json'), '--addr', 'sa2.service.consul:9095', '--debug-addr', ':8103'),
8103, 'sa2.service.consul:9495',
('./bin/boulder', 'boulder-sa', '--config', os.path.join(config_dir, 'sa.json'), '--addr', 'sa2.service.consul:9495', '--debug-addr', ':8103'),
None),
Service('ct-test-srv',
4500, None,
('./bin/ct-test-srv', '--config', 'test/ct-test-srv/ct-test-srv.json'), None),
Service('boulder-publisher-1',
8009, 'publisher1.service.consul:9091',
('./bin/boulder', 'boulder-publisher', '--config', os.path.join(config_dir, 'publisher.json'), '--addr', 'publisher1.service.consul:9091', '--debug-addr', ':8009'),
8009, 'publisher1.service.consul:9391',
('./bin/boulder', 'boulder-publisher', '--config', os.path.join(config_dir, 'publisher.json'), '--addr', 'publisher1.service.consul:9391', '--debug-addr', ':8009'),
None),
Service('boulder-publisher-2',
8109, 'publisher2.service.consul:9091',
('./bin/boulder', 'boulder-publisher', '--config', os.path.join(config_dir, 'publisher.json'), '--addr', 'publisher2.service.consul:9091', '--debug-addr', ':8109'),
8109, 'publisher2.service.consul:9491',
('./bin/boulder', 'boulder-publisher', '--config', os.path.join(config_dir, 'publisher.json'), '--addr', 'publisher2.service.consul:9491', '--debug-addr', ':8109'),
None),
Service('mail-test-srv',
9380, None,
@ -51,20 +52,20 @@ SERVICES = (
('./bin/boulder', 'ocsp-responder', '--config', os.path.join(config_dir, 'ocsp-responder.json'), '--addr', '0.0.0.0:4002', '--debug-addr', ':8005'),
('boulder-ra-1', 'boulder-ra-2')),
Service('boulder-va-1',
8004, 'va1.service.consul:9092',
('./bin/boulder', 'boulder-va', '--config', os.path.join(config_dir, 'va.json'), '--addr', 'va1.service.consul:9092', '--debug-addr', ':8004'),
8004, 'va1.service.consul:9392',
('./bin/boulder', 'boulder-va', '--config', os.path.join(config_dir, 'va.json'), '--addr', 'va1.service.consul:9392', '--debug-addr', ':8004'),
('boulder-remoteva-a', 'boulder-remoteva-b')),
Service('boulder-va-2',
8104, 'va2.service.consul:9092',
('./bin/boulder', 'boulder-va', '--config', os.path.join(config_dir, 'va.json'), '--addr', 'va2.service.consul:9092', '--debug-addr', ':8104'),
8104, 'va2.service.consul:9492',
('./bin/boulder', 'boulder-va', '--config', os.path.join(config_dir, 'va.json'), '--addr', 'va2.service.consul:9492', '--debug-addr', ':8104'),
('boulder-remoteva-a', 'boulder-remoteva-b')),
Service('boulder-ca-1',
8001, 'ca1.service.consul:9093',
('./bin/boulder', 'boulder-ca', '--config', os.path.join(config_dir, 'ca.json'), '--addr', 'ca1.service.consul:9093', '--debug-addr', ':8001'),
8001, 'ca1.service.consul:9393',
('./bin/boulder', 'boulder-ca', '--config', os.path.join(config_dir, 'ca.json'), '--addr', 'ca1.service.consul:9393', '--debug-addr', ':8001'),
('boulder-sa-1', 'boulder-sa-2')),
Service('boulder-ca-2',
8101, 'ca2.service.consul:9093',
('./bin/boulder', 'boulder-ca', '--config', os.path.join(config_dir, 'ca.json'), '--addr', 'ca2.service.consul:9093', '--debug-addr', ':8101'),
8101, 'ca2.service.consul:9493',
('./bin/boulder', 'boulder-ca', '--config', os.path.join(config_dir, 'ca.json'), '--addr', 'ca2.service.consul:9493', '--debug-addr', ':8101'),
('boulder-sa-1', 'boulder-sa-2')),
Service('akamai-test-srv',
6789, None,
@ -72,7 +73,7 @@ SERVICES = (
None),
Service('akamai-purger',
9666, None,
('./bin/boulder', 'akamai-purger', '--config', os.path.join(config_dir, 'akamai-purger.json'), '--debug-addr', ':9666'),
('./bin/boulder', 'akamai-purger', '--addr', ':9399', '--config', os.path.join(config_dir, 'akamai-purger.json'), '--debug-addr', ':9666'),
('akamai-test-srv',)),
Service('s3-test-srv',
7890, None,
@ -80,31 +81,31 @@ SERVICES = (
None),
Service('crl-storer',
9667, None,
('./bin/boulder', 'crl-storer', '--config', os.path.join(config_dir, 'crl-storer.json'), '--addr', ':9109', '--debug-addr', ':9667'),
('./bin/boulder', 'crl-storer', '--config', os.path.join(config_dir, 'crl-storer.json'), '--addr', ':9309', '--debug-addr', ':9667'),
('s3-test-srv',)),
Service('crl-updater',
8021, None,
('./bin/boulder', 'crl-updater', '--config', os.path.join(config_dir, 'crl-updater.json'), '--debug-addr', ':8021'),
('boulder-ca-1', 'boulder-ca-2', 'boulder-sa-1', 'boulder-sa-2', 'crl-storer')),
Service('boulder-ra-1',
8002, 'ra1.service.consul:9094',
('./bin/boulder', 'boulder-ra', '--config', os.path.join(config_dir, 'ra.json'), '--addr', 'ra1.service.consul:9094', '--debug-addr', ':8002'),
8002, 'ra1.service.consul:9394',
('./bin/boulder', 'boulder-ra', '--config', os.path.join(config_dir, 'ra.json'), '--addr', 'ra1.service.consul:9394', '--debug-addr', ':8002'),
('boulder-sa-1', 'boulder-sa-2', 'boulder-ca-1', 'boulder-ca-2', 'boulder-va-1', 'boulder-va-2', 'akamai-purger', 'boulder-publisher-1', 'boulder-publisher-2')),
Service('boulder-ra-2',
8102, 'ra2.service.consul:9094',
('./bin/boulder', 'boulder-ra', '--config', os.path.join(config_dir, 'ra.json'), '--addr', 'ra2.service.consul:9094', '--debug-addr', ':8102'),
8102, 'ra2.service.consul:9494',
('./bin/boulder', 'boulder-ra', '--config', os.path.join(config_dir, 'ra.json'), '--addr', 'ra2.service.consul:9494', '--debug-addr', ':8102'),
('boulder-sa-1', 'boulder-sa-2', 'boulder-ca-1', 'boulder-ca-2', 'boulder-va-1', 'boulder-va-2', 'akamai-purger', 'boulder-publisher-1', 'boulder-publisher-2')),
Service('bad-key-revoker',
8020, None,
('./bin/boulder', 'bad-key-revoker', '--config', os.path.join(config_dir, 'bad-key-revoker.json'), '--debug-addr', ':8020'),
('boulder-ra-1', 'boulder-ra-2', 'mail-test-srv')),
Service('nonce-service-taro',
8111, 'nonce1.service.consul:9101',
('./bin/boulder', 'nonce-service', '--config', os.path.join(config_dir, 'nonce-a.json'), '--addr', '10.77.77.77:9101', '--debug-addr', ':8111',),
8111, 'nonce1.service.consul:9301',
('./bin/boulder', 'nonce-service', '--config', os.path.join(config_dir, 'nonce-a.json'), '--addr', '10.77.77.77:9301', '--debug-addr', ':8111',),
None),
Service('nonce-service-zinc',
8112, 'nonce2.service.consul:9101',
('./bin/boulder', 'nonce-service', '--config', os.path.join(config_dir, 'nonce-b.json'), '--addr', '10.88.88.88:9101', '--debug-addr', ':8112',),
8112, 'nonce2.service.consul:9401',
('./bin/boulder', 'nonce-service', '--config', os.path.join(config_dir, 'nonce-b.json'), '--addr', '10.88.88.88:9401', '--debug-addr', ':8112',),
None),
Service('boulder-wfe2',
4001, None,
@ -258,7 +259,7 @@ def startChallSrv():
'--defaultIPv4', os.environ.get("FAKE_DNS"),
'-defaultIPv6', '',
'--dns01', ':8053,:8054',
'--doh', '10.77.77.77:8443,10.88.88.88:8443',
'--doh', '10.77.77.77:8343,10.88.88.88:8443',
'--doh-cert', 'test/grpc-creds/10.77.77.77/cert.pem',
'--doh-cert-key', 'test/grpc-creds/10.77.77.77/key.pem',
'--management', ':8055',