proxy: Fall back to plaintext communication when a TLS handshake fails (#1173)

This branch modifies the proxy's logic for opening a connection so
that when an attempted TLS handshake fails, the proxy will retry that
connection without TLS.

This is implemented by changing the `UpgradeToTls` case in the `Future`
implementation for `Connecting`, so that rather than simply wrapping
a poll to the TLS upgrade future with `try_ready!` (and thus failing
the future if the upgrade future fails), we reset the state of the
future to the `Plaintext` state and continue looping. The `tls_status`
field of the future is changed to `ReasonForNoTls::HandshakeFailed`,
and the `Plaintext` state is changed so that if its `tls_status` is
`HandshakeFailed`, it will no longer attempt to upgrade to TLS when the
plaintext connection is successfully established.

Closes #1084 

Signed-off-by: Eliza Weisman <eliza@buoyant.io>

proxy/src/connection.rs

@@ -9,6 +9,7 @@ use tokio::{
     net::{TcpListener, TcpStream, ConnectFuture},
     reactor::Handle,
 };
+
 use conditional::Conditional;
 use ctx::transport::TlsStatus;
 use config::Addr;
@@ -24,9 +25,13 @@ pub fn connect(addr: &SocketAddr,
                tls: tls::ConditionalConnectionConfig<tls::ClientConfig>)
     -> Connecting
 {
-    Connecting::Plaintext {
+    let state = ConnectingState::Plaintext {
         connect: TcpStream::connect(addr),
         tls: Some(tls),
+    };
+    Connecting {
+        addr: *addr,
+        state,
     }
 }
 
@@ -43,10 +48,15 @@ struct ConditionallyUpgradeServerToTlsInner {
 }
 
 /// A socket that is in the process of connecting.
-pub enum Connecting {
+pub struct Connecting {
+    addr: SocketAddr,
+    state: ConnectingState,
+}
+
+enum ConnectingState {
     Plaintext {
         connect: ConnectFuture,
-        tls: Option<tls::ConditionalConnectionConfig<tls::ClientConfig>>,
+        tls: Option<tls::ConditionalConnectionConfig<tls::ClientConfig>>
     },
     UpgradeToTls(tls::UpgradeClientToTls),
 }
@@ -69,7 +79,7 @@ pub struct Connection {
     peek_buf: BytesMut,
 
     /// Whether or not the connection is secured with TLS.
-    tls_status: TlsStatus,
+    pub tls_status: TlsStatus,
 }
 
 /// A trait describing that a type can peek bytes.
@@ -252,24 +262,47 @@ impl Future for Connecting {
 
     fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
         loop {
-            *self = match self {
-                Connecting::Plaintext { connect, tls } => {
+            self.state = match &mut self.state {
+                ConnectingState::Plaintext { connect, tls } => {
                     let plaintext_stream = try_ready!(connect.poll());
+                    trace!("Connecting: state=plaintext; tls={:?};",tls);
                     set_nodelay_or_warn(&plaintext_stream);
                     match tls.take().expect("Polled after ready") {
                         Conditional::Some(config) => {
-                            let upgrade_to_tls = tls::Connection::connect(
+                            trace!("plaintext connection established; trying to upgrade");
+                            let upgrade = tls::Connection::connect(
                                 plaintext_stream, &config.identity, config.config);
-                            Connecting::UpgradeToTls(upgrade_to_tls)
+                            ConnectingState::UpgradeToTls(upgrade)
                         },
                         Conditional::None(why) => {
+                            trace!("plaintext connection established; no TLS ({:?})", why);
                             return Ok(Async::Ready(Connection::plain(plaintext_stream, why)));
                         },
                     }
                 },
-                Connecting::UpgradeToTls(upgrading) => {
-                    let tls_stream = try_ready!(upgrading.poll());
-                    return Ok(Async::Ready(Connection::tls(BoxedIo::new(tls_stream))));
+                ConnectingState::UpgradeToTls(upgrade) => {
+                    match upgrade.poll() {
+                        Ok(Async::NotReady) => return Ok(Async::NotReady),
+                        Ok(Async::Ready(tls_stream)) => {
+                            let conn = Connection::tls(BoxedIo::new(tls_stream));
+                            return Ok(Async::Ready(conn));
+                        },
+                        Err(e) => {
+                            debug!(
+                                "TLS handshake with {:?} failed: {}\
+                                    -> falling back to plaintext",
+                                self.addr, e,
+                            );
+                            let connect = TcpStream::connect(&self.addr);
+                            // TODO: emit a `HandshakeFailed` telemetry event.
+                            let reason = tls::ReasonForNoTls::HandshakeFailed;
+                            // Reset self to try the plaintext connection.
+                            ConnectingState::Plaintext {
+                                connect,
+                                tls: Some(Conditional::None(reason))
+                            }
+                        }
+                    }
                 },
             };
         }

proxy/src/telemetry/metrics/labels.rs

@@ -381,6 +381,8 @@ impl fmt::Display for ctx::transport::TlsStatus {
         match *self {
             Conditional::Some(()) => f.pad(",tls=\"true\""),
             Conditional::None(tls::ReasonForNoTls::NoConfig) => f.pad(",tls=\"no_config\""),
+            Conditional::None(tls::ReasonForNoTls::HandshakeFailed) =>
+                f.pad("tls=\"handshake_failed\""),
             Conditional::None(tls::ReasonForNoTls::Disabled) |
             Conditional::None(tls::ReasonForNoTls::InternalTraffic) |
             Conditional::None(tls::ReasonForNoTls::NoIdentity(_)) |

proxy/src/telemetry/sensor/mod.rs

@@ -69,7 +69,7 @@ impl Sensors {
 
     pub fn connect<C>(&self, connect: C, ctx: &Arc<ctx::transport::Client>) -> Connect<C>
     where
-        C: tokio_connect::Connect,
+        C: tokio_connect::Connect<Connected = ::connection::Connection>,
     {
         Connect::new(connect, &self.0, ctx)
     }

proxy/src/telemetry/sensor/transport.rs

@@ -6,7 +6,7 @@ use std::time::Instant;
 use tokio_connect;
 use tokio::io::{AsyncRead, AsyncWrite};
 
-use connection::Peek;
+use connection::{self, Peek};
 use ctx;
 use telemetry::event;
 
@@ -191,7 +191,10 @@ impl<T: AsyncRead + AsyncWrite + Peek> Peek for Transport<T> {
 
 // === impl Connect ===
 
-impl<C: tokio_connect::Connect> Connect<C> {
+impl<C> Connect<C>
+where
+    C: tokio_connect::Connect<Connected = connection::Connection>,
+{
     /// Returns a `Connect` to `addr` and `handle`.
     pub(super) fn new(
         underlying: C,
@@ -206,7 +209,10 @@ impl<C: tokio_connect::Connect> Connect<C> {
     }
 }
 
-impl<C: tokio_connect::Connect> tokio_connect::Connect for Connect<C> {
+impl<C> tokio_connect::Connect for Connect<C>
+where
+    C: tokio_connect::Connect<Connected = connection::Connection>,
+{
     type Connected = Transport<C::Connected>;
     type Error = C::Error;
     type Future = Connecting<C>;
@@ -222,14 +228,23 @@ impl<C: tokio_connect::Connect> tokio_connect::Connect for Connect<C> {
 
 // === impl Connecting ===
 
-impl<C: tokio_connect::Connect> Future for Connecting<C> {
+impl<C> Future for Connecting<C>
+where
+    C: tokio_connect::Connect<Connected = connection::Connection>,
+{
     type Item = Transport<C::Connected>;
     type Error = C::Error;
 
     fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
         let io = try_ready!(self.underlying.poll());
         debug!("client connection open");
-        let ctx = Arc::new(Arc::clone(&self.ctx).into());
+        let ctx = ctx::transport::Client::new(
+            &self.ctx.proxy,
+            &self.ctx.remote,
+            self.ctx.metadata.clone(),
+            io.tls_status,
+        );
+        let ctx = Arc::new(ctx.into());
         let trans = Transport::open(io, Instant::now(), &self.handle, ctx);
         Ok(trans.into())
     }

proxy/src/transport/tls/config.rs

@@ -103,6 +103,9 @@ pub enum ReasonForNoTls {
     /// The connection isn't TLS or it is TLS but not intended to be handled
     /// by the proxy.
     NotProxyTls,
+
+    /// We fell back to plaintext because the TLS handshake failed.
+    HandshakeFailed,
 }
 
 #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]

proxy/src/transport/tls/mod.rs

@@ -33,4 +33,5 @@ pub use self::{
     },
     dns_name::{DnsName, InvalidDnsName},
     identity::Identity,
+    rustls::TLSError as Error,
 };