Add attributes to netty connection failure span (#3115)
This commit is contained in:
parent
ed88cca533
commit
e16cf3001f
|
@ -12,7 +12,6 @@ import static net.bytebuddy.matcher.ElementMatchers.isMethod;
|
||||||
import static net.bytebuddy.matcher.ElementMatchers.named;
|
import static net.bytebuddy.matcher.ElementMatchers.named;
|
||||||
import static net.bytebuddy.matcher.ElementMatchers.takesArgument;
|
import static net.bytebuddy.matcher.ElementMatchers.takesArgument;
|
||||||
|
|
||||||
import io.opentelemetry.api.trace.SpanKind;
|
|
||||||
import io.opentelemetry.context.Context;
|
import io.opentelemetry.context.Context;
|
||||||
import io.opentelemetry.context.Scope;
|
import io.opentelemetry.context.Scope;
|
||||||
import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
|
import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
|
||||||
|
@ -70,8 +69,7 @@ public class ChannelFutureListenerInstrumentation implements TypeInstrumentation
|
||||||
}
|
}
|
||||||
Scope parentScope = parentContext.makeCurrent();
|
Scope parentScope = parentContext.makeCurrent();
|
||||||
if (channelTraceContext.createConnectionSpan()) {
|
if (channelTraceContext.createConnectionSpan()) {
|
||||||
Context errorContext = tracer().startSpan("CONNECT", SpanKind.CLIENT);
|
tracer().connectionFailure(parentContext, future.getChannel(), cause);
|
||||||
tracer().endExceptionally(errorContext, cause);
|
|
||||||
}
|
}
|
||||||
return parentScope;
|
return parentScope;
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,6 +7,8 @@ package io.opentelemetry.javaagent.instrumentation.netty.v3_8.client;
|
||||||
|
|
||||||
import static io.opentelemetry.api.trace.SpanKind.CLIENT;
|
import static io.opentelemetry.api.trace.SpanKind.CLIENT;
|
||||||
import static io.opentelemetry.javaagent.instrumentation.netty.v3_8.client.NettyResponseInjectAdapter.SETTER;
|
import static io.opentelemetry.javaagent.instrumentation.netty.v3_8.client.NettyResponseInjectAdapter.SETTER;
|
||||||
|
import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.NetTransportValues.IP_TCP;
|
||||||
|
import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.NetTransportValues.IP_UDP;
|
||||||
import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.HOST;
|
import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.HOST;
|
||||||
|
|
||||||
import io.opentelemetry.api.trace.SpanBuilder;
|
import io.opentelemetry.api.trace.SpanBuilder;
|
||||||
|
@ -14,11 +16,14 @@ import io.opentelemetry.context.Context;
|
||||||
import io.opentelemetry.context.propagation.TextMapSetter;
|
import io.opentelemetry.context.propagation.TextMapSetter;
|
||||||
import io.opentelemetry.instrumentation.api.tracer.HttpClientTracer;
|
import io.opentelemetry.instrumentation.api.tracer.HttpClientTracer;
|
||||||
import io.opentelemetry.instrumentation.api.tracer.net.NetPeerAttributes;
|
import io.opentelemetry.instrumentation.api.tracer.net.NetPeerAttributes;
|
||||||
|
import io.opentelemetry.semconv.trace.attributes.SemanticAttributes;
|
||||||
import java.net.InetSocketAddress;
|
import java.net.InetSocketAddress;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.URISyntaxException;
|
import java.net.URISyntaxException;
|
||||||
import org.checkerframework.checker.nullness.qual.Nullable;
|
import org.checkerframework.checker.nullness.qual.Nullable;
|
||||||
|
import org.jboss.netty.channel.Channel;
|
||||||
import org.jboss.netty.channel.ChannelHandlerContext;
|
import org.jboss.netty.channel.ChannelHandlerContext;
|
||||||
|
import org.jboss.netty.channel.socket.DatagramChannel;
|
||||||
import org.jboss.netty.handler.codec.http.HttpHeaders;
|
import org.jboss.netty.handler.codec.http.HttpHeaders;
|
||||||
import org.jboss.netty.handler.codec.http.HttpRequest;
|
import org.jboss.netty.handler.codec.http.HttpRequest;
|
||||||
import org.jboss.netty.handler.codec.http.HttpResponse;
|
import org.jboss.netty.handler.codec.http.HttpResponse;
|
||||||
|
@ -46,6 +51,17 @@ public class NettyHttpClientTracer
|
||||||
return context;
|
return context;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void connectionFailure(Context parentContext, Channel channel, Throwable throwable) {
|
||||||
|
SpanBuilder spanBuilder = spanBuilder(parentContext, "CONNECT", CLIENT);
|
||||||
|
spanBuilder.setAttribute(
|
||||||
|
SemanticAttributes.NET_TRANSPORT, channel instanceof DatagramChannel ? IP_UDP : IP_TCP);
|
||||||
|
NetPeerAttributes.INSTANCE.setNetPeer(
|
||||||
|
spanBuilder, (InetSocketAddress) channel.getRemoteAddress());
|
||||||
|
|
||||||
|
Context context = withClientSpan(parentContext, spanBuilder.startSpan());
|
||||||
|
tracer().endExceptionally(context, throwable);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected String method(HttpRequest httpRequest) {
|
protected String method(HttpRequest httpRequest) {
|
||||||
return httpRequest.getMethod().getName();
|
return httpRequest.getMethod().getName();
|
||||||
|
|
|
@ -108,7 +108,7 @@ class Netty38ClientTest extends HttpClientTest<Request> implements AgentTestTrai
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
boolean hasClientSpanAttributes(URI uri) {
|
boolean hasClientSpanHttpAttributes(URI uri) {
|
||||||
switch (uri.toString()) {
|
switch (uri.toString()) {
|
||||||
case "http://localhost:61/": // unopened port
|
case "http://localhost:61/": // unopened port
|
||||||
case "http://www.google.com:81/": // dropped request
|
case "http://www.google.com:81/": // dropped request
|
||||||
|
|
|
@ -13,7 +13,6 @@ import static net.bytebuddy.matcher.ElementMatchers.named;
|
||||||
import static net.bytebuddy.matcher.ElementMatchers.takesArgument;
|
import static net.bytebuddy.matcher.ElementMatchers.takesArgument;
|
||||||
|
|
||||||
import io.netty.channel.ChannelFuture;
|
import io.netty.channel.ChannelFuture;
|
||||||
import io.opentelemetry.api.trace.SpanKind;
|
|
||||||
import io.opentelemetry.context.Context;
|
import io.opentelemetry.context.Context;
|
||||||
import io.opentelemetry.context.Scope;
|
import io.opentelemetry.context.Scope;
|
||||||
import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
|
import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
|
||||||
|
@ -62,8 +61,7 @@ public class ChannelFutureListenerInstrumentation implements TypeInstrumentation
|
||||||
|
|
||||||
Scope parentScope = parentContext.makeCurrent();
|
Scope parentScope = parentContext.makeCurrent();
|
||||||
if (tracer().shouldStartSpan(parentContext)) {
|
if (tracer().shouldStartSpan(parentContext)) {
|
||||||
Context errorContext = tracer().startSpan("CONNECT", SpanKind.CLIENT);
|
tracer().connectionFailure(parentContext, future.channel(), cause);
|
||||||
tracer().endExceptionally(errorContext, cause);
|
|
||||||
}
|
}
|
||||||
return parentScope;
|
return parentScope;
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,8 +8,12 @@ package io.opentelemetry.javaagent.instrumentation.netty.v4_0.client;
|
||||||
import static io.netty.handler.codec.http.HttpHeaders.Names.HOST;
|
import static io.netty.handler.codec.http.HttpHeaders.Names.HOST;
|
||||||
import static io.opentelemetry.api.trace.SpanKind.CLIENT;
|
import static io.opentelemetry.api.trace.SpanKind.CLIENT;
|
||||||
import static io.opentelemetry.javaagent.instrumentation.netty.common.client.NettyResponseInjectAdapter.SETTER;
|
import static io.opentelemetry.javaagent.instrumentation.netty.common.client.NettyResponseInjectAdapter.SETTER;
|
||||||
|
import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.NetTransportValues.IP_TCP;
|
||||||
|
import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.NetTransportValues.IP_UDP;
|
||||||
|
|
||||||
|
import io.netty.channel.Channel;
|
||||||
import io.netty.channel.ChannelHandlerContext;
|
import io.netty.channel.ChannelHandlerContext;
|
||||||
|
import io.netty.channel.socket.DatagramChannel;
|
||||||
import io.netty.handler.codec.http.HttpHeaders;
|
import io.netty.handler.codec.http.HttpHeaders;
|
||||||
import io.netty.handler.codec.http.HttpRequest;
|
import io.netty.handler.codec.http.HttpRequest;
|
||||||
import io.netty.handler.codec.http.HttpResponse;
|
import io.netty.handler.codec.http.HttpResponse;
|
||||||
|
@ -18,6 +22,7 @@ import io.opentelemetry.context.Context;
|
||||||
import io.opentelemetry.context.propagation.TextMapSetter;
|
import io.opentelemetry.context.propagation.TextMapSetter;
|
||||||
import io.opentelemetry.instrumentation.api.tracer.HttpClientTracer;
|
import io.opentelemetry.instrumentation.api.tracer.HttpClientTracer;
|
||||||
import io.opentelemetry.instrumentation.api.tracer.net.NetPeerAttributes;
|
import io.opentelemetry.instrumentation.api.tracer.net.NetPeerAttributes;
|
||||||
|
import io.opentelemetry.semconv.trace.attributes.SemanticAttributes;
|
||||||
import java.net.InetSocketAddress;
|
import java.net.InetSocketAddress;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.URISyntaxException;
|
import java.net.URISyntaxException;
|
||||||
|
@ -46,6 +51,16 @@ public class NettyHttpClientTracer
|
||||||
return context;
|
return context;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void connectionFailure(Context parentContext, Channel channel, Throwable throwable) {
|
||||||
|
SpanBuilder spanBuilder = spanBuilder(parentContext, "CONNECT", CLIENT);
|
||||||
|
spanBuilder.setAttribute(
|
||||||
|
SemanticAttributes.NET_TRANSPORT, channel instanceof DatagramChannel ? IP_UDP : IP_TCP);
|
||||||
|
NetPeerAttributes.INSTANCE.setNetPeer(spanBuilder, (InetSocketAddress) channel.remoteAddress());
|
||||||
|
|
||||||
|
Context context = withClientSpan(parentContext, spanBuilder.startSpan());
|
||||||
|
tracer().endExceptionally(context, throwable);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected String method(HttpRequest httpRequest) {
|
protected String method(HttpRequest httpRequest) {
|
||||||
return httpRequest.getMethod().name();
|
return httpRequest.getMethod().name();
|
||||||
|
|
|
@ -92,7 +92,7 @@ class Netty40ClientTest extends HttpClientTest<DefaultFullHttpRequest> implement
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
boolean hasClientSpanAttributes(URI uri) {
|
boolean hasClientSpanHttpAttributes(URI uri) {
|
||||||
switch (uri.toString()) {
|
switch (uri.toString()) {
|
||||||
case "http://localhost:61/": // unopened port
|
case "http://localhost:61/": // unopened port
|
||||||
case "http://www.google.com:81/": // dropped request
|
case "http://www.google.com:81/": // dropped request
|
||||||
|
|
|
@ -63,8 +63,7 @@ public class ChannelFutureListenerInstrumentation implements TypeInstrumentation
|
||||||
|
|
||||||
Scope parentScope = parentContext.makeCurrent();
|
Scope parentScope = parentContext.makeCurrent();
|
||||||
if (tracer().shouldStartSpan(parentContext, SpanKind.CLIENT)) {
|
if (tracer().shouldStartSpan(parentContext, SpanKind.CLIENT)) {
|
||||||
Context errorContext = tracer().startSpan("CONNECT", SpanKind.CLIENT);
|
tracer().connectionFailure(parentContext, future.channel(), cause);
|
||||||
tracer().endExceptionally(errorContext, cause);
|
|
||||||
}
|
}
|
||||||
return parentScope;
|
return parentScope;
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,8 +8,12 @@ package io.opentelemetry.javaagent.instrumentation.netty.v4_1.client;
|
||||||
import static io.netty.handler.codec.http.HttpHeaderNames.HOST;
|
import static io.netty.handler.codec.http.HttpHeaderNames.HOST;
|
||||||
import static io.opentelemetry.api.trace.SpanKind.CLIENT;
|
import static io.opentelemetry.api.trace.SpanKind.CLIENT;
|
||||||
import static io.opentelemetry.javaagent.instrumentation.netty.common.client.NettyResponseInjectAdapter.SETTER;
|
import static io.opentelemetry.javaagent.instrumentation.netty.common.client.NettyResponseInjectAdapter.SETTER;
|
||||||
|
import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.NetTransportValues.IP_TCP;
|
||||||
|
import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.NetTransportValues.IP_UDP;
|
||||||
|
|
||||||
|
import io.netty.channel.Channel;
|
||||||
import io.netty.channel.ChannelHandlerContext;
|
import io.netty.channel.ChannelHandlerContext;
|
||||||
|
import io.netty.channel.socket.DatagramChannel;
|
||||||
import io.netty.handler.codec.http.HttpHeaders;
|
import io.netty.handler.codec.http.HttpHeaders;
|
||||||
import io.netty.handler.codec.http.HttpRequest;
|
import io.netty.handler.codec.http.HttpRequest;
|
||||||
import io.netty.handler.codec.http.HttpResponse;
|
import io.netty.handler.codec.http.HttpResponse;
|
||||||
|
@ -18,6 +22,7 @@ import io.opentelemetry.context.Context;
|
||||||
import io.opentelemetry.context.propagation.TextMapSetter;
|
import io.opentelemetry.context.propagation.TextMapSetter;
|
||||||
import io.opentelemetry.instrumentation.api.tracer.HttpClientTracer;
|
import io.opentelemetry.instrumentation.api.tracer.HttpClientTracer;
|
||||||
import io.opentelemetry.instrumentation.api.tracer.net.NetPeerAttributes;
|
import io.opentelemetry.instrumentation.api.tracer.net.NetPeerAttributes;
|
||||||
|
import io.opentelemetry.semconv.trace.attributes.SemanticAttributes;
|
||||||
import java.net.InetSocketAddress;
|
import java.net.InetSocketAddress;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.URISyntaxException;
|
import java.net.URISyntaxException;
|
||||||
|
@ -46,6 +51,16 @@ public class NettyHttpClientTracer
|
||||||
return context;
|
return context;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void connectionFailure(Context parentContext, Channel channel, Throwable throwable) {
|
||||||
|
SpanBuilder spanBuilder = spanBuilder(parentContext, "CONNECT", CLIENT);
|
||||||
|
spanBuilder.setAttribute(
|
||||||
|
SemanticAttributes.NET_TRANSPORT, channel instanceof DatagramChannel ? IP_UDP : IP_TCP);
|
||||||
|
NetPeerAttributes.INSTANCE.setNetPeer(spanBuilder, (InetSocketAddress) channel.remoteAddress());
|
||||||
|
|
||||||
|
Context context = withClientSpan(parentContext, spanBuilder.startSpan());
|
||||||
|
tracer().endExceptionally(context, throwable);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected String method(HttpRequest httpRequest) {
|
protected String method(HttpRequest httpRequest) {
|
||||||
return httpRequest.method().name();
|
return httpRequest.method().name();
|
||||||
|
|
|
@ -110,7 +110,7 @@ class Netty41ClientTest extends HttpClientTest<DefaultFullHttpRequest> implement
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
boolean hasClientSpanAttributes(URI uri) {
|
boolean hasClientSpanHttpAttributes(URI uri) {
|
||||||
switch (uri.toString()) {
|
switch (uri.toString()) {
|
||||||
case "http://localhost:61/": // unopened port
|
case "http://localhost:61/": // unopened port
|
||||||
case "http://www.google.com:81/": // dropped request
|
case "http://www.google.com:81/": // dropped request
|
||||||
|
|
|
@ -99,7 +99,7 @@ class RatpackHttpClientTest extends HttpClientTest<Void> implements AgentTestTra
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
boolean hasClientSpanAttributes(URI uri) {
|
boolean hasClientSpanHttpAttributes(URI uri) {
|
||||||
switch (uri.toString()) {
|
switch (uri.toString()) {
|
||||||
case "http://localhost:61/": // unopened port
|
case "http://localhost:61/": // unopened port
|
||||||
case "http://www.google.com:81/": // dropped request
|
case "http://www.google.com:81/": // dropped request
|
||||||
|
|
|
@ -84,7 +84,7 @@ abstract class AbstractReactorNettyHttpClientTest extends HttpClientTest<HttpCli
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
boolean hasClientSpanAttributes(URI uri) {
|
boolean hasClientSpanHttpAttributes(URI uri) {
|
||||||
switch (uri.toString()) {
|
switch (uri.toString()) {
|
||||||
case "http://localhost:61/": // unopened port
|
case "http://localhost:61/": // unopened port
|
||||||
case "http://www.google.com:81/": // dropped request
|
case "http://www.google.com:81/": // dropped request
|
||||||
|
|
|
@ -84,7 +84,7 @@ abstract class AbstractReactorNettyHttpClientTest extends HttpClientTest<HttpCli
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
boolean hasClientSpanAttributes(URI uri) {
|
boolean hasClientSpanHttpAttributes(URI uri) {
|
||||||
switch (uri.toString()) {
|
switch (uri.toString()) {
|
||||||
case "http://localhost:61/": // unopened port
|
case "http://localhost:61/": // unopened port
|
||||||
case "http://www.google.com:81/": // dropped request
|
case "http://www.google.com:81/": // dropped request
|
||||||
|
|
|
@ -821,6 +821,7 @@ abstract class HttpClientTest<REQUEST> extends InstrumentationSpecification {
|
||||||
void clientSpan(TraceAssert trace, int index, Object parentSpan, String method = "GET", URI uri = server.address.resolve("/success"), Integer responseCode = 200, Throwable exception = null, String httpFlavor = "1.1") {
|
void clientSpan(TraceAssert trace, int index, Object parentSpan, String method = "GET", URI uri = server.address.resolve("/success"), Integer responseCode = 200, Throwable exception = null, String httpFlavor = "1.1") {
|
||||||
def userAgent = userAgent()
|
def userAgent = userAgent()
|
||||||
def extraAttributes = extraAttributes()
|
def extraAttributes = extraAttributes()
|
||||||
|
def hasClientSpanHttpAttributes = hasClientSpanHttpAttributes(uri)
|
||||||
trace.span(index) {
|
trace.span(index) {
|
||||||
if (parentSpan == null) {
|
if (parentSpan == null) {
|
||||||
hasNoParent()
|
hasNoParent()
|
||||||
|
@ -835,7 +836,6 @@ abstract class HttpClientTest<REQUEST> extends InstrumentationSpecification {
|
||||||
} else if (responseCode >= 400) {
|
} else if (responseCode >= 400) {
|
||||||
status ERROR
|
status ERROR
|
||||||
}
|
}
|
||||||
if (hasClientSpanAttributes(uri)) {
|
|
||||||
attributes {
|
attributes {
|
||||||
"${SemanticAttributes.NET_TRANSPORT.key}" IP_TCP
|
"${SemanticAttributes.NET_TRANSPORT.key}" IP_TCP
|
||||||
if (uri.port == UNUSABLE_PORT || uri.host == "192.0.2.1" || (uri.host == "www.google.com" && uri.port == 81)) {
|
if (uri.port == UNUSABLE_PORT || uri.host == "192.0.2.1" || (uri.host == "www.google.com" && uri.port == 81)) {
|
||||||
|
@ -854,6 +854,7 @@ abstract class HttpClientTest<REQUEST> extends InstrumentationSpecification {
|
||||||
} else {
|
} else {
|
||||||
"${SemanticAttributes.NET_PEER_IP.key}" { it == null || it == "127.0.0.1" || it == uri.host } // Optional
|
"${SemanticAttributes.NET_PEER_IP.key}" { it == null || it == "127.0.0.1" || it == uri.host } // Optional
|
||||||
}
|
}
|
||||||
|
if (hasClientSpanHttpAttributes) {
|
||||||
"${SemanticAttributes.HTTP_URL.key}" { it == "${uri}" || it == "${removeFragment(uri)}" }
|
"${SemanticAttributes.HTTP_URL.key}" { it == "${uri}" || it == "${removeFragment(uri)}" }
|
||||||
"${SemanticAttributes.HTTP_METHOD.key}" method
|
"${SemanticAttributes.HTTP_METHOD.key}" method
|
||||||
if (uri.host == "www.google.com") {
|
if (uri.host == "www.google.com") {
|
||||||
|
@ -864,6 +865,7 @@ abstract class HttpClientTest<REQUEST> extends InstrumentationSpecification {
|
||||||
if (userAgent) {
|
if (userAgent) {
|
||||||
"${SemanticAttributes.HTTP_USER_AGENT.key}" { it.startsWith(userAgent) }
|
"${SemanticAttributes.HTTP_USER_AGENT.key}" { it.startsWith(userAgent) }
|
||||||
}
|
}
|
||||||
|
}
|
||||||
if (responseCode) {
|
if (responseCode) {
|
||||||
"${SemanticAttributes.HTTP_STATUS_CODE.key}" responseCode
|
"${SemanticAttributes.HTTP_STATUS_CODE.key}" responseCode
|
||||||
}
|
}
|
||||||
|
@ -886,7 +888,6 @@ abstract class HttpClientTest<REQUEST> extends InstrumentationSpecification {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
void serverSpan(TraceAssert traces, int index, Object parentSpan = null,
|
void serverSpan(TraceAssert traces, int index, Object parentSpan = null,
|
||||||
@ClosureParams(value = SimpleType, options = ['io.opentelemetry.instrumentation.test.asserts.AttributesAssert'])
|
@ClosureParams(value = SimpleType, options = ['io.opentelemetry.instrumentation.test.asserts.AttributesAssert'])
|
||||||
|
@ -921,7 +922,7 @@ abstract class HttpClientTest<REQUEST> extends InstrumentationSpecification {
|
||||||
spanAssert.errorEvent(errorType, message)
|
spanAssert.errorEvent(errorType, message)
|
||||||
}
|
}
|
||||||
|
|
||||||
boolean hasClientSpanAttributes(URI uri) {
|
boolean hasClientSpanHttpAttributes(URI uri) {
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue