BrowserMob 0 response metrics (#7)

* BrowserMob 0 response metrics

* Fix failed tests

* Fix metrics names

* Release snapshot version

Author: kirillbilchenko

README.md

@@ -395,7 +395,7 @@ When you build the latest code from source, you'll have access to the latest sna
     <dependency>
         <groupId>net.lightbody.bmp</groupId>
         <artifactId>browsermob-core</artifactId>
-        <version>2.1.10-SNAPSHOT</version>
+        <version>2.1.11-SNAPSHOT</version>
         <scope>test</scope>
     </dependency>
 ```

browsermob-core/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <artifactId>browsermob-proxy</artifactId>
         <groupId>net.lightbody.bmp</groupId>
-        <version>2.1.10-SNAPSHOT</version>
+        <version>2.1.11-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

browsermob-core/src/main/java/net/lightbody/bmp/filters/HarCaptureFilter.java

@@ -2,6 +2,8 @@
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.io.BaseEncoding;
+import com.timgroup.statsd.NonBlockingStatsDClient;
+import com.timgroup.statsd.StatsDClient;
 import io.netty.buffer.ByteBuf;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.handler.codec.http.HttpContent;
@@ -44,8 +46,12 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import static net.lightbody.bmp.filters.StatsDMetricsFilter.*;
+import static net.lightbody.bmp.filters.StatsDMetricsFilter.getStatsDPort;
+
 public class HarCaptureFilter extends HttpsAwareFiltersAdapter {
     private static final Logger log = LoggerFactory.getLogger(HarCaptureFilter.class);
+    private static final StatsDClient statsDClient = new NonBlockingStatsDClient("automated_tests", getStatsDHost(), getStatsDPort());
 
     /**
      * The currently active HAR at the time the current request is received.
@@ -123,16 +129,16 @@ public class HarCaptureFilter extends HttpsAwareFiltersAdapter {
      * <p/>
      * Regardless of the CaptureTypes specified in <code>dataToCapture</code>, the HarCaptureFilter will always capture:
      * <ul>
-     *     <li>Request and response sizes</li>
-     *     <li>HTTP request and status lines</li>
-     *     <li>Page timing information</li>
+     * <li>Request and response sizes</li>
+     * <li>HTTP request and status lines</li>
+     * <li>Page timing information</li>
      * </ul>
      *
      * @param originalRequest the original HttpRequest from the HttpFiltersSource factory
-     * @param har a reference to the ProxyServer's current HAR file at the time this request is received (can be null if HAR capture is not required)
-     * @param currentPageRef the ProxyServer's currentPageRef at the time this request is received from the client
-     * @param dataToCapture the data types to capture for this request. null or empty set indicates only basic information will be
-     *                      captured (see {@link net.lightbody.bmp.proxy.CaptureType} for information on data collected for each CaptureType)
+     * @param har             a reference to the ProxyServer's current HAR file at the time this request is received (can be null if HAR capture is not required)
+     * @param currentPageRef  the ProxyServer's currentPageRef at the time this request is received from the client
+     * @param dataToCapture   the data types to capture for this request. null or empty set indicates only basic information will be
+     *                        captured (see {@link net.lightbody.bmp.proxy.CaptureType} for information on data collected for each CaptureType)
      */
     public HarCaptureFilter(HttpRequest originalRequest, ChannelHandlerContext ctx, Har har, String currentPageRef, Set<CaptureType> dataToCapture) {
         super(originalRequest, ctx);
@@ -199,6 +205,8 @@ public HttpResponse clientToProxyRequest(HttpObject httpObject) {
             HarResponse defaultHarResponse = HarCaptureUtil.createHarResponseForFailure();
             defaultHarResponse.setError(HarCaptureUtil.getNoResponseReceivedErrorMessage());
             harEntry.setResponse(defaultHarResponse);
+            statsDClient.increment(getProxyPrefix().concat(prepareMetric(harEntry.getRequest().getUrl()))
+                    .concat("." + harEntry.getResponse().getStatus()).concat(".client_proxy_connection_fail"));
 
             captureQueryParameters(httpRequest);
             // not capturing user agent: in many cases, it doesn't make sense to capture at the HarLog level, since the proxy could be
@@ -278,6 +286,8 @@ public void serverToProxyResponseTimedOut() {
         // replace any existing HarResponse that was created if the server sent a partial response
         HarResponse response = HarCaptureUtil.createHarResponseForFailure();
         harEntry.setResponse(response);
+        statsDClient.increment(getProxyPrefix().concat(prepareMetric(harEntry.getRequest().getUrl()))
+                .concat("." + harEntry.getResponse().getStatus()).concat(".response_timeout"));
 
         response.setError(HarCaptureUtil.getResponseTimedOutErrorMessage());
 
@@ -408,7 +418,7 @@ protected void captureRequestContent(HttpRequest httpRequest, byte[] fullMessage
 
         Charset charset;
         try {
-             charset = BrowserMobHttpUtil.readCharsetInContentTypeHeader(contentType);
+            charset = BrowserMobHttpUtil.readCharsetInContentTypeHeader(contentType);
         } catch (UnsupportedCharsetException e) {
             log.warn("Found unsupported character set in Content-Type header '{}' in HTTP request to {}. Content will not be captured in HAR.", contentType, httpRequest.getUri(), e);
             return;
@@ -651,6 +661,8 @@ public InetSocketAddress proxyToServerResolutionStarted(String resolvingServerHo
     public void proxyToServerResolutionFailed(String hostAndPort) {
         HarResponse response = HarCaptureUtil.createHarResponseForFailure();
         harEntry.setResponse(response);
+        statsDClient.increment(getProxyPrefix().concat(prepareMetric(harEntry.getRequest().getUrl()))
+                .concat("." + harEntry.getResponse().getStatus()).concat(".server_resolution_fail"));
 
         response.setError(HarCaptureUtil.getResolutionFailedErrorMessage(hostAndPort));
 
@@ -692,6 +704,8 @@ public void proxyToServerConnectionStarted() {
     @Override
     public void proxyToServerConnectionFailed() {
         HarResponse response = HarCaptureUtil.createHarResponseForFailure();
+        statsDClient.increment(getProxyPrefix().concat(prepareMetric(harEntry.getRequest().getUrl()))
+                .concat("." + harEntry.getResponse().getStatus()).concat(".server_connection_fail"));
         harEntry.setResponse(response);
 
         response.setError(HarCaptureUtil.getConnectionFailedErrorMessage());

browsermob-core/src/main/java/net/lightbody/bmp/filters/HttpConnectHarCaptureFilter.java

@@ -1,6 +1,8 @@
 package net.lightbody.bmp.filters;
 
 import com.google.common.cache.CacheBuilder;
+import com.timgroup.statsd.NonBlockingStatsDClient;
+import com.timgroup.statsd.StatsDClient;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.handler.codec.http.HttpObject;
 import io.netty.handler.codec.http.HttpRequest;
@@ -23,6 +25,8 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 
+import static net.lightbody.bmp.filters.StatsDMetricsFilter.*;
+
 /**
  * This filter captures HAR data for HTTP CONNECT requests. CONNECTs are "meta" requests that must be made before HTTPS
  * requests, but are not populated as separate requests in the HAR. Most information from HTTP CONNECTs (such as SSL
@@ -31,10 +35,10 @@
  * static methods. This filter also handles HTTP CONNECT errors and creates HAR entries for those errors, since there
  * would otherwise not be any record in the HAR of the error (if the CONNECT fails, there will be no subsequent "real"
  * request in which to record the error).
- *
  */
 public class HttpConnectHarCaptureFilter extends HttpsAwareFiltersAdapter implements ModifiedRequestAwareFilter {
     private static final Logger log = LoggerFactory.getLogger(HttpConnectHarCaptureFilter.class);
+    private static final StatsDClient statsDClient = new NonBlockingStatsDClient("automated_tests", getStatsDHost(), getStatsDPort());
 
     /**
      * The currently active HAR at the time the current request is received.
@@ -104,7 +108,7 @@ public class HttpConnectHarCaptureFilter extends HttpsAwareFiltersAdapter implem
     /**
      * Stores SSL connection timing information from HTTP CONNNECT requests. This timing information is stored in the first HTTP request
      * after the CONNECT, not in the CONNECT itself, so it needs to be stored across requests.
-     *
+     * <p>
      * This is the only state stored across multiple requests.
      */
     private static final ConcurrentMap<InetSocketAddress, HttpConnectTiming> httpConnectTimes =
@@ -330,6 +334,8 @@ private HarEntry createHarEntryForFailedCONNECT(String errorMessage) {
 
         HarResponse response = HarCaptureUtil.createHarResponseForFailure();
         harEntry.setResponse(response);
+        statsDClient.increment(getProxyPrefix().concat(prepareMetric(harEntry.getRequest().getUrl()))
+                .concat("." + harEntry.getResponse().getStatus()).concat(".failed_connect_request"));
 
         response.setError(errorMessage);
 

browsermob-core/src/main/java/net/lightbody/bmp/filters/StatsDMetricsFilter.java

@@ -43,15 +43,10 @@ public HttpObject serverToProxyResponse(HttpObject httpObject) {
             if (status > 399 || status == 0) {
                 String metric;
                 String url = HTTP_RESPONSE_STACK.pop();
-                try {
-                    URI uri = new URI(url);
-                    metric = "proxy.".concat(
-                            prepareMetric(uri.getHost().concat(uri.getPath())).concat(String.format(".%s", status)));
-                    client.increment(metric);
-                    HTTP_RESPONSE_STACK.clear();
-                } catch (URISyntaxException e) {
-                    e.printStackTrace();
-                }
+                metric = getProxyPrefix().concat(
+                        prepareMetric(url)).concat(String.format(".%s", status));
+                client.increment(metric);
+                HTTP_RESPONSE_STACK.clear();
             }
         }
         return super.serverToProxyResponse(httpObject);
@@ -65,8 +60,19 @@ public static int getStatsDPort() {
         return StringUtils.isEmpty(System.getenv("STATSD_PORT")) ? 8125 : NumberUtils.toInt(System.getenv("STATSD_PORT"));
     }
 
-    private String prepareMetric(String initialUrl) {
-        return initialUrl.replaceAll("/", "_").replaceAll("\\.", "_");
+    public static String getProxyPrefix() {
+        return "proxy.";
+    }
+
+    public static String prepareMetric(String initialUrl) {
+        URI uri = null;
+        try {
+            uri = new URI(initialUrl);
+        } catch (URISyntaxException e) {
+            e.printStackTrace();
+        }
+        return uri.getHost().concat(uri.getPath()).replaceAll("/", "_")
+                .replaceAll("\\.", "_");
     }
 
 }

browsermob-dist/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>browsermob-proxy</artifactId>
         <groupId>net.lightbody.bmp</groupId>
-        <version>2.1.10-SNAPSHOT</version>
+        <version>2.1.11-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

browsermob-legacy/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>browsermob-proxy</artifactId>
         <groupId>net.lightbody.bmp</groupId>
-        <version>2.1.10-SNAPSHOT</version>
+        <version>2.1.11-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

browsermob-rest/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>browsermob-proxy</artifactId>
         <groupId>net.lightbody.bmp</groupId>
-        <version>2.1.10-SNAPSHOT</version>
+        <version>2.1.11-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

mitm/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>browsermob-proxy</artifactId>
         <groupId>net.lightbody.bmp</groupId>
-        <version>2.1.10-SNAPSHOT</version>
+        <version>2.1.11-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>net.lightbody.bmp</groupId>
     <artifactId>browsermob-proxy</artifactId>
-    <version>2.1.10-SNAPSHOT</version>
+    <version>2.1.11-SNAPSHOT</version>
     <modules>
         <module>browsermob-core</module>
         <module>browsermob-legacy</module>