adding cause tag for transient poll failures. (#775)

Author: mindaugasbarcauskas

src/main/java/com/uber/cadence/internal/metrics/MetricsTag.java

@@ -24,4 +24,5 @@ public class MetricsTag {
   public static final String WORKFLOW_TYPE = "WorkflowType";
   public static final String REQUEST_TYPE = "RequestType";
   public static final String VERSION = "Version";
+  public static final String CAUSE = "Cause";
 }

src/main/java/com/uber/cadence/internal/metrics/MetricsTagValue.java

@@ -20,4 +20,6 @@
 public class MetricsTagValue {
   public static final String REQUEST_TYPE_NORMAL = "normal";
   public static final String REQUEST_TYPE_LONG_POLL = "long-poll";
+  public static final String SERVICE_BUSY = "serviceBusy";
+  public static final String INTERNAL_SERVICE_ERROR = "internalServiceError";
 }

src/main/java/com/uber/cadence/internal/worker/ActivityPollTask.java

@@ -17,19 +17,24 @@
 
 package com.uber.cadence.internal.worker;
 
+import com.google.common.collect.ImmutableMap;
 import com.uber.cadence.InternalServiceError;
 import com.uber.cadence.PollForActivityTaskRequest;
 import com.uber.cadence.PollForActivityTaskResponse;
 import com.uber.cadence.ServiceBusyError;
 import com.uber.cadence.TaskList;
 import com.uber.cadence.TaskListMetadata;
+import com.uber.cadence.internal.metrics.MetricsTag;
 import com.uber.cadence.internal.metrics.MetricsType;
 import com.uber.cadence.serviceclient.IWorkflowService;
 import com.uber.m3.tally.Stopwatch;
 import org.apache.thrift.TException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static com.uber.cadence.internal.metrics.MetricsTagValue.INTERNAL_SERVICE_ERROR;
+import static com.uber.cadence.internal.metrics.MetricsTagValue.SERVICE_BUSY;
+
 final class ActivityPollTask extends ActivityPollTaskBase {
 
   private static final Logger log = LoggerFactory.getLogger(ActivityPollTask.class);
@@ -66,10 +71,18 @@ protected PollForActivityTaskResponse pollTask() throws TException {
     PollForActivityTaskResponse result;
     try {
       result = service.PollForActivityTask(pollRequest);
-    } catch (InternalServiceError | ServiceBusyError e) {
-      options.getMetricsScope().counter(MetricsType.ACTIVITY_POLL_TRANSIENT_FAILED_COUNTER).inc(1);
+    } catch (InternalServiceError e) {
+      options.getMetricsScope()
+              .tagged(ImmutableMap.of(MetricsTag.CAUSE, INTERNAL_SERVICE_ERROR))
+              .counter(MetricsType.ACTIVITY_POLL_TRANSIENT_FAILED_COUNTER).inc(1);
+      throw e;
+    } catch (ServiceBusyError e) {
+      options.getMetricsScope()
+              .tagged(ImmutableMap.of(MetricsTag.CAUSE, SERVICE_BUSY))
+              .counter(MetricsType.ACTIVITY_POLL_TRANSIENT_FAILED_COUNTER).inc(1);
       throw e;
-    } catch (TException e) {
+    }
+    catch (TException e) {
       options.getMetricsScope().counter(MetricsType.ACTIVITY_POLL_FAILED_COUNTER).inc(1);
       throw e;
     }

src/main/java/com/uber/cadence/internal/worker/WorkflowPollTask.java

@@ -30,11 +30,15 @@
 import com.uber.m3.tally.Stopwatch;
 import com.uber.m3.util.Duration;
 import com.uber.m3.util.ImmutableMap;
+
 import java.util.Objects;
 import org.apache.thrift.TException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static com.uber.cadence.internal.metrics.MetricsTagValue.INTERNAL_SERVICE_ERROR;
+import static com.uber.cadence.internal.metrics.MetricsTagValue.SERVICE_BUSY;
+
 final class WorkflowPollTask implements Poller.PollTask<PollForDecisionTaskResponse> {
 
   private static final Logger log = LoggerFactory.getLogger(WorkflowWorker.class);
@@ -77,8 +81,15 @@ public PollForDecisionTaskResponse poll() throws TException {
     PollForDecisionTaskResponse result;
     try {
       result = service.PollForDecisionTask(pollRequest);
-    } catch (InternalServiceError | ServiceBusyError e) {
-      metricScope.counter(MetricsType.DECISION_POLL_TRANSIENT_FAILED_COUNTER).inc(1);
+    } catch (InternalServiceError e) {
+      metricScope
+              .tagged(ImmutableMap.of(MetricsTag.CAUSE, INTERNAL_SERVICE_ERROR))
+              .counter(MetricsType.DECISION_POLL_TRANSIENT_FAILED_COUNTER).inc(1);
+      throw e;
+    } catch (ServiceBusyError e) {
+      metricScope
+              .tagged(ImmutableMap.of(MetricsTag.CAUSE, SERVICE_BUSY))
+              .counter(MetricsType.DECISION_POLL_TRANSIENT_FAILED_COUNTER).inc(1);
       throw e;
     } catch (TException e) {
       metricScope.counter(MetricsType.DECISION_POLL_FAILED_COUNTER).inc(1);