Improved tracing of failures, user exceptions are now tracked

with a userException member.

12 files changed, 206 insertions, 80 deletions

diff --git a/java/src/Ice/AsyncResult.java b/java/src/Ice/AsyncResult.java
index e70aac96a0b..cff5677dc2d 100644
--- a/java/src/Ice/AsyncResult.java
+++ b/java/src/Ice/AsyncResult.java
@@ -329,6 +329,15 @@ public class AsyncResult
                 __error(exc);
             }
         }
+
+        if(_observer != null)
+        {
+            Ice.ObjectPrx proxy = getProxy();
+            if(proxy == null || !proxy.ice_isTwoway())
+            {
+                _observer.detach();
+            }
+        }
     }
 
     public void 
@@ -344,6 +353,11 @@ public class AsyncResult
         }
     }
 
+    public Ice.Instrumentation.InvocationObserver __getObserver()
+    {
+        return _observer;
+    }
+    
     public final void __sentAsync()
     {
         //
diff --git a/java/src/Ice/ObjectPrxHelperBase.java b/java/src/Ice/ObjectPrxHelperBase.java
index 032e41c50e3..bd5e7b0ccd7 100644
--- a/java/src/Ice/ObjectPrxHelperBase.java
+++ b/java/src/Ice/ObjectPrxHelperBase.java
@@ -235,22 +235,35 @@ public class ObjectPrxHelperBase implements ObjectPrx, java.io.Serializable
     end_ice_isA(AsyncResult __result)
     {
         AsyncResult.__check(__result, this, __ice_isA_name);
-        if(!__result.__wait())
+        boolean __ok = __result.__wait();
+        try
         {
-            try
+            if(!__ok)
             {
-                __result.__throwUserException();
+                try
+                {
+                    __result.__throwUserException();
+                }
+                catch(UserException __ex)
+                {
+                    throw new UnknownUserException(__ex.ice_name(), __ex);
+                }
             }
-            catch(UserException __ex)
+            boolean __ret;
+            IceInternal.BasicStream __is = __result.__startReadParams();
+            __ret = __is.readBool();
+            __result.__endReadParams();
+            return __ret;
+        }
+        catch(Ice.LocalException ex)
+        {
+            InvocationObserver obsv = __result.__getObserver();
+            if(obsv != null)
             {
-                throw new UnknownUserException(__ex.ice_name(), __ex);
+                obsv.failed(ex.ice_name());
             }
+            throw ex;
         }
-        boolean __ret;
-        IceInternal.BasicStream __is = __result.__startReadParams();
-        __ret = __is.readBool();
-        __result.__endReadParams();
-        return __ret;
     }
 
     /**
@@ -585,22 +598,35 @@ public class ObjectPrxHelperBase implements ObjectPrx, java.io.Serializable
     end_ice_ids(AsyncResult __result)
     {
         AsyncResult.__check(__result, this, __ice_ids_name);
-        if(!__result.__wait())
+        boolean __ok = __result.__wait();
+        try
         {
-            try
+            if(!__ok)
             {
-                __result.__throwUserException();
+                try
+                {
+                    __result.__throwUserException();
+                }
+                catch(UserException __ex)
+                {
+                    throw new UnknownUserException(__ex.ice_name(), __ex);
+                }
             }
-            catch(UserException __ex)
+            String[] __ret = null;
+            IceInternal.BasicStream __is = __result.__startReadParams();
+            __ret = StringSeqHelper.read(__is);
+            __result.__endReadParams();
+            return __ret;
+        }
+        catch(Ice.LocalException ex)
+        {
+            InvocationObserver obsv = __result.__getObserver();
+            if(obsv != null)
             {
-                throw new UnknownUserException(__ex.ice_name(), __ex);
+                obsv.failed(ex.ice_name());
             }
+            throw ex;
         }
-        String[] __ret = null;
-        IceInternal.BasicStream __is = __result.__startReadParams();
-        __ret = StringSeqHelper.read(__is);
-        __result.__endReadParams();
-        return __ret;
     }
 
     /**
@@ -769,22 +795,35 @@ public class ObjectPrxHelperBase implements ObjectPrx, java.io.Serializable
     end_ice_id(AsyncResult __result)
     {
         AsyncResult.__check(__result, this, __ice_id_name);
-        if(!__result.__wait())
+        boolean __ok = __result.__wait();
+        try
         {
-            try
+            if(!__ok)
             {
-                __result.__throwUserException();
+                try
+                {
+                    __result.__throwUserException();
+                }
+                catch(UserException __ex)
+                {
+                    throw new UnknownUserException(__ex.ice_name(), __ex);
+                }
             }
-            catch(UserException __ex)
+            String __ret = null;
+            IceInternal.BasicStream __is = __result.__startReadParams();
+            __ret = __is.readString();
+            __result.__endReadParams();
+            return __ret;
+        }
+        catch(Ice.LocalException ex)
+        {
+            InvocationObserver obsv = __result.__getObserver();
+            if(obsv != null)
             {
-                throw new UnknownUserException(__ex.ice_name(), __ex);
+                obsv.failed(ex.ice_name());
             }
+            throw ex;
         }
-        String __ret = null;
-        IceInternal.BasicStream __is = __result.__startReadParams();
-        __ret = __is.readString();
-        __result.__endReadParams();
-        return __ret;
     }
 
     /**
@@ -1035,9 +1074,21 @@ public class ObjectPrxHelperBase implements ObjectPrx, java.io.Serializable
         boolean ok = __result.__wait();
         if(_reference.getMode() == IceInternal.Reference.ModeTwoway)
         {
-            if(outParams != null)
+            try
             {
-                outParams.value = __result.__readParamEncaps();
+                if(outParams != null)
+                {
+                    outParams.value = __result.__readParamEncaps();
+                }
+            }
+            catch(Ice.LocalException ex)
+            {
+                InvocationObserver obsv = __result.__getObserver();
+                if(obsv != null)
+                {
+                    obsv.failed(ex.ice_name());
+                }
+                throw ex;
             }
         }
         return ok;
@@ -2023,7 +2074,8 @@ public class ObjectPrxHelperBase implements ObjectPrx, java.io.Serializable
     }
 
     public final int
-    __handleException(_ObjectDel delegate, LocalException ex, Ice.IntHolder interval, int cnt, InvocationObserver obsv)
+    __handleException(_ObjectDel delegate, LocalException ex, Ice.IntHolder interval, int cnt, 
+                      InvocationObserver obsv)
     {
         //
         // Only _delegate needs to be mutex protected here.
@@ -2091,10 +2143,6 @@ public class ObjectPrxHelperBase implements ObjectPrx, java.io.Serializable
             }
             throw ex.get();
         }
-        else if(obsv != null)
-        {
-            obsv.retried();
-        }
     }
 
     public final int
@@ -2117,10 +2165,6 @@ public class ObjectPrxHelperBase implements ObjectPrx, java.io.Serializable
                     _delegate = null;
                 }
             }
-            if(obsv != null)
-            {
-                obsv.retried();
-            }
             return cnt;
         }
     }
@@ -2208,18 +2252,30 @@ public class ObjectPrxHelperBase implements ObjectPrx, java.io.Serializable
         boolean ok = __result.__wait();
         if(_reference.getMode() == IceInternal.Reference.ModeTwoway)
         {
-            if(!ok)
+            try
             {
-                try
+                if(!ok)
                 {
-                    __result.__throwUserException();
+                    try
+                    {
+                        __result.__throwUserException();
+                    }
+                    catch(UserException __ex)
+                    {
+                        throw new UnknownUserException(__ex.ice_name(), __ex);
+                    }
                 }
-                catch(UserException __ex)
+                __result.__readEmptyParams();
+            }
+            catch(Ice.LocalException ex)
+            {
+                InvocationObserver obsv = __result.__getObserver();
+                if(obsv != null)
                 {
-                    throw new UnknownUserException(__ex.ice_name(), __ex);
+                    obsv.failed(ex.ice_name());
                 }
+                throw ex;
             }
-            __result.__readEmptyParams();
         }
     }
 
diff --git a/java/src/IceInternal/BatchOutgoing.java b/java/src/IceInternal/BatchOutgoing.java
index 8dfeefcf49d..69bb30285a5 100644
--- a/java/src/IceInternal/BatchOutgoing.java
+++ b/java/src/IceInternal/BatchOutgoing.java
@@ -53,11 +53,6 @@ public final class BatchOutgoing implements OutgoingMessageCallback
                     }
                 }
                 
-                if(_remoteObserver != null)
-                {
-                    _remoteObserver.detach();
-                    _remoteObserver = null;
-                }
                 if(_exception != null)
                 {
                     throw _exception;
@@ -81,11 +76,22 @@ public final class BatchOutgoing implements OutgoingMessageCallback
         {
             _sent = true;
         }
+        if(_remoteObserver != null)
+        {
+            _remoteObserver.detach();
+            _remoteObserver = null;
+        }
     }
     
     public synchronized void
     finished(Ice.LocalException ex, boolean sent)
     {
+        if(_remoteObserver != null)
+        {
+            _remoteObserver.failed(ex.ice_name());
+            _remoteObserver.detach();
+            _remoteObserver = null;
+        }
         _exception = ex;
         notify();
     }
diff --git a/java/src/IceInternal/BatchOutgoingAsync.java b/java/src/IceInternal/BatchOutgoingAsync.java
index 6278faf9412..17ae33b2aa1 100644
--- a/java/src/IceInternal/BatchOutgoingAsync.java
+++ b/java/src/IceInternal/BatchOutgoingAsync.java
@@ -40,6 +40,7 @@ public class BatchOutgoingAsync extends Ice.AsyncResult implements OutgoingAsync
     {
         if(_remoteObserver != null)
         {
+            _remoteObserver.failed(exc.ice_name());
             _remoteObserver.detach();
             _remoteObserver = null;
         }
diff --git a/java/src/IceInternal/CommunicatorBatchOutgoingAsync.java b/java/src/IceInternal/CommunicatorBatchOutgoingAsync.java
index b64c56b8d99..014d6e041e6 100644
--- a/java/src/IceInternal/CommunicatorBatchOutgoingAsync.java
+++ b/java/src/IceInternal/CommunicatorBatchOutgoingAsync.java
@@ -64,6 +64,7 @@ public class CommunicatorBatchOutgoingAsync extends Ice.AsyncResult
             {
                 if(_remoteObserver != null)
                 {
+                    _remoteObserver.failed(ex.ice_name());
                     _remoteObserver.detach();
                     _remoteObserver = null;
                 }
@@ -110,13 +111,6 @@ public class CommunicatorBatchOutgoingAsync extends Ice.AsyncResult
             {
                 return;
             }
-            
-            if(_observer != null)
-            {
-                _observer.detach();
-                _observer = null;
-            }
-
             _state |= Done | OK | Sent;
             _monitor.notifyAll();
         }
diff --git a/java/src/IceInternal/Incoming.java b/java/src/IceInternal/Incoming.java
index a990559bd32..bd5d7715a6f 100644
--- a/java/src/IceInternal/Incoming.java
+++ b/java/src/IceInternal/Incoming.java
@@ -160,7 +160,7 @@ final public class Incoming extends IncomingBase implements Ice.Request
 
                         if(_observer != null)
                         {
-                            _observer.failed(ex.ice_name());
+                            _observer.userException();
                         }
 
                         if(_response)
diff --git a/java/src/IceInternal/IncomingBase.java b/java/src/IceInternal/IncomingBase.java
index 6d1ec52dde6..64cd5912b91 100644
--- a/java/src/IceInternal/IncomingBase.java
+++ b/java/src/IceInternal/IncomingBase.java
@@ -114,6 +114,11 @@ public class IncomingBase
     public void 
     __endWriteParams(boolean ok)
     {
+        if(!ok && _observer != null)
+        {
+            _observer.userException();
+        }
+
         if(_response)
         {
             int save = _os.pos();
@@ -139,6 +144,11 @@ public class IncomingBase
     public void 
     __writeParamEncaps(byte[] v, boolean ok)
     {
+        if(!ok && _observer != null)
+        {
+            _observer.userException();
+        }
+
         if(_response)
         {
             assert(_os.size() == Protocol.headerSize + 4); // Reply status position.
@@ -158,10 +168,6 @@ public class IncomingBase
     public void
     __writeUserException(Ice.UserException ex, Ice.FormatType format)
     {
-        if(_observer != null)
-        {
-            _observer.failed(ex.ice_name());
-        }
         BasicStream __os = __startWriteParams(format);
         __os.writeUserException(ex);
         __endWriteParams(false);
@@ -270,7 +276,7 @@ public class IncomingBase
 
             if(_observer != null)
             {
-                _observer.failed(ex.ice_name());
+                _observer.userException();
             }
 
             //
@@ -558,7 +564,7 @@ public class IncomingBase
     protected Ice.Object _servant;
     protected Ice.ServantLocator _locator;
     protected Ice.LocalObjectHolder _cookie;
-    protected Ice.Instrumentation.Observer _observer;
+    protected Ice.Instrumentation.DispatchObserver _observer;
 
     protected boolean _response;
     protected byte _compress;
diff --git a/java/src/IceInternal/Outgoing.java b/java/src/IceInternal/Outgoing.java
index 9a4d0fba420..90848d502e6 100644
--- a/java/src/IceInternal/Outgoing.java
+++ b/java/src/IceInternal/Outgoing.java
@@ -321,6 +321,10 @@ public final class Outgoing implements OutgoingMessageCallback
             
             case ReplyStatus.replyUserException:
             {
+                if(_observer != null)
+                {
+                    _observer.userException();
+                }
                 _state = StateUserException; // The state must be set last, in case there is an exception.
                 break;
             }
@@ -440,6 +444,7 @@ public final class Outgoing implements OutgoingMessageCallback
         assert(_state <= StateInProgress);
         if(_remoteObserver != null)
         {
+            _remoteObserver.failed(ex.ice_name());
             _remoteObserver.detach();
             _remoteObserver = null;
         }
@@ -529,10 +534,6 @@ public final class Outgoing implements OutgoingMessageCallback
         }
         catch(Ice.UserException ex)
         {
-            if(_observer != null)
-            {
-                _observer.failed(ex.ice_name());
-            }
             _is.endReadEncaps();
             throw ex;
         }
diff --git a/java/src/IceInternal/OutgoingAsync.java b/java/src/IceInternal/OutgoingAsync.java
index 04fb8b0ebf3..cb44b4ea4dc 100644
--- a/java/src/IceInternal/OutgoingAsync.java
+++ b/java/src/IceInternal/OutgoingAsync.java
@@ -138,11 +138,6 @@ public class OutgoingAsync extends Ice.AsyncResult implements OutgoingAsyncMessa
     public void __sent()
     {
         __sentInternal();
-        if(_observer != null && !_proxy.ice_isTwoway())
-        {
-            _observer.detach();
-            _observer = null;
-        }
     }
 
     public void __finished(Ice.LocalException exc, boolean sent)
@@ -152,6 +147,7 @@ public class OutgoingAsync extends Ice.AsyncResult implements OutgoingAsyncMessa
             assert((_state & Done) == 0);
             if(_remoteObserver != null)
             {
+                _remoteObserver.failed(exc.ice_name());
                 _remoteObserver.detach();
                 _remoteObserver = null;
             }
@@ -196,6 +192,7 @@ public class OutgoingAsync extends Ice.AsyncResult implements OutgoingAsyncMessa
         
         if(_remoteObserver != null)
         {
+            _remoteObserver.failed(exc.get().ice_name());
             _remoteObserver.detach();
             _remoteObserver = null;
         }
@@ -249,8 +246,16 @@ public class OutgoingAsync extends Ice.AsyncResult implements OutgoingAsyncMessa
                 switch(replyStatus)
                 {
                     case ReplyStatus.replyOK:
+                    {
+                        break;
+                    }
+
                     case ReplyStatus.replyUserException:
                     {
+                        if(_observer != null)
+                        {
+                            _observer.userException();
+                        }
                         break;
                     }
 
diff --git a/java/src/IceMX/CommunicatorObserverI.java b/java/src/IceMX/CommunicatorObserverI.java
index 678b232bfd2..c5f724fe5e0 100644
--- a/java/src/IceMX/CommunicatorObserverI.java
+++ b/java/src/IceMX/CommunicatorObserverI.java
@@ -181,7 +181,7 @@ public class CommunicatorObserverI implements Ice.Instrumentation.CommunicatorOb
         private Ice.EndpointInfo _endpointInfo;
     };
 
-    static private final class DispatchHelper extends MetricsHelper<Metrics>
+    static private final class DispatchHelper extends MetricsHelper<DispatchMetrics>
     {
         static private final AttributeResolver _attributes = new AttributeResolver()
             {
@@ -607,7 +607,7 @@ public class CommunicatorObserverI implements Ice.Instrumentation.CommunicatorOb
 
         _connections = new ObserverFactory<ConnectionMetrics, ConnectionObserverI>(metrics, "Connection", 
                                                                                    ConnectionMetrics.class);
-        _dispatch = new ObserverFactory<Metrics, ObserverI>(metrics, "Dispatch", Metrics.class);
+        _dispatch = new ObserverFactory<DispatchMetrics, DispatchObserverI>(metrics, "Dispatch", DispatchMetrics.class);
         _invocations = new ObserverFactory<InvocationMetrics, InvocationObserverI>(metrics, "Invocation", 
                                                                                    InvocationMetrics.class);
         _threads = new ObserverFactory<ThreadMetrics, ThreadObserverI>(metrics, "Thread", ThreadMetrics.class);
@@ -711,14 +711,14 @@ public class CommunicatorObserverI implements Ice.Instrumentation.CommunicatorOb
         return null;
     }
     
-    public Ice.Instrumentation.Observer
+    public Ice.Instrumentation.DispatchObserver
     getDispatchObserver(Ice.Current c)
     {
         if(_dispatch.isEnabled())
         {
             try
             {
-                return _dispatch.getObserver(new DispatchHelper(c), ObserverI.class);
+                return _dispatch.getObserver(new DispatchHelper(c), DispatchObserverI.class);
             }
             catch(Exception ex)
             {
@@ -754,7 +754,7 @@ public class CommunicatorObserverI implements Ice.Instrumentation.CommunicatorOb
 
     final private IceInternal.MetricsAdminI _metrics;
     final private ObserverFactory<ConnectionMetrics, ConnectionObserverI> _connections;
-    final private ObserverFactory<Metrics, ObserverI> _dispatch;
+    final private ObserverFactory<DispatchMetrics, DispatchObserverI> _dispatch;
     final private ObserverFactory<InvocationMetrics, InvocationObserverI> _invocations;
     final private ObserverFactory<ThreadMetrics, ThreadObserverI> _threads;
     final private ObserverFactory<Metrics, ObserverI> _connects;
diff --git a/java/src/IceMX/DispatchObserverI.java b/java/src/IceMX/DispatchObserverI.java
new file mode 100644
index 00000000000..bb41ea3a102
--- /dev/null
+++ b/java/src/IceMX/DispatchObserverI.java
@@ -0,0 +1,28 @@
+// **********************************************************************
+//
+// Copyright (c) 2003-2012 ZeroC, Inc. All rights reserved.
+//
+// This copy of Ice is licensed to you under the terms described in the
+// ICE_LICENSE file included in this distribution.
+//
+// **********************************************************************
+
+package IceMX;
+
+public class DispatchObserverI extends Observer<DispatchMetrics> implements Ice.Instrumentation.DispatchObserver
+{
+    public void
+    userException()
+    {
+        forEach(_userException);
+    }
+
+    final MetricsUpdate<DispatchMetrics> _userException = new MetricsUpdate<DispatchMetrics>()
+        {
+            public void
+            update(DispatchMetrics v)
+            {
+                ++v.userException;
+            }
+        };
+}
\ No newline at end of file
diff --git a/java/src/IceMX/InvocationObserverI.java b/java/src/IceMX/InvocationObserverI.java
index 288b0d19730..8c2d6cf1d32 100644
--- a/java/src/IceMX/InvocationObserverI.java
+++ b/java/src/IceMX/InvocationObserverI.java
@@ -107,6 +107,12 @@ public class InvocationObserverI extends Observer<InvocationMetrics> implements
     };
 
     public void
+    userException()
+    {
+        forEach(_userException);
+    }
+
+    public void
     retried()
     {
         forEach(_incrementRetry);
@@ -126,4 +132,13 @@ public class InvocationObserverI extends Observer<InvocationMetrics> implements
                 ++v.retry;
             }
         };
+
+    final MetricsUpdate<InvocationMetrics> _userException = new MetricsUpdate<InvocationMetrics>()
+        {
+            public void
+            update(InvocationMetrics v)
+            {
+                ++v.userException;
+            }
+        };
 }
\ No newline at end of file