Connection timeout when network interface is down

javiroman

Hi!

I'm getting a weird behavior with a simple method synchronous call when the NIC interface is hanged. If you invoke a remote method and the server side of the Ice application lose the NIC configuration, the server never returns, and the client is hanged waiting for the server response.

Ice doesn't report any error (even with all the traces on), only when the TCP keepalive counter is reached (7200 seconds) the Ice runtime raise an connection error.

How can we handle these kinds of situations?

You can reproduce this error with the following sample:

client.py:

#!/usr/bin/env python

import sys, traceback, Ice
import os, time
import threading

Ice.loadSlice("contract.ice")
import Test

CONFIG_FILE = "client.cfg"
VERSION="0.0.1"

class ApplicationClient(Ice.Application):
        def __init__(self):
                print "constructor cliente"

        def run(self, args):
                ic = self.communicator()

                try:   
                        end_point1 = "ObjectIdentity1:default -p 10000"
                        end_point2 = "ObjectIdentity2:default -p 10000"
                        e_servant1 = Test.InterfaceServant1Prx.checkedCast(ic.stringToProxy(end_point1))
                        e_servant2 = Test.InterfaceServant2Prx.checkedCast(ic.stringToProxy(end_point2))
                except Ice.NotRegisteredException:
                        print "Execpcion!!!"
                        traceback.print_exc()
                        return -1

                print "invoked doMethod1()"
                e_servant1.doMethod1()
                print "invoked doMethod2()"
                e_servant1.doMethod2()

                print "invoked doMethod3()"
                e_servant2.doMethod3()

                if (ic):
                        try:   
                                self.communicator().destroy()
                        except:
                                traceback.print_exc()
                                status=1
                return 0

def main():
        print "-------------------------------------------------------"
        print "launched Client client_uno %s" % VERSION
        print "-------------------------------------------------------"

        app = ApplicationClient()

        if not os.path.exists(CONFIG_FILE):
                return(app.main(sys.argv))
        else:
                data = Ice.InitializationData()
                #data.logger = MyLogger()
                data.properties = Ice.createProperties()
                data.properties.load(CONFIG_FILE)

                return(app.main(sys.argv, CONFIG_FILE, data))

if __name__ == "__main__":
        sys.exit(main())

server.py

#!/usr/bin/env python
 
import sys, traceback, Ice
import os, time
        
Ice.loadSlice("contract.ice")
import Test

CONFIG_FILE = "server.cfg"
VERSION="0.0.1" 
         
class InterfaceServant1I(Test.InterfaceServant1):
        def __init__(self):
                print "InterfaceServant_1 constructor"
        
        def doMethod1(self, current=None):
                print "doMethod_1: sleeping 60 seconds ..."
                time.sleep(10)

        def doMethod2(self, current=None):
                print "doMethod_2: sleeping 60 seconds ..."
                time.sleep(60)

class InterfaceServant2I(Test.InterfaceServant2):
        def __init__(self):
                print "InterfaceServant_2 constructor"
        
        def doMethod3(self, current=None):
                print "doMethod_3: sleeping 5 seconds ..."
                time.sleep(60)

class ApplicationServer(Ice.Application):
        def __init__(self, service_name="Servidor"):
                print "ApplicationServer constructor"
                self.service_name = service_name

        def run(self, args):
                ic = self.communicator()
                #adapter = ic.createObjectAdapter("HelloAdapter")
                adapter = ic.createObjectAdapterWithEndpoints("SimpleAdapter", "default -p 10000")

                serv_thread1 = InterfaceServant1I()
                ident1 = ic.stringToIdentity("ObjectIdentity1")
                adapter.add(serv_thread1, ident1)

                serv_thread2 = InterfaceServant2I()
                ident2 = ic.stringToIdentity("ObjectIdentity2")
                adapter.add(serv_thread2, ident2)

                adapter.activate()

                print "ApplicationServer running ..."

                self.communicator().waitForShutdown()

def main():
        print "-------------------------------------------------------"
        print "launched Servant server_uno %s" % VERSION
        print "-------------------------------------------------------"

        app = ApplicationServer()

        if not os.path.exists(CONFIG_FILE):
                return(app.main(sys.argv))
        else:
                data = Ice.InitializationData()
                data.properties = Ice.createProperties()
                data.properties.load(CONFIG_FILE)

                return(app.main(sys.argv, CONFIG_FILE, data))

if __name__=="__main__":
        sys.exit(main())

If your run server.py and client.py, if you disable the loopback interface (ifconfig lo down) when some remote method is invoked, we get the above error. Even if you raise again the interface (ifconfig lo up) the client and server never return, only when the main TCP timeout (two hours) is raised.

Thank for your help.

Note: we are using Ice 3.4.0 (Python 2.4)

Best regards.

denn

Ice 3.4.0 is rather old.
Might be time to upgrade and try the upcoming 3.5 version