Python framework - Memory leak

[pipiche]

I'm developing a python plugin, and some of the users are complaining since Domoticz 2023.2 on memory leaks.

I have used a bit of tracemalloc in the code, and it looks like the following line is just getting more and more memory

Code: Select all

Devices[Unit_].Touch()

PS/
I'm using the legacy framework


[Edit] Additional testing

start Domoticz with no plugin started

VIRT: 177 M
RES: 31104
SHR: 25600

start a plugin

VIRT: 250M
RES: 73108
SHR: 34560

stoping plugin

VIRT: 243 M
RES: 73364
SHR: 34560

Looks like the memory is not released when plugin stop , and if I restart and then stop, I see more less an increase in

VIRT: +5M
RES: +10000
SSHR: no change

[pipiche]

Here is a small plugin which is producing such memory consumption.
The plugin is creating about 100 Unit (Devices), and every second is using the Touch API.

You will see that VIRT and RES are increasing over the time.

Code: Select all

"""
<plugin key="MemoryLeak" name="Memory Leak on iTouch" author="pipiche" version="0.1" externallink="">
    <params>
        <param field="Mode6" label="Debugging" width="150px"  default="None" required="true">
            <description><br/><h3>Plugin debug</h3>This debugging option has been moved to the WebUI > Tools > Debug<br/></description>
                <options>
                    <option label="None" value="0"  default="true" />
                    <option label="Python Only" value="2"/>
                    <option label="Basic Debugging" value="62"/>
                    <option label="Basic+Messages" value="126"/>
                    <option label="Connections Only" value="16"/>
                    <option label="Connections+Queue" value="144"/>
                    <option label="All" value="-1"/>
                </options>
        </param>
    </params>
</plugin>
"""

import Domoticz

try:
    from Domoticz import Devices, Images, Parameters, Settings
except ImportError:
    pass


class BasePlugin:

    def __init__(self):
        self.plugin_ready = False
        return

    def onStart(self):
        Domoticz.Log('onStart called')

        for unit in range(1,100):
            if unit in Devices:
                continue
            myDev = Domoticz.Device(DeviceID="MemoryLeak", Name="MemoryLeak %s" % unit, Unit=unit, Type=244, Subtype=73, )
            myDev.Create()

        # Set plugin heartbeat to 1s
        Domoticz.Heartbeat(1)

    def onHeartbeat(self):
        Domoticz.Log('onHeartbeat called')
        for unit in Devices:
            Domoticz.Log('Touch %s' % unit)
            Devices[unit].Touch()

    def onCommand(self, Unit, Command, Level, Hue):
        Domoticz.Debug('onCommand called')

    def onStop(self):
        Domoticz.Debug('onStop called')


global _plugin
_plugin = BasePlugin()


def onStart():
    global _plugin
    _plugin.onStart()


def onStop():
    global _plugin
    _plugin.onStop()


def onHeartbeat():
    global _plugin
    _plugin.onHeartbeat()


def onCommand(Unit, Command, Level, Hue):
    global _plugin
    _plugin.onCommand(Unit, Command, Level, Hue)

[waltervl]

I will ask a not so nice question:
Do you see the same behavior with the DomoticzEx framework touch function?

[pipiche]

just update the here above plugin with DomoticzEx

[pipiche]

Here is the code.
You can switch from DomoticzEx to Domoticz by commenting lines 24,25 and de-commenting lines 26,27

I see the same memory consumption

Code: Select all

"""
<plugin key="MemoryLeak" name="Memory Leak on iTouch" author="pipiche" version="0.1" externallink="">
    <params>
        <param field="Mode6" label="Debugging" width="150px"  default="None" required="true">
            <description><br/><h3>Plugin debug</h3>This debugging option has been moved to the WebUI > Tools > Debug<br/></description>
                <options>
                    <option label="None" value="0"  default="true" />
                    <option label="Python Only" value="2"/>
                    <option label="Basic Debugging" value="62"/>
                    <option label="Basic+Messages" value="126"/>
                    <option label="Connections Only" value="16"/>
                    <option label="Connections+Queue" value="144"/>
                    <option label="All" value="-1"/>
                </options>
        </param>
    </params>
</plugin>
"""

HEARTBEAT_FEQ = 1
NUMBER_UNIT = 100


import DomoticzEx as Domoticz
DOMOTICZ_EXTENDED_API = True
# import Domoticz
# DOMOTICZ_EXTENDED_API = False


class BasePlugin:

    def __init__(self):
        self.plugin_ready = False
        return

    def onStart(self):
        Domoticz.Log('onStart called %s' % DOMOTICZ_EXTENDED_API)

        for unit in range(1, NUMBER_UNIT):
            device_create_unit(self, Devices, "MemoryLeak", "MemoryLeak %s" % unit, unit)

        # Set plugin heartbeat to 1s
        Domoticz.Heartbeat(HEARTBEAT_FEQ)

    def onHeartbeat(self):
        Domoticz.Log('onHeartbeat called')

        if DOMOTICZ_EXTENDED_API:
            for deviceId in Devices:
                for unit in Devices[ deviceId ].Units:
                    device_touch_api(self, Devices, deviceId, unit)
        else:
            for unit in Devices:
                device_touch_api(self, Devices, "MemoryLeak", unit)

    def onCommand(self, Unit, Command, Level, Hue):
        Domoticz.Debug('onCommand called')

    def onStop(self):
        Domoticz.Debug('onStop called')


def device_create_unit(self, Devices, deviceId, Name, unit):

    if DOMOTICZ_EXTENDED_API:
        if deviceId in Devices and unit in Devices[deviceId].Units:
            return
    elif not DOMOTICZ_EXTENDED_API and unit in Devices:
        return
    
    Domoticz.Log('device_create_unit %s/%s' % (deviceId ,unit))
    domoticz_device_api_class = Domoticz.Unit if DOMOTICZ_EXTENDED_API else Domoticz.Device

    myDev = domoticz_device_api_class(DeviceID=deviceId, Name=Name, Unit=unit, Type=244, Subtype=73, )
    myDev.Create()


def device_touch_api(self, Devices, deviceId, unit):
    
    if DOMOTICZ_EXTENDED_API:
        if "MemoryLeak" not in Devices:
            return
        Domoticz.Log('device_touch_api %s/%s' % (deviceId ,unit))
        Devices[ deviceId ].Units[ unit ].Touch()
        return
    else:
        # Legacy
        Domoticz.Log('device_touch_api %s' % unit)
        Devices[unit].Touch()
    
global _plugin
_plugin = BasePlugin()


def onStart():
    global _plugin
    _plugin.onStart()


def onStop():
    global _plugin
    _plugin.onStop()


def onHeartbeat():
    global _plugin
    _plugin.onHeartbeat()


def onCommand(Unit, Command, Level, Hue):
    global _plugin
    _plugin.onCommand(Unit, Command, Level, Hue)

[pipiche]

https://github.com/domoticz/domoticz/issues/5881