Export data from DB to Excel [SOLVED]
Moderator: leecollings
-
- Posts: 497
- Joined: Friday 22 May 2015 12:21
- Target OS: Raspberry Pi / ODroid
- Domoticz version: 4.11083
- Location: Asten NB Nederland
- Contact:
Export data from DB to Excel
Hi There,
I have made an Excel sheet with all the date from outside temp, kWh usage and solar production.
At the moment I download the database and copy the data from there to my excel sheet.
Is the a way to do this automatically?
Curious if anyone has ever done this.
greetz
Jan
I have made an Excel sheet with all the date from outside temp, kWh usage and solar production.
At the moment I download the database and copy the data from there to my excel sheet.
Is the a way to do this automatically?
Curious if anyone has ever done this.
greetz
Jan
Last edited by JuanUil on Monday 30 December 2019 17:46, edited 1 time in total.
Your mind is like a parachute,
It only works when it is opened!
RPI4 several Fibaro, KaKu, Neocoolcam switches, Z-Wave, Zigbee2Mqtt, Ikea bulbs and remote, Zigbee temp nodes
It only works when it is opened!
RPI4 several Fibaro, KaKu, Neocoolcam switches, Z-Wave, Zigbee2Mqtt, Ikea bulbs and remote, Zigbee temp nodes
- waaren
- Posts: 6028
- Joined: Tuesday 03 January 2017 14:18
- Target OS: Linux
- Domoticz version: Beta
- Location: Netherlands
- Contact:
Re: Export data from DB to Excel
If a CSV file works for you then a bash- or dzVents script can do this for you. Just let me know the ID's or names, the field names and the frequency and I will have a look.
Debian buster, bullseye on RPI-4, Intel NUC.
dz Beta, Z-Wave, RFLink, RFXtrx433e, P1, Youless, Hue, Yeelight, Xiaomi, MQTT
==>> dzVents wiki
dz Beta, Z-Wave, RFLink, RFXtrx433e, P1, Youless, Hue, Yeelight, Xiaomi, MQTT
==>> dzVents wiki
-
- Posts: 497
- Joined: Friday 22 May 2015 12:21
- Target OS: Raspberry Pi / ODroid
- Domoticz version: 4.11083
- Location: Asten NB Nederland
- Contact:
Re: Export data from DB to Excel
Hi Waaren,
I Think I can find a way to import the csv to my workbook.
So The ID's
44 for the kWh used
48 for the solarproduction
72 for the outside temp
happy oud en nieuw
Jan
I Think I can find a way to import the csv to my workbook.
So The ID's
44 for the kWh used
48 for the solarproduction
72 for the outside temp
happy oud en nieuw
Jan
Your mind is like a parachute,
It only works when it is opened!
RPI4 several Fibaro, KaKu, Neocoolcam switches, Z-Wave, Zigbee2Mqtt, Ikea bulbs and remote, Zigbee temp nodes
It only works when it is opened!
RPI4 several Fibaro, KaKu, Neocoolcam switches, Z-Wave, Zigbee2Mqtt, Ikea bulbs and remote, Zigbee temp nodes
- waaren
- Posts: 6028
- Joined: Tuesday 03 January 2017 14:18
- Target OS: Linux
- Domoticz version: Beta
- Location: Netherlands
- Contact:
Re: Export data from DB to Excel
can you try this ? it collects the date from the daily history tables in the domoticz database.
Code: Select all
--[[ -- sqliteDump.lua
Dumps daily data as csv file
requires sqlite3
install command on linux: sudo apt install sqlite3
install command on openwrt: opkg install sqlite3-cli
install command on synology: sudo /opt/bin/opkg install sqlite3-cli
History:
20191231
first public release
]]--
return
{
on =
{
timer =
{
"at 01:07",
},
},
logging =
{
level = domoticz.LOG_DEBUG,
marker = "sqliteDump"
},
execute = function(dz)
-- ======================= enter you settings below this line ==================
local path = "/home/pi/domoticz/" -- full qualified path to your domoticz directory
local database = "domoticz.db" -- database filename + extension
local sqlite = "/usr/bin/sqlite3" -- location of your sqlite3 tool -- which sqlite3
local allTemperatures = "72" -- comma separated list of temperature devices to export
local allUsages = "44" -- comma separated list
local allSolar = "48" -- comma separated list
local csvFile = "/home/pi/domoticz2excel.csv"
-- ======================= No modification needed below this line ==================
local copy = "safeCopy.db"
local baseCommand = "sudo " .. sqlite .. " -header -csv " .. copy
local closure = " >> " .. csvFile
local collects = {}
collects.meters = 'select a.id as meter, a.name as naam, b.Value as value, b.counter as counter from devicestatus a , meter_calendar b where a.id = b.devicerowid and a.id in (' .. allUsages .. ',' .. allSolar ..');'
collects.temperatures = 'select a.id as temperature, a.name as naam, b.temp_min as min, b.temp_max as max, b.temp_avg as avg, b.date as date from devicestatus a , temperature_calendar b where a.id = b.devicerowid and a.id in (' .. allTemperatures .. ') order by a.id;'
local function osCommand(cmd)
dz.log('Executing Command: ' .. cmd,dz.LOG_DEBUG)
local fileHandle = assert(io.popen(cmd .. ' 2>&1 || echo ::ERROR::', 'r'))
local commandOutput = assert(fileHandle:read('*a'))
local returnTable = {fileHandle:close()}
if commandOutput:find '::ERROR::' then -- something went wrong
dz.log('Error ==>> ' .. tostring(commandOutput:match('^(.*)%s+::ERROR::') or ' ... but no error message ' ) ,dz.LOG_DEBUG)
else -- all is fine!!
dz.log('ReturnCode: ' .. returnTable[3] .. '\ncommandOutput:\n' .. commandOutput, dz.LOG_DEBUG)
end
return commandOutput,returnTable[3] -- rc[3] contains returnCode
end
local function sqliteDump()
local result, rc
if dz.utils.fileExists(path .. database) then
if dz.utils.fileExists(sqlite) then
result, rc = osCommand('cp ' .. path .. database .. ' ' .. copy)
if rc ~= 0 then return rc, result end
result, rc = osCommand('rm -f ' .. csvFile)
if rc ~= 0 then return rc, result end
for type, sql in pairs (collects) do
result, rc = osCommand(baseCommand .. ' "' .. sql .. '" ' .. closure)
if rc ~= 0 then return rc, result end
end
result, rc = osCommand('rm -f ' .. copy)
if rc ~= 0 then return rc, result end
else
return -1,"sqlite3 not installed / not found"
end
else
return -1,"wrong path to database"
end
return 0, commandOutput
end
-- main program
local rc, result = sqliteDump()
if rc ~= 0 then dz.log(result,dz.LOG_ERROR) end
end
}
Debian buster, bullseye on RPI-4, Intel NUC.
dz Beta, Z-Wave, RFLink, RFXtrx433e, P1, Youless, Hue, Yeelight, Xiaomi, MQTT
==>> dzVents wiki
dz Beta, Z-Wave, RFLink, RFXtrx433e, P1, Youless, Hue, Yeelight, Xiaomi, MQTT
==>> dzVents wiki
-
- Posts: 497
- Joined: Friday 22 May 2015 12:21
- Target OS: Raspberry Pi / ODroid
- Domoticz version: 4.11083
- Location: Asten NB Nederland
- Contact:
Re: Export data from DB to Excel
Hi Waaren,
Thank you for your effort!!!
I am not at home today so I will try it tomorrow and let you know.
Happy oud en nieuw
Thank you for your effort!!!
I am not at home today so I will try it tomorrow and let you know.
Happy oud en nieuw
Your mind is like a parachute,
It only works when it is opened!
RPI4 several Fibaro, KaKu, Neocoolcam switches, Z-Wave, Zigbee2Mqtt, Ikea bulbs and remote, Zigbee temp nodes
It only works when it is opened!
RPI4 several Fibaro, KaKu, Neocoolcam switches, Z-Wave, Zigbee2Mqtt, Ikea bulbs and remote, Zigbee temp nodes
-
- Posts: 497
- Joined: Friday 22 May 2015 12:21
- Target OS: Raspberry Pi / ODroid
- Domoticz version: 4.11083
- Location: Asten NB Nederland
- Contact:
Re: Export data from DB to Excel
Hi Waaren,
First of all: Best wishes for 2020!!!
I have tried your program and it makes a domoticz2csv file in home/pi.
The problem is that I can't open it.
In Filezilla it gives:
So I cannot check.
I have tried to change the rights of the file but was not succesful
Jan
First of all: Best wishes for 2020!!!
I have tried your program and it makes a domoticz2csv file in home/pi.
The problem is that I can't open it.
In Filezilla it gives:
Code: Select all
Fout: /home/pi/Documents/test.csv: open for read: permission denied
Fout: Bestandsoverdracht mislukt
I have tried to change the rights of the file but was not succesful
Jan
Your mind is like a parachute,
It only works when it is opened!
RPI4 several Fibaro, KaKu, Neocoolcam switches, Z-Wave, Zigbee2Mqtt, Ikea bulbs and remote, Zigbee temp nodes
It only works when it is opened!
RPI4 several Fibaro, KaKu, Neocoolcam switches, Z-Wave, Zigbee2Mqtt, Ikea bulbs and remote, Zigbee temp nodes
- waaren
- Posts: 6028
- Joined: Tuesday 03 January 2017 14:18
- Target OS: Linux
- Domoticz version: Beta
- Location: Netherlands
- Contact:
Re: Export data from DB to Excel
Hi Jan,
also a happy new year for you.
What is the owner of the file ? And what are the permissions ?
For now I add a chmod statement to the script.
Code: Select all
--[[ -- sqliteDump.lua
Dumps daily data as csv file
requires sqlite3
install command on linux: sudo apt install sqlite3
install command on openwrt: opkg install sqlite3-cli
install command on synology: sudo /opt/bin/opkg install sqlite3-cli
History:
20191231 first public release
20200102 Add sudo chmod to make csvfile readable for all
]]--
return
{
on =
{
timer =
{
"at 01:07",
},
},
logging =
{
level = domoticz.LOG_DEBUG,
marker = "sqliteDump"
},
execute = function(dz)
-- ======================= enter you settings below this line ==================
local path = "/home/pi/domoticz/" -- full qualified path to your domoticz directory
local database = "domoticz.db" -- database filename + extension
local sqlite = "/usr/bin/sqlite3" -- location of your sqlite3 tool -- which sqlite3
local allTemperatures = "72" -- comma separated list of temperature devices to export
local allUsages = "44" -- comma separated list
local allSolar = "48" -- comma separated list
local csvFile = "/home/pi/domoticz2excel.csv"
-- ======================= No modification needed below this line ==================
local copy = "safeCopy.db"
local baseCommand = "sudo " .. sqlite .. " -header -csv " .. copy
local closure = " >> " .. csvFile
local collects = {}
collects.meters = 'select a.id as meter, a.name as naam, b.Value as value, b.counter as counter from devicestatus a , meter_calendar b where a.id = b.devicerowid and a.id in (' .. allUsages .. ',' .. allSolar ..');'
collects.temperatures = 'select a.id as temperature, a.name as naam, b.temp_min as min, b.temp_max as max, b.temp_avg as avg, b.date as date from devicestatus a , temperature_calendar b where a.id = b.devicerowid and a.id in (' .. allTemperatures .. ') order by a.id;'
local function osCommand(cmd)
dz.log('Executing Command: ' .. cmd,dz.LOG_DEBUG)
local fileHandle = assert(io.popen(cmd .. ' 2>&1 || echo ::ERROR::', 'r'))
local commandOutput = assert(fileHandle:read('*a'))
local returnTable = {fileHandle:close()}
if commandOutput:find '::ERROR::' then -- something went wrong
dz.log('Error ==>> ' .. tostring(commandOutput:match('^(.*)%s+::ERROR::') or ' ... but no error message ' ) ,dz.LOG_DEBUG)
else -- all is fine!!
dz.log('ReturnCode: ' .. returnTable[3] .. '\ncommandOutput:\n' .. commandOutput, dz.LOG_DEBUG)
end
return commandOutput,returnTable[3] -- rc[3] contains returnCode
end
local function sqliteDump()
local result, rc
if dz.utils.fileExists(path .. database) then
if dz.utils.fileExists(sqlite) then
result, rc = osCommand('cp ' .. path .. database .. ' ' .. copy)
if rc ~= 0 then return rc, result end
result, rc = osCommand('rm -f ' .. csvFile)
if rc ~= 0 then return rc, result end
for type, sql in pairs (collects) do
result, rc = osCommand(baseCommand .. ' "' .. sql .. '" ' .. closure)
if rc ~= 0 then return rc, result end
end
result, rc = osCommand('rm -f ' .. copy)
if rc ~= 0 then return rc, result end
result, rc = osCommand('sudo chmod 777 ' .. csvFile)
if rc ~= 0 then return rc, result end
else
return -1,"sqlite3 not installed / not found"
end
else
return -1,"wrong path to database"
end
return 0, commandOutput
end
-- main program
local rc, result = sqliteDump()
if rc ~= 0 then dz.log(result,dz.LOG_ERROR) end
end
}
Debian buster, bullseye on RPI-4, Intel NUC.
dz Beta, Z-Wave, RFLink, RFXtrx433e, P1, Youless, Hue, Yeelight, Xiaomi, MQTT
==>> dzVents wiki
dz Beta, Z-Wave, RFLink, RFXtrx433e, P1, Youless, Hue, Yeelight, Xiaomi, MQTT
==>> dzVents wiki
-
- Posts: 497
- Joined: Friday 22 May 2015 12:21
- Target OS: Raspberry Pi / ODroid
- Domoticz version: 4.11083
- Location: Asten NB Nederland
- Contact:
Re: Export data from DB to Excel
The owner is root and normaly I am pi, but it is working now!!
Fabulous, thank you verry much!!
I am going to make a macro to import the csv to my workbook now.
If you are interested I will show you the result.
Greetings
Jan
Fabulous, thank you verry much!!
I am going to make a macro to import the csv to my workbook now.
If you are interested I will show you the result.
Greetings
Jan
Your mind is like a parachute,
It only works when it is opened!
RPI4 several Fibaro, KaKu, Neocoolcam switches, Z-Wave, Zigbee2Mqtt, Ikea bulbs and remote, Zigbee temp nodes
It only works when it is opened!
RPI4 several Fibaro, KaKu, Neocoolcam switches, Z-Wave, Zigbee2Mqtt, Ikea bulbs and remote, Zigbee temp nodes
- Egregius
- Posts: 2582
- Joined: Thursday 09 April 2015 12:19
- Target OS: Linux
- Domoticz version: v2024.7
- Location: Beitem, BE
- Contact:
Re: Export data from DB to Excel
Isn't it a better idea to put the data in a MySQL database instead of Excel?
How is Excel gonna handle the data after several years?
How is Excel gonna handle the data after several years?
-
- Posts: 497
- Joined: Friday 22 May 2015 12:21
- Target OS: Raspberry Pi / ODroid
- Domoticz version: 4.11083
- Location: Asten NB Nederland
- Contact:
Re: Export data from DB to Excel
I have data from my Youless from 2015 till now.
The csv file made from the program of Waaren is gathering them perfectly.
The advandtage from excel is that you can make nice graphs and predictions.
The csv file made from the program of Waaren is gathering them perfectly.
The advandtage from excel is that you can make nice graphs and predictions.
Your mind is like a parachute,
It only works when it is opened!
RPI4 several Fibaro, KaKu, Neocoolcam switches, Z-Wave, Zigbee2Mqtt, Ikea bulbs and remote, Zigbee temp nodes
It only works when it is opened!
RPI4 several Fibaro, KaKu, Neocoolcam switches, Z-Wave, Zigbee2Mqtt, Ikea bulbs and remote, Zigbee temp nodes
- waaren
- Posts: 6028
- Joined: Tuesday 03 January 2017 14:18
- Target OS: Linux
- Domoticz version: Beta
- Location: Netherlands
- Contact:
Re: Export data from DB to Excel
That would be nice. Thx in advance !
Debian buster, bullseye on RPI-4, Intel NUC.
dz Beta, Z-Wave, RFLink, RFXtrx433e, P1, Youless, Hue, Yeelight, Xiaomi, MQTT
==>> dzVents wiki
dz Beta, Z-Wave, RFLink, RFXtrx433e, P1, Youless, Hue, Yeelight, Xiaomi, MQTT
==>> dzVents wiki
-
- Posts: 10
- Joined: Saturday 10 October 2020 16:48
- Target OS: Raspberry Pi / ODroid
- Domoticz version: 2020.2
- Location: Netherlands
- Contact:
Re: Export data from DB to Excel
Very interested in this script. I tried it and it runs but the csv file remains empty. The data i am trying to export is my P1 power meter (idx 17) and p1 gas meter (idx 21).
That way I can use excel to calculate the actual power usage when combined with the solar inverter download (until i can get this automated in Domoticz as well, see https://domoticz.com/forum/viewtopic.php?f=14&t=34272)
I just changed the idx numbers for allusages to 17, and allsolar to 21 to test. No other changes. So what else needs to be amended to get the export going?
That way I can use excel to calculate the actual power usage when combined with the solar inverter download (until i can get this automated in Domoticz as well, see https://domoticz.com/forum/viewtopic.php?f=14&t=34272)
I just changed the idx numbers for allusages to 17, and allsolar to 21 to test. No other changes. So what else needs to be amended to get the export going?
- waaren
- Posts: 6028
- Joined: Tuesday 03 January 2017 14:18
- Target OS: Linux
- Domoticz version: Beta
- Location: Netherlands
- Contact:
Re: Export data from DB to Excel
The P1 history is in another table ( MultiMeter_Calendar ) and was not implemented yet in this script. I updated the script to include it (see below)
Code: Select all
--[[ -- sqliteDump.lua
Dumps daily data as csv file
requires sqlite3
install command on linux: sudo apt install sqlite3
install command on openwrt: opkg install sqlite3-cli
install command on synology: sudo /opt/bin/opkg install sqlite3-cli
History:
20191231 first public release
20200102 Add sudo chmod to make csvfile readable for all
20201023 Add option to use MultiMeter
]]--
return
{
on =
{
timer =
{
'at 01:07',
'at 14:47',
},
devices =
{
'sqlDump', -- test trigger can be ignored
},
},
logging =
{
level = domoticz.LOG_DEBUG, -- set to domoticz.LOG_ERROR when all is OK
marker = 'sqliteDump',
},
execute = function(dz)
-- ======================= enter you settings below this line ==================
local path = '/opt/domoticz/' -- full qualified path to your domoticz directory
local database = 'domoticz.db' -- database filename + extension
local sqlite = '/usr/bin/sqlite3' -- location of your sqlite3 tool; check with 'which sqlite3'
local csvFile = '/usr/local/domotica/data/domoticz2excel.csv'
-- set to '' for the types you don't need
local allTemperatures = '152,148' -- comma separated list of Temperature sensors to export
local allMeters = '36,1168,2872' -- comma separated list of Solar + Meters + Usage sensors
local allMultiMeters = '35' -- comma separated list P1 energy sensor
-- ======================= No modification needed below this line ==================
local copy = 'safeCopy.db'
local baseCommand = 'sudo ' .. sqlite .. ' -header -csv ' .. copy
local closure = ' >> ' .. csvFile
local collects = {}
collects.meters = 'SELECT a.ID Meter, a.Name Name, b.Value Value, b.Counter Counter, b.Date Date FROM DeviceStatus a , Meter_Calendar b ' ..
' WHERE a.ID = b.DeviceRowID AND a.ID IN (' .. allMeters .. ');'
collects.Temperatures = 'SELECT a.ID Temperature, a.Name Name, b.Temp_Min min, b.Temp_Max max, b.Temp_Avg avg, b.Date Date ' ..
' FROM DeviceStatus a , Temperature_Calendar b WHERE a.ID = b.DeviceRowID AND a.ID IN (' .. allTemperatures .. ') ORDER BY a.ID;'
collects.MultiMeters = 'SELECT a.Name, b.* FROM DeviceStatus a , MultiMeter_Calendar b WHERE a.ID = b.DeviceRowID AND a.ID IN (' .. allMultiMeters .. ') ORDER BY b.DeviceRowID;'
local function osCommand(cmd)
dz.log('Executing Command: ' .. cmd,dz.LOG_DEBUG)
local fileHandle = assert(io.popen(cmd .. ' 2>&1 || echo ::ERROR::', 'r'))
local commandOutput = assert(fileHandle:read('*a'))
local returnTable = {fileHandle:close()}
if commandOutput:find '::ERROR::' then -- something went wrong
dz.log('Error ==>> ' .. tostring(commandOutput:match('^(.*)%s+::ERROR::') or ' ... but no error message ' ) ,dz.LOG_DEBUG)
else -- all is fine!!
dz.log('ReturnCode: ' .. returnTable[3] .. '\ncommandOutput:\n' .. commandOutput, dz.LOG_DEBUG)
end
return commandOutput,returnTable[3] -- rc[3] contains returnCode
end
local function sqliteDump()
local result, rc
if dz.utils.fileExists(path .. database) then
if dz.utils.fileExists(sqlite) then
result, rc = osCommand('cp ' .. path .. database .. ' ' .. copy)
if rc ~= 0 then return rc, result end
result, rc = osCommand('rm -f ' .. csvFile)
if rc ~= 0 then return rc, result end
for _, sql in pairs (collects) do
result, rc = osCommand(baseCommand .. ' "' .. sql .. '" ' .. closure)
if rc ~= 0 then return rc, result end
end
result, rc = osCommand('rm -f ' .. copy)
if rc ~= 0 then return rc, result end
result, rc = osCommand('sudo chmod 777 ' .. csvFile)
if rc ~= 0 then return rc, result end
else
return -1,'sqlite3 not installed / not found'
end
else
return -1,'wrong path to database'
end
return 0, commandOutput
end
-- main program
local rc, result = sqliteDump()
if rc ~= 0 then dz.log(result,dz.LOG_ERROR) end
end
}
Debian buster, bullseye on RPI-4, Intel NUC.
dz Beta, Z-Wave, RFLink, RFXtrx433e, P1, Youless, Hue, Yeelight, Xiaomi, MQTT
==>> dzVents wiki
dz Beta, Z-Wave, RFLink, RFXtrx433e, P1, Youless, Hue, Yeelight, Xiaomi, MQTT
==>> dzVents wiki
-
- Posts: 10
- Joined: Saturday 10 October 2020 16:48
- Target OS: Raspberry Pi / ODroid
- Domoticz version: 2020.2
- Location: Netherlands
- Contact:
Re: Export data from DB to Excel
@waaren thanks. With a few small modifications (paths and smart meter idx) this works and i get a CSV dump of the P1 meter per day.
How can I change the code so that I get the values for each day (so every measurement)? These are needed to calculate the actual usage per counter (high / normal), and the percentage of used solar energy - and with that the return on investment of the solar panels.
For others that are interested: I used the paths below for the raspberry pi Domoticz install. Note that the change is the reference to the home directory.
How can I change the code so that I get the values for each day (so every measurement)? These are needed to calculate the actual usage per counter (high / normal), and the percentage of used solar energy - and with that the return on investment of the solar panels.
For others that are interested: I used the paths below for the raspberry pi Domoticz install. Note that the change is the reference to the home directory.
Code: Select all
local path = "/home/pi/domoticz/" -- full qualified path to your domoticz directory
local database = 'domoticz.db' -- database filename + extension
local sqlite = '/usr/bin/sqlite3' -- location of your sqlite3 tool; check with 'which sqlite3'
local csvFile = "/home/pi/domoticz2excel.csv"
Last edited by fzoet on Monday 26 October 2020 14:20, edited 1 time in total.
- waaren
- Posts: 6028
- Joined: Tuesday 03 January 2017 14:18
- Target OS: Linux
- Domoticz version: Beta
- Location: Netherlands
- Contact:
Re: Export data from DB to Excel
Change the query strings tofzoet wrote: ↑Monday 26 October 2020 12:33 How can I change the code so that I get the values for each day (so every measurement)? These are needed to calculate the actual usage per counter (high / normal), and the percentage of used solar energy - and with that the return on investment of the solar panels.
Code: Select all
collects.meters = 'SELECT a.ID Meter, a.Name Name, b.Value Value, b.Usage Usage, b.Date Date '..
' FROM DeviceStatus a , Meter b WHERE a.ID = b.DeviceRowID AND a.ID IN (' .. allMeters .. ')'
collects.Temperatures = 'SELECT a.ID ID, a.Name Name, b.Temperature Temperature, b.Date Date ' ..
' FROM DeviceStatus a , Temperature b WHERE a.ID = b.DeviceRowID AND ' ..
' a.ID IN ( '.. allTemperatures .. ') ORDER BY a.ID '
collects.MultiMeters = 'SELECT a.Name, b.* FROM DeviceStatus a , MultiMeter b WHERE a.ID = b.DeviceRowID ' ..
' AND a.ID IN (' .. allMultiMeters .. ') ORDER BY b.DeviceRowID'
Debian buster, bullseye on RPI-4, Intel NUC.
dz Beta, Z-Wave, RFLink, RFXtrx433e, P1, Youless, Hue, Yeelight, Xiaomi, MQTT
==>> dzVents wiki
dz Beta, Z-Wave, RFLink, RFXtrx433e, P1, Youless, Hue, Yeelight, Xiaomi, MQTT
==>> dzVents wiki
-
- Posts: 10
- Joined: Saturday 10 October 2020 16:48
- Target OS: Raspberry Pi / ODroid
- Domoticz version: 2020.2
- Location: Netherlands
- Contact:
Re: Export data from DB to Excel
Thanks @waaren.
As the file will be containing the past 24 hours and overwrite the previous file, I changed the filename to contain yesterday's date and run the timer at 00:04.
Question:
is there a way to dump this daily data from all the past months at once? - to get the backlog
As the file will be containing the past 24 hours and overwrite the previous file, I changed the filename to contain yesterday's date and run the timer at 00:04.
Code: Select all
local path = "/home/pi/domoticz/" -- full qualified path to your domoticz directory
local database = 'domoticz.db' -- database filename + extension
local sqlite = '/usr/bin/sqlite3' -- location of your sqlite3 tool; check with 'which sqlite3'
local csvFile = "/home/pi/domoticz2excel"
local csvFile = csvFile .. "_" .. os.date("%Y-%m-%d",os.time()-24*60*60) .. ".csv"
is there a way to dump this daily data from all the past months at once? - to get the backlog
- waaren
- Posts: 6028
- Joined: Tuesday 03 January 2017 14:18
- Target OS: Linux
- Domoticz version: Beta
- Location: Netherlands
- Contact:
Re: Export data from DB to Excel
Only if you have daily backups that go back that far. The short log sensors (288 rows a day / sensor) are collated and averaged into the long term storage (1 row / day) once a day just after midnight.
The retention period for short log sensors is max. 7 days
Debian buster, bullseye on RPI-4, Intel NUC.
dz Beta, Z-Wave, RFLink, RFXtrx433e, P1, Youless, Hue, Yeelight, Xiaomi, MQTT
==>> dzVents wiki
dz Beta, Z-Wave, RFLink, RFXtrx433e, P1, Youless, Hue, Yeelight, Xiaomi, MQTT
==>> dzVents wiki
-
- Posts: 10
- Joined: Saturday 10 October 2020 16:48
- Target OS: Raspberry Pi / ODroid
- Domoticz version: 2020.2
- Location: Netherlands
- Contact:
Re: Export data from DB to Excel
@waaren. Thanks, yes I feared that from what I have already read about domoticz. Thanks for the confirmation.
-
- Posts: 10
- Joined: Saturday 10 October 2020 16:48
- Target OS: Raspberry Pi / ODroid
- Domoticz version: 2020.2
- Location: Netherlands
- Contact:
Re: Export data from DB to Excel [SOLVED]
Here is the last full code, as it works in my setup. Thanks @waaren for all your contributions - the credits fully go to you!
Code: Select all
--[[ -- sqliteDump.lua
Dumps daily data as csv file
requires sqlite3
install command on linux: sudo apt install sqlite3
install command on openwrt: opkg install sqlite3-cli
install command on synology: sudo /opt/bin/opkg install sqlite3-cli
History:
20191231 first public release
20200102 Add sudo chmod to make csvfile readable for all
20201023 Add option to use MultiMeter
20201026 Added option to get daily data, and included yesterdays' date to csv filename
]]--
return
{
on =
{
timer =
{
'at 00:04',
--'at 17:16',
},
devices =
{
'sqlDump', -- test trigger can be ignored
},
},
logging =
{
level = domoticz.LOG_DEBUG, -- set to domoticz.LOG_ERROR when all is OK
marker = 'sqliteDump',
},
execute = function(dz)
-- ======================= enter you settings below this line ==================
local path = "/home/pi/domoticz/" -- full qualified path to your domoticz directory
local database = 'domoticz.db' -- database filename + extension
local sqlite = '/usr/bin/sqlite3' -- location of your sqlite3 tool; check with 'which sqlite3'
local csvFile = "/home/pi/domoticz2excel" --full path to the csv filename for the dump, without the .csv extension (will be added by script, together with date)
-- set to '' for the types you don't need
local allTemperatures = '152,148' -- comma separated list of Temperature sensors to export
local allMeters = '36,1168,2872' -- comma separated list of Solar + Meters + Usage sensors
local allMultiMeters = '17' -- comma separated list P1 energy sensor
-- ======================= No modification needed below this line ==================
local csvFile = csvFile .. "_" .. os.date("%Y-%m-%d",os.time()-24*60*60) .. ".csv"
local copy = 'safeCopy.db'
local baseCommand = 'sudo ' .. sqlite .. ' -header -csv ' .. copy
local closure = ' >> ' .. csvFile
local collects = {}
--==Query for daily data==
--[[
collects.meters = 'SELECT a.ID Meter, a.Name Name, b.Value Value, b.Counter Counter, b.Date Date FROM DeviceStatus a , Meter_Calendar b ' ..
' WHERE a.ID = b.DeviceRowID AND a.ID IN (' .. allMeters .. ');'
collects.Temperatures = 'SELECT a.ID Temperature, a.Name Name, b.Temp_Min min, b.Temp_Max max, b.Temp_Avg avg, b.Date Date ' ..
' FROM DeviceStatus a , Temperature_Calendar b WHERE a.ID = b.DeviceRowID AND a.ID IN (' .. allTemperatures .. ') ORDER BY a.ID;'
collects.MultiMeters = 'SELECT a.Name, b.* FROM DeviceStatus a , MultiMeter_Calendar b WHERE a.ID = b.DeviceRowID AND a.ID IN (' .. allMultiMeters .. ') ORDER BY b.DeviceRowID;'
]]--
--==end query daily data
--==Query for minute data
collects.meters = 'SELECT a.ID Meter, a.Name Name, b.Value Value, b.Usage Usage, b.Date Date '..
' FROM DeviceStatus a , Meter b WHERE a.ID = b.DeviceRowID AND a.ID IN (' .. allMeters .. ')'
collects.Temperatures = 'SELECT a.ID ID, a.Name Name, b.Temperature Temperature, b.Date Date ' ..
' FROM DeviceStatus a , Temperature b WHERE a.ID = b.DeviceRowID AND ' ..
' a.ID IN ( '.. allTemperatures .. ') ORDER BY a.ID '
collects.MultiMeters = 'SELECT a.Name, b.* FROM DeviceStatus a , MultiMeter b WHERE a.ID = b.DeviceRowID ' ..
' AND a.ID IN (' .. allMultiMeters .. ') ORDER BY b.DeviceRowID'
--==end query minute data
local function osCommand(cmd)
dz.log('Executing Command: ' .. cmd,dz.LOG_DEBUG)
local fileHandle = assert(io.popen(cmd .. ' 2>&1 || echo ::ERROR::', 'r'))
local commandOutput = assert(fileHandle:read('*a'))
local returnTable = {fileHandle:close()}
if commandOutput:find '::ERROR::' then -- something went wrong
dz.log('Error ==>> ' .. tostring(commandOutput:match('^(.*)%s+::ERROR::') or ' ... but no error message ' ) ,dz.LOG_DEBUG)
else -- all is fine!!
dz.log('ReturnCode: ' .. returnTable[3] .. '\ncommandOutput:\n' .. commandOutput, dz.LOG_DEBUG)
end
return commandOutput,returnTable[3] -- rc[3] contains returnCode
end
local function sqliteDump()
local result, rc
if dz.utils.fileExists(path .. database) then
if dz.utils.fileExists(sqlite) then
result, rc = osCommand('cp ' .. path .. database .. ' ' .. copy)
if rc ~= 0 then return rc, result end
result, rc = osCommand('rm -f ' .. csvFile)
if rc ~= 0 then return rc, result end
for _, sql in pairs (collects) do
result, rc = osCommand(baseCommand .. ' "' .. sql .. '" ' .. closure)
if rc ~= 0 then return rc, result end
end
result, rc = osCommand('rm -f ' .. copy)
if rc ~= 0 then return rc, result end
result, rc = osCommand('sudo chmod 777 ' .. csvFile)
if rc ~= 0 then return rc, result end
else
return -1,'sqlite3 not installed / not found'
end
else
return -1,'wrong path to database'
end
return 0, commandOutput
end
-- main program
local rc, result = sqliteDump()
if rc ~= 0 then dz.log(result,dz.LOG_ERROR) end
end
}
Who is online
Users browsing this forum: No registered users and 0 guests