Commit 2dcba2cc authored by Alan Mitchell's avatar Alan Mitchell

Finalized Okofen data collection Periodic Script.

Changed sensor ID naming, started using a routine to identify changed
values to save reading space.
parent 5f90a713
......@@ -2,6 +2,7 @@
Script to collect sensor readings from an OkoFEN pellet boiler.
from datetime import datetime, timedelta, date
import sys
import StringIO
import urlparse
import traceback
......@@ -11,6 +12,18 @@ import requests
import pandas as pd
import pytz
# Parameters that are State-type values instead of continuous analog values.
# The name of the parameter here should be the PXXX number, or the sensor ID
# for a sensor that does not have a PXXX number. e.g. the "Boiler 1" sensor
# would be shown as 'boiler_1'.
# Compiled RegEx used to find parameter number in a parameter name
P_REGEX = re.compile('^P\d{3}\s')
def run(url= '', site_id='', tz_data='US/Alaska', last_date_loaded='2016-01-01', **kwargs):
......@@ -88,18 +101,17 @@ def run(url= '', site_id='', tz_data='US/Alaska', last_date_loaded='2016-01-01',
# clean up the column names
cols = []
for col in df.columns:
col = re.sub('^P\d{3}\s', '', col) # remove PXXX identifier
col = col.replace('.', '').replace(' ', '_').replace('/', '_').lower()
match_res = P_REGEX.match(col)
if match_res:
# there was a parameter number in the column name
col =
# no parameter number, so make a clean column name
col = col.replace('.', '').replace(' ', '_').replace('/', '_').lower()
cols[0] = 'datetime'
df.columns = cols
# *** TESTING ONLY ***
# Take every third record
# Drop this eventually and replace with data aggregation and selection
every_third = [x % 3 == 0 for x in range(len(df))]
df = df[every_third]
# make a list of UNIX timestamps and then dispose of the datetime
# column since it is not a sensor reading column.
tstamps = [ts_from_datestr(dt, tz_data) for dt in df['datetime']]
......@@ -110,7 +122,11 @@ def run(url= '', site_id='', tz_data='US/Alaska', last_date_loaded='2016-01-01',
for col in df.columns:
sensor_id = '%s_%s' % (site_id, col)
sensor_ids.append(sensor_id) # list used later to return info on sensors read
new_reads = zip(tstamps, (sensor_id,) * len(df), df[col].values)
# get a set of filtered sensor readings, filtered to show significant changes.
filtered_ts, filtered_vals = find_changes(tstamps,
state_change=(col in STATE_PARAMS))
new_reads = zip(filtered_ts, (sensor_id,) * len(filtered_ts), filtered_vals)
readings += new_reads
# successfully loaded this date, so update the tracking variable
......@@ -142,3 +158,46 @@ def ts_from_datestr(datestr, tzname):
dt = datetime.strptime(datestr, '%Y-%m-%d %H:%M')
dt_aware = tz.localize(dt)
return calendar.timegm(dt_aware.utctimetuple())
def find_changes(times, vals, state_change=False, min_change=0.02, max_spacing=600):
"""Finds significant analog or state changes in a set of sensor readings for one
sensor, and returns those filtered readings. This is primarily used to reduce
the number of sensor readings stored.
times: Iterable of UNIX timestamps (seconds past Epoch) for the sensor readings
vals: Iterable of sensor reading values
state_change: If True, treats the sensor readings as State values, not continuous
analog values. For this type of reading, every change is considered significant
and is returned in the filtered list.
min_change: This determines what is considered a significant change for an analog
sensor reading. This argument is expressed as the fraction of the difference
between the minimum and maximum sensor reading in the input set.
max_spacing: If no significant change has occurred in 'max_spacing' number of seconds
then a sensor reading is returned anyway.
A two-tuple, the first element being a list of timestamps of the filtered readings,
and the second element being a list of filtered readings.
if state_change:
# any change at all counts as a change. Use smallest float as trigger
chg_trigger = sys.float_info.min
chg_trigger = (max(vals) - min(vals)) * min_change # 2% change trigger reading
if chg_trigger == 0.0:
chg_trigger = sys.float_info.min # must be some change to record a reading
# Always include first point
filtered_ts = [times[0]]
filtered_vals = [vals[0]]
for ts, val in zip(times[1:], vals[1:]):
if abs(val - filtered_vals[-1]) >= chg_trigger or (ts - filtered_ts[-1]) >= max_spacing:
return filtered_ts, filtered_vals
\ No newline at end of file
Building [0] Sensor ID [1] Title [2] Unit Label [3] Sensor Group [4] Sort Order [5] Is Calc'd Field [6] Calc or Transform Function [7] Function Parameters [8]
Homer Strawbale hmr_StrHmT Straw Home Temperature deg F Space Conditions, Temperature 10
Homer Strawbale hmr_GreenHsT Greenhouse Temperature deg F Space Conditions, Temperature 20
Homer Strawbale hmr_BoilerRmT Boiler Room Temp deg F Space Conditions, Temperature 30
Homer Strawbale hmr_ArptTemp Homer Airport Temperature, F deg F Weather 10
Homer Strawbale hmr_ArptWind Homer Airport Wind, mph mph Weather 20
Homer Strawbale hmr_OutdoorT Site Outdoor Temperature, F deg F Weather 30
Homer Strawbale hmr_StrArrBtuMeter Straw Array Btu/hr Btu/hour Solar Thermal 10
Homer Strawbale hmr_StrArrInT Straw Array Inlet Temp deg F Solar Thermal 20
Homer Strawbale hmr_StrArrOutT Straw Array Outlet Temp deg F Solar Thermal 30
Homer Strawbale hmr_StrArrFlowMeter Straw Array Flow gpm gpm Solar Thermal 40
Homer Strawbale hmr_GrnArrBtuMeter Greenhouse Array Btu/hr Btu/hour Solar Thermal 50
Homer Strawbale hmr_GrnArrInT Greenhouse Array Inlet Temp deg F Solar Thermal 60
Homer Strawbale hmr_GrnArrOutT Greenhouse Array Outlet Temp deg F Solar Thermal 70
Homer Strawbale hmr_GrnArrFlowMeter Greenhouse Array Flow gpm gpm Solar Thermal 80
Homer Strawbale hmr_PoolT HS Tank Temp deg F Heat Storage 10
Homer Strawbale hmr_PoolSoilT Soil Temp Beside Tank deg F Heat Storage 20
Homer Strawbale hmr_PL750_bottom PL750 Tank Temp Bottom deg F Heat Storage 30
Homer Strawbale hmr_PL750_mid PL750 Tank Temp Mid deg F Heat Storage 40
Homer Strawbale hmr_PL750_top PL750 Tank Temp Top deg F Heat Storage 50
Homer Strawbale hmr_PriToPoolReturnT PL750 Heat Dump Temp deg F Heat Storage 60
Homer Strawbale hmr_PriLoopT Primary Loop Temp deg F Heating System 10
Homer Strawbale hmr_PoolToPriT Strawbale Mixed Temp deg F Heating System 20
Homer Strawbale hmr_MixedT Greenhouse Mixed Temp deg F Heating System 30
Homer Strawbale hmr_BoilerStackT Boiler Stack Temp deg F Heating System 40
Haines Senior Center HainesSrCtr_boiler_1 Boiler Supply Temperature deg F Pellet Boiler 10
Haines Senior Center HainesSrCtr_P241 Boiler Status, P241 code Pellet Boiler 20
Haines Senior Center HainesSrCtr_P107 Flue/Flame Temperature, P107 deg F Pellet Boiler 30
Haines Senior Center HainesSrCtr_P109 Chip Temperature, P109 deg F Pellet Boiler 40
Haines Senior Center HainesSrCtr_P112 Burner Startups, P112 count Pellet Boiler 50
Haines Senior Center HainesSrCtr_P115 Boiler Off Time, P115 minutes Pellet Boiler 60
Haines Senior Center HainesSrCtr_P116 Auger Pulse, P116 seconds Pellet Boiler 70
Haines Senior Center HainesSrCtr_P117 Auger Pulse Off, P117 seconds Pellet Boiler 80
Haines Senior Center HainesSrCtr_P124 Flame Temperature Setpoint, P124 deg F Pellet Boiler 90
Haines Senior Center HainesSrCtr_P184 Speed AV, P184 u / minute Pellet Boiler 100
Haines Senior Center HainesSrCtr_P185 A. Pulse / Vac On, P185 units Pellet Boiler 110
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment