Commit eb8ecaeb authored by Alan Mitchell's avatar Alan Mitchell

Merge branch 'mesonet_api'; Allows for Weather data from Mesonet sites.

parents c9c46d8d 5c19b312
##################################################
# Django settings for BMON project #
##################################################
from .settings_common import *
import logging
#----------------- Settings Specific to the Monitoring App ----------------------
# ----------------- Settings Specific to the Monitoring App ----------------------
# The key needed to store sensor readings into the database.
# This is a REQUIRED setting. You can have multiple valid storage keys
# by listing them in a tuple or list: ('key1', 'key2') or ['key1', 'key2'].
# You can load https://bms.ahfc.us/make-store-key/ in a browser to generate
# a suitable random store key. Any sensors that post data to this site will
# need to include thise Store Key when they post the data.
# need to include thise Store Key when they post the data.
# See bmsapp/views.storereading() and bmsapp/views.storereadings() for details
# of how the Store Key is included in a reading post.
BMSAPP_STORE_KEY = 'PutStorageKeyHere'
# Store key used with old URL pattern.
# Store key used with old URL pattern.
# *** NOT USED WITH NEW INSTALLS. LEAVE COMMENTED OUT ***
# BMSAPP_STORE_KEY_OLD = ''
......@@ -30,18 +31,18 @@ BMSAPP_HEADER = 'XYZ Remote Monitoring'
# First item in tuple is Text that will be shown for the link.
# Second item is the name of the template that will be rendered to produce the page.
# 'reports' is a special name that will cause the main reports/charts page to be
# rendered. 'custom-reports' is also special and will cause a page showing the
# available custom reports. For other names in this position, there must be a corresponding
# rendered. 'custom-reports' is also special and will cause a page showing the
# available custom reports. For other names in this position, there must be a corresponding
# [template name].html file present in the templates/bmsapp directory. The custom
# template cannot match any of the URLs listed in urls.py.
# The third item (optional) is True if this item should be the default index page for
# the application.
BMSAPP_NAV_LINKS = ( ('Map', 'map'),
('Data Charts and Reports', 'reports', True),
('Custom Reports', 'custom-reports'),
('Training Videos and Project Reports', 'training-anthc'),
('System Administrator', 'admin-reports'),
)
BMSAPP_NAV_LINKS = (('Map', 'map'),
('Graphs/Reports', 'reports', True),
('Custom Reports', 'custom-reports'),
('Training', 'training-anthc'),
('Sys Admin', 'admin-reports'),
)
# The number of hours before a sensor is considered to be inactive (not posting data).
BMSAPP_SENSOR_INACTIVITY = 2.0 # Hours
......@@ -57,10 +58,14 @@ BMSAPP_PUSHOVER_APP_TOKEN = '123456789012345678901234567890'
# Levels in order from least to greatest severity are: DEBUG, INFO, WARNING, ERROR, CRITICAL
BMSAPP_LOG_LEVEL = logging.INFO
# Put a Weather Underground API key here, if you are acquiring temperature or wind
# data from any Weather Underground sites (getWUtemperature, getWUwindSpeed).
# See http://www.wunderground.com/weather/api for details on acuiring a key for your use.
BMSAPP_WU_API_KEY = ''
# Put a Mesonet API Token here, if you are acquiring temperature or wind data
# using the Mesonet API (getAllMesonetTemperature, getAllMesonetWindSpeed).
# See https://developers.synopticdata.com/signup to obtain a token.
BMSAPP_MESONET_API_TOKEN = ''
# Weather Underground API is no longer supported due to changes in their service
# See http://www.wunderground.com/weather/api for details
# BMSAPP_WU_API_KEY = ''
# Enter URL, Username and Password for the ARIS api here if you are using the
# getUsageFromARIS calculation function to import building energy usage info
......@@ -99,7 +104,7 @@ LANGUAGE_CODE = 'en-us'
# The Names and Emails of people who should be emailed in the case of an
# application exception. Everyone appearing in the list will be notified; add
# additional tuples for each Admin. Unlike the sample below, remove the # symbol
# additional tuples for each Admin. Unlike the sample below, remove the # symbol
# in front of each tuple.
# See documentation at https://docs.djangoproject.com/en/1.7/ref/settings/#std:setting-ADMINS
# NOTE: You can also view the error log for the BMON application by browsing to the page:
......@@ -108,7 +113,7 @@ ADMINS = (
# ('Admin Name Here', 'Admin Email Address Here'),
)
# The following email settings need to be filled out for sending out alerts from
# The following email settings need to be filled out for sending out alerts from
# the BMON app.
# For the Webfaction hosting service, see documentation at:
# http://docs.webfaction.com/software/django/getting-started.html#configuring-django-to-send-email-messages
......@@ -116,17 +121,19 @@ ADMINS = (
# https://docs.djangoproject.com/en/1.7/ref/settings/#std:setting-EMAIL_HOST
# the SMTP server used to send mail, 'smtp.webfaction.com' or the Webfaction hosting service
EMAIL_HOST = 'smtp.webfaction.com'
EMAIL_HOST = 'smtp.webfaction.com'
EMAIL_HOST_USER = 'mailbox_username'
EMAIL_HOST_PASSWORD = 'mailbox_password'
# If you are using the Webfaction smtp server, the two FROM adddresses below need to
# be email addresses set up in your Webfaction account (you can use the same address
# for both, if desired). The Webfaction server will not send emails with a FROM address
# that is not on the Webfaction host.
DEFAULT_FROM_EMAIL = 'valid_from_email_address' # this will be the FROM for alert messages
SERVER_EMAIL = 'valid_from_email_for_error_messages' # this is the FROM for error messages
# this will be the FROM for alert messages
DEFAULT_FROM_EMAIL = 'valid_from_email_address'
# this is the FROM for error messages
SERVER_EMAIL = 'valid_from_email_for_error_messages'
# If DEBUG=True, a detailed error traceback is displayed in the browser when an error
# If DEBUG=True, a detailed error traceback is displayed in the browser when an error
# occurs. This setting should be False for production use for security reasons, but if
# errors are occurring with use of the application, setting to True provides valuable
# debug information.
......@@ -138,8 +145,6 @@ DEBUG = False
BMSAPP_STATIC_APP_NAME = 'bmon_static'
# Import settings that are generally common to all installs of BMON.
from .settings_common import *
# ----- If you need to override any of the settings in the 'settings_common.py' file
# ----- do so below this point in this file.
......@@ -123,6 +123,89 @@ class CalcReadingFuncs_01(calcreadings.CalcReadingFuncs_base):
return [int(time.time())], [wind_val]
else:
return [], []
def getLatestMesonetObservation(self, stnList, parameter, min_val=-float("inf"), max_val=float("inf")):
"""** No parameters are sensor reading arrays **
Returns just one record of information, timestamped with the reading time
"""
obs = internetwx.getMesonetObservation(stnList)
# Testing...
#print('obs: {}'.format(obs))
observed_val = float(obs[parameter + '_value_1']['value'])
time_val = int(obs[parameter + '_value_1']['date_time'])
if observed_val >= min_val and observed_val <= max_val:
return [time_val], [observed_val]
else:
return [], []
def getAllMesonetObservations(self, stnID, parameter, min_val=-float("inf"), max_val=float("inf"), request_interval_hours=2, since=None):
"""** No parameters are sensor reading arrays **
Returns a list of observations from a Mesonet station.
"""
# determine the timestamp of the last entry in the database for this calculated field.
last_calc_rec = self.db.last_read(self.calc_id)
# use 0 ts if no records
last_ts = int(last_calc_rec['ts']) if last_calc_rec else 0
# check to see if the request interval has elapsed
hours_elapsed = (time.time() - last_ts) / 60 / 60
print('hours_elapsed = {}'.format(hours_elapsed))
if hours_elapsed < request_interval_hours:
return [], []
if since:
# constrain this value to greater or equal to 'since'
last_ts = max(last_ts, bmsapp.data_util.datestr_to_ts(since))
else:
# constrain this value to greater or equal to 'reach_back'
last_ts = max(last_ts, int(time.time() - self.reach_back))
# Get the observations from the MesonetAPI
obs = internetwx.getMesonetTimeseries(stnID, parameter, last_ts)
obs_dict = dict(zip(obs['date_time'], obs[parameter + '_set_1']))
time_vals = []
observed_vals = []
for time_val, observed_val in obs_dict.items():
if observed_val >= min_val and observed_val <= max_val:
time_vals.append(int(time_val))
observed_vals.append(float(observed_val))
return time_vals, observed_vals
def getMesonetTemperature(self, stn, stn2=None):
"""** No parameters are sensor reading arrays **
Returns a temperature (deg F) from a Mesonet station.
'stn' is the primary station to use. 'stn2' is a backup station.
"""
return self.getLatestMesonetObservation([stn, stn2], 'air_temp', -120.0, 150.0)
def getMesonetWindSpeed(self, stn, stn2=None):
"""** No parameters are sensor reading arrays **
Returns a wind speed (mph) from a Mesonet station.
'stn' is the primary station to use. 'stn2' is a backup station.
"""
return self.getLatestMesonetObservation([stn, stn2], 'wind_speed', 0.0, 150.0)
def getAllMesonetTemperature(self, stn, request_interval_hours=2, since=None):
"""** No parameters are sensor reading arrays **
Returns a list of temperature (deg F) observations from a Mesonet station.
"""
return self.getAllMesonetObservations(stn, 'air_temp', -120.0, 150.0, request_interval_hours, since)
def getAllMesonetWindSpeed(self, stn, request_interval_hours=2, since=None):
"""** No parameters are sensor reading arrays **
Returns a list of wind speed (mph) observations from a Mesonet station.
"""
return self.getAllMesonetObservations(stn, 'wind_speed', 0.0, 150.0, request_interval_hours, since)
def runtimeFromOnOff(self, onOffID, runtimeInterval=30, state_xform_func=None):
"""** No parameters are sensor reading arrays **
......
"""Allows acquisition of Internet weather data.
"""
import urllib.request, urllib.error, urllib.parse, time, json, urllib.request, urllib.parse, urllib.error
import urllib.request
import urllib.error
import urllib.parse
import time
import json
import urllib.request
import urllib.parse
import urllib.error
from django.conf import settings
from metar import Metar
from .cache import Cache
# cache for storing NWS observations
_nws_cache = Cache()
_nws_cache = Cache()
def getWeatherObservation(stnCode):
"""Returns a current weather observation from an NWS weather station, using the metar
......@@ -31,7 +39,8 @@ def getWeatherObservation(stnCode):
# try 3 times in case of download errors.
for i in range(3):
try:
read_str = urllib.request.urlopen(URL % stnCode).read().decode('utf-8')
read_str = urllib.request.urlopen(
URL % stnCode).read().decode('utf-8')
break
except:
# wait before retrying
......@@ -41,14 +50,17 @@ def getWeatherObservation(stnCode):
# retries must have failed if there is no 'read_str' variable.
raise Exception('Could not access %s.' % stnCode)
obs = Metar.Metar('\n'.join( read_str.splitlines()[1:] )) # second line onward
# second line onward
obs = Metar.Metar('\n'.join(read_str.splitlines()[1:]))
_nws_cache.store(stnCode, obs)
return obs
# cache for storing Weather Underground observations
_wu_cache = Cache()
def getWUobservation(stnList):
"""Returns a current weather observation (dictionary) retrieved from weather underground.
Google Weather Underground API for more info.
......@@ -60,23 +72,109 @@ def getWUobservation(stnList):
# ignore None stations
if stn is None:
continue
# retrieve from cache, if there.
obs = _wu_cache.get(stn)
if obs is None:
# not in cache; download from weather underground.
wu_key = getattr(settings, 'BMSAPP_WU_API_KEY', None)
if wu_key:
url = 'http://api.wunderground.com/api/%s/conditions/q/%s.json' % (wu_key, urllib.parse.quote(stn))
url = 'http://api.wunderground.com/api/%s/conditions/q/%s.json' % (
wu_key, urllib.parse.quote(stn))
json_str = urllib.request.urlopen(url).read().decode('utf-8')
obs = json.loads(json_str)
_wu_cache.store(stn, obs)
else:
raise ValueError('No Weather Underground API key in Settings File.')
raise ValueError(
'No Weather Underground API key in Settings File.')
if 'current_observation' in obs:
return obs['current_observation']
# No stations were successful
raise ValueError("No stations with data.")
# cache for storing Mesonet observations
_mesonet_cache = Cache()
def getMesonetObservation(stnList):
"""Returns a current weather observation (dictionary) retrieved from MesonetAPI.
See https://developers.synopticdata.com/mesonet/ for more information.
'stnList' is a list of stations. Results will be returned only from the first
valid station with current data..
"""
for stn in stnList:
# ignore None stations
if stn is None:
continue
# retrieve from cache, if there.
obs = _mesonet_cache.get(stn)
if obs is None:
# not in cache; download from internet.
api_token = getattr(settings, 'BMSAPP_MESONET_API_TOKEN', None)
params = {'token': api_token,
'stid': stn,
'vars': 'air_temp,wind_speed,relative_humidity',
'units': 'english',
'status': 'active',
'within': 60,
'timeformat': '%s'
}
query_string = urllib.parse.urlencode(params)
url = 'https://api.synopticdata.com/v2/stations/latest?' + query_string
if api_token:
# print('URL: {}'.format(url))
json_str = urllib.request.urlopen(url).read().decode('utf-8')
obs = json.loads(json_str)
_mesonet_cache.store(stn, obs)
else:
raise ValueError('No Mesonet API key in Settings File.')
if 'STATION' in obs:
return obs['STATION'][0]['OBSERVATIONS']
# No stations were successful
raise ValueError(obs['SUMMARY']['RESPONSE_MESSAGE'])
def getMesonetTimeseries(stnID, parameter, last_ts):
"""Returns weather observations (dictionary) retrieved from MesonetAPI.
See https://developers.synopticdata.com/mesonet/ for more information.
'stnID' is the Mesonet Station ID.
Returns a list of timestamps and values.
"""
api_token = getattr(settings, 'BMSAPP_MESONET_API_TOKEN', None)
params = {'token': api_token,
'stid': stnID,
'start': time.strftime(r'%Y%m%d%H%M', time.gmtime(last_ts + 1)),
'end': time.strftime(r'%Y%m%d%H%M', time.gmtime()),
'vars': parameter,
'hfmetars': 0,
'units': 'english',
'timeformat': '%s'
}
query_string = urllib.parse.urlencode(params)
url = 'https://api.synopticdata.com/v2/stations/timeseries?' + query_string
if api_token:
json_str = urllib.request.urlopen(url).read().decode('utf-8')
obs = json.loads(json_str)
else:
raise ValueError('No Mesonet API key in Settings File.')
if 'STATION' in obs:
return obs['STATION'][0]['OBSERVATIONS']
else:
print(url)
print(obs)
raise ValueError(obs['SUMMARY']['RESPONSE_MESSAGE'])
.. _calculated-fields:
Calculated Fields
=================
......@@ -182,42 +181,54 @@ appropriate unit and title changes elsewhere.
--------------
The Weather Underground service has a broader variety of weather
stations, including personal weather stations. To gather temperature or
wind data from this service, you must first acquire a `Weather
Underground API Key <http://www.wunderground.com/weather/api/>`_ and enter
The MesonetAPI service includes a larger set of weather stations.
To gather temperature or wind data from this service, you must first acquire a
`Mesonet API Token <https://developers.synopticdata.com/signup/>`_ and enter
that key into the :ref:`BMON Settings File <how-to-install-BMON-on-a-web-server>`
as the ``BMSAPP_WU_API_KEY`` setting (restarting the Django web
as the ``BMSAPP_MESONET_API_TOKEN`` setting (restarting the Django web
application after changing a setting is necessary).
There is currently no charge for limited use of the API up to 5,000 requests
and 5 million service units per month. Beyond that there is charge of
5 cents per thousand requests, and 15 cents per million Service Units.
If either your Requests or Service Units exceed the free tier levels,
you will be charged a $5.00 monthly service fee, in addition to the rated
charges for any usage above the free tier levels. See the `Mesonet Pricing
Page <https://developers.synopticdata.com/mesonet/pricing/>`_ for more information.
Here is an example configuration for acquiring temperature data from the
service:
.. image:: /_static/calc_ex2.png
:align: center
+-------------------+---------------------------------------------+
| Calculated Field | |
+===================+=============================================+
|| Transform or || ``getAllMesonetTemperature`` |
|| Calculated Field | |
|| Function Name: | |
+-------------------+---------------------------------------------+
|| Function || ``stn: F2072`` |
|| Parameters in || ``request_interval_hours: 2`` |
|| YAML form: || ``since: 6/1/2019`` |
+-------------------+---------------------------------------------+
The key differences from the National Weather Service configuration are:
* ``getWUtemperature`` must be entered into the
* ``getAllMesonetTemperature`` must be entered into the
``Transform or Calculated Field Function Name`` box. If you are
acquiring wind speed data, then the correct entry is
``getWUwindSpeed``. Capitalization must be as shown.
``getAllMesonetWindSpeed``. Capitalization must be as shown.
* The ``Function Parameters`` box must contain a ``stn`` entry for the
main weather station you want data from and an optional ``stn2`` code
for a weather station to use as a backup in case the primary station
is not available. An example entry is:
::
stn: pws:KAKANCHO124
stn2: pws:MD0691
For information on how to form station codes, see the `Weather Underground API
documentation <http://www.wunderground.com/weather/api/d/docs?d=data/index>`_
for the ``query`` parameter. In this example, two personal weather
stations are being used with station IDs of ``KAKANCH0124`` and
``MD0691``.
weather station you want data from. To find station codes, refer to
the `Mesonet map <http://www.wrh.noaa.gov/map/?&zoom=5&scroll_zoom=false&center=62.0,-150.0&boundaries=false,false,false,false,false,false,false,false,false&tab=observation&obs=true&obs_type=weather&elements=temp,wind,gust&temp_filter=-80,130&gust_filter=0,150&rh_filter=0,100&elev_filter=-300,14000&precip_filter=0.01,18&obs_popup=false&obs_density=60&obs_provider=ALL>`_.
* The ``Function Parameters`` box may contain an additional entry for the
``request_interval_hours`` which specifies the minimum interval at which
data is updated. To stay within the limit of 5,000 requests per month, the
interval can be 0.5 for up to three calculated sensors or 2.0 for up to 13.
To estimate the minimum interval you can take the total number of fields
that will use the mesonet API and multiply by 0.15. The default is two hours.
* The ``Function Parameters`` box may also contain an additional ``since`` entry
which specifies the earliest date or date/time to retieve data for.
Converting On/Off Events into Runtime Fraction
----------------------------------------------
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment