Merge branch 'mesonet_api'; Allows for Weather data from Mesonet sites.

bmon/settings_example.py

 ##################################################
 # Django settings for BMON project               #
 ##################################################
+from .settings_common import *
 import logging
 
-#----------------- Settings Specific to the Monitoring App ----------------------
+# ----------------- Settings Specific to the Monitoring App ----------------------
 
 # The key needed to store sensor readings into the database.
 # This is a REQUIRED setting.  You can have multiple valid storage keys
 # by listing them in a tuple or list:  ('key1', 'key2') or ['key1', 'key2'].
 # You can load https://bms.ahfc.us/make-store-key/ in a browser to generate
 # a suitable random store key.  Any sensors that post data to this site will
-# need to include thise Store Key when they post the data.  
+# need to include thise Store Key when they post the data.
 # See bmsapp/views.storereading() and bmsapp/views.storereadings() for details
 # of how the Store Key is included in a reading post.
 BMSAPP_STORE_KEY = 'PutStorageKeyHere'
 
-# Store key used with old URL pattern. 
+# Store key used with old URL pattern.
 # *** NOT USED WITH NEW INSTALLS.  LEAVE COMMENTED OUT ***
 # BMSAPP_STORE_KEY_OLD = ''
 
@@ -30,18 +31,18 @@ BMSAPP_HEADER = 'XYZ Remote Monitoring'
 #     First item in tuple is Text that will be shown for the link.
 #     Second item is the name of the template that will be rendered to produce the page.
 #          'reports' is a special name that will cause the main reports/charts page to be
-#          rendered. 'custom-reports' is also special and will cause a page showing the 
-#          available custom reports. For other names in this position, there must be a corresponding 
+#          rendered. 'custom-reports' is also special and will cause a page showing the
+#          available custom reports. For other names in this position, there must be a corresponding
 #          [template name].html file present in the templates/bmsapp directory.  The custom
 #          template cannot match any of the URLs listed in urls.py.
 #     The third item (optional) is True if this item should be the default index page for
 #         the application.
-BMSAPP_NAV_LINKS = ( ('Map', 'map'),
-                     ('Data Charts and Reports', 'reports', True),
-                     ('Custom Reports', 'custom-reports'),
-                     ('Training Videos and Project Reports', 'training-anthc'),
-                     ('System Administrator', 'admin-reports'),
-                   )
+BMSAPP_NAV_LINKS = (('Map', 'map'),
+                    ('Graphs/Reports', 'reports', True),
+                    ('Custom Reports', 'custom-reports'),
+                    ('Training', 'training-anthc'),
+                    ('Sys Admin', 'admin-reports'),
+                    )
 
 # The number of hours before a sensor is considered to be inactive (not posting data).
 BMSAPP_SENSOR_INACTIVITY = 2.0   # Hours
@@ -57,10 +58,14 @@ BMSAPP_PUSHOVER_APP_TOKEN = '123456789012345678901234567890'
 # Levels in order from least to greatest severity are:  DEBUG, INFO, WARNING, ERROR, CRITICAL
 BMSAPP_LOG_LEVEL = logging.INFO
 
-# Put a Weather Underground API key here, if you are acquiring temperature or wind
-# data from any Weather Underground sites (getWUtemperature, getWUwindSpeed).  
-# See http://www.wunderground.com/weather/api for details on acuiring a key for your use.
-BMSAPP_WU_API_KEY = ''
+# Put a Mesonet API Token here, if you are acquiring temperature or wind data
+# using the Mesonet API (getAllMesonetTemperature, getAllMesonetWindSpeed).
+# See https://developers.synopticdata.com/signup to obtain a token.
+BMSAPP_MESONET_API_TOKEN = ''
+
+# Weather Underground API is no longer supported due to changes in their service
+# See http://www.wunderground.com/weather/api for details
+# BMSAPP_WU_API_KEY = ''
 
 # Enter URL, Username and Password for the ARIS api here if you are using the
 # getUsageFromARIS calculation function to import building energy usage info
@@ -99,7 +104,7 @@ LANGUAGE_CODE = 'en-us'
 
 # The Names and Emails of people who should be emailed in the case of an
 # application exception.  Everyone appearing in the list will be notified; add
-# additional tuples for each Admin.  Unlike the sample below, remove the # symbol 
+# additional tuples for each Admin.  Unlike the sample below, remove the # symbol
 # in front of each tuple.
 # See documentation at https://docs.djangoproject.com/en/1.7/ref/settings/#std:setting-ADMINS
 # NOTE:  You can also view the error log for the BMON application by browsing to the page:
@@ -108,7 +113,7 @@ ADMINS = (
     # ('Admin Name Here', 'Admin Email Address Here'),
 )
 
-# The following email settings need to be filled out for sending out alerts from 
+# The following email settings need to be filled out for sending out alerts from
 # the BMON app.
 # For the Webfaction hosting service, see documentation at:
 # http://docs.webfaction.com/software/django/getting-started.html#configuring-django-to-send-email-messages
@@ -116,17 +121,19 @@ ADMINS = (
 # https://docs.djangoproject.com/en/1.7/ref/settings/#std:setting-EMAIL_HOST
 
 # the SMTP server used to send mail, 'smtp.webfaction.com' or the Webfaction hosting service
-EMAIL_HOST = 'smtp.webfaction.com'   
+EMAIL_HOST = 'smtp.webfaction.com'
 EMAIL_HOST_USER = 'mailbox_username'
 EMAIL_HOST_PASSWORD = 'mailbox_password'
 # If you are using the Webfaction smtp server, the two FROM adddresses below need to
 # be email addresses set up in your Webfaction account (you can use the same address
 # for both, if desired).  The Webfaction server will not send emails with a FROM address
 # that is not on the Webfaction host.
-DEFAULT_FROM_EMAIL = 'valid_from_email_address'      # this will be the FROM for alert messages
-SERVER_EMAIL = 'valid_from_email_for_error_messages'   # this is the FROM for error messages
+# this will be the FROM for alert messages
+DEFAULT_FROM_EMAIL = 'valid_from_email_address'
+# this is the FROM for error messages
+SERVER_EMAIL = 'valid_from_email_for_error_messages'
 
-# If DEBUG=True, a detailed error traceback is displayed in the browser when an error 
+# If DEBUG=True, a detailed error traceback is displayed in the browser when an error
 # occurs.  This setting should be False for production use for security reasons, but if
 # errors are occurring with use of the application, setting to True provides valuable
 # debug information.
@@ -138,8 +145,6 @@ DEBUG = False
 BMSAPP_STATIC_APP_NAME = 'bmon_static'
 
 # Import settings that are generally common to all installs of BMON.
-from .settings_common import *
 
 # ----- If you need to override any of the settings in the 'settings_common.py' file
 # ----- do so below this point in this file.
-

bmsapp/calcs/calcfuncs01.py

@@ -123,6 +123,89 @@ class CalcReadingFuncs_01(calcreadings.CalcReadingFuncs_base):
             return [int(time.time())], [wind_val]
         else:
             return [], []
+
+    def getLatestMesonetObservation(self, stnList, parameter, min_val=-float("inf"), max_val=float("inf")):
+        """** No parameters are sensor reading arrays **
+
+        Returns just one record of information, timestamped with the reading time
+        """
+        obs = internetwx.getMesonetObservation(stnList)
+
+        # Testing...
+        #print('obs: {}'.format(obs))
+
+        observed_val = float(obs[parameter + '_value_1']['value'])
+        time_val = int(obs[parameter + '_value_1']['date_time'])
+        if observed_val >= min_val and observed_val <= max_val:
+            return [time_val], [observed_val]
+        else:
+            return [], []
+
+    def getAllMesonetObservations(self, stnID, parameter, min_val=-float("inf"), max_val=float("inf"), request_interval_hours=2, since=None):
+        """** No parameters are sensor reading arrays **
+
+        Returns a list of observations from a Mesonet station.
+        """
+        # determine the timestamp of the last entry in the database for this calculated field.
+        last_calc_rec = self.db.last_read(self.calc_id)
+        # use 0 ts if no records
+        last_ts = int(last_calc_rec['ts']) if last_calc_rec else 0
+
+        # check to see if the request interval has elapsed
+        hours_elapsed = (time.time() - last_ts) / 60 / 60
+        print('hours_elapsed = {}'.format(hours_elapsed))
+        if hours_elapsed < request_interval_hours:
+            return [], []
+
+        if since:
+            # constrain this value to greater or equal to 'since'
+            last_ts = max(last_ts, bmsapp.data_util.datestr_to_ts(since))
+        else:
+            # constrain this value to greater or equal to 'reach_back'
+            last_ts = max(last_ts, int(time.time() - self.reach_back))
+
+        # Get the observations from the MesonetAPI
+        obs = internetwx.getMesonetTimeseries(stnID, parameter, last_ts)
+
+        obs_dict = dict(zip(obs['date_time'], obs[parameter + '_set_1']))
+        time_vals = []
+        observed_vals = []
+        for time_val, observed_val in obs_dict.items():
+            if observed_val >= min_val and observed_val <= max_val:
+                time_vals.append(int(time_val))
+                observed_vals.append(float(observed_val))
+
+        return time_vals, observed_vals
+
+    def getMesonetTemperature(self, stn, stn2=None):
+        """** No parameters are sensor reading arrays **
+
+        Returns a temperature (deg F) from a Mesonet station.
+        'stn' is the primary station to use.  'stn2' is a backup station.
+        """
+        return self.getLatestMesonetObservation([stn, stn2], 'air_temp', -120.0, 150.0)
+
+    def getMesonetWindSpeed(self, stn, stn2=None):
+        """** No parameters are sensor reading arrays **
+
+        Returns a wind speed (mph) from a Mesonet station.
+        'stn' is the primary station to use.  'stn2' is a backup station.
+        """
+        return self.getLatestMesonetObservation([stn, stn2], 'wind_speed', 0.0, 150.0)
+
+    def getAllMesonetTemperature(self, stn, request_interval_hours=2, since=None):
+        """** No parameters are sensor reading arrays **
+
+        Returns a list of temperature (deg F) observations from a Mesonet station.
+        """
+        return self.getAllMesonetObservations(stn, 'air_temp', -120.0, 150.0, request_interval_hours, since)
+
+    def getAllMesonetWindSpeed(self, stn, request_interval_hours=2, since=None):
+        """** No parameters are sensor reading arrays **
+
+        Returns a list of wind speed (mph) observations from a Mesonet station.
+        """
+        return self.getAllMesonetObservations(stn, 'wind_speed', 0.0, 150.0, request_interval_hours, since)
     
     def runtimeFromOnOff(self, onOffID, runtimeInterval=30, state_xform_func=None):
         """** No parameters are sensor reading arrays **

bmsapp/calcs/internetwx.py

 """Allows acquisition of Internet weather data.
 """
 
-import urllib.request, urllib.error, urllib.parse, time, json, urllib.request, urllib.parse, urllib.error
+import urllib.request
+import urllib.error
+import urllib.parse
+import time
+import json
+import urllib.request
+import urllib.parse
+import urllib.error
 from django.conf import settings
 from metar import Metar
 from .cache import Cache
 
 # cache for storing NWS observations
-_nws_cache = Cache()   
+_nws_cache = Cache()
+
 
 def getWeatherObservation(stnCode):
     """Returns a current weather observation from an NWS weather station, using the metar 
@@ -31,7 +39,8 @@ def getWeatherObservation(stnCode):
         # try 3 times in case of download errors.
         for i in range(3):
             try:
-                read_str = urllib.request.urlopen(URL % stnCode).read().decode('utf-8')
+                read_str = urllib.request.urlopen(
+                    URL % stnCode).read().decode('utf-8')
                 break
             except:
                 # wait before retrying
@@ -41,14 +50,17 @@ def getWeatherObservation(stnCode):
             # retries must have failed if there is no 'read_str' variable.
             raise Exception('Could not access %s.' % stnCode)
 
-        obs = Metar.Metar('\n'.join( read_str.splitlines()[1:] ))  # second line onward
+        # second line onward
+        obs = Metar.Metar('\n'.join(read_str.splitlines()[1:]))
         _nws_cache.store(stnCode, obs)
 
     return obs
 
+
 # cache for storing Weather Underground observations
 _wu_cache = Cache()
 
+
 def getWUobservation(stnList):
     """Returns a current weather observation (dictionary) retrieved from weather underground.
     Google Weather Underground API for more info.  
@@ -60,23 +72,109 @@ def getWUobservation(stnList):
         # ignore None stations
         if stn is None:
             continue
-        
+
         # retrieve from cache, if there.
         obs = _wu_cache.get(stn)
-    
+
         if obs is None:
             # not in cache; download from weather underground.
             wu_key = getattr(settings, 'BMSAPP_WU_API_KEY', None)
             if wu_key:
-                url = 'http://api.wunderground.com/api/%s/conditions/q/%s.json' % (wu_key, urllib.parse.quote(stn))
+                url = 'http://api.wunderground.com/api/%s/conditions/q/%s.json' % (
+                    wu_key, urllib.parse.quote(stn))
                 json_str = urllib.request.urlopen(url).read().decode('utf-8')
                 obs = json.loads(json_str)
                 _wu_cache.store(stn, obs)
             else:
-                raise ValueError('No Weather Underground API key in Settings File.')
+                raise ValueError(
+                    'No Weather Underground API key in Settings File.')
 
         if 'current_observation' in obs:
             return obs['current_observation']
-    
+
     # No stations were successful
     raise ValueError("No stations with data.")
+
+
+# cache for storing Mesonet observations
+_mesonet_cache = Cache()
+
+
+def getMesonetObservation(stnList):
+    """Returns a current weather observation (dictionary) retrieved from MesonetAPI.
+    See https://developers.synopticdata.com/mesonet/ for more information.
+
+    'stnList' is a list of stations. Results will be returned only from the first
+    valid station with current data..
+    """
+
+    for stn in stnList:
+        # ignore None stations
+        if stn is None:
+            continue
+
+        # retrieve from cache, if there.
+        obs = _mesonet_cache.get(stn)
+
+        if obs is None:
+            # not in cache; download from internet.
+            api_token = getattr(settings, 'BMSAPP_MESONET_API_TOKEN', None)
+            params = {'token': api_token,
+                      'stid': stn,
+                      'vars': 'air_temp,wind_speed,relative_humidity',
+                      'units': 'english',
+                      'status': 'active',
+                      'within': 60,
+                      'timeformat': '%s'
+                      }
+            query_string = urllib.parse.urlencode(params)
+            url = 'https://api.synopticdata.com/v2/stations/latest?' + query_string
+
+            if api_token:
+                # print('URL: {}'.format(url))
+                json_str = urllib.request.urlopen(url).read().decode('utf-8')
+                obs = json.loads(json_str)
+                _mesonet_cache.store(stn, obs)
+            else:
+                raise ValueError('No Mesonet API key in Settings File.')
+
+        if 'STATION' in obs:
+            return obs['STATION'][0]['OBSERVATIONS']
+
+    # No stations were successful
+    raise ValueError(obs['SUMMARY']['RESPONSE_MESSAGE'])
+
+
+def getMesonetTimeseries(stnID, parameter, last_ts):
+    """Returns weather observations (dictionary) retrieved from MesonetAPI.
+    See https://developers.synopticdata.com/mesonet/ for more information.
+
+    'stnID' is the Mesonet Station ID.
+
+    Returns a list of timestamps and values.
+    """
+    api_token = getattr(settings, 'BMSAPP_MESONET_API_TOKEN', None)
+    params = {'token': api_token,
+              'stid': stnID,
+              'start': time.strftime(r'%Y%m%d%H%M', time.gmtime(last_ts + 1)),
+              'end': time.strftime(r'%Y%m%d%H%M', time.gmtime()),
+              'vars': parameter,
+              'hfmetars': 0,
+              'units': 'english',
+              'timeformat': '%s'
+              }
+    query_string = urllib.parse.urlencode(params)
+    url = 'https://api.synopticdata.com/v2/stations/timeseries?' + query_string
+
+    if api_token:
+        json_str = urllib.request.urlopen(url).read().decode('utf-8')
+        obs = json.loads(json_str)
+    else:
+        raise ValueError('No Mesonet API key in Settings File.')
+
+    if 'STATION' in obs:
+        return obs['STATION'][0]['OBSERVATIONS']
+    else:
+        print(url)
+        print(obs)
+        raise ValueError(obs['SUMMARY']['RESPONSE_MESSAGE'])

docs/source/calculated-fields.rst

 .. _calculated-fields:
-
 Calculated Fields
 =================
 
@@ -182,42 +181,54 @@ appropriate unit and title changes elsewhere.
 
 --------------
 
-The Weather Underground service has a broader variety of weather
-stations, including personal weather stations. To gather temperature or
-wind data from this service, you must first acquire a `Weather
-Underground API Key <http://www.wunderground.com/weather/api/>`_ and enter
+The MesonetAPI service includes a larger set of weather stations.
+To gather temperature or wind data from this service, you must first acquire a 
+`Mesonet API Token <https://developers.synopticdata.com/signup/>`_ and enter
 that key into the :ref:`BMON Settings File <how-to-install-BMON-on-a-web-server>` 
-as the ``BMSAPP_WU_API_KEY`` setting (restarting the Django web
+as the ``BMSAPP_MESONET_API_TOKEN`` setting (restarting the Django web
 application after changing a setting is necessary).
 
+There is currently no charge for limited use of the API up to 5,000 requests
+and 5 million service units per month. Beyond that  there is charge of 
+5 cents per thousand requests, and 15 cents per million Service Units.
+If either your Requests or Service Units exceed the free tier levels, 
+you will be charged a $5.00 monthly service fee, in addition to the rated 
+charges for any usage above the free tier levels. See the `Mesonet Pricing
+Page <https://developers.synopticdata.com/mesonet/pricing/>`_ for more information.
+
 Here is an example configuration for acquiring temperature data from the
 service:
 
-.. image:: /_static/calc_ex2.png
-    :align: center
++-------------------+---------------------------------------------+
+| Calculated Field  |                                             |
++===================+=============================================+
+|| Transform or     || ``getAllMesonetTemperature``               |
+|| Calculated Field |                                             |
+|| Function Name:   |                                             |
++-------------------+---------------------------------------------+
+|| Function         || ``stn: F2072``                             |
+|| Parameters in    || ``request_interval_hours: 2``              |
+|| YAML form:       || ``since: 6/1/2019``                        |
++-------------------+---------------------------------------------+
 
 The key differences from the National Weather Service configuration are:
 
-*  ``getWUtemperature`` must be entered into the
+*  ``getAllMesonetTemperature`` must be entered into the
    ``Transform or Calculated Field Function Name`` box. If you are
    acquiring wind speed data, then the correct entry is
-   ``getWUwindSpeed``. Capitalization must be as shown.
+   ``getAllMesonetWindSpeed``. Capitalization must be as shown.
 *  The ``Function Parameters`` box must contain a ``stn`` entry for the
-   main weather station you want data from and an optional ``stn2`` code
-   for a weather station to use as a backup in case the primary station
-   is not available. An example entry is:
-
-::
-
-        stn: pws:KAKANCHO124
-        stn2: pws:MD0691
-
-For information on how to form station codes, see the `Weather Underground API
-documentation <http://www.wunderground.com/weather/api/d/docs?d=data/index>`_
-for the ``query`` parameter. In this example, two personal weather
-stations are being used with station IDs of ``KAKANCH0124`` and
-``MD0691``.
-
+   weather station you want data from. To find station codes, refer to
+   the `Mesonet map <http://www.wrh.noaa.gov/map/?&zoom=5&scroll_zoom=false&center=62.0,-150.0&boundaries=false,false,false,false,false,false,false,false,false&tab=observation&obs=true&obs_type=weather&elements=temp,wind,gust&temp_filter=-80,130&gust_filter=0,150&rh_filter=0,100&elev_filter=-300,14000&precip_filter=0.01,18&obs_popup=false&obs_density=60&obs_provider=ALL>`_.
+*  The ``Function Parameters`` box may contain an additional entry for the
+   ``request_interval_hours`` which specifies the minimum interval at which
+   data is updated. To stay within the limit of 5,000 requests per month, the
+   interval can be 0.5 for up to three calculated sensors or 2.0 for up to 13.
+   To estimate the minimum interval you can take the total number of fields
+   that will use the mesonet API and multiply by 0.15. The default is two hours.
+*  The ``Function Parameters`` box may also contain an additional ``since`` entry
+   which specifies the earliest date or date/time to retieve data for.
+   
 Converting On/Off Events into Runtime Fraction
 ----------------------------------------------