PR for #336 add User-Agent and Accept-Encoding headers to all Probes

GeoHealthCheck/plugins/probe/http.py

@@ -88,4 +88,6 @@ def get_request_headers(self):
         # request_headers =
         #       self.REQUEST_HEADERS['content-type'].format(**content_type)
         # Hmm seems simpler
-        return {'content-type': self._parameters['content_type']}
+        headers = Probe.get_request_headers(self)
+        return headers.update(
+            {'Content-Type': self._parameters['content_type']})

GeoHealthCheck/probe.py

@@ -9,6 +9,7 @@
 from plugin import Plugin
 from result import ProbeResult
 from util import create_requests_retry_session
+from GeoHealthCheck import __version__
 
 LOGGER = logging.getLogger(__name__)
 
@@ -37,7 +38,16 @@ class Probe(Plugin):
 
     REQUEST_HEADERS = {}
     """
-    `dict` of optional requests headers.
+    `dict` of optional HTTP request headers.
+    """
+
+    STANDARD_REQUEST_HEADERS = {
+        'User-Agent': 'GeoHealthCheck '
+                      '{} (https://geohealthcheck.org)'.format(__version__),
+        'Accept-Encoding': 'deflate, gzip;q=1.0, *;q=0.5'
+    }
+    """
+    `dict` of HTTP headers to add to each HTTP request.
     """
 
     REQUEST_TEMPLATE = ''
@@ -232,8 +242,14 @@ def after_request(self):
 
     def get_request_headers(self):
         if not self._resource:
-            return dict()
+            return Probe.STANDARD_REQUEST_HEADERS
+
         headers = Plugin.copy(self.REQUEST_HEADERS)
+
+        # Add standard headers like User-Agent
+        headers.update(Probe.STANDARD_REQUEST_HEADERS)
+
+        # May add optional Auth header(s)
         return self._resource.add_auth_header(headers)
 
     def perform_request(self):

pavement.py

@@ -76,7 +76,7 @@ def setup():
         config_file.copy(config_site)
 
     # setup deps
-    sh('pip install -r requirements.txt')
+    # sh('pip install -r requirements.txt')
 
     skin = 'http://github.com/BlackrockDigital/startbootstrap-sb-admin-2/archive/v3.3.7+1.zip'  # noqa
 
@@ -136,14 +136,16 @@ def setup():
     zipfile_obj.extractall(options.base.static_lib / 'leaflet')
 
     # install html5shiv to static/lib
+    cdn_url = 'https://cdn.jsdelivr.net/npm'
     with open(path(options.base.static_lib / 'html5shiv.min.js'), 'w') as f:
-        url = 'http://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js'
+        url = \
+            '{}/[email protected]/html5shiv.min.js'.format(cdn_url)
         content = urlopen(url).read().decode()
         f.write(content)
 
     # install respond to static/lib
     with open(path(options.base.static_lib / 'respond.min.js'), 'w') as f:
-        url = 'http://oss.maxcdn.com/respond/1.4.2/respond.min.js'
+        url = '{}/respond.min.js@1.4.2/respond.min.js'.format(cdn_url)
         content = urlopen(url).read().decode()
         f.write(content)