Merge pull request #2125 from DanCech/multiTag

Multi series tag/delete support

Author: DanCech

docs/tags.rst

@@ -113,14 +113,19 @@ The default settings (above) will connect to a local Redis server on the default
 HTTP(S) TagDB
 ^^^^^^^^^^^^^
 
-The HTTP(S) TagDB is used to delegate all tag operations to an external server that implements the Graphite tagging HTTP API.  It can be used in clustered graphite scenarios, or with custom data stores.  It is selected by setting ``TAGDB='graphite.tags.http.HttpTagDB'`` in `local_settings.py`.  There are 3 additional config settings for the HTTP(S) TagDB::
+The HTTP(S) TagDB is used to delegate all tag operations to an external server that implements the Graphite tagging HTTP API.  It can be used in clustered graphite scenarios, or with custom data stores.  It is selected by setting ``TAGDB='graphite.tags.http.HttpTagDB'`` in `local_settings.py`.  There are 4 additional config settings for the HTTP(S) TagDB::
 
     TAGDB_HTTP_URL = 'https://another.server'
     TAGDB_HTTP_USER = ''
     TAGDB_HTTP_PASSWORD = ''
+    TAGDB_HTTP_AUTOCOMPLETE = False
 
 The ``TAGDB_HTTP_URL`` is required. ``TAGDB_HTTP_USER`` and ``TAGDB_HTTP_PASSWORD`` are optional and if specified will be used to send a Basic Authorization header in all requests.
 
+``TAGDB_HTTP_AUTOCOMPLETE`` is also optional, if set to ``True`` auto-complete requests will be forwarded to the remote TagDB, otherwise calls to `/tags/findSeries`, `/tags` & `/tags/<tag>` will be used to provide auto-complete functionality.
+
+If ``REMOTE_STORE_FORWARD_HEADERS`` is defined, those headers will also be forwarded to the remote TagDB.
+
 Adding Series to the TagDB
 --------------------------
 Normally `carbon` will take care of this, it submits all new series to the TagDB, and periodically re-submits all series to ensure that the TagDB is kept up to date.  There are 2 `carbon` configuration settings related to tagging; the `GRAPHITE_URL` setting specifies the url of your graphite-web installation (default `http://127.0.0.1:8000`), and the `TAG_UPDATE_INTERVAL` setting specifies how often each series should be re-submitted to the TagDB (default is every 100th update).
@@ -136,6 +141,22 @@ Series can be submitted via HTTP POST using command-line tools such as ``curl``
 
 This endpoint returns the canonicalized version of the path, with the tags sorted in alphabetical order.
 
+To add multiple series with a single HTTP request, use the ``/tags/tagMultiSeries`` endpoint, which support multiple ``path`` parameters:
+
+.. code-block:: none
+
+    $ curl -X POST "http://graphite/tags/tagMultiSeries" \
+      --data-urlencode 'path=disk.used;rack=a1;datacenter=dc1;server=web01' \
+      --data-urlencode 'path=disk.used;rack=a1;datacenter=dc1;server=web02' \
+      --data-urlencode 'pretty=1'
+
+    [
+      "disk.used;datacenter=dc1;rack=a1;server=web01",
+      "disk.used;datacenter=dc1;rack=a1;server=web02"
+    ]
+
+This endpoint returns a list of the canonicalized paths, in the same order they are specified.
+
 Exploring Tags
 --------------
 You can use the HTTP api to get lists of defined tags, values for each tag, and to find series using the same logic as the `seriesByTag <functions.html#graphite.render.functions.seriesByTag>`_ function.
@@ -309,3 +330,13 @@ Series can be deleted via HTTP POST to the `/tags/delSeries` endpoint:
       --data-urlencode 'path=disk.used;datacenter=dc1;rack=a1;server=web01'
 
     true
+
+To delete multiple series at once pass multiple ``path`` parameters:
+
+.. code-block:: none
+
+    $ curl -X POST "http://graphite/tags/delSeries" \
+      --data-urlencode 'path=disk.used;datacenter=dc1;rack=a1;server=web01' \
+      --data-urlencode 'path=disk.used;datacenter=dc1;rack=a1;server=web02'
+
+    true

webapp/graphite/storage.py

@@ -54,7 +54,7 @@ def __init__(self, finders=None, tagdb=None):
 
         if tagdb is None:
             tagdb = settings.TAGDB
-        self.tagdb = get_tagdb(tagdb) if tagdb else None
+        self.tagdb = get_tagdb(tagdb)
 
     def get_finders(self, local=False):
         for finder in self.finders:

webapp/graphite/tags/base.py

@@ -147,12 +147,26 @@ def tag_series(self, series, requestContext=None):
     Enter series into database.  Accepts a series string, upserts into the TagDB and returns the canonicalized series name.
     """
 
+  def tag_multi_series(self, seriesList, requestContext=None):
+    """
+    Enter series into database.  Accepts a list of series strings, upserts into the TagDB and returns a list of canonicalized series names.
+    """
+    return [self.tag_series(series, requestContext) for series in seriesList]
+
   @abc.abstractmethod
   def del_series(self, series, requestContext=None):
     """
     Remove series from database.  Accepts a series string and returns True
     """
 
+  def del_multi_series(self, seriesList, requestContext=None):
+    """
+    Remove series from database.  Accepts a list of series strings, removes them from the TagDB and returns True
+    """
+    for series in seriesList:
+      self.del_series(series, requestContext)
+    return True
+
   def auto_complete_tags(self, exprs, tagPrefix=None, limit=None, requestContext=None):
     """
     Return auto-complete suggestions for tags based on the matches for the specified expressions, optionally filtered by tag prefix

webapp/graphite/tags/http.py

@@ -1,6 +1,5 @@
 from __future__ import absolute_import
 
-from urllib import quote
 import json
 
 from graphite.http_pool import http
@@ -19,10 +18,7 @@ def __init__(self, settings, *args, **kwargs):
     self.username = settings.TAGDB_HTTP_USER
     self.password = settings.TAGDB_HTTP_PASSWORD
 
-  def request(self, method, url, fields=None, requestContext=None):
-    if not fields:
-      fields = {}
-
+  def request(self, method, url, fields, requestContext=None):
     headers = requestContext.get('forwardHeaders') if requestContext else {}
     if 'Authorization' not in headers and self.username and self.password:
       headers['Authorization'] = 'Basic ' + ('%s:%s' % (self.username, self.password)).encode('base64')
@@ -35,8 +31,11 @@ def request(self, method, url, fields=None, requestContext=None):
       timeout=self.settings.REMOTE_FIND_TIMEOUT,
     )
 
+    if result.status == 400:
+      raise ValueError(json.loads(result.data.decode('utf-8')).get('error'))
+
     if result.status != 200:
-      raise Exception('HTTP Error from remote tagdb: %s' % result.status)
+      raise Exception('HTTP Error from remote tagdb: %s %s' % (result.status, result.data))
 
     return json.loads(result.data.decode('utf-8'))
 
@@ -51,8 +50,9 @@ def find_series_cachekey(self, tags, requestContext=None):
 
   def _find_series(self, tags, requestContext=None):
     return self.request(
-      'GET',
-      '/tags/findSeries?' + '&'.join([('expr=%s' % quote(tag)) for tag in tags]),
+      'POST',
+      '/tags/findSeries',
+      {'expr': tags},
       requestContext=requestContext,
     )
 
@@ -83,9 +83,15 @@ def list_values(self, tag, valueFilter=None, limit=None, requestContext=None):
   def tag_series(self, series, requestContext=None):
     return self.request('POST', '/tags/tagSeries', {'path': series}, requestContext)
 
+  def tag_multi_series(self, seriesList, requestContext=None):
+    return self.request('POST', '/tags/tagMultiSeries', {'path': seriesList}, requestContext)
+
   def del_series(self, series, requestContext=None):
     return self.request('POST', '/tags/delSeries', {'path': series}, requestContext)
 
+  def del_multi_series(self, seriesList, requestContext=None):
+    return self.request('POST', '/tags/delSeries', {'path': seriesList}, requestContext)
+
   def auto_complete_tags(self, exprs, tagPrefix=None, limit=None, requestContext=None):
     """
     Return auto-complete suggestions for tags based on the matches for the specified expressions, optionally filtered by tag prefix
@@ -97,10 +103,13 @@ def auto_complete_tags(self, exprs, tagPrefix=None, limit=None, requestContext=N
     if limit is None:
       limit = self.settings.TAGDB_AUTOCOMPLETE_LIMIT
 
-    url = '/tags/autoComplete/tags?tagPrefix=' + quote(tagPrefix or '') + '&limit=' + quote(str(limit)) + \
-      '&' + '&'.join([('expr=%s' % quote(expr or '')) for expr in exprs])
+    fields = {
+      'tagPrefix': tagPrefix or '',
+      'limit': str(limit),
+      'expr': exprs,
+    }
 
-    return self.request('GET', url)
+    return self.request('POST', '/tags/autoComplete/tags', fields, requestContext)
 
   def auto_complete_values(self, exprs, tag, valuePrefix=None, limit=None, requestContext=None):
     """
@@ -113,7 +122,11 @@ def auto_complete_values(self, exprs, tag, valuePrefix=None, limit=None, request
     if limit is None:
       limit = self.settings.TAGDB_AUTOCOMPLETE_LIMIT
 
-    url = '/tags/autoComplete/values?tag=' + quote(tag or '') + '&valuePrefix=' + quote(valuePrefix or '') + \
-      '&limit=' + quote(str(limit)) + '&' + '&'.join([('expr=%s' % quote(expr or '')) for expr in exprs])
+    fields = {
+      'tag': tag or '',
+      'valuePrefix': valuePrefix or '',
+      'limit': str(limit),
+      'expr': exprs,
+    }
 
-    return self.request('GET', url)
+    return self.request('POST', '/tags/autoComplete/values', fields, requestContext)

webapp/graphite/tags/urls.py

@@ -17,6 +17,7 @@
 
 urlpatterns = [
   url('tagSeries', views.tagSeries, name='tagSeries'),
+  url('tagMultiSeries', views.tagMultiSeries, name='tagMultiSeries'),
   url('delSeries', views.delSeries, name='delSeries'),
   url('findSeries', views.findSeries, name='findSeries'),
   url('autoComplete/tags', views.autoCompleteTags, name='tagAutoCompleteTags'),