Groupby (#15)

* Initial version of allowing grouping for visualization for Gremlin

* Added code to allow for grouping Gremlin output.  When not specified it will group by the label (if it exists).  Yo can also specify the property to groupby using the switch --groupby or -g followed by the property name

* Initial version of allowing grouping for visualization for Gremlin

* Added code to allow for grouping Gremlin output.  When not specified it will group by the label (if it exists).  Yo can also specify the property to groupby using the switch --groupby or -g followed by the property name

* Fixed several issues that were causing CI tests to fail as well as found an incorrect case of grouping which was fixed and a test added

* Fixed merge conflict that was missed in the last push

* Fixed unit test test_add_vertex_with_callback which broke becasue the group field was not in the expected results

* Addressed comments from akline PR review which included mainly cleaning up the code to simplify the group assignment and adjusting the switch to better match the standard.  This required updating the unit tests as well

* Fixed merge issues

* Updated per PR feedback to simplify the groupby assignment to GremlinNetwork and bumped versions

* Updated grouping to properly handle highlighting etc on selection and search as well as added option to ignore grouping

* Added the functionality to sort the values in the details box by key

* Updated Air-Routes-Visualization notebook to discuss the group by functionality

Co-authored-by: Dave Bechberger <[email protected]>

Author: bechbd

src/graph_notebook/__init__.py

@@ -3,4 +3,4 @@
 SPDX-License-Identifier: Apache-2.0
 """
 
-__version__ = '2.0.4'
+__version__ = '2.0.5'

src/graph_notebook/magics/graph_magic.py

@@ -294,9 +294,12 @@ def gremlin(self, line, cell, local_ns: dict = None):
         parser.add_argument('query_mode', nargs='?', default='query',
                             help='query mode (default=query) [query|explain|profile]')
         parser.add_argument('-p', '--path-pattern', default='', help='path pattern')
+        parser.add_argument('-g', '--group-by', default='T.label', help='Property used to group nodes (e.g. code, T.region) default is T.label')
         parser.add_argument('--store-to', type=str, default='', help='store query result to this variable')
+        parser.add_argument('--ignore-groups', action='store_true', help="Ignore all grouping options")
         args = parser.parse_args(line.split())
         mode = str_to_query_mode(args.query_mode)
+        logger.debug(f'Arguments {args}')
 
         tab = widgets.Tab()
         if mode == QueryMode.EXPLAIN:
@@ -345,7 +348,12 @@ def gremlin(self, line, cell, local_ns: dict = None):
             children.append(table_output)
 
             try:
-                gn = GremlinNetwork()
+                logger.debug(f'groupby: {args.group_by}')
+                if args.ignore_groups:
+                    gn = GremlinNetwork()
+                else:
+                    gn = GremlinNetwork(group_by_property=args.group_by)
+
                 if args.path_pattern == '':
                     gn.add_results(query_res)
                 else:

src/graph_notebook/network/gremlin/GremlinNetwork.py

@@ -6,13 +6,17 @@
 import hashlib
 import json
 import uuid
+import logging
 from enum import Enum
 
 from graph_notebook.network.EventfulNetwork import EventfulNetwork
 from gremlin_python.process.traversal import T
 from gremlin_python.structure.graph import Path, Vertex, Edge
 from networkx import MultiDiGraph
 
+logging.basicConfig()
+logger = logging.getLogger(__file__)
+
 T_LABEL = 'T.label'
 T_ID = 'T.id'
 
@@ -84,10 +88,12 @@ class GremlinNetwork(EventfulNetwork):
     You can find more details on this in our design doc for visualization here: https://quip-amazon.com/R1jbA8eECdDd
     """
 
-    def __init__(self, graph: MultiDiGraph = None, callbacks=None, label_max_length=DEFAULT_LABEL_MAX_LENGTH):
+    def __init__(self, graph: MultiDiGraph = None, callbacks=None, label_max_length=DEFAULT_LABEL_MAX_LENGTH,
+                 group_by_property=T_LABEL):
         if graph is None:
             graph = MultiDiGraph()
         self.label_max_length = label_max_length
+        self.group_by_property = group_by_property
         super().__init__(graph, callbacks)
 
     def add_results_with_pattern(self, results, pattern_list: list):
@@ -264,20 +270,24 @@ def add_vertex(self, v):
         if type(v) is Vertex:
             node_id = v.id
             title = v.label
+            group = v.label
             label = title if len(title) <= self.label_max_length else title[:self.label_max_length - 3] + '...'
-            data = {'label': label, 'title': title, 'properties': {'id': node_id, 'label': title}}
+            data = {'label': label, 'title': title, 'group': group, 'properties': {'id': node_id, 'label': title}}
         elif type(v) is dict:
             properties = {}
 
             title = ''
             label = ''
+            group = ''
             for k in v:
                 if str(k) == T_LABEL:
                     title = str(v[k])
                     label = title if len(title) <= self.label_max_length else title[:self.label_max_length - 3] + '...'
                 elif str(k) == T_ID:
                     node_id = str(v[k])
                 properties[k] = v[k]
+                if str(k) == self.group_by_property:
+                    group = str(v[k])
 
             # handle when there is no id in a node. In this case, we will generate one which
             # is consistently regenerated so that duplicate dicts will be dedubed to the same vertex.
@@ -291,12 +301,13 @@ def add_vertex(self, v):
                     title += str(v[key])
                 label = title if len(title) <= self.label_max_length else title[:self.label_max_length - 3] + '...'
 
-            data = {'properties': properties, 'label': label, 'title': title}
+            data = {'properties': properties, 'label': label, 'title': title, 'group': group}
         else:
             node_id = str(v)
             title = str(v)
             label = title if len(title) <= self.label_max_length else title[:self.label_max_length - 3] + '...'
-            data = {'title': title, 'label': label}
+            data = {'title': title, 'label': label, 'group': ''}
+
         self.add_node(node_id, data)
 
     def add_path_edge(self, edge, from_id='', to_id='', data=None):

src/graph_notebook/notebooks/02-Visualization/Air-Routes-Gremlin.ipynb

@@ -279,6 +279,37 @@
     "         by(keys))"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Color Results by Group\n",
+    "\n",
+    "Run the query below, select the Graph tab, and you will see that the vertices in the graph are now colored by the country."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%gremlin -g country\n",
+    "g.V().has('airport','code','CZM').\n",
+    "  out('route').\n",
+    "  path().\n",
+    "    by(valueMap('code','city','region','desc','lat','lon', 'country').\n",
+    "       order(local).\n",
+    "         by(keys))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "To group vertices together by color we us the `-g` or `--group-by` switch for the `%%gremlin` command.  This specifies the property of the vertex to use to group items together.  If no property is specified than items will be grouped by the `T.label` property if it exists.  If it does not exist, or the property specified does not exist for a particular vertex then the vertex is added to a default group.  Grouping of items is enabled by default but can be disabled using the `--ignore-groups` switch.  "
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -853,6 +884,55 @@
     "  limit(5)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Changing Group Colors and Adding Icons\n",
+    "\n",
+    "One of the features that is also available is the ability to change the color, add an image, or associate a particular icon representation for a group.  Run the two cells below and you will see that all airports in Mexico are shown with the Mexican flag, all airports in the US are shown as a blue flag, and all airports in Canada are shown in red."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%graph_notebook_vis_options\n",
+    "{\n",
+    "  \"groups\": {\n",
+    "      \"['CA']\": {\"color\": \"red\"},\n",
+    "    \"['MX']\": {\"shape\": \"image\", \n",
+    "               \"image\":\"https://cdn.countryflags.com/thumbs/mexico/flag-round-250.png\"},\n",
+    "    \n",
+    "    \"['US']\": {\n",
+    "      \"shape\": \"icon\",\n",
+    "      \"icon\": {\n",
+    "        \"face\": \"FontAwesome\",\n",
+    "        \"code\": \"\\uf024\",\n",
+    "        \"color\": \"blue\"\n",
+    "      }\n",
+    "  }\n",
+    "}\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%gremlin -g country\n",
+    "g.V().has('airport','code','CZM').\n",
+    "  out('route').\n",
+    "  path().\n",
+    "    by(valueMap('code','city','region','desc','lat','lon', 'country').\n",
+    "       order(local).\n",
+    "         by(keys))"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -973,7 +1053,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.5"
+   "version": "3.7.7"
   }
  },
  "nbformat": 4,

src/graph_notebook/widgets/package.json

@@ -1,6 +1,6 @@
 {
   "name": "graph_notebook_widgets",
-  "version": "2.0.4",
+  "version": "2.0.5",
   "author": "amazon",
   "description": "A Custom Jupyter Library for rendering NetworkX MultiDiGraphs using vis-network",
   "dependencies": {

src/graph_notebook/widgets/src/force_widget.ts

@@ -488,12 +488,13 @@ export class ForceView extends DOMWidgetView {
     }
 
     if (nodeID !== undefined && nodeID !== null && node.id === nodeID) {
-      node.color = this.searchMatchColorNode;
+      if (!node.group) {
+        node.color = this.searchMatchColorNode;
+      }
     } else {
       node.color = this.visOptions.nodes.color;
     }
-
-    node.font = { color: "black" };
+    node.borderWidth = 0
     this.nodeDataset.update(node);
     this.vis?.stopSimulation();
     this.selectedNodeID = "";
@@ -566,7 +567,10 @@ export class ForceView extends DOMWidgetView {
    */
   buildTableRows(data: DynamicObject): Array<HTMLElement> {
     const rows: Array<HTMLElement> = new Array<HTMLElement>();
-    Object.entries(data).forEach((entry: Array<any>) => {
+    const sorted = Object.entries(data).sort((a, b) => {
+        return a[0].localeCompare(b[0]);
+    });
+    sorted.forEach((entry: Array<any>) => {
       const row = document.createElement("tr");
       const property = document.createElement("td");
       const value = document.createElement("td");
@@ -607,15 +611,20 @@ export class ForceView extends DOMWidgetView {
     if (node === null) {
       return;
     }
-
     if (node.label !== undefined && node.label !== "") {
       this.detailsText.innerText = "Details - " + node.title;
     } else {
       this.detailsText.innerText = "Details";
     }
 
     this.buildGraphPropertiesTable(node);
-    node.font = { color: "white" };
+    if (node.group) {
+        node.font = { bold: true };
+        node.opacity = 1
+        node.borderWidth= 3
+    } else {
+        node.font = { color: "white" };
+    }
     this.nodeDataset.update(node);
     this.vis?.stopSimulation();
     this.selectedNodeID = nodeID;
@@ -672,11 +681,9 @@ export class ForceView extends DOMWidgetView {
       this.nodeDataset.forEach((item, id) => {
         if (this.search(text, item, 0)) {
           const nodeID = id.toString();
-          // if (selectedNodes[nodeID])
           nodeUpdate.push({
             id: nodeID,
             borderWidth: 3,
-            color: this.searchMatchColorNode,
           });
           nodeIDs[id.toString()] = true;
         }
@@ -693,7 +700,18 @@ export class ForceView extends DOMWidgetView {
           edgeIDs[id.toString()] = true;
         }
       });
-    }
+    } else {
+        //Reset the opacity and border width
+        this.nodeDataset.forEach((item, id) => {
+          const nodeID = id.toString();
+          nodeUpdate.push({
+            id: nodeID,
+            opacity: 1,
+            borderWidth: 0
+          });
+          nodeIDs[id.toString()] = true;
+        })
+      };
 
     // check current matched nodes and clear all nodes which are no longer matches
     this.nodeIDSearchMatches.forEach((value) => {
@@ -706,10 +724,7 @@ export class ForceView extends DOMWidgetView {
           borderWidth: selected
             ? this.visOptions["nodes"]["borderWidthSelected"]
             : 0,
-          color: selected
-            ? this.visOptions["nodes"]["color"]["highlight"]
-            : this.searchMatchColorNode,
-        });
+          opacity: 0.35        });
       }
     });