Merge pull request #40 from aiven/parse-replicas

Added parsing of nodes and replicas

Author: ftisiot

README.md

@@ -100,5 +100,8 @@ Contact
 Bug reports and patches are very welcome, please post them as GitHub issues and pull requests at https://github.com/aiven/metadata-parser . 
 To report any possible vulnerabilities or other serious issues please see our [security](SECURITY.md) policy.
 
+All images under the `src` folder and shown in `nx.html` file are taken from https://www.flaticon.com/
+All images under the `src/services` folder are a property of Aiven
+
 
 Apache, Apache Kafka, Kafka, Apache Flink, Flink, Apache Cassandra, and Cassandra are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries. ClickHouse is a registered trademark of ClickHouse, Inc. https://clickhouse.com. M3, M3 Aggregator, M3 Coordinator, OpenSearch, PostgreSQL, MySQL, InfluxDB, Grafana, Terraform, and Kubernetes are trademarks and property of their respective owners. *Redis is a trademark of Redis Ltd. and the Redis box logo is a mark of Redis Ltd. Any rights therein are reserved to Redis Ltd. Any use by Aiven is for referential purposes only and does not indicate any sponsorship, endorsement or affiliation between Redis and Aiven.  All product and service names used in this website are for identification purposes only and do not imply endorsement.

img/images.txt

@@ -0,0 +1 @@
+All images are taken from https://www.flaticon.com/

img/servers.png

@@ -110,7 +110,7 @@ def explore(self, service_type, service_name, project):
             f"Don't know how to explore RUNNING {service_type}"
             f"service {service_name}"
         )
-        print("AAAAA" + traceback.format_exc())
+
         return nodes, edges
 
     try:
@@ -136,6 +136,15 @@ def explore(self, service_type, service_name, project):
                 "label": service_name,
             }
         )
+
+        # Getting components
+
+        (new_nodes, new_edges) = explore_components(
+            service_name, service_type, service["components"]
+        )
+        nodes = nodes + new_nodes
+        edges = edges + new_edges
+
         # Getting integrations
 
         (new_nodes, new_edges) = integration.explore_integrations(
@@ -348,3 +357,44 @@ def explore_ext_endpoints(self, project):
         )
 
     return nodes, edges
+
+
+def explore_components(service_name, service_type, components):
+    nodes = []
+    edges = []
+
+    global SERVICE_MAP
+
+    for component in components:
+        nodes.append(
+            {
+                "id": service_type
+                + "~"
+                + service_name
+                + "~node~"
+                + component.get("host"),
+                "host": component.get("host"),
+                "port": component.get("port"),
+                "privatelink_connection_id": component.get(
+                    "privatelink_connection_id"
+                ),
+                "route": component.get("route"),
+                "usage": component.get("usage"),
+                "type": "service_node",
+                "service_type": service_type,
+                "label": component.get("host"),
+            }
+        )
+        edges.append(
+            {
+                "from": service_type
+                + "~"
+                + service_name
+                + "~node~"
+                + component.get("host"),
+                "to": service_name,
+                "label": "partition",
+            }
+        )
+        SERVICE_MAP[component.get("host")] = service_name
+    return nodes, edges

src/explore_service.py

@@ -28,6 +28,7 @@
 colors["backup"] = "cccccc"
 colors["consumer_group"] = "#0000cc"
 colors["partition"] = "#0000cc"
+colors["service_nodes"] = "#0000cc"
 
 
 sizes = {}
@@ -56,6 +57,7 @@
 sizes["backup"] = 10
 sizes["consumer_group"] = 10
 sizes["partition"] = 10
+sizes["service_node"] = 10
 
 
 images = {}
@@ -85,6 +87,7 @@
 images["backup"] = "img/database.png"
 images["consumer_group"] = "img/user.png"
 images["partition"] = "img/layout.png"
+images["service_node"] = "img/servers.png"
 
 
 def pyviz_graphy(nodes, edges):
@@ -160,7 +163,7 @@ def pyviz_graphy(nodes, edges):
 
     print("Writing NX file")
     try:
-        nt = Network(height="1200px", width="1600px", font_color="#000000")
+        nt = Network(height="1000px", width="1000px", font_color="#000000")
         nt.from_nx(g)
         # nt.show_buttons()
         nt.show_buttons()
@@ -169,35 +172,4 @@ def pyviz_graphy(nodes, edges):
     except Exception as err:
         print(f"Error writing NX file: {err.__class__.__name__} {err}")
 
-    """
-    not_services =  (n for n in g if g.nodes[n]['type'] != 'service')
-    services =  (n for n in g if g.nodes[n]['type'] == 'service')
-    
-    g_connected = node_connected_component(g.to_undirected().subgraph(not_services),'pg~demo-pg~schema~public~table~pasta')
-    #g_connected = (n for n in g_connected if g_connected[n]['type'] in ('service',''))
-    g_subgraph = g.subgraph(g_connected)
-    nt = Network(height='1200px', width='800px', font_color="#000000")
-    nt.from_nx(g_subgraph)
-    nt.show('filtered.html')
-    content = '<table><tr><td ><div id = "mynetwork"></div></td><td ><div><table style="order-spacing: 0px; border-collapse: collapse; width: 100%; max-width: 100%; margin-bottom: 15px; background-color: transparent; text-align: left;">' \
-              '<tr><th style="font-weight: bold; border: 1px solid #cccccc; padding: 8px;"><b>Img</b></th><th style="font-weight: bold; border: 1px solid #cccccc; padding: 8px;"><b>Node type</b></th><th style="font-weight: bold; border: 1px solid #cccccc; padding: 8px;"><b>Node Label</b></th><th style="font-weight: bold; border: 1px solid #cccccc; padding: 8px;"><b>Service type</b></td><th style="font-weight: bold; border: 1px solid #cccccc; padding: 8px;"><b>Service Name</b></th></tr>'
-    for node in g_subgraph.nodes:
-        
-        content = content + '<tr><td style="border: 1px solid #cccccc; padding: 8px;"><img src="'+g_subgraph.nodes[node]["image"]+'" width="20px" ></img></td><td style="border: 1px solid #cccccc; padding: 8px;">'+g_subgraph.nodes[node]["type"]+'</td><td style="border: 1px solid #cccccc; padding: 8px;">'+g_subgraph.nodes[node]["label"]+'</td><td style="border: 1px solid #cccccc; padding: 8px;">'+g_subgraph.nodes[node]["service_type"]+'</td><td style="border: 1px solid #cccccc; padding: 8px;">'+g_subgraph.nodes[node]["id"].split('~')[1]+'</td></tr>'
-    content = content + '</table></div></table>'
-    file = open("filtered.html", "r")
-    
-        
-    replacement = ""
-    # using the for loop
-    for line in file:
-        line = line.strip()
-        changes = line.replace('<div id = "mynetwork"></div>', content)
-        replacement = replacement + changes + "\n"
-
-    file.close()
-    # opening the file in write mode
-    fout = open("filtered.html", "w")
-    fout.write(replacement)
-    fout.close()
-    """
+    ## All images are taken from https://www.flaticon.com/