initial commit

Signed-off-by: Zhenxing Shen <[email protected]>

Author: s-zx

config/opensearch_dashboards.yml

@@ -1,5 +1,5 @@
 # OpenSearch Dashboards is served by a back end server. This setting specifies the port to use.
-#server.port: 5601
+# server.port: 5602
 
 # Specifies the address to which the OpenSearch Dashboards server will bind. IP addresses and host names are both valid values.
 # The default is 'localhost', which usually means remote machines will not be able to connect.
@@ -379,3 +379,28 @@
 
 # Set the value to true to enable the new UI for savedQueries in Discover
 # data.savedQueriesNewUI.enabled: true
+# for olly2
+workspace.enabled: true
+savedObjects.permission.enabled: true
+data_source.enabled: true
+uiSettings:
+  overrides:
+    'home:useNewHomePage': true
+    'courier:batchSearches': false
+    'query:enhancements:enabled': true
+  defaults:
+    'theme:darkMode': false
+assistant.alertInsight.enabled: true
+assistant.chat.enabled: true
+assistant.smartAnomalyDetector.enabled: true
+assistant.text2viz.enabled: true
+queryEnhancements.queryAssist.summary.enabled: true
+observability.query_assist.enabled: true
+# opensearch.hosts: ['http://k8s-ruanyl-opensear-0b8ea918c5-897697169.us-west-1.elb.amazonaws.com:80']
+opensearch.hosts:
+  ['http://k8s-zxshende-opensear-89396f29f2-469603532.us-east-1.elb.amazonaws.com:80']
+# opensearch.hosts: ['http://localhost:9200']
+opensearch.username: 'admin'
+opensearch.password: 'myStrongPassword123!'
+opensearchDashboards.dashboardAdmin.users: ['admin']
+server.port: 5602

src/core/public/chrome/utils.ts

@@ -292,7 +292,6 @@ export async function searchNavigationLinks(
         const originalFullLink = allSearchAbleLinks.find((fullLink) => fullLink.id === link.id);
         return originalFullLink || null;
       })
-      // .filter(Boolean) as (ChromeRegistrationNavLink & ChromeNavLink)[];
       .filter(Boolean) as Array<ChromeRegistrationNavLink & ChromeNavLink>;
 
     console.log('Final Semantic Search Result (from backend): ', finalResult);

src/plugins/workspace/public/plugin.ts

@@ -345,6 +345,8 @@ export class WorkspacePlugin
       title: i18n.translate('workspace.settings.workspaceDetail', {
         defaultMessage: 'Workspace Detail',
       }),
+      description:
+        "Define and manage a workspace's details, including its name, description, use case, icon color, and privacy settings.",
       navLinkStatus: core.chrome.navGroup.getNavGroupEnabled()
         ? AppNavLinkStatus.visible
         : AppNavLinkStatus.hidden,
@@ -362,6 +364,7 @@ export class WorkspacePlugin
       title: i18n.translate('workspace.settings.workspaceCollaborators', {
         defaultMessage: 'Collaborators',
       }),
+      description: 'Manage workspace access and permissions.',
       navLinkStatus: core.chrome.navGroup.getNavGroupEnabled()
         ? AppNavLinkStatus.visible
         : AppNavLinkStatus.hidden,

src/plugins/workspace/server/routes/index.ts

@@ -116,6 +116,7 @@ const updateWorkspaceAttributesSchema = schema.object({
 
 // Declare a variable outside the route handler to cache the model.
 let semanticExtractor: any = null;
+let answerer: any = null;
 
 export function registerRoutes({
   client,
@@ -367,13 +368,19 @@ export function registerRoutes({
       try {
         const { query, links } = req.body;
 
-        // Filter links to ensure they have a description for the semantic search
-        let linksWithDescription = links.filter((link) => !!link.description);
-        linksWithDescription = Array.from(new Set(linksWithDescription.map((link) => link.id))).map(
-          (id) => linksWithDescription.find((link) => link.id === id)!
-        );
-
         console.log('-------------Enter semanticSearch (Node.js)-------------');
+        if (!answerer) {
+          const startTime = performance.now();
+          answerer = await pipeline(
+            'question-answering',
+            'Xenova/tiny-random-RoFormerForQuestionAnswering'
+          );
+          const endTime = performance.now();
+          const loadingTimeMs = endTime - startTime;
+
+          console.log('Answer loaded and ready for inference.');
+          console.log(`Answer loading took: ${loadingTimeMs.toFixed(2)} ms`);
+        }
 
         // Load the model only once and reuse it later
         if (!semanticExtractor) {
@@ -393,7 +400,7 @@ export function registerRoutes({
 
         // Generate embeddings for links
         const linkEmbeddings = await Promise.all(
-          linksWithDescription.map(async (link) => {
+          links.map(async (link) => {
             const titleOutput = await semanticExtractor(link.title, {
               pooling: 'mean',
               normalize: true,
@@ -409,13 +416,13 @@ export function registerRoutes({
               descriptionEmbedding = Array.from(descOutput.data) as number[];
             }
 
-            const WEIGHT_TITLE = 0.7;
-            const WEIGHT_DESC = 0.3;
+            const titleWeight = 0.7;
+            const descWeight = 0.3;
 
             let combinedEmbedding: number[] = [];
 
             combinedEmbedding = titleEmbedding.map(
-              (val, i) => val * WEIGHT_TITLE + descriptionEmbedding![i] * WEIGHT_DESC
+              (val, i) => val * titleWeight + descriptionEmbedding![i] * descWeight
             );
 
             return { ...link, embedding: combinedEmbedding };
@@ -430,18 +437,38 @@ export function registerRoutes({
         // Calculate scores and sort
         const scored = linkEmbeddings.map((link) => ({
           ...link,
-          // score: cosineSimilarity(queryEmbedding as number[], link.embedding as number[]),
           score: cos_sim(queryEmbedding as number[], link.embedding as number[]),
         }));
 
         scored.sort((a, b) => b.score - a.score);
 
         const semanticSearchResult = scored
-          .slice(0, 8)
-          .filter((item) => item.score > 0.08)
+          .slice(0, 5)
+          .filter((item) => item.score > 0.2)
           .map(({ embedding, ...rest }) => rest);
         console.log('semanticSearchResult: ', semanticSearchResult);
 
+        const qaResults = [];
+
+        for (const link of links) {
+          const answer = await answerer(query, link.description);
+          if (answer && answer.answer) {
+            qaResults.push({
+              linkId: link.id,
+              title: link.title,
+              description: link.description,
+              answer: answer.answer,
+              score: answer.score,
+            });
+          }
+        }
+
+        qaResults.sort((a, b) => b.score - a.score);
+
+        const finalQaResults = qaResults.slice(0, 5).filter((item) => item.score > 0.1);
+
+        console.log('Question Answering: ', finalQaResults);
+
         return res.ok({ body: semanticSearchResult });
       } catch (error) {
         console.error('Error during semantic search:', error);