WIP

package.json

@@ -234,6 +234,8 @@
     "@osd/ui-framework": "1.0.0",
     "@osd/ui-shared-deps": "1.0.0",
     "@reduxjs/toolkit": "^1.6.1",
+    "@tanstack/react-table": "^8.21.3",
+    "@tanstack/react-virtual": "^3.13.12",
     "@types/ndjson": "^2.0.4",
     "@types/yauzl": "^2.9.1",
     "@xyflow/react": "^12.8.2",

src/plugins/dataset_management/public/components/field_editor/field_editor.tsx

@@ -25,6 +25,7 @@ import {
   EuiSpacer,
   EuiText,
   EUI_MODAL_CONFIRM_BUTTON,
+  EuiButtonEmpty,
 } from '@elastic/eui';
 
 import { i18n } from '@osd/i18n';

src/plugins/explore/public/application/context/dataset_context/dataset_context.tsx

@@ -10,7 +10,7 @@ import { useOpenSearchDashboards } from '../../../../../opensearch_dashboards_re
 import { ExploreServices } from '../../../types';
 import { RootState } from '../../utils/state_management/store';
 
-interface DatasetContextValue {
+export interface DatasetContextValue {
   dataset: DataView | undefined;
   isLoading: boolean | null;
   error: string | null;

src/plugins/explore/public/application/utils/hooks/use_page_initialization.ts

@@ -18,12 +18,16 @@ import {
   clearLastExecutedData,
   setEditorMode,
   setUsingRegexPatterns,
+  setDefaultColumnNames,
+  resetEphemeralLogsState,
+  setVisibleColumnNames,
 } from '../state_management/slices';
 import { executeQueries } from '../state_management/actions/query_actions';
 import { ExploreFlavor } from '../../../../common';
 import { useSetEditorText } from '../../hooks';
 import { EditorMode } from '../state_management/types';
 import { getVisualizationBuilder } from '../../../components/visualizations/visualization_builder';
+import { getDefaultColumnNames } from '../state_management/utils/get_default_columns';
 
 export const useInitPage = () => {
   const dispatch = useDispatch();
@@ -35,67 +39,74 @@ export const useInitPage = () => {
   const visualizationBuilder = getVisualizationBuilder();
 
   useEffect(() => {
-    if (savedExplore && !error) {
-      if (savedExplore.id) {
-        // Deserialize state from saved object
-        const { title } = savedExplore;
+    const loadSaveExplore = async () => {
+      if (savedExplore && !error) {
+        if (savedExplore.id) {
+          // Deserialize state from saved object
+          const { title } = savedExplore;
 
-        // Update browser title and breadcrumbs
-        chrome.docTitle.change(title);
-        chrome.setBreadcrumbs([{ text: 'Explore', href: '#/' }, { text: title }]);
+          // Update browser title and breadcrumbs
+          chrome.docTitle.change(title);
+          chrome.setBreadcrumbs([{ text: 'Explore', href: '#/' }, { text: title }]);
 
-        // Sync query from saved object to data plugin (explore doesn't use filters)
-        const searchSourceFields = savedExplore.kibanaSavedObjectMeta;
-        const queryFromUrl = services.osdUrlStateStorage?.get('_q') ?? {};
-        if (searchSourceFields?.searchSourceJSON) {
-          const searchSource = JSON.parse(searchSourceFields.searchSourceJSON);
-          const queryFromSavedSearch = searchSource.query;
-          const query = { ...queryFromSavedSearch, ...queryFromUrl };
-          if (query) {
-            dispatch(setQueryState(query));
-            setEditorText(query.query);
+          // Sync query from saved object to data plugin (explore doesn't use filters)
+          const searchSourceFields = savedExplore.kibanaSavedObjectMeta;
+          const queryFromUrl = services.osdUrlStateStorage?.get('_q') ?? {};
+          if (searchSourceFields?.searchSourceJSON) {
+            const searchSource = JSON.parse(searchSourceFields.searchSourceJSON);
+            const queryFromSavedSearch = searchSource.query;
+            const query = { ...queryFromSavedSearch, ...queryFromUrl };
+            if (query) {
+              dispatch(setQueryState(query));
+              setEditorText(query.query);
+            }
+            const defaultColumnNames = await getDefaultColumnNames(services, query.dataset);
+            dispatch(setDefaultColumnNames(defaultColumnNames));
+            dispatch(setVisibleColumnNames(savedExplore.columns || defaultColumnNames));
           }
-        }
 
-        // Update savedSearch to store just the ID (like discover)
-        // TODO: remove this once legacy state is not consumed any more
-        dispatch(setSavedSearch(savedExplore.id));
+          // Update savedSearch to store just the ID (like discover)
+          // TODO: remove this once legacy state is not consumed any more
+          dispatch(setSavedSearch(savedExplore.id));
 
-        // Init vis state and ui state
-        const visualization = savedExplore.visualization;
-        const uiState = savedExplore.uiState;
-        if (visualization) {
-          const { chartType, params, axesMapping } = JSON.parse(visualization);
-          visualizationBuilder.setVisConfig({ type: chartType, styles: params, axesMapping });
-        }
-        if (uiState) {
-          const { activeTab } = JSON.parse(uiState);
-          dispatch(setActiveTab(activeTab));
-        }
+          // Init vis state and ui state
+          const visualization = savedExplore.visualization;
+          const uiState = savedExplore.uiState;
+          if (visualization) {
+            const { chartType, params, axesMapping } = JSON.parse(visualization);
+            visualizationBuilder.setVisConfig({ type: chartType, styles: params, axesMapping });
+          }
+          if (uiState) {
+            const { activeTab } = JSON.parse(uiState);
+            dispatch(setActiveTab(activeTab));
+          }
 
-        // Add to recently accessed
-        chrome.recentlyAccessed.add(
-          `/app/explore/${savedExplore.type ?? ExploreFlavor.Logs}#/view/${savedExplore.id}`,
-          title,
-          savedExplore.id,
-          { type: 'explore' }
-        );
+          // Add to recently accessed
+          chrome.recentlyAccessed.add(
+            `/app/explore/${savedExplore.type ?? ExploreFlavor.Logs}#/view/${savedExplore.id}`,
+            title,
+            savedExplore.id,
+            { type: 'explore' }
+          );
 
-        dispatch(clearLastExecutedData());
-        dispatch(setEditorMode(EditorMode.Query));
-        dispatch(clearResults());
-        dispatch(clearQueryStatusMap());
-        dispatch(setUsingRegexPatterns(false));
-        dispatch(executeQueries({ services }));
+          dispatch(clearLastExecutedData());
+          dispatch(setEditorMode(EditorMode.Query));
+          dispatch(clearResults());
+          dispatch(clearQueryStatusMap());
+          dispatch(setUsingRegexPatterns(false));
+          dispatch(resetEphemeralLogsState());
+          dispatch(executeQueries({ services }));
+        }
       }
-    }
-    if (error) {
-      // Navigate to management page for invalid IDs
-      // TODO: need to confirm the UI behavior for invalid ID, the current logic is copied from useSavedExplore hook
-      if (error.includes('Not found')) {
-        chrome.setBreadcrumbs([{ text: 'Explore', href: '#/' }, { text: 'Error' }]);
+      if (error) {
+        // Navigate to management page for invalid IDs
+        // TODO: need to confirm the UI behavior for invalid ID, the current logic is copied from useSavedExplore hook
+        if (error.includes('Not found')) {
+          chrome.setBreadcrumbs([{ text: 'Explore', href: '#/' }, { text: 'Error' }]);
+        }
       }
-    }
+    };
+    loadSaveExplore();
   }, [
     chrome,
     data.query.queryString,

src/plugins/explore/public/application/utils/state_management/actions/columns/add_visible_column_name/add_visible_column_name.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { AppDispatch, RootState } from '../../../store';
+import { SOURCE_COLUMN_ID_AND_NAME } from '../../../../../../components/results_table/table_constants';
+import { setVisibleColumnNames } from '../../../slices';
+
+export const addVisibleColumnName = (columnId: string) => (
+  dispatch: AppDispatch,
+  getState: () => RootState
+) => {
+  const {
+    tab: {
+      logs: { visibleColumnNames },
+    },
+  } = getState();
+
+  if (visibleColumnNames.includes(columnId)) {
+    return;
+  }
+
+  // filter out _SOURCE since we are adding columns
+  const newColumns = visibleColumnNames.filter((column) => column !== SOURCE_COLUMN_ID_AND_NAME);
+  newColumns.push(columnId);
+  dispatch(setVisibleColumnNames(newColumns));
+};

src/plugins/explore/public/application/utils/state_management/actions/columns/add_visible_column_name/index.ts

@@ -0,0 +1,6 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export * from './add_visible_column_name';

src/plugins/explore/public/application/utils/state_management/actions/columns/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export * from './reconcile_visible_columns_with_dataset';
+export * from './add_visible_column_name';
+export * from './remove_visible_column_name';

src/plugins/explore/public/application/utils/state_management/actions/columns/reconcile_visible_columns_with_dataset/index.ts

@@ -0,0 +1,6 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export * from './reconcile_visible_columns_with_dataset';

src/plugins/explore/public/application/utils/state_management/actions/columns/reconcile_visible_columns_with_dataset/reconcile_visible_columns_with_dataset.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { ExploreServices } from '../../../../../../types';
+import { AppDispatch, RootState } from '../../../store';
+import { Dataset, DEFAULT_DATA } from '../../../../../../../../data/common';
+import { setVisibleColumnNames } from '../../../slices';
+import { addSourceOrTimeToFields } from '../../../utils/add_source_or_time_to_fields';
+
+export const reconcileVisibleColumnsWithDataset = (
+  services: ExploreServices,
+  dataset?: Dataset
+) => async (dispatch: AppDispatch, getState: () => RootState) => {
+  const { dataViews } = services;
+  const {
+    tab: {
+      logs: { visibleColumnNames, defaultColumnNames },
+    },
+  } = getState();
+
+  if (!dataset || !visibleColumnNames.length) {
+    dispatch(setVisibleColumnNames(defaultColumnNames));
+    return;
+  }
+
+  const dataView = await dataViews.get(
+    dataset.id,
+    dataset.type !== DEFAULT_DATA.SET_TYPES.INDEX_PATTERN
+  );
+
+  const fieldsNameFromDataset = dataView.fields.getAll().map((field) => field.name);
+
+  // filter out any columns from previous visibleColumns that is not in the dataset and remove duplicates
+  const filteredColumnNames = [
+    ...new Set(visibleColumnNames.filter((column) => fieldsNameFromDataset.includes(column))),
+  ];
+
+  const correctedFilteredColumns = addSourceOrTimeToFields(filteredColumnNames, dataset);
+  dispatch(setVisibleColumnNames(correctedFilteredColumns));
+};

src/plugins/explore/public/application/utils/state_management/actions/columns/remove_visible_column_name/index.ts

@@ -0,0 +1,6 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export * from './remove_visible_column_name';

src/plugins/explore/public/application/utils/state_management/actions/columns/remove_visible_column_name/remove_visible_column_name.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { AppDispatch, RootState } from '../../../store';
+import { setVisibleColumnNames } from '../../../slices';
+
+export const removeVisibleColumnName = (columnId: string) => (
+  dispatch: AppDispatch,
+  getState: () => RootState
+) => {
+  const {
+    query: { dataset },
+    tab: {
+      logs: { visibleColumnNames, defaultColumnNames },
+    },
+  } = getState();
+
+  const newColumns = visibleColumnNames.filter((col) => col !== columnId);
+  const newColumnsWithoutTime = newColumns.filter((col) => col !== dataset?.timeFieldName);
+
+  if (!newColumnsWithoutTime.length) {
+    dispatch(setVisibleColumnNames(defaultColumnNames));
+  } else {
+    dispatch(setVisibleColumnNames(newColumns));
+  }
+};

src/plugins/explore/public/application/utils/state_management/actions/query_editor/run_query/run_query.ts

@@ -7,13 +7,10 @@ import { AppDispatch, RootState } from '../../../store';
 import {
   clearResults,
   setQueryStringWithHistory,
-  setActiveTab,
   setQueryExecutionButtonStatus,
-} from '../../../slices';
-import {
   clearQueryStatusMap,
   setIsQueryEditorDirty,
-} from '../../../slices/query_editor/query_editor_slice';
+} from '../../../slices';
 import { executeQueries } from '../../query_actions';
 import { ExploreServices } from '../../../../../../types';
 import { detectAndSetOptimalTab } from '../../detect_optimal_tab';

src/plugins/explore/public/application/utils/state_management/middleware/dataset_change_middleware.ts

@@ -17,13 +17,18 @@ import {
   setSummaryAgentIsAvailable,
   setPatternsField,
   setUsingRegexPatterns,
+  resetEphemeralLogsState,
+  setDefaultColumnNames,
+  clearQueryStatusMap,
+  setBreakdownField,
 } from '../slices';
-import { clearQueryStatusMap, setBreakdownField } from '../slices/query_editor/query_editor_slice';
 import { executeQueries } from '../actions/query_actions';
 import { getPromptModeIsAvailable } from '../../get_prompt_mode_is_available';
 import { getSummaryAgentIsAvailable } from '../../get_summary_agent_is_available';
 import { detectAndSetOptimalTab } from '../actions/detect_optimal_tab';
 import { resetLegacyStateActionCreator } from '../actions/reset_legacy_state';
+import { getDefaultColumnNames } from '../utils/get_default_columns';
+import { reconcileVisibleColumnsWithDataset } from '../actions/columns';
 
 /**
  * Middleware to handle dataset changes and trigger necessary side effects
@@ -62,6 +67,14 @@ export const createDatasetChangeMiddleware = (
       store.dispatch(setBreakdownField(undefined));
       store.dispatch((resetLegacyStateActionCreator(services) as unknown) as AnyAction);
 
+      const defaultColumnNames = await getDefaultColumnNames(services, dataset);
+
+      store.dispatch(resetEphemeralLogsState());
+      store.dispatch(setDefaultColumnNames(defaultColumnNames));
+      store.dispatch(
+        (reconcileVisibleColumnsWithDataset(services, dataset) as unknown) as AnyAction
+      );
+
       const [newPromptModeIsAvailable, newSummaryAgentIsAvailable] = await Promise.allSettled([
         getPromptModeIsAvailable(services),
         getSummaryAgentIsAvailable(services, currentDataset?.dataSource?.id || ''),