Merge pull request #718 from SlideRuleEarth/carlos-dev4

Fix support for geo parquet file exports #717

Author: cugarteblair

web-client/src/components/SrExportDialog.vue

@@ -51,7 +51,7 @@ import { useToast } from 'primevue/usetoast';
 import { db } from '@/db/SlideRuleDb';
 import { createDuckDbClient } from '@/utils/SrDuckDb';
 import { useSrParquetCfgStore } from '@/stores/srParquetCfgStore';
-import {getFetchUrlAndOptions} from "@/utils/fetchUtils";
+import {getArrowFetchUrlAndOptions} from "@/utils/fetchUtils";
 import { exportCsvStreamed, getWritableFileStream } from "@/utils/SrParquetUtils";
 
 
@@ -212,43 +212,85 @@ async function exportParquet(fileName: string): Promise<boolean> {
 }
 
 async function exportGeoParquet(fileName: string) : Promise<boolean>  {
-    const { url, options } = await getFetchUrlAndOptions(props.reqId,true);
+    // Check if file already has geo metadata
+    const duckDbClient = await createDuckDbClient();
+    const metadata = await duckDbClient.getAllParquetMetadata(fileName);
 
-    const wb = await getWritableFileStream(`${fileName}_GEO.parquet`, 'application/octet-stream');
-    if (!wb) return false;
+    const hasGeoMetadata = metadata && 'geo' in metadata;
 
-    try {
-        console.log('Fetching GeoParquet export:', url, options);
-        const response = await fetch(url, {
-            method: 'POST',
-            body: options.body,
-            headers: { 'Content-Type': 'application/json', ...(options.headers || {}) },
-        });
+    if (hasGeoMetadata) {
+        // File already has geo metadata, export as-is using exportParquet pattern
+        console.log('File already has geo metadata, exporting as-is');
+        const opfsRoot = await navigator.storage.getDirectory();
+        const directoryHandle = await opfsRoot.getDirectoryHandle('SlideRule');
+        const fileHandle = await directoryHandle.getFileHandle(fileName);
+        const srcFile = await fileHandle.getFile();
 
-        if (!response.ok || !response.body) throw new Error(`Export failed with status ${response.status}`);
+        const wb = await getWritableFileStream(fileName, 'application/octet-stream');
+        if (!wb) return false;
 
-        if (wb.writable) {
-            // 🚀 Fast path: let the streams connect directly
-            await response.body.pipeTo(wb.writable);
-        } else {
-            // fallback: manual loop (same as version 1)
-            // exportGeoParquet manual loop branch
-            const reader = response.body.getReader();
-            const writer = wb.getWriter();
-            while (true) {
-                const { done, value } = await reader.read();
-                if (done) break;
-                if (value) {
-                    await writer.ready;
-                    await writer.write(value.slice());
+        try {
+            if (wb.writable && typeof srcFile.stream === 'function') {
+                await srcFile.stream().pipeTo(wb.writable);
+            } else {
+                const reader = srcFile.stream().getReader();
+                const writer = wb.getWriter();
+                while (true) {
+                    const { done, value } = await reader.read();
+                    if (done) break;
+                    if (value) {
+                        await writer.ready;
+                        await writer.write(value.slice());
+                    }
                 }
+                await writer.close();
             }
-            await writer.close();
+            return true;
+        } catch (err) {
+            console.error('Stream copy failed:', err);
+            try { await wb.abort(err); } catch {}
+            return false;
+        }
+    } else {
+        // No geo metadata, fetch from server with as_geo: true
+        console.log('No geo metadata found, fetching GeoParquet from server');
+        const { url, options } = await getArrowFetchUrlAndOptions(props.reqId, true);
+
+        const baseFileName = fileName.replace(/\.parquet$/i, '');
+        const wb = await getWritableFileStream(`${baseFileName}_GEO.parquet`, 'application/octet-stream');
+        if (!wb) return false;
+
+        try {
+            console.log('Fetching GeoParquet export:', url, options);
+            const response = await fetch(url, {
+                method: 'POST',
+                body: options.body,
+                headers: { 'Content-Type': 'application/json', ...(options.headers || {}) },
+            });
+
+            if (!response.ok || !response.body) throw new Error(`Export failed with status ${response.status}`);
+
+            if (wb.writable) {
+                await response.body.pipeTo(wb.writable);
+            } else {
+                const reader = response.body.getReader();
+                const writer = wb.getWriter();
+                while (true) {
+                    const { done, value } = await reader.read();
+                    if (done) break;
+                    if (value) {
+                        await writer.ready;
+                        await writer.write(value.slice());
+                    }
+                }
+                await writer.close();
+            }
+            return true;
+        } catch (err) {
+            console.error('Failed to write GeoParquet file:', err);
+            try { await wb.abort(err); } catch {}
+            return false;
         }
-        return true;
-    } catch (err) {
-        await wb.abort(err);
-        return false;
     }
 }
 

web-client/src/components/SrImportParquetFile.vue

@@ -154,7 +154,7 @@ const customUploader = async (event: any) => {
     upload_progress.value = 90;
 
     const metadata = await duckDbClient.getAllParquetMetadata(opfsFile.name);
-
+    console.log('SrImportParquetFile Extracted metadata:', metadata);
     // Metadata validations that DELETE the file on failure:
     if (!metadata || !('sliderule' in metadata)) {
       toast.add({ severity: 'error', summary: 'Invalid File Format', detail: 'SlideRule metadata missing.', life: 5000 });

web-client/src/utils/fetchUtils.ts

@@ -27,7 +27,7 @@ export function forceGeoParquet(inputReqParms: any): any {
     return inputReqParms;
 }
 
-export async function getFetchUrlAndOptions(
+export async function getArrowFetchUrlAndOptions(
     reqid: number,
     forceAsGeo: boolean = false
 ): Promise<{ url: string; options: RequestInit }> {
@@ -40,9 +40,9 @@ export async function getFetchUrlAndOptions(
         parm = forceGeoParquet(parm);
     }
     const host = sysConfigStore.getOrganization() && (sysConfigStore.getOrganization() + '.' + sysConfigStore.getDomain()) || sysConfigStore.getDomain();
-    const api_path = `source/${api}`;
+    const api_path = `arrow/${api}`;
     const url = 'https://' + host + '/' + api_path;
-    //console.log('source url:', url);
+    //console.log('getArrowFetchUrlAndOptions source url:', url);
     // Setup Request Options
     let body = null;
     const options: RequestInit = {

web-client/tests/unit/SrExportDialog.spec.ts

@@ -50,6 +50,7 @@ vi.mock('@/utils/SrDuckDb', () => ({
     getTotalRowCount: duckRowCount,
     queryForColNames: duckCols,
     query: duckQuery,
+    getAllParquetMetadata: vi.fn().mockResolvedValue(undefined),
   }),
 }));
 
@@ -61,7 +62,7 @@ vi.mock('@/stores/srParquetCfgStore', () => ({
 }));
 
 vi.mock('@/utils/fetchUtils', () => ({
-  getFetchUrlAndOptions: vi.fn().mockResolvedValue({
+  getArrowFetchUrlAndOptions: vi.fn().mockResolvedValue({
     url: '/fake/export',
     options: { body: JSON.stringify({ ok: true }), headers: {} },
   }),