Merge pull request #1397 from leggedrobotics/dev

Release 0.43.3

Author: wp99cp

backend/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kleinkram-backend",
-    "version": "0.43.2",
+    "version": "0.43.3",
     "description": "",
     "author": "",
     "private": true,

backend/src/endpoints/project/project.controller.ts

@@ -198,7 +198,7 @@ export class ProjectController {
 
 // TODO: this controller should get removed at some point,
 // filtered and recent will effectively be replaced by `GET /projects`
-// for the getDefaultRights endpoint we should make a seperate controller that
+// for the getDefaultRights endpoint we should make a separate controller that
 // does all the access control stuff
 @Controller('oldProject')
 export class OldProjectController {

backend/src/endpoints/queue/queue.controller.ts

@@ -63,6 +63,16 @@ export class QueueController {
         };
     }
 
+    @Post('recalculateHashes')
+    @AdminOnly()
+    @OutputDto(null)
+    async recalculateHashes(): Promise<{
+        success: boolean;
+        fileCount: number;
+    }> {
+        return await this.queueService.recalculateHashes();
+    }
+
     @Get('active')
     @LoggedIn()
     @OutputDto(null)

backend/src/services/queue.service.ts

@@ -4,6 +4,7 @@ import { InjectRepository } from '@nestjs/typeorm';
 import {
     FindOptionsWhere,
     In,
+    IsNull,
     LessThan,
     Like,
     MoreThan,
@@ -164,6 +165,41 @@ export class QueueService implements OnModuleInit {
         return {};
     }
 
+    /**
+     * re-calculates the hashes for all files without a valid hash
+     */
+    async recalculateHashes(): Promise<{
+        success: boolean;
+        fileCount: number;
+    }> {
+        // search for files with no hash (empty or null) and state OK
+        const files = await this.fileRepository.find({
+            where: [
+                { hash: IsNull(), state: FileState.OK },
+                { hash: '', state: FileState.OK },
+            ],
+            relations: ['mission', 'mission.project'],
+        });
+
+        logger.debug(`Add ${files.length} files to queue for hash calculation`);
+        logger.debug(JSON.stringify(files.map((file) => file.uuid)));
+
+        for (const file of files) {
+            try {
+                await this.fileQueue.add('extractHashFromMinio', {
+                    file_uuid: file.uuid,
+                });
+            } catch (error: any) {
+                logger.error(error);
+            }
+        }
+
+        return {
+            success: true,
+            fileCount: files.length,
+        };
+    }
+
     async confirmUpload(uuid: string, md5: string): Promise<void> {
         const queue = await this.queueRepository.findOneOrFail({
             where: { identifier: uuid },
@@ -191,12 +227,6 @@ export class QueueService implements OnModuleInit {
         file.size = fileInfo.size;
         await this.fileRepository.save(file);
 
-        // abort if autoConvert is disabled
-        if (!file.mission?.project?.autoConvert) {
-            logger.debug(`Skip auto convert to mcap`);
-            return;
-        }
-
         queue.state = QueueState.AWAITING_PROCESSING;
         await this.queueRepository.save(queue);
 

cli/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = kleinkram
-version = 0.43.2
+version = 0.43.3
 description = give me your bags
 long_description = file: README.md
 long_description_content_type = text/markdown

common/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kleinkram-common",
-    "version": "0.43.2",
+    "version": "0.43.3",
     "license": "MIT",
     "scripts": {
         "seed:config": "ts-node ./node_modules/typeorm-seeding/dist/cli.js config",

docs/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kleinkram-docs",
-    "version": "0.43.2",
+    "version": "0.43.3",
     "license": "MIT",
     "target": "es2022",
     "scripts": {

frontend/package.json

@@ -1,6 +1,6 @@
 {
     "name": "kleinkram-frontend",
-    "version": "0.43.2",
+    "version": "0.43.3",
     "description": "Data storage of ROS bags",
     "productName": "Kleinkram",
     "author": "Johann Schwabe <[email protected]>",

frontend/src/components/user-profile/admin-settings.vue

@@ -28,6 +28,21 @@
                 There is no confirmation!
             </div>
         </div>
+
+        <div style="width: 300px; margin-left: 20px">
+            <q-btn
+                label="Recalculate Hashes"
+                class="button-border bg-button-primary full-width"
+                icon="sym_o_fingerprint"
+                flat
+                @click="recalculateHashes"
+            />
+            <div class="help-text q-pt-sm">
+                This will extract the MD5 hash from the file (from minio) and
+                store it in the database. This action cannot be undone. There is
+                no confirmation!
+            </div>
+        </div>
     </div>
 </template>
 <script setup lang="ts">
@@ -57,4 +72,15 @@ async function resetFileSizes(): Promise<void> {
         timeout: 2000,
     });
 }
+
+async function recalculateHashes(): Promise<void> {
+    const { data } = await axios.post('queue/recalculateHashes');
+
+    $q.notify({
+        message: `Recalculating hashes started. ${data.fileCount} files to process`,
+        color: 'positive',
+        position: 'bottom',
+        timeout: 2000,
+    });
+}
 </script>

package.json

@@ -1,6 +1,6 @@
 {
     "name": "kleinkram",
-    "version": "0.43.2",
+    "version": "0.43.3",
     "main": "index.js",
     "repository": "[email protected]:leggedrobotics/GrandTourDatasets.git",
     "author": "Johann Schwabe <[email protected]>",