Train with Poses

.gitignore

@@ -8,3 +8,13 @@ tools/_jupyter/.ipynb_checkpoints
 training
 validation
 testing
+void-dataset
+test_data
+pretrained
+trained_kbnet/
+setup/__pycache__
+trained_kbnet_orb_poses/
+trained_kbnet_orb_posenet_structureloss/
+training_results_orb/
+evaluation_results/
+

bash/void/run_kbnet_orb.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+export CUDA_VISIBLE_DEVICES=0
+
+python src/run_kbnet.py \
+--is_orb_data 1 \
+--to_scale_depth 0 \
+--image_path testing/orb/test_image.txt \
+--sparse_depth_path testing/orb/test_sparse_depth.txt \
+--intrinsics_path testing/orb/test_intrinsics.txt \
+--ground_truth_path testing/orb/test_ground_truth.txt \
+--input_channels_image 3 \
+--input_channels_depth 2 \
+--normalized_image_range 0 1 \
+--outlier_removal_kernel_size 7 \
+--outlier_removal_threshold 1.5 \
+--min_pool_sizes_sparse_to_dense_pool 15 17 19 \
+--max_pool_sizes_sparse_to_dense_pool 23 27 \
+--n_convolution_sparse_to_dense_pool 3 \
+--n_filter_sparse_to_dense_pool 8 \
+--n_filters_encoder_image 48 96 192 384 384 \
+--n_filters_encoder_depth 16 32 64 128 128 \
+--resolutions_backprojection 0 1 2 3 \
+--n_filters_decoder 256 128 128 64 12 \
+--deconv_type up \
+--weight_initializer xavier_normal \
+--activation_func leaky_relu \
+--min_predict_depth 0.01 \
+--max_predict_depth 8.0 \
+--min_evaluate_depth 0.02 \
+--max_evaluate_depth 10.0 \
+--save_outputs \
+--depth_model_restore_path orb_model.pth \
+--output_path \
+evaluation_results/orb/670/orb_trSmteB/ \
+--device gpu

bash/void/run_kbnet_void1500.sh

@@ -28,8 +28,7 @@ python src/run_kbnet.py \
 --min_evaluate_depth 0.2 \
 --max_evaluate_depth 5.0 \
 --save_outputs \
---depth_model_restore_path \
-pretrained_models/void/kbnet-void1500.pth \
+--depth_model_restore_path traine_with_poses.pth \
 --output_path \
-pretrained_models/void/evaluation_results/void1500 \
---device gpu
+evaluation_results/void1500_poses \
+--device gpu

bash/void/train_kbnet_orb.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+export CUDA_VISIBLE_DEVICES=0
+
+python src/train_kbnet.py \
+--train_image_path training/orb/train_image.txt \
+--train_sparse_depth_path training/orb/train_sparse_depth.txt \
+--train_pose_path training/orb/train_pose.txt \
+--train_intrinsics_path training/orb/train_intrinsics.txt \
+--val_image_path validation/orb/val_image.txt \
+--val_sparse_depth_path validation/orb/val_sparse_depth.txt \
+--val_intrinsics_path validation/orb/val_intrinsics.txt \
+--val_ground_truth_path validation/orb/val_ground_truth.txt \
+--is_orb_data 1 \
+--pose_in_world_frame 0 \
+--to_scale_depth 0 \
+--n_batch 4 \
+--n_height 480 \
+--n_width 848 \
+--input_channels_image 3 \
+--input_channels_depth 2 \
+--normalized_image_range 0 1 \
+--outlier_removal_kernel_size 7 \
+--outlier_removal_threshold 1.5 \
+--min_pool_sizes_sparse_to_dense_pool 25 27 29 \
+--max_pool_sizes_sparse_to_dense_pool 33 37 \
+--n_convolution_sparse_to_dense_pool 3  \
+--n_filter_sparse_to_dense_pool 8 \
+--n_filters_encoder_image 48 96 192 384 384 \
+--n_filters_encoder_depth 16 32 64 128 128 \
+--resolutions_backprojection 0 1 2 3 \
+--n_filters_decoder 256 128 128 64 12 \
+--deconv_type up \
+--min_predict_depth 0.01 \
+--max_predict_depth 8.0 \
+--weight_initializer xavier_normal \
+--activation_func leaky_relu \
+--learning_rates 1e-4 5e-5  \
+--learning_schedule 100 150 \
+--augmentation_probabilities 0.00 \
+--augmentation_schedule -1 \
+--augmentation_random_crop_type horizontal vertical anchored \
+--augmentation_random_remove_points 0.30 0.60 \
+--augmentation_random_noise_type none \
+--augmentation_random_noise_spread -1 \
+--w_color 0.15 \
+--w_structure 0.95 \
+--w_sparse_depth 2.0 \
+--w_smoothness 2.0 \
+--w_weight_decay_depth 0.00 \
+--w_weight_decay_pose 0.00 \
+--min_evaluate_depth 0.2 \
+--max_evaluate_depth 5.0 \
+--n_summary 1000 \
+--n_checkpoint 1000 \
+--validation_start 1000 \
+--checkpoint_path trained_kbnet/orb/kbnet_model \
+--device gpu \
+--n_thread 16

bash/void/train_kbnet_void1500.sh

@@ -6,10 +6,12 @@ python src/train_kbnet.py \
 --train_image_path training/void/void_train_image_1500.txt \
 --train_sparse_depth_path training/void/void_train_sparse_depth_1500.txt \
 --train_intrinsics_path training/void/void_train_intrinsics_1500.txt \
+--train_pose_path training/void/void_train_pose_1500.txt \
 --val_image_path testing/void/void_test_image_1500.txt \
 --val_sparse_depth_path testing/void/void_test_sparse_depth_1500.txt \
 --val_intrinsics_path testing/void/void_test_intrinsics_1500.txt \
 --val_ground_truth_path testing/void/void_test_ground_truth_1500.txt \
+--pose_in_world_frame 0 \
 --n_batch 8 \
 --n_height 480 \
 --n_width 640 \
@@ -52,4 +54,4 @@ python src/train_kbnet.py \
 --validation_start 5000 \
 --checkpoint_path trained_kbnet/void1500/kbnet_model \
 --device gpu \
---n_thread 8
+--n_thread 16

run_kbnet_void1500.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+export CUDA_VISIBLE_DEVICES=0
+
+python src/run_kbnet.py \
+--image_path /run/user/1000/gvfs/sftp:host=data.ciirc.cvut.cz,user=madharak/nfs/datasets/SPRING/ARI/Depth\ Completion\ Datasets/void-dataset/data/void_release/void_1500/data/birthplace_of_internet/image/1552097915.5257.png \
+--sparse_depth_path /run/user/1000/gvfs/sftp:host=data.ciirc.cvut.cz,user=madharak/nfs/datasets/SPRING/ARI/Depth\ Completion\ Datasets/void-dataset/data/void_release/void_1500/data/birthplace_of_internet/sparse_depth/1552097915.5257.png \
+--intrinsics_path /run/user/1000/gvfs/sftp:host=data.ciirc.cvut.cz,user=madharak/nfs/datasets/SPRING/ARI/Depth\ Completion\ Datasets/void-dataset/data/void_release/void_1500/data/birthplace_of_internet/K.txt \
+--ground_truth_path /run/user/1000/gvfs/sftp:host=data.ciirc.cvut.cz,user=madharak/nfs/datasets/SPRING/ARI/Depth\ Completion\ Datasets/void-dataset/data/void_release/void_1500/data/birthplace_of_internet/ground_truth/1552097915.5257.png  \
+--input_channels_image 3 \
+--input_channels_depth 2 \
+--normalized_image_range 0 1 \
+--outlier_removal_kernel_size 7 \
+--outlier_removal_threshold 1.5 \
+--min_pool_sizes_sparse_to_dense_pool 15 17 \
+--max_pool_sizes_sparse_to_dense_pool 23 27 29 \
+--n_convolution_sparse_to_dense_pool 3 \
+--n_filter_sparse_to_dense_pool 8 \
+--n_filters_encoder_image 48 96 192 384 384 \
+--n_filters_encoder_depth 16 32 64 128 128 \
+--resolutions_backprojection 0 1 2 3 \
+--n_filters_decoder 256 128 128 64 12 \
+--deconv_type up \
+--weight_initializer xavier_normal \
+--activation_func leaky_relu \
+--min_predict_depth 0.1 \
+--max_predict_depth 8.0 \
+--min_evaluate_depth 0.2 \
+--max_evaluate_depth 5.0 \
+--save_outputs \
+--depth_model_restore_path \
+pretrained/pretrained_models/void/kbnet-void1500.pth \
+--output_path \
+pretrained_models/void/evaluation_results/void1500 \
+--device gpu

setup/get_training_data.py

@@ -0,0 +1,64 @@
+import cv2
+import numpy as np 
+import glob
+import os
+import sys
+from pathlib import Path
+from sklearn.cluster import MiniBatchKMeans
+sys.path.append('../')
+import data_utils
+
+N_INIT_CORNER = 1500
+
+outer_folder = Path(sys.argv[1])
+d435_image_folder = Path(str(outer_folder)+'/d435_color')
+color_image_list = glob.glob(str(d435_image_folder)+"*.png")
+
+validity_map_folder = Path(str(outer_folder)+'/validity_map')
+validity_map_folder.mkdir(parents=True, exist_ok=True)
+
+sparse_depth_folder = Path(str(outer_folder)+'/sparse_depth')
+sparse_depth_folder.mkdir(parents=True, exist_ok=True)
+
+
+depth_folder = Path(str(outer_folder) + '/d435_depth')
+depth_image_list = glob.glob(str(depth_folder)+"*.png")
+
+assert len(color_image_list) == len(depth_image_list), "Not same images for depth and color"
+
+for it,image_name in enumerate(color_image_list):
+    sparse_depth_filename_without_ext = Path(image_name).stem
+    sparse_depth_filename = str(sparse_depth_folder) + '/' + sparse_depth_filename_without_ext + '.png'
+
+    image = cv2.imread(image_name, cv2.IMREAD_COLOR)
+    depth_image = data_utils.load_depth(depth_image_list[it])
+    sparse_depth = np.zeros_like(depth_image)
+
+    corners = cv2.cornerHarris(image, blockSize=5, ksize=3, k=0.04)
+    corners = corners.ravel()
+    corner_locs = np.argsort(corners)[0:N_INIT_CORNER]
+
+    corner_map = np.zeros_like(corners)
+    corner_map[corner_locs] = 1
+
+    corner_locs = np.unravel_index(corner_locs, (image.shape[0], image.shape[1]))
+    corner_locs = np.transpose(np.array([corner_locs[0], corner_locs[1]]))
+
+
+    kmeans = MiniBatchKMeans(
+        n_clusters=384,
+        max_iter=2,
+        n_init=1,
+        init_size=None,
+        random_state=1,
+        reassignment_ratio=1e-11)
+
+    kmeans.fit(corner_locs)
+    corner_locs = kmeans.cluster_centers_.astype(np.uint16)
+
+    validity_map = np.zeros_like(image).astype(np.int16)
+    validity_map[corner_locs[:, 0], corner_locs[:, 1]] = 1
+
+    sparse_depth[np.where(validity_map==1)] = dense_depth[np.where(validity_map==1)] 
+    data_utils.save_depth(sparse_depth, sparse_depth_filename)
+