feat: 20823 Added progress output to export command (#20825) (#20835)

Signed-off-by: Ivan Malygin <[email protected]>

Author: imalygin

hedera-state-validator/src/main/java/com/hedera/statevalidation/ExportCommand.java

@@ -46,13 +46,17 @@ public void run() {
         }
 
         MerkleNodeState state;
+        System.out.println("Initializing the state");
+        long start = System.currentTimeMillis();
         try {
             DeserializedSignedState deserializedSignedState = StateResolver.initState();
             state = deserializedSignedState.reservedSignedState().get().getState();
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
 
+        System.out.printf("State has been initialized in %d seconds. \n", (System.currentTimeMillis() - start) / 1000);
+
         ((VirtualMap) state.getRoot()).getDataSource().stopAndDisableBackgroundCompaction();
 
         final boolean sorted = Boolean.parseBoolean(System.getProperty("sorted", "false"));
@@ -70,6 +74,8 @@ public void run() {
             final JsonExporter exporter = new JsonExporter(resultDir, state, serviceName, stateName);
             exporter.export();
         }
+
+        System.out.printf("Total time is  %d seconds. \n", (System.currentTimeMillis() - start) / 1000);
     }
 
     private List<Pair<String, String>> prepareServiceNameAndStateKeys() {

hedera-state-validator/src/main/java/com/hedera/statevalidation/exporters/JsonExporter.java

@@ -31,6 +31,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * This class exports the state into JSON file(s)
@@ -54,6 +55,9 @@ public class JsonExporter {
 
     private final boolean allStates;
 
+    private final AtomicLong objectsProcessed = new AtomicLong(0);
+    private long totalNumber;
+
     public JsonExporter(File resultDir, MerkleNodeState state, String serviceName, String stateKeyName) {
         this.resultDir = resultDir;
         this.state = state;
@@ -75,14 +79,16 @@ public JsonExporter(File resultDir, MerkleNodeState state, String serviceName, S
     public void export() {
         final long startTimestamp = System.currentTimeMillis();
         final VirtualMap vm = (VirtualMap) state.getRoot();
+        System.out.println("Start exporting state");
         List<CompletableFuture<Void>> futures = traverseVmInParallel(vm);
         CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
-        System.out.println("Export time: " + (System.currentTimeMillis() - startTimestamp) + "ms");
+        System.out.println("Export time: " + ((System.currentTimeMillis() - startTimestamp) / 1000) + " seconds");
         executorService.close();
     }
 
     private List<CompletableFuture<Void>> traverseVmInParallel(final VirtualMap virtualMap) {
         VirtualMapMetadata metadata = virtualMap.getMetadata();
+        totalNumber = metadata.getSize();
         List<CompletableFuture<Void>> futures = new ArrayList<>();
         for (int i = 0; i < writingParallelism; i++) {
             String fileName;
@@ -137,6 +143,10 @@ private void processRange(String fileName, long start, long end) {
                                         .formatted(path, keyToJson(stateKey.key()), valueToJson(stateValue.value())));
                     }
                     emptyFile = false;
+                    long currentObjCount = objectsProcessed.incrementAndGet();
+                    if (currentObjCount % MAX_OBJ_PER_FILE == 0) {
+                        System.out.printf("%s objects of %s are processed\n", currentObjCount, totalNumber);
+                    }
                 } catch (ParseException e) {
                     throw new RuntimeException(e);
                 }

hedera-state-validator/src/main/java/com/hedera/statevalidation/exporters/SortedJsonExporter.java

@@ -35,6 +35,7 @@
 import java.util.concurrent.ConcurrentSkipListSet;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.LongStream;
 
 /**
@@ -54,6 +55,9 @@ public class SortedJsonExporter {
     private final Map<Integer, Set<Pair<Long, Bytes>>> keysByExpectedStateIds;
     private final Map<Integer, Pair<String, String>> nameByStateId;
 
+    private final AtomicLong objectsProcessed = new AtomicLong(0);
+    private long totalNumber;
+
     public SortedJsonExporter(File resultDir, MerkleNodeState state, String serviceName, String stateKeyName) {
         this(resultDir, state, List.of(Pair.of(serviceName, stateKeyName)));
     }
@@ -109,6 +113,8 @@ public SortedJsonExporter(
     public void export() {
         final long startTimestamp = System.currentTimeMillis();
         final VirtualMap vm = (VirtualMap) state.getRoot();
+        totalNumber = vm.size();
+        System.out.println("Collecting keys from the state...");
         collectKeys(vm);
         keysByExpectedStateIds.forEach((key, values) -> {
             if (values.isEmpty()) {
@@ -119,7 +125,7 @@ public void export() {
 
         List<CompletableFuture<Void>> futures = traverseVmInParallel();
         CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
-        System.out.println("Export time: " + (System.currentTimeMillis() - startTimestamp) + "ms");
+        System.out.println("Export time: " + ((System.currentTimeMillis() - startTimestamp) / 1000) + " seconds");
         executorService.close();
     }
 
@@ -201,6 +207,10 @@ private void processRange(final List<Pair<Long, Bytes>> keys, String fileName, i
                 } catch (ParseException e) {
                     throw new RuntimeException(e);
                 }
+                long currentObjCount = objectsProcessed.incrementAndGet();
+                if (currentObjCount % MAX_OBJ_PER_FILE == 0) {
+                    System.out.printf("%s objects of %s are processed\n", currentObjCount, totalNumber);
+                }
             }
         } catch (IOException e) {
             throw new RuntimeException(e);