Graph Serialization, Prefetching, Branch Prediction and more

Algorithms/CSA/CSA.h

@@ -133,7 +133,9 @@ class CSA {
     inline void scanConnections(const ConnectionId begin, const ConnectionId end) noexcept {
         for (ConnectionId i = begin; i < end; i++) {
             const Connection& connection = data.connections[i];
-            if (targetStop != noStop && connection.departureTime > arrivalTime[targetStop]) break;
+
+            // This branch got mispredicted a lot!
+            if (targetStop != noStop && connection.departureTime > arrivalTime[targetStop]) [[unlikely]] break;
             if (connectionIsReachable(connection, i)) {
                 profiler.countMetric(METRIC_CONNECTIONS);
                 arrivalByTrip(connection.arrivalStopId, connection.arrivalTime, connection.tripId);

Algorithms/CSA/DijkstraCSA.h

@@ -166,7 +166,7 @@ class DijkstraCSA {
         for (ConnectionId i = begin; i < end; i++) {
             const Connection& connection = data.connections[i];
             runDijkstra(connection.departureTime);
-            if (targetStop != noStop && connection.departureTime > arrivalTime[targetStop]) break;
+            if (targetStop != noStop && connection.departureTime > arrivalTime[targetStop]) [[unlikely]] break;
             if (connectionIsReachable(connection, i)) {
                 profiler.countMetric(METRIC_CONNECTIONS);
                 arrivalByTrip(connection.arrivalStopId, connection.arrivalTime, connection.tripId);

Algorithms/CSA/HLCSA.h

@@ -142,7 +142,7 @@ class HLCSA {
     inline void scanConnections(const ConnectionId begin, const ConnectionId end) noexcept {
         for (ConnectionId i = begin; i < end; i++) {
             const Connection& connection = data.connections[i];
-            if (targetVertex != noVertex && connection.departureTime > arrivalTime[targetVertex]) break;
+            if (targetVertex != noVertex && connection.departureTime > arrivalTime[targetVertex]) [[unlikely]] break;
             if (connectionIsReachable(connection, i)) {
                 profiler.countMetric(METRIC_CONNECTIONS);
                 arrivalByTrip(connection.arrivalStopId, connection.arrivalTime, connection.tripId);

Algorithms/CSA/ULTRACSA.h

@@ -152,7 +152,7 @@ class ULTRACSA {
     inline void scanConnections(const ConnectionId begin, const ConnectionId end) noexcept {
         for (ConnectionId i = begin; i < end; i++) {
             const Connection& connection = data.connections[i];
-            if (targetStop != noStop && connection.departureTime > arrivalTime[targetStop]) break;
+            if (targetStop != noStop && connection.departureTime > arrivalTime[targetStop]) [[unlikely]] break;
             if (connectionIsReachable(connection, i)) {
                 profiler.countMetric(METRIC_CONNECTIONS);
                 arrivalByTrip(connection.arrivalStopId, connection.arrivalTime, connection.tripId);

Algorithms/DepthFirstSearch.h

@@ -117,7 +117,7 @@ class DepthFirstSearch {
 
 template<typename GRAPH, typename OPERATION>
 struct SimpleOperation {
-    SimpleOperation(const GRAPH& graph, const OPERATION& operation) : graph(graph), operation(operation) {}
+    SimpleOperation(const GRAPH& graph, const OPERATION& operation) : operation(operation), graph(graph) {}
     inline void operator()(const Edge edge, const Vertex) {operation(graph.get(ToVertex, edge));}
     OPERATION operation;
     const GRAPH& graph;

Algorithms/RAPTOR/RAPTOR.h

@@ -220,7 +220,17 @@ class RAPTOR {
     }
 
     inline void collectRoutesServingUpdatedStops() noexcept {
-        for (const StopId stop : stopsUpdatedByTransfer) {
+        auto& valuesToLoopOver = stopsUpdatedByTransfer.getValues();
+        for (size_t i = 0; i < valuesToLoopOver.size(); ++i) {
+
+            #ifdef ENABLE_PREFETCH
+            if (i + 4 < valuesToLoopOver.size()) {
+                __builtin_prefetch(&(previousRound()[valuesToLoopOver[i + 4]]));
+                __builtin_prefetch(&(data.routeSegments[data.firstRouteSegmentOfStop[valuesToLoopOver[i + 4]]]));
+            }
+            #endif
+
+            const StopId stop = valuesToLoopOver[i];
             Assert(data.isStop(stop), "Stop " << stop << " is out of range!");
             const int arrivalTime = previousRound()[stop].arrivalTime;
             Assert(arrivalTime < never, "Updated stop has arrival time = never!");
@@ -240,7 +250,19 @@ class RAPTOR {
 
     inline void scanRoutes() noexcept {
         stopsUpdatedByRoute.clear();
-        for (const RouteId route : routesServingUpdatedStops.getKeys()) {
+        auto& valuesToLoopOver = routesServingUpdatedStops.getKeys();
+
+        for (size_t i = 0; i < valuesToLoopOver.size(); ++i) {
+
+            #ifdef ENABLE_PREFETCH
+            if (i + 4 < valuesToLoopOver.size()) {
+                __builtin_prefetch(data.stopArrayOfRoute(valuesToLoopOver[i + 4]));
+                __builtin_prefetch(data.firstTripOfRoute(valuesToLoopOver[i + 4]));
+                __builtin_prefetch(data.lastTripOfRoute(valuesToLoopOver[i + 4]));
+            }
+            #endif
+
+            const RouteId route = valuesToLoopOver[i];
             profiler.countMetric(METRIC_ROUTES);
             StopIndex stopIndex = routesServingUpdatedStops[route];
             const size_t tripSize = data.numberOfStopsInRoute(route);
@@ -276,7 +298,17 @@ class RAPTOR {
     inline void relaxTransfers() noexcept {
         stopsUpdatedByTransfer.clear();
         routesServingUpdatedStops.clear();
-        for (const StopId stop : stopsUpdatedByRoute) {
+
+        auto& valuesToLoopOver = stopsUpdatedByRoute.getValues();
+        for (size_t i = 0; i < valuesToLoopOver.size(); ++i) {
+
+            #ifdef ENABLE_PREFETCH
+            if (i + 4 < valuesToLoopOver.size()) {
+                __builtin_prefetch(SeparateRouteAndTransferEntries ? &previousRound()[valuesToLoopOver[i + 4]].arrivalTime : &currentRound()[valuesToLoopOver[i + 4]].arrivalTime);
+                data.transferGraph.prefetchBeginOut(valuesToLoopOver[i + 4]);
+            }
+            #endif
+
             const int earliestArrivalTime = SeparateRouteAndTransferEntries ? previousRound()[stop].arrivalTime : currentRound()[stop].arrivalTime;
             for (const Edge edge : data.transferGraph.edgesFrom(stop)) {
                 if constexpr (INITIAL_TRANSFERS && PreventDirectWalking) {

Algorithms/TripBased/Preprocessing/StopEventGraphBuilder.h

@@ -111,6 +111,12 @@ class StopEventGraphBuilder {
 
     inline void reduceTransfers(const TripId trip) noexcept {
         timestamp++;
+
+        if (timestamp == 0) {
+            labels.clear();
+            labels.resize(data.numberOfStops());
+        }
+
         const StopId* stops = data.stopArrayOfTrip(trip);
         for (StopIndex i = StopIndex(data.numberOfStopsInTrip(trip) - 1); i > 0; i--) {
             const int arrivalTime = data.getStopEvent(trip, i).arrivalTime;

Algorithms/TripBased/Query/TransitiveQuery.h

@@ -198,7 +198,17 @@ class TransitiveQuery {
             }
         }
         reachedRoutes.sort();
-        for (const RouteId route : reachedRoutes) {
+        auto& valuesToLoopOver = reachedRoutes.getValues();
+
+        for (size_t i = 0; i < valuesToLoopOver.size(); ++i) {
+            #ifdef ENABLE_PREFETCH
+            if (i + 4 < valuesToLoopOver.size()) {
+                __builtin_prefetch(&(routeLabels[valuesToLoopOver[i + 4]]));
+                __builtin_prefetch(&(data.firstTripOfRoute[valuesToLoopOver[i + 4]]));
+            }
+            #endif
+
+            const RouteId route = valuesToLoopOver[i];
             const RouteLabel& label = routeLabels[route];
             const StopIndex endIndex = label.end();
             const TripId firstTrip = data.firstTripOfRoute[route];
@@ -225,17 +235,31 @@ class TransitiveQuery {
             targetLabels.emplace_back(targetLabels.back());
             // Evaluate final transfers in order to check if the target is reachable
             for (size_t i = roundBegin; i < roundEnd; i++) {
+                #ifdef ENABLE_PREFETCH
+                if (i + 4 < roundEnd) {
+                    __builtin_prefetch(&(queue[i + 4]));
+                    __builtin_prefetch(&(data.arrivalEvents[queue[i + 4].begin]));
+                }
+                #endif
+
                 const TripLabel& label = queue[i];
                 profiler.countMetric(METRIC_SCANNED_TRIPS);
                 for (StopEventId j = label.begin; j < label.end; j++) {
                     profiler.countMetric(METRIC_SCANNED_STOPS);
-                    if (data.arrivalEvents[j].arrivalTime >= minArrivalTime) break;
+                    if (data.arrivalEvents[j].arrivalTime >= minArrivalTime) [[unlikely]] break;
                     const int timeToTarget = transferToTarget[data.arrivalEvents[j].stop];
                     if (timeToTarget != INFTY) addTargetLabel(data.arrivalEvents[j].arrivalTime + timeToTarget, i);
                 }
             }
             // Find the range of transfers for each trip
             for (size_t i = roundBegin; i < roundEnd; i++) {
+                #ifdef ENABLE_PREFETCH
+                if (i + 4 < roundEnd) {
+                    __builtin_prefetch(&(queue[i + 4]));
+                    __builtin_prefetch(&(data.arrivalEvents[queue[i + 4].begin]));
+                }
+                #endif
+
                 TripLabel& label = queue[i];
                 for (StopEventId j = label.begin; j < label.end; j++) {
                     if (data.arrivalEvents[j].arrivalTime >= minArrivalTime) label.end = j;
@@ -270,7 +294,7 @@ class TransitiveQuery {
     inline void enqueue(const Edge edge, const size_t parent) noexcept {
         profiler.countMetric(METRIC_ENQUEUES);
         const EdgeLabel& label = edgeLabels[edge];
-        if (reachedIndex.alreadyReached(label.trip, label.stopEvent - label.firstEvent)) return;
+        if (reachedIndex.alreadyReached(label.trip, label.stopEvent - label.firstEvent)) [[likely]] return;
         queue[queueSize] = TripLabel(label.stopEvent, StopEventId(label.firstEvent + reachedIndex(label.trip)), parent);
         queueSize++;
         Assert(queueSize <= queue.size(), "Queue is overfull!");
@@ -373,4 +397,4 @@ class TransitiveQuery {
     Profiler profiler;
 };
 
-}
+}

CMakeLists.txt

@@ -8,6 +8,13 @@ set(CMAKE_CXX_FLAGS "-pipe -march=native -Wfatal-errors")
 set(CMAKE_CXX_FLAGS_DEBUG "-rdynamic -Werror -Wpedantic -pedantic-errors -Wall -Wextra -Wparentheses -D_GLIBCXX_DEBUG -g -fno-omit-frame-pointer -O0")
 set(CMAKE_CXX_FLAGS_RELEASE "-ffast-math -DNDEBUG -O3")
 
+# Add an option for enabling prefetch
+option(USE_PREFETCH "Enable prefetch optimization" OFF)
+
+if(USE_PREFETCH)
+    add_compile_definitions(ENABLE_PREFETCH)
+endif()
+
 #Libraries
 set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
 set(THREADS_PREFER_PTHREAD_FLAG TRUE)

DataStructures/Graph/Classes/DynamicGraph.h

@@ -745,6 +745,30 @@ class DynamicGraphImplementation {
     }
 
     // IO:
+    inline void serialize(IO::Serialization& serialize) const {
+        serialize(vertexAttributes, edgeAttributes);
+    }
+
+    inline void deserialize(IO::Deserialization& deserialize) {
+        clear();
+        deserialize(vertexAttributes, edgeAttributes);
+        for (const Vertex vertex : vertices())
+            edgeCount += outDegree(vertex);
+        Assert(satisfiesInvariants());
+    }
+
+    inline void serialize(const std::string& fileName) const {
+        IO::serialize(fileName, vertexAttributes, edgeAttributes);
+    }
+
+    inline void deserialize(const std::string& fileName) {
+        clear();
+        IO::deserialize(fileName, vertexAttributes, edgeAttributes);
+        for (const Vertex vertex : vertices())
+            edgeCount += outDegree(vertex);
+        Assert(satisfiesInvariants());
+    }
+
     inline void writeBinary(const std::string& fileName, const std::string& separator = ".") const noexcept {
         vertexAttributes.serialize(fileName, separator);
         edgeAttributes.serialize(fileName, separator);

DataStructures/Graph/Classes/EdgeList.h

@@ -481,6 +481,24 @@ class EdgeListImplementation {
     }
 
     // IO:
+    inline void serialize(IO::Serialization& serialize) const {
+        serialize(vertexAttributes, edgeAttributes);
+    }
+
+    inline void deserialize(IO::Deserialization& deserialize) {
+        clear();
+        deserialize(vertexAttributes, edgeAttributes);
+    }
+
+    inline void serialize(const std::string& fileName) const {
+        IO::serialize(fileName, vertexAttributes, edgeAttributes);
+    }
+
+    inline void deserialize(const std::string& fileName) {
+        clear();
+        IO::deserialize(fileName, vertexAttributes, edgeAttributes);
+    }
+
     inline void writeBinary(const std::string& fileName, const std::string& separator = ".") const noexcept {
         vertexAttributes.serialize(fileName, separator);
         edgeAttributes.serialize(fileName, separator);

DataStructures/Graph/Classes/StaticGraph.h

@@ -179,10 +179,16 @@ class StaticGraphImplementation {
 
     // Manipulation:
     inline void clear() noexcept {
+        beginOut.clear();
         vertexAttributes.clear();
         edgeAttributes.clear();
     }
 
+    inline void removeEdges() noexcept {
+        std::fill(beginOut.begin(), beginOut.end(), Edge(0));
+        edgeAttributes.clear();
+    }
+
     inline void reserve(const size_t numVertices, const size_t numEdges) noexcept {
         beginOut.reserve(numVertices + 1);
         vertexAttributes.reserve(numVertices);
@@ -511,6 +517,26 @@ class StaticGraphImplementation {
     }
 
     // IO:
+    inline void serialize(IO::Serialization& serialize) const {
+        serialize(beginOut, vertexAttributes, edgeAttributes);
+    }
+
+    inline void deserialize(IO::Deserialization& deserialize) {
+        clear();
+        deserialize(beginOut, vertexAttributes, edgeAttributes);
+        AssertMsg(satisfiesInvariants(), "Invariants not satisfied!");
+    }
+
+    inline void serialize(const std::string& fileName) const {
+        IO::serialize(fileName, beginOut, vertexAttributes, edgeAttributes);
+    }
+
+    inline void deserialize(const std::string& fileName) {
+        clear();
+        IO::deserialize(fileName, beginOut, vertexAttributes, edgeAttributes);
+        AssertMsg(satisfiesInvariants(), "Invariants not satisfied!");
+    }
+
     inline void writeBinary(const std::string& fileName, const std::string& separator = ".") const noexcept {
         IO::serialize(fileName + separator + "beginOut", beginOut);
         vertexAttributes.serialize(fileName, separator);
@@ -749,6 +775,12 @@ class StaticGraphImplementation {
         return true;
     }
 
+    inline void prefetchBeginOut(const Vertex vertex) const noexcept {
+        AssertMsg(isVertex(vertex), "Vertex is not valid!");
+
+        __builtin_prefetch(&beginOut[vertex]);
+    }
+
 private:
     std::vector<Edge> beginOut;
     VertexAttributes vertexAttributes;