Improve small sort performance

Use optimal in-place sorting for ranges of size 5 and smaller. This first minimizes maximum and average number of comparisons (these two goals never conflict at this size), then minimizes comparisons for already sorted inputs, then minimizes comparisons for reverse sorted inputs.

With this new code, there is not an advantage to the hybrid approach I made for sorting 6 elements, so that is currently disabled. There is still work to be done here to optimize small ranges that are larger than the hand-coded algorithms.

For ranges that are large, have non-trivial element types, or use a comparison function other than `std::less` and `std::greater`, assume comparisons are expensive and use an algorithm that minimizes comparisons, otherwise use an algorithm that performs best for types like `int`.

These small sorting algorithms perform much better than existing standard library implementations for all types tested.

Author: davidstone

source/containers/CMakeLists.txt

@@ -15,6 +15,7 @@ target_sources(containers PUBLIC
 		algorithms/sort/dereference_all.cpp
 		algorithms/sort/double_buffered_ska_sort.cpp
 		algorithms/sort/fixed_size_merge_sort.cpp
+		algorithms/sort/low_high_ref.cpp
 		algorithms/sort/inplace_radix_sort.cpp
 		algorithms/sort/is_sorted.cpp
 		algorithms/sort/merge_relocate_second_range.cpp
@@ -29,6 +30,7 @@ target_sources(containers PUBLIC
 		algorithms/sort/sort_exactly_4.cpp
 		algorithms/sort/sort_exactly_5.cpp
 		algorithms/sort/sort_exactly_6.cpp
+		algorithms/sort/sort_exactly_n.cpp
 		algorithms/sort/to_radix_sort_key.cpp
 		algorithms/accumulate.cpp
 		algorithms/adjacent.cpp

source/containers/algorithms/sort/cheaply_sortable.cpp

@@ -14,7 +14,12 @@ namespace containers {
 
 // Size determined experimentally, but also the size of a cache line. Maybe it
 // should be `std::hardware_destructive_interference_size`?
-export template<typename T>
-concept cheaply_sortable = std::is_trivially_copyable_v<T> and sizeof(T) <= 64_bi;
+// This incorrectly considers `std::string_view` cheaply_sortable. This ideally
+// would have something like `has_default_compare`.
+export template<typename T, typename Compare>
+concept cheaply_sortable =
+	std::is_trivially_copyable_v<T> and
+	sizeof(T) <= 64_bi and
+	(std::same_as<Compare, std::less<>> or std::same_as<Compare, std::greater<>>);
 
 } // namespace containers

source/containers/algorithms/sort/chunked_insertion_sort.cpp

@@ -3,18 +3,18 @@
 // (See accompanying file LICENSE_1_0.txt or copy at
 // http://www.boost.org/LICENSE_1_0.txt)
 
+module;
+
+#include <bounded/assert.hpp>
+
 export module containers.algorithms.sort.chunked_insertion_sort;
 
 import containers.algorithms.sort.merge_relocate_second_range;
 import containers.algorithms.sort.small_size_optimized_sort;
-import containers.algorithms.sort.sort_exactly_1;
-import containers.algorithms.sort.sort_exactly_2;
-import containers.algorithms.sort.sort_exactly_3;
-import containers.algorithms.sort.sort_exactly_4;
-import containers.algorithms.sort.sort_exactly_5;
-import containers.algorithms.sort.sort_exactly_6;
+import containers.algorithms.sort.sort_exactly_n;
 
 import containers.begin_end;
+import containers.integer_range;
 import containers.offset_type;
 import containers.range;
 import containers.size;
@@ -29,64 +29,31 @@ using namespace bounded::literal;
 
 namespace containers {
 
-template<typename Iterator>
-constexpr auto maybe_sort_exactly_n_relocate(Iterator it, auto const size, auto out, auto const compare) -> Iterator {
-	constexpr auto max_size = numeric_traits::max_value<offset_type<Iterator>>;
-	if constexpr (size <= max_size) {
-		return ::containers::sort_exactly_n_relocate(it, size, out, compare);
-	} else {
-		std::unreachable();
-	}
-}
-
-template<typename Iterator>
-constexpr auto runtime_sort_exactly_n_relocate(Iterator it, auto const size, auto out, auto const compare) -> Iterator {
-	auto do_sort = [&](auto const constant_size) {
-		return ::containers::maybe_sort_exactly_n_relocate(it, constant_size, out, compare);
-	};
-	switch (static_cast<std::size_t>(size)) {
-		case 0:
-			return it;
-		case 1:
-			return do_sort(1_bi);
-		case 2:
-			return do_sort(2_bi);
-		case 3:
-			return do_sort(3_bi);
-		case 4:
-			return do_sort(4_bi);
-		default:
-			std::unreachable();
-	}
-}
-
 struct chunked_insertion_sort_t {
 	template<typename Compare = std::less<>>
 	static constexpr auto operator()(range auto && r, Compare const compare = Compare()) -> void {
 		constexpr auto chunk_size = 4_bi;
+		auto const size = ::containers::size(r);
 		auto it = containers::begin(r);
-		auto const last = containers::end(r);
-		#if 1
-		auto const initial_sort_size = bounded::min(chunk_size, ::containers::size(r));
+		auto const initial_sort_size = size % chunk_size;
 		::containers::small_size_optimized_sort(
 			subrange(it, initial_sort_size),
 			compare,
 			[](auto &&, auto) { std::unreachable(); }
 		);
 		it += initial_sort_size;
-		#endif
 		auto buffer = containers::uninitialized_array<std::remove_reference_t<decltype(*it)>, chunk_size>();
-		while (it != last) {
-			auto const count = bounded::min(chunk_size, last - it);
-			auto next_it = ::containers::runtime_sort_exactly_n_relocate(it, count, buffer.data(), compare);
+		for (auto _ : containers::integer_range(size / chunk_size)) {
+			auto const next_it = ::containers::sort_exactly_n_relocate(it, chunk_size, buffer.data(), compare);
 			merge_relocate_second_range(
 				subrange(containers::begin(r), it),
-				subrange(buffer.data(), count),
+				buffer,
 				next_it,
 				compare
 			);
 			it = next_it;
 		}
+		BOUNDED_ASSERT(it == containers::end(r));
 	}
 };
 export constexpr auto chunked_insertion_sort = chunked_insertion_sort_t();

source/containers/algorithms/sort/dereference_all.cpp

@@ -5,26 +5,35 @@
 
 export module containers.algorithms.sort.dereference_all;
 
+import containers.array;
+import containers.iter_value_t;
+
 import bounded;
 import std_module;
 export import tv;
 
 namespace containers {
 
-template<typename T, std::size_t>
-using type_ignore_size = T;
+export template<typename T, auto size>
+using element_refs = containers::array<std::reference_wrapper<T>, size>;
 
 export template<typename Iterator>
 constexpr auto dereference_all(Iterator it, auto const size) {
-	using T = decltype(*it);
-	static_assert(std::is_lvalue_reference_v<T>);
-	return [&]<std::size_t... indexes>(std::index_sequence<indexes...>) {
-		// Must use brace initialization for guaranteed evaluation order. Uses a
-		// tuple instead of an array (even though all types are the same)
-		// because we cannot have arrays of references. Cannot use CTAD because
-		// we want to deduce references for the values.
-		return tv::tuple<type_ignore_size<T, indexes>..., Iterator>{(static_cast<void>(indexes), *it++)..., std::move(it)};
-	}(bounded::make_index_sequence(size));
+	auto const [...indexes] = bounded::index_sequence_struct(size);
+	using value_type = iter_value_t<Iterator>;
+	using elements_t = element_refs<value_type, size>;
+	struct result_t {
+		elements_t elements;
+		Iterator last;
+	};
+	// Must use brace initialization for guaranteed evaluation order. Uses a
+	// tuple instead of an array (even though all types are the same)
+	// because we cannot have arrays of references. Cannot use CTAD because
+	// we want to deduce references for the values.
+	return result_t{
+		elements_t{(static_cast<void>(indexes), *it++)...},
+		std::move(it)
+	};
 }
 
 } // namespace containers

source/containers/algorithms/sort/fixed_size_merge_sort.cpp

@@ -5,11 +5,7 @@
 
 export module containers.algorithms.sort.fixed_size_merge_sort;
 
-import containers.algorithms.sort.sort_exactly_1;
-import containers.algorithms.sort.sort_exactly_2;
-import containers.algorithms.sort.sort_exactly_3;
-import containers.algorithms.sort.sort_exactly_4;
-import containers.algorithms.sort.sort_exactly_5;
+import containers.algorithms.sort.sort_exactly_n;
 
 import containers.algorithms.uninitialized;
 
@@ -22,7 +18,7 @@ import std_module;
 using namespace bounded::literal;
 namespace containers {
 
-export constexpr auto fixed_size_merge_sort(auto it, auto const size1, auto const size2, auto const compare) {
+export constexpr auto fixed_size_merge_sort(auto it, auto const size1, auto const size2, auto const compare) -> void {
 	static_assert(size1 <= size2);
 	static_assert(size1 > 0_bi);
 	static_assert(size2 > 0_bi);

source/containers/algorithms/sort/low_high_ref.cpp

@@ -0,0 +1,22 @@
+// Copyright David Stone 2023.
+// Distributed under the Boost Software License, Version 1.0.
+// (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+export module containers.algorithms.sort.low_high_ref;
+
+namespace containers {
+
+export template<typename T>
+struct low_high_ref {
+	constexpr low_high_ref(T & low_, T & high_):
+		low(low_),
+		high(high_)
+	{
+	}
+
+	T & low;
+	T & high;
+};
+
+} // namespace containers

source/containers/algorithms/sort/small_size_optimized_sort.cpp

@@ -10,12 +10,7 @@ module;
 export module containers.algorithms.sort.small_size_optimized_sort;
 
 import containers.algorithms.sort.is_sorted;
-import containers.algorithms.sort.sort_exactly_1;
-import containers.algorithms.sort.sort_exactly_2;
-import containers.algorithms.sort.sort_exactly_3;
-import containers.algorithms.sort.sort_exactly_4;
-import containers.algorithms.sort.sort_exactly_5;
-import containers.algorithms.sort.sort_exactly_6;
+import containers.algorithms.sort.sort_exactly_n;
 
 import containers.begin_end;
 import containers.range;
@@ -30,13 +25,12 @@ namespace containers {
 
 using namespace bounded::literal;
 
-export constexpr auto max_small_sort_size = 6_bi;
-
 export template<range Range>
 inline constexpr auto small_size_optimized_sort(Range && r, auto const compare, auto const sort_large_range) -> void {
+	constexpr auto min_size = numeric_traits::min_value<range_size_t<Range>>;
 	constexpr auto max_size = numeric_traits::max_value<range_size_t<Range>>;
 	auto do_sort = [&](auto const count) {
-		if constexpr (count <= max_size) {
+		if constexpr (min_size <= count and count <= max_size) {
 			::containers::sort_exactly_n(containers::begin(r), count, compare);
 			BOUNDED_ASSERT(::containers::is_sorted(r, compare));
 		} else {
@@ -59,11 +53,8 @@ inline constexpr auto small_size_optimized_sort(Range && r, auto const compare,
 		case 5:
 			do_sort(5_bi);
 			return;
-		case 6:
-			do_sort(6_bi);
-			return;
 		default:
-			if constexpr (max_size >= max_small_sort_size) {
+			if constexpr (max_size > 5_bi) {
 				sort_large_range(r, compare);
 				return;
 			} else {

source/containers/algorithms/sort/sort.cpp

@@ -18,6 +18,7 @@ import containers.algorithms.advance;
 import containers.algorithms.partition;
 
 import containers.begin_end;
+import containers.iter_value_t;
 import containers.legacy_iterator;
 import containers.range;
 import containers.range_size_t;
@@ -54,12 +55,14 @@ constexpr auto introsort_impl(Depth const depth) {
 		auto median = first + length / 2_bi;
 		auto const before_last = containers::prev(last);
 		auto median_of = [&](auto... its) {
-			::containers::sort_exactly_n_impl(*its..., compare);
+			::containers::sort_exactly_n_in_place_impl(
+				*its...,
+				compare
+			);
 		};
 		if (length >= 1000_bi) {
 			median_of(first, first + length / 4_bi, median, median + length / 4_bi, before_last);
 		} else {
-			static_assert(max_small_sort_size >= 3_bi);
 			median_of(first, median, before_last);
 		}
 		median = iterator_partition(first, median, last, compare);

source/containers/algorithms/sort/sort_exactly_1.cpp

@@ -5,17 +5,31 @@
 
 export module containers.algorithms.sort.sort_exactly_1;
 
+import containers.iterator;
+
 import bounded;
+import std_module;
+
+using namespace bounded::literal;
 
 namespace containers {
 
-export constexpr auto sort_exactly_n([[maybe_unused]] auto const it, bounded::constant_t<1>, [[maybe_unused]] auto const compare) -> void {
+// Stable
+// Average compares: 0 (weight 0)
+// Max compares: 0
+// Sorted compares: 0
+// Reversed compares: 0
+export template<iterator Iterator, typename Compare>
+constexpr auto sort_exactly_n_in_place(Iterator, bounded::constant_t<1>, Compare) -> void {
 }
 
-export constexpr auto sort_exactly_n_relocate(auto it, bounded::constant_t<1>, auto const out, [[maybe_unused]] auto const compare) {
-	bounded::relocate_at(*out, *it);
-	++it;
-	return it;
+export template<typename T, typename Compare>
+constexpr auto sort_exactly_n_relocate_impl(
+	T & x0,
+	auto const out,
+	Compare
+) -> void {
+	bounded::relocate_at(*out, x0);
 }
 
 } // namespace containers

source/containers/algorithms/sort/sort_exactly_2.cpp

@@ -5,34 +5,43 @@
 
 export module containers.algorithms.sort.sort_exactly_2;
 
-import containers.algorithms.sort.dereference_all;
 import containers.algorithms.sort.relocate_in_order;
 
 import bounded;
 import std_module;
 
+using namespace bounded::literal;
+
 namespace containers {
 
-// stable, 1 compare (average 1), 0-3 relocates (average 1.5)
-export constexpr auto sort_exactly_n_impl(auto & x0, auto & x1, auto const compare) -> void {
+// Stable
+// Average compares: 1 (weight 1)
+// Max compares: 1
+// Sorted compares: 1
+// Reversed compares: 1
+export template<typename T>
+constexpr auto sort_exactly_n_in_place_impl(
+	T & x0,
+	T & x1,
+	auto const compare
+) -> void {
 	if (compare(x1, x0)) {
 		std::ranges::swap(x0, x1);
 	}
 }
 
-export constexpr auto sort_exactly_n(auto it, bounded::constant_t<2> const size, auto const compare) -> void {
-	auto [x0, x1, last] = ::containers::dereference_all(std::move(it), size);
-	sort_exactly_n_impl(x0, x1, compare);
-}
-
-export constexpr auto sort_exactly_n_relocate(auto it, bounded::constant_t<2> const size, auto const out, auto const compare) {
-	auto [x0, x1, last] = ::containers::dereference_all(std::move(it), size);
+export template<typename T>
+constexpr auto sort_exactly_n_relocate_impl(
+	T & x0,
+	T & x1,
+	auto const out,
+	auto const compare
+) -> void {
 	if (compare(x1, x0)) {
-		relocate_in_order(out, x1, x0);
+		::containers::relocate_in_order(out, x1, x0);
 	} else {
-		relocate_in_order(out, x0, x1);
+		::containers::relocate_in_order(out, x0, x1);
 	}
-	return last;
 }
 
 } // namespace containers