Skip to content

Commit f33c400

Browse files
author
John Wellbelove
committed
Added get_token_list to string_utilities to retrieve multiple tokens
1 parent 95a4b10 commit f33c400

File tree

4 files changed

+196
-5
lines changed

4 files changed

+196
-5
lines changed

‎include/etl/cyclic_value.h‎

Lines changed: 34 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,36 @@ SOFTWARE.
4646

4747
namespace etl
4848
{
49+
namespace detail
50+
{
51+
//Helper: only performs the v < F check when has_lower == true ––
52+
template <bool has_lower, typename U, U F>
53+
struct lower_bound_impl;
54+
55+
// Specialisation for First != 0: enforce v >= First
56+
template <typename U, U F>
57+
struct lower_bound_impl<true, U, F>
58+
{
59+
static ETL_CONSTEXPR auto apply(U& v) ETL_NOEXCEPT -> void
60+
{
61+
if (v < F)
62+
{
63+
v = F;
64+
}
65+
}
66+
};
67+
68+
// Specialisation for First == 0: no-op
69+
template <typename U, U F>
70+
struct lower_bound_impl<false, U, F>
71+
{
72+
static ETL_CONSTEXPR auto apply(U&) ETL_NOEXCEPT -> void
73+
{
74+
// nothing
75+
}
76+
};
77+
} // namespace detail
78+
4979
//***************************************************************************
5080
/// Provides a value that cycles between two limits.
5181
//***************************************************************************
@@ -113,10 +143,9 @@ namespace etl
113143
{
114144
value_ = Last;
115145
}
116-
else if (value_ < First)
117-
{
118-
value_ = First;
119-
}
146+
147+
// lower-bound clamp only if First != 0
148+
detail::lower_bound_impl<(First != 0), T, First>::apply(value_);
120149

121150
value = value_;
122151
}
@@ -351,7 +380,7 @@ namespace etl
351380
cyclic_value(T first_, T last_)
352381
: value(first_)
353382
, first_value(first_)
354-
, last_value(last_)
383+
, last_value(last_)
355384
{
356385
}
357386

‎include/etl/string_utilities.h‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -756,6 +756,45 @@ namespace etl
756756
return etl::optional<TStringView>(view);
757757
}
758758

759+
//***************************************************************************
760+
/// get_token_list
761+
///\brief Splits a string of tokens to a set of views, according to a set of delimiters.
762+
/// The tokenisation stops if:
763+
/// 1. The end of the input text is reached.
764+
/// 2. The max_size() of the output container is reached.
765+
/// 3. The number of tokens found reaches max_n_tokens.
766+
/// The input container must define <code>const_pointer</code>.
767+
/// The output container must define <code>value_type</code>.
768+
/// The output container must define the member function <code>max_size</code> that returns the maximum size of the container.
769+
/// The output container must define the member function <code>push_back</code> that pushes the view on to the back of the container.
770+
///\param input The input string.
771+
///\param output A reference to an output container of string views.
772+
///\param delimiters A pointer to a string of valid delimiters.
773+
///\param ignore_empty_tokens If <b>true</b> then empty tokens are ignored.
774+
///\param max_n_tokens The maximum number of tokens to collect. Default tokenise everything.
775+
///\return Returns <b>true</b> if all tokens were added to the list, otherwise <b>false</b>.
776+
//***************************************************************************
777+
template <typename TInput, typename TOutput>
778+
bool get_token_list(const TInput& input, TOutput& output, typename TInput::const_pointer delimiters, bool ignore_empty_tokens, size_t max_n_tokens = etl::integral_limits<size_t>::max)
779+
{
780+
typedef typename TOutput::value_type string_view_t;
781+
782+
etl::optional<string_view_t> token;
783+
784+
size_t count = 0;
785+
while ((count != output.max_size()) &&
786+
(count != max_n_tokens) &&
787+
(token = etl::get_token(input, delimiters, token, ignore_empty_tokens)))
788+
{
789+
output.push_back(token.value());
790+
++count;
791+
}
792+
793+
bool all_tokens_found = (token.has_value() == false);
794+
795+
return all_tokens_found;
796+
}
797+
759798
//***************************************************************************
760799
/// pad_left
761800
//***************************************************************************

‎support/Release notes.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
Updates:
55
Added etl::tuple
6+
Added get_token_list to string_utilities to retrieve multiple tokens
67
#757 Add etl::chrono time/date classes
78
#1085 Add The Remaining Math Operators To ETL_TYPEDEF
89
#1084 Add 'type' Alias to Strong Types
@@ -29,6 +30,7 @@ Fixes:
2930
#1041 Incorrect rounding of scaled natural numbers in scaled_rounding #1041
3031

3132
Pull Requests:
33+
#1088 Fix numeric overflow in bip buffer's get_write_reserve
3234
#1090 Fix copy_s() for C++03
3335
#1089 Cleanup fixes for C++03
3436
#1083 Add take() to etl::span

‎test/test_string_utilities.cpp‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,9 @@ SOFTWARE.
3434
#include "etl/vector.h"
3535

3636
#include <string>
37+
#include <string_view>
38+
#include <vector>
39+
#include <list>
3740

3841
#undef STR
3942
#define STR(x) x
@@ -54,6 +57,9 @@ namespace
5457
using Vector = etl::vector<String, 15>;
5558
using SizeType = etl::istring::size_type;
5659

60+
using VectorOfViews7 = etl::vector<StringView, 7>;
61+
using VectorOfViews5 = etl::vector<StringView, 5>;
62+
5763
#if ETL_USING_CPP17
5864
constexpr auto Whitespace = etl::whitespace_v<String::value_type>;
5965
#else
@@ -1085,6 +1091,121 @@ namespace
10851091
CHECK_EQUAL(0U, tokens.size());
10861092
}
10871093

1094+
//*************************************************************************
1095+
TEST(test_split_to_vector_of_string_view_all_tokens_captured_ignore_empty_tokens)
1096+
{
1097+
String text(STR(",,,The,cat,sat,,on,the,mat"));
1098+
VectorOfViews7 views;
1099+
1100+
bool all_views_found = etl::get_token_list(text, views, STR(","), true);
1101+
1102+
CHECK_TRUE(all_views_found);
1103+
CHECK_EQUAL(6, views.size());
1104+
CHECK_EQUAL(std::string("The"), std::string(views[0].begin(), views[0].end()));
1105+
CHECK_EQUAL(std::string("cat"), std::string(views[1].begin(), views[1].end()));
1106+
CHECK_EQUAL(std::string("sat"), std::string(views[2].begin(), views[2].end()));
1107+
CHECK_EQUAL(std::string("on"), std::string(views[3].begin(), views[3].end()));
1108+
CHECK_EQUAL(std::string("the"), std::string(views[4].begin(), views[4].end()));
1109+
CHECK_EQUAL(std::string("mat"), std::string(views[5].begin(), views[5].end()));
1110+
}
1111+
1112+
//*************************************************************************
1113+
TEST(test_get_token_list_to_vector_of_string_view_all_but_1_tokens_captured_ignore_empty_tokens)
1114+
{
1115+
String text(STR(",,,The,cat,sat,,on,the,mat"));
1116+
VectorOfViews5 views;
1117+
1118+
bool all_views_found = etl::get_token_list(text, views, STR(","), true);
1119+
1120+
CHECK_FALSE(all_views_found);
1121+
CHECK_EQUAL(5, views.size());
1122+
CHECK_EQUAL(std::string("The"), std::string(views[0].begin(), views[0].end()));
1123+
CHECK_EQUAL(std::string("cat"), std::string(views[1].begin(), views[1].end()));
1124+
CHECK_EQUAL(std::string("sat"), std::string(views[2].begin(), views[2].end()));
1125+
CHECK_EQUAL(std::string("on"), std::string(views[3].begin(), views[3].end()));
1126+
CHECK_EQUAL(std::string("the"), std::string(views[4].begin(), views[4].end()));
1127+
}
1128+
1129+
#if ETL_USING_CPP17
1130+
//*************************************************************************
1131+
TEST(test_get_token_list_to_std_vector_of_std_string_view_all_tokens_captured_ignore_empty_tokens)
1132+
{
1133+
std::string text(STR(",,,The,cat,sat,,on,the,mat"));
1134+
std::vector<std::string_view> views;
1135+
1136+
bool all_views_found = etl::get_token_list(text, views, STR(","), true);
1137+
1138+
CHECK_TRUE(all_views_found);
1139+
CHECK_EQUAL(6, views.size());
1140+
CHECK_EQUAL(std::string("The"), std::string(views[0].begin(), views[0].end()));
1141+
CHECK_EQUAL(std::string("cat"), std::string(views[1].begin(), views[1].end()));
1142+
CHECK_EQUAL(std::string("sat"), std::string(views[2].begin(), views[2].end()));
1143+
CHECK_EQUAL(std::string("on"), std::string(views[3].begin(), views[3].end()));
1144+
CHECK_EQUAL(std::string("the"), std::string(views[4].begin(), views[4].end()));
1145+
CHECK_EQUAL(std::string("mat"), std::string(views[5].begin(), views[5].end()));
1146+
}
1147+
#endif
1148+
1149+
//*************************************************************************
1150+
TEST(test_get_token_list_to_vector_of_string_view_all_tokens_captured_ignore_empty_tokens_maximum_of_3_tokens)
1151+
{
1152+
String text(STR(",,,The,cat,sat,,on,the,mat"));
1153+
VectorOfViews7 views;
1154+
1155+
bool all_views_found = etl::get_token_list(text, views, STR(","), true, 3);
1156+
1157+
CHECK_FALSE(all_views_found);
1158+
CHECK_EQUAL(3, views.size());
1159+
CHECK_EQUAL(std::string("The"), std::string(views[0].begin(), views[0].end()));
1160+
CHECK_EQUAL(std::string("cat"), std::string(views[1].begin(), views[1].end()));
1161+
CHECK_EQUAL(std::string("sat"), std::string(views[2].begin(), views[2].end()));
1162+
}
1163+
1164+
#if ETL_USING_CPP17
1165+
//*************************************************************************
1166+
TEST(test_get_token_list_to_std_vector_of_std_string_view_all_tokens_captured_ignore_empty_tokens_maximum_of_3_tokens)
1167+
{
1168+
std::string text(STR(",,,The,cat,sat,,on,the,mat"));
1169+
std::vector<std::string_view> views;
1170+
1171+
bool all_views_found = etl::get_token_list(text, views, STR(","), true, 3);
1172+
1173+
CHECK_FALSE(all_views_found);
1174+
CHECK_EQUAL(3, views.size());
1175+
CHECK_EQUAL(std::string("The"), std::string(views[0].begin(), views[0].end()));
1176+
CHECK_EQUAL(std::string("cat"), std::string(views[1].begin(), views[1].end()));
1177+
CHECK_EQUAL(std::string("sat"), std::string(views[2].begin(), views[2].end()));
1178+
}
1179+
#endif
1180+
1181+
#if ETL_USING_CPP17
1182+
//*************************************************************************
1183+
TEST(test_get_token_list_to_std_list_of_std_string_view_all_tokens_captured_ignore_empty_tokens)
1184+
{
1185+
std::string text(STR(",,,The,cat,sat,,on,the,mat"));
1186+
std::list<std::string_view> views;
1187+
1188+
bool all_views_found = etl::get_token_list(text, views, STR(","), true);
1189+
1190+
CHECK_TRUE(all_views_found);
1191+
CHECK_EQUAL(6, views.size());
1192+
1193+
auto itr = views.begin();
1194+
1195+
CHECK_EQUAL(std::string("The"), std::string(itr->begin(), itr->end()));
1196+
++itr;
1197+
CHECK_EQUAL(std::string("cat"), std::string(itr->begin(), itr->end()));
1198+
++itr;
1199+
CHECK_EQUAL(std::string("sat"), std::string(itr->begin(), itr->end()));
1200+
++itr;
1201+
CHECK_EQUAL(std::string("on"), std::string(itr->begin(), itr->end()));
1202+
++itr;
1203+
CHECK_EQUAL(std::string("the"), std::string(itr->begin(), itr->end()));
1204+
++itr;
1205+
CHECK_EQUAL(std::string("mat"), std::string(itr->begin(), itr->end()));
1206+
}
1207+
#endif
1208+
10881209
//*************************************************************************
10891210
TEST(test_pad_left)
10901211
{

0 commit comments

Comments
 (0)