7#ifndef __UTFITERATOR_H__
8#define __UTFITERATOR_H__
12#if U_SHOW_CPLUSPLUS_API || U_SHOW_CPLUSPLUS_HEADER_API || !defined(UTYPES_H)
15#if __has_include(<version>)
18#if defined(__cpp_lib_ranges)
141#ifndef U_HIDE_DRAFT_API
176namespace U_HEADER_ONLY_NAMESPACE {
179#if U_CPLUSPLUS_VERSION >= 20
182template<
typename Iter>
186template<
typename Iter>
190template<
typename Iter>
194template<
typename Iter>
198template<
typename Range>
199constexpr bool range = std::ranges::range<Range>;
204template<
typename Iter>
208template<
typename Iter>
212template<
typename Iter>
215 std::forward_iterator_tag,
216 typename std::iterator_traits<Iter>::iterator_category>;
219template<
typename Iter>
222 std::bidirectional_iterator_tag,
223 typename std::iterator_traits<Iter>::iterator_category>;
226template<
typename Range,
typename =
void>
230template<
typename Range>
233 std::void_t<decltype(std::declval<Range>().begin()),
234 decltype(std::declval<Range>().end())>> : std::true_type {};
237template<
typename Range>
246template <
typename... Args>
253template<
typename CP32,
bool skipSurrogates>
255 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
279 if (skipSurrogates && c_ == 0xd800) {
307template<
typename CP32>
309 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
338template<
typename CP32>
340 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
372template<
typename CP32,
typename UnitIter,
typename =
void>
374 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
400 UnitIter
begin()
const {
return start_; }
407 UnitIter
end()
const {
return limit_; }
415#if U_CPLUSPLUS_VERSION >= 20
421 template<std::contiguous_iterator Iter = UnitIter>
422 std::basic_string_view<Unit>
stringView()
const {
423 return std::basic_string_view<Unit>(
begin(),
end());
431 template<typename Iter = UnitIter, typename Unit = typename std::iterator_traits<Iter>::value_type>
432 std::enable_if_t<std::is_pointer_v<Iter> ||
433 std::is_same_v<Iter, typename std::basic_string<Unit>::iterator> ||
434 std::is_same_v<Iter, typename std::basic_string<Unit>::const_iterator> ||
435 std::is_same_v<Iter, typename std::basic_string_view<Unit>::iterator> ||
436 std::is_same_v<Iter, typename std::basic_string_view<Unit>::const_iterator>,
437 std::basic_string_view<Unit>>
439 return std::basic_string_view<Unit>(&*start_, len_);
454template<
typename CP32,
typename UnitIter>
455class UnsafeCodeUnits<
458 std::enable_if_t<!prv::forward_iterator<UnitIter>>> {
459 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
468 uint8_t
length()
const {
return len_; }
492template<
typename CP32,
typename UnitIter,
typename =
void>
517template<
typename CP32,
typename UnitIter>
521 std::enable_if_t<!prv::forward_iterator<UnitIter>>> :
522 public UnsafeCodeUnits<CP32, UnitIter> {
527 CodeUnits(
const CodeUnits &other) =
default;
541 typename UnitIter,
typename LimitIter = UnitIter,
typename =
void>
555template<
typename CP32, UTFIllFormedBehavior behavior,
typename UnitIter,
typename LimitIter>
559 std::enable_if_t<sizeof(typename prv::iter_value_t<UnitIter>) == 1>> {
560 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
562 "For 8-bit strings, the SURROGATE option does not have an equivalent.");
574 U_FORCE_INLINE static void inc(UnitIter &p,
const LimitIter &limit) {
580 if ((0xe0 <= b && b < 0xf0)) {
585 }
else if (b < 0xe0) {
615 if (0xe0 <= b2 && b2 <= 0xf4) {
634 UnitIter &p0, UnitIter &p,
const LimitIter &limit) {
635 constexpr bool isMultiPass = prv::forward_iterator<UnitIter>;
637 CP32 c = uint8_t(*p);
640 if constexpr (isMultiPass) {
641 return {c, 1,
true, p0, p};
657 (c = (c << 6) | (t & 0x3f), ++length, ++p != limit) &&
658 (t = *p - 0x80) <= 0x3f) &&
660 (c = (c << 6) | t, ++length, ++p != limit)
662 c >= 0xc2 && (c &= 0x1f, 1)) &&
664 (t = *p - 0x80) <= 0x3f) {
668 if constexpr (isMultiPass) {
669 return {c, length,
true, p0, p};
671 return {c, length,
true};
674 if constexpr (isMultiPass) {
675 return {sub(), length,
false, p0, p};
677 return {sub(), length,
false};
681 U_FORCE_INLINE static CodeUnits<CP32, UnitIter> decAndRead(UnitIter start, UnitIter &p) {
684 CP32 c = uint8_t(*--p);
686 return {c, 1,
true, p, p0};
694 c = ((b1 - 0xc0) << 6) | (c & 0x3f);
695 return {c, 2,
true, p, p0};
696 }
else if (b1 < 0xf0 ?
701 return {sub(), 2,
false, p, p0};
707 if (0xe0 <= b2 && b2 <= 0xf4) {
712 c = (b2 << 12) | ((b1 & 0x3f) << 6) | c;
713 return {c, 3,
true, p, p0};
718 return {sub(), 3,
false, p, p0};
722 if (0xf0 <= b3 && b3 <= 0xf4) {
726 c = (b3 << 18) | ((b2 & 0x3f) << 12) | ((b1 & 0x3f) << 6) | c;
727 return {c, 4,
true, p, p0};
733 return {sub(), 1,
false, p, p0};
738template<
typename CP32, UTFIllFormedBehavior behavior,
typename UnitIter,
typename LimitIter>
742 std::enable_if_t<sizeof(typename prv::iter_value_t<UnitIter>) == 2>> {
743 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
757 U_FORCE_INLINE static void inc(UnitIter &p,
const LimitIter &limit) {
775 UnitIter &p0, UnitIter &p,
const LimitIter &limit) {
776 constexpr bool isMultiPass = prv::forward_iterator<UnitIter>;
778 CP32 c =
static_cast<CP32
>(*p);
781 if constexpr (isMultiPass) {
782 return {c, 1,
true, p0, p};
791 if constexpr (isMultiPass) {
792 return {c, 2,
true, p0, p};
797 if constexpr (isMultiPass) {
798 return {sub(c), 1,
false, p0, p};
800 return {sub(c), 1,
false};
806 U_FORCE_INLINE static CodeUnits<CP32, UnitIter> decAndRead(UnitIter start, UnitIter &p) {
809 CP32 c =
static_cast<CP32
>(*--p);
811 return {c, 1,
true, p, p0};
818 return {c, 2,
true, p, p0};
820 return {sub(c), 1,
false, p, p0};
827template<
typename CP32, UTFIllFormedBehavior behavior,
typename UnitIter,
typename LimitIter>
831 std::enable_if_t<sizeof(typename prv::iter_value_t<UnitIter>) == 4>> {
832 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
835 U_FORCE_INLINE static CP32 sub(
bool forSurrogate, CP32 surrogate) {
842 return forSurrogate ? surrogate : 0xfffd;
855 UnitIter &p0, UnitIter &p,
const LimitIter &) {
856 constexpr bool isMultiPass = prv::forward_iterator<UnitIter>;
860 if (uc < 0xd800 || (0xe000 <= uc && uc <= 0x10ffff)) {
861 if constexpr (isMultiPass) {
862 return {c, 1,
true, p0, p};
867 if constexpr (isMultiPass) {
868 return {sub(uc < 0xe000, c), 1,
false, p0, p};
870 return {sub(uc < 0xe000, c), 1,
false};
875 U_FORCE_INLINE static CodeUnits<CP32, UnitIter> decAndRead(UnitIter , UnitIter &p) {
879 if (uc < 0xd800 || (0xe000 <= uc && uc <= 0x10ffff)) {
880 return {c, 1,
true, p, p0};
882 return {sub(uc < 0xe000, c), 1,
false, p, p0};
889template<
typename CP32,
typename UnitIter,
typename =
void>
893template<
typename CP32,
typename UnitIter>
897 std::enable_if_t<sizeof(typename prv::iter_value_t<UnitIter>) == 1>> {
898 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
911 U_FORCE_INLINE static UnsafeCodeUnits<CP32, UnitIter> readAndInc(UnitIter &p0, UnitIter &p) {
912 constexpr bool isMultiPass = prv::forward_iterator<UnitIter>;
914 CP32 c = uint8_t(*p);
917 if constexpr (isMultiPass) {
918 return {c, 1, p0, p};
922 }
else if (c < 0xe0) {
923 c = ((c & 0x1f) << 6) | (*p & 0x3f);
925 if constexpr (isMultiPass) {
926 return {c, 2, p0, p};
930 }
else if (c < 0xf0) {
933 c = uint16_t(c << 12) | ((*p & 0x3f) << 6);
937 if constexpr (isMultiPass) {
938 return {c, 3, p0, p};
943 c = ((c & 7) << 18) | ((*p & 0x3f) << 12);
945 c |= (*p & 0x3f) << 6;
949 if constexpr (isMultiPass) {
950 return {c, 4, p0, p};
957 U_FORCE_INLINE static UnsafeCodeUnits<CP32, UnitIter> decAndRead(UnitIter &p) {
960 CP32 c = uint8_t(*--p);
962 return {c, 1, p, p0};
967 for (uint8_t shift = 6;;) {
971 c |= uint32_t{b} << shift;
974 c |= (uint32_t{b} & 0x3f) << shift;
980 return {c, count, p, p0};
985template<
typename CP32,
typename UnitIter>
989 std::enable_if_t<sizeof(typename prv::iter_value_t<UnitIter>) == 2>> {
990 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
1008 U_FORCE_INLINE static UnsafeCodeUnits<CP32, UnitIter> readAndInc(UnitIter &p0, UnitIter &p) {
1009 constexpr bool isMultiPass = prv::forward_iterator<UnitIter>;
1011 CP32 c =
static_cast<CP32
>(*p);
1014 if constexpr (isMultiPass) {
1015 return {c, 1, p0, p};
1023 if constexpr (isMultiPass) {
1024 return {c, 2, p0, p};
1031 U_FORCE_INLINE static UnsafeCodeUnits<CP32, UnitIter> decAndRead(UnitIter &p) {
1034 CP32 c =
static_cast<CP32
>(*--p);
1036 return {c, 1, p, p0};
1040 return {c, 2, p, p0};
1046template<
typename CP32,
typename UnitIter>
1050 std::enable_if_t<sizeof(typename prv::iter_value_t<UnitIter>) == 4>> {
1051 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
1061 U_FORCE_INLINE static UnsafeCodeUnits<CP32, UnitIter> readAndInc(UnitIter &p0, UnitIter &p) {
1062 constexpr bool isMultiPass = prv::forward_iterator<UnitIter>;
1065 if constexpr (isMultiPass) {
1066 return {c, 1, p0, p};
1072 U_FORCE_INLINE static UnsafeCodeUnits<CP32, UnitIter> decAndRead(UnitIter &p) {
1075 return {c, 1, p, p0};
1107 typename UnitIter,
typename LimitIter = UnitIter,
typename =
void>
1109 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
1110 using Impl = UTFImpl<CP32, behavior, UnitIter, LimitIter>;
1134 prv::bidirectional_iterator<UnitIter>,
1135 std::bidirectional_iterator_tag,
1136 std::forward_iterator_tag>;
1152 p_(p), start_(start), limit_(limit), units_(0, 0, false, p, p) {}
1165 p_(p), start_(p), limit_(limit), units_(0, 0, false, p, p) {}
1220 !std::is_same_v<Sentinel, UTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
1223 return iter.
base() == s;
1226#if U_CPLUSPLUS_VERSION < 20
1239 !std::is_same_v<Sentinel, UTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
1242 return iter.
base() == s;
1252 !std::is_same_v<Sentinel, UTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
1263 !std::is_same_v<Sentinel, UTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
1277 return state_ <= 0 ? p_ : units_.begin();
1289 units_ = Impl::readAndInc(p0, p_, limit_);
1306 units_ = Impl::readAndInc(p0, p_, limit_);
1309 return Proxy(units_);
1322 }
else if (state_ == 0) {
1323 Impl::inc(p_, limit_);
1346 }
else if (state_ == 0) {
1348 units_ = Impl::readAndInc(p0, p_, limit_);
1369 template <
typename Iter = UnitIter>
1371 std::enable_if_t<prv::bidirectional_iterator<Iter>,
UTFIterator&>
1375 p_ = units_.begin();
1377 units_ = Impl::decAndRead(start_, p_);
1389 template<
typename Iter = UnitIter>
1391 std::enable_if_t<prv::bidirectional_iterator<Iter>,
UTFIterator>
1399 friend class std::reverse_iterator<
UTFIterator<CP32, behavior, UnitIter>>;
1402 mutable UnitIter p_;
1414 mutable int8_t state_ = 0;
1419template<
typename CP32, UTFIllFormedBehavior behavior,
typename UnitIter,
typename LimitIter>
1422 UnitIter, LimitIter,
1423 std::enable_if_t<!prv::forward_iterator<UnitIter>>> {
1424 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
1425 using Impl = UTFImpl<CP32, behavior, UnitIter, LimitIter>;
1432 explicit Proxy(CodeUnits<CP32, UnitIter> &units) : units_(units) {}
1433 CodeUnits<CP32, UnitIter>& operator*() U_LIFETIME_BOUND {
return units_; }
1434 CodeUnits<CP32, UnitIter>* operator->() U_LIFETIME_BOUND {
return &units_; }
1436 CodeUnits<CP32, UnitIter> units_;
1440 using value_type = CodeUnits<CP32, UnitIter>;
1459 return p_ == other.p_ && ahead_ == other.ahead_;
1467 !std::is_same_v<Sentinel, UTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
1470 return !iter.ahead_ && iter.p_ == s;
1473#if U_CPLUSPLUS_VERSION < 20
1476 !std::is_same_v<Sentinel, UTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
1479 return !iter.ahead_ && iter.p_ == s;
1484 !std::is_same_v<Sentinel, UTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
1490 !std::is_same_v<Sentinel, UTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
1497 units_ = Impl::readAndInc(p_, p_, limit_);
1505 units_ = Impl::readAndInc(p_, p_, limit_);
1508 return Proxy(units_);
1516 Impl::inc(p_, limit_);
1526 units_ = Impl::readAndInc(p_, p_, limit_);
1529 return Proxy(units_);
1534 mutable UnitIter p_;
1540 mutable CodeUnits<CP32, UnitIter> units_ = {0, 0,
false};
1544 mutable bool ahead_ =
false;
1554template<
typename CP32, UTFIllFormedBehavior behavior,
typename UnitIter>
1555class std::reverse_iterator<U_HEADER_ONLY_NAMESPACE::UTFIterator<CP32, behavior, UnitIter>> {
1556 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
1557 using Impl = U_HEADER_ONLY_NAMESPACE::UTFImpl<CP32, behavior, UnitIter>;
1564 explicit Proxy(CodeUnits_ units) : units_(units) {}
1565 CodeUnits_& operator*() U_LIFETIME_BOUND {
return units_; }
1566 CodeUnits_* operator->() U_LIFETIME_BOUND {
return &units_; }
1572 using value_type = CodeUnits_;
1573 using reference = value_type;
1574 using pointer = Proxy;
1576 using iterator_category = std::bidirectional_iterator_tag;
1579 p_(iter.base()), start_(iter.start_), limit_(iter.limit_),
1580 units_(0, 0, false, p_, p_) {}
1581 U_FORCE_INLINE reverse_iterator() : p_{}, start_{}, limit_{}, units_(0, 0, false, p_, p_) {}
1583 U_FORCE_INLINE reverse_iterator(reverse_iterator &&src)
noexcept =
default;
1584 U_FORCE_INLINE reverse_iterator &operator=(reverse_iterator &&src)
noexcept =
default;
1586 U_FORCE_INLINE reverse_iterator(
const reverse_iterator &other) =
default;
1587 U_FORCE_INLINE reverse_iterator &operator=(
const reverse_iterator &other) =
default;
1590 return getLogicalPosition() == other.getLogicalPosition();
1596 start_, getLogicalPosition(), limit_);
1601 units_ = Impl::decAndRead(start_, p_);
1609 units_ = Impl::decAndRead(start_, p_);
1612 return Proxy(units_);
1619 }
else if (state_ == 0) {
1620 Impl::dec(start_, p_);
1623 p_ = units_.begin();
1632 reverse_iterator result(*
this);
1635 }
else if (state_ == 0) {
1636 units_ = Impl::decAndRead(start_, p_);
1637 reverse_iterator result(*
this);
1642 reverse_iterator result(*
this);
1644 p_ = units_.begin();
1656 units_ = Impl::readAndInc(p0, p_, limit_);
1662 reverse_iterator result(*
this);
1669 return state_ >= 0 ? p_ : units_.end();
1673 mutable UnitIter p_;
1680 mutable CodeUnits_ units_;
1685 mutable int8_t state_ = 0;
1689namespace U_HEADER_ONLY_NAMESPACE {
1714 typename UnitIter,
typename LimitIter = UnitIter>
1715auto utfIterator(UnitIter start, UnitIter p, LimitIter limit) {
1716 return UTFIterator<CP32, behavior, UnitIter, LimitIter>(
1717 std::move(start), std::move(p), std::move(limit));
1741 typename UnitIter,
typename LimitIter = UnitIter>
1742auto utfIterator(UnitIter p, LimitIter limit) {
1743 return UTFIterator<CP32, behavior, UnitIter, LimitIter>(
1744 std::move(p), std::move(limit));
1771template<
typename CP32, UTFIllFormedBehavior behavior,
typename UnitIter>
1772auto utfIterator(UnitIter p) {
1773 return UTFIterator<CP32, behavior, UnitIter>(std::move(p));
1803template<
typename CP32, UTFIllFormedBehavior behavior,
typename Range>
1804#if defined(__cpp_lib_ranges)
1805 requires std::ranges::range<Range>
1807class UTFStringCodePoints {
1808 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
1814 UTFStringCodePoints() =
default;
1821 template<
typename R = Range,
typename = std::enable_if_t<!std::is_reference_v<R>>>
1822 explicit UTFStringCodePoints(Range unitRange) : unitRange(std::move(unitRange)) {}
1831 template<
typename R = Range,
typename = std::enable_if_t<std::is_reference_v<R>>,
typename =
void>
1832 explicit UTFStringCodePoints(Range unitRange U_LIFETIME_BOUND) : unitRange(unitRange) {}
1835 UTFStringCodePoints(
const UTFStringCodePoints &other) =
default;
1838 UTFStringCodePoints &operator=(
const UTFStringCodePoints &other) =
default;
1845 return utfIterator<CP32, behavior>(unitRange.begin(), unitRange.end());
1852 template<
typename R = Range,
typename = std::enable_if_t<prv::range<const R>>>
1853 auto begin()
const {
1854 return utfIterator<CP32, behavior>(unitRange.begin(), unitRange.end());
1862 using UnitIter =
decltype(unitRange.begin());
1863 using LimitIter =
decltype(unitRange.end());
1864 if constexpr (!std::is_same_v<UnitIter, LimitIter>) {
1866 return unitRange.end();
1867 }
else if constexpr (prv::bidirectional_iterator<UnitIter>) {
1868 return utfIterator<CP32, behavior>(unitRange.begin(), unitRange.end(), unitRange.end());
1871 return utfIterator<CP32, behavior>(unitRange.end(), unitRange.end());
1879 template<
typename R = Range,
typename = std::enable_if_t<prv::range<const R>>>
1881 using UnitIter =
decltype(unitRange.begin());
1882 using LimitIter =
decltype(unitRange.end());
1883 if constexpr (!std::is_same_v<UnitIter, LimitIter>) {
1885 return unitRange.end();
1886 }
else if constexpr (prv::bidirectional_iterator<UnitIter>) {
1887 return utfIterator<CP32, behavior>(unitRange.begin(), unitRange.end(), unitRange.end());
1890 return utfIterator<CP32, behavior>(unitRange.end(), unitRange.end());
1898 auto rbegin()
const {
1899 return std::make_reverse_iterator(end());
1907 return std::make_reverse_iterator(begin());
1917 auto front()
const {
1937template<
typename CP32, UTFIllFormedBehavior behavior>
1938struct UTFStringCodePointsAdaptor
1940 __cpp_lib_bind_back >= 2022'02
1941 : std::ranges::range_adaptor_closure<UTFStringCodePointsAdaptor<CP32, behavior>>
1945 template<
typename Range>
1946 auto operator()(Range &&unitRange)
const {
1947#if defined(__cpp_lib_ranges) && __cpp_lib_ranges >= 2021'10
1948 return UTFStringCodePoints<CP32, behavior, std::ranges::views::all_t<Range>>(
1949 std::forward<Range>(unitRange));
1951 if constexpr (prv::is_basic_string_view_v<std::decay_t<Range>>) {
1954 return UTFStringCodePoints<CP32, behavior, std::decay_t<Range>>(
1955 std::forward<Range>(unitRange));
1957 return UTFStringCodePoints<CP32, behavior, Range>(std::forward<Range>(unitRange));
1977template<
typename CP32, UTFIllFormedBehavior behavior>
1978constexpr UTFStringCodePointsAdaptor<CP32, behavior> utfStringCodePoints;
2003template<
typename CP32,
typename UnitIter,
typename =
void>
2004class UnsafeUTFIterator {
2005 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
2006 using Impl = UnsafeUTFImpl<CP32, UnitIter>;
2012 explicit Proxy(UnsafeCodeUnits<CP32, UnitIter> &units) : units_(units) {}
2013 UnsafeCodeUnits<CP32, UnitIter>& operator*() U_LIFETIME_BOUND {
return units_; }
2014 UnsafeCodeUnits<CP32, UnitIter>* operator->() U_LIFETIME_BOUND {
return &units_; }
2016 UnsafeCodeUnits<CP32, UnitIter> units_;
2021 using value_type = UnsafeCodeUnits<CP32, UnitIter>;
2023 using reference = value_type;
2025 using pointer = Proxy;
2027 using difference_type = prv::iter_difference_t<UnitIter>;
2029 using iterator_category = std::conditional_t<
2030 prv::bidirectional_iterator<UnitIter>,
2031 std::bidirectional_iterator_tag,
2032 std::forward_iterator_tag>;
2043 U_FORCE_INLINE explicit UnsafeUTFIterator(UnitIter p) : p_(p), units_(0, 0, p, p) {}
2049 U_FORCE_INLINE UnsafeUTFIterator() : p_{}, units_(0, 0, p_, p_) {}
2052 U_FORCE_INLINE UnsafeUTFIterator(UnsafeUTFIterator &&src)
noexcept =
default;
2054 U_FORCE_INLINE UnsafeUTFIterator &operator=(UnsafeUTFIterator &&src)
noexcept =
default;
2057 U_FORCE_INLINE UnsafeUTFIterator(
const UnsafeUTFIterator &other) =
default;
2059 U_FORCE_INLINE UnsafeUTFIterator &operator=(
const UnsafeUTFIterator &other) =
default;
2067 return base() == other.base();
2084 !std::is_same_v<Sentinel, UnsafeUTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
2086 operator==(
const UnsafeUTFIterator &iter,
const Sentinel &s) {
2087 return iter.base() == s;
2090#if U_CPLUSPLUS_VERSION < 20
2099 !std::is_same_v<Sentinel, UnsafeUTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
2101 operator==(
const Sentinel &s,
const UnsafeUTFIterator &iter) {
2102 return iter.base() == s;
2112 !std::is_same_v<Sentinel, UnsafeUTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
2114 operator!=(
const UnsafeUTFIterator &iter,
const Sentinel &s) {
return !(iter == s); }
2123 !std::is_same_v<Sentinel, UnsafeUTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
2125 operator!=(
const Sentinel &s,
const UnsafeUTFIterator &iter) {
return !(iter == s); }
2137 return state_ <= 0 ? p_ : units_.begin();
2146 U_FORCE_INLINE UnsafeCodeUnits<CP32, UnitIter> operator*()
const {
2149 units_ = Impl::readAndInc(p0, p_);
2166 units_ = Impl::readAndInc(p0, p_);
2169 return Proxy(units_);
2178 U_FORCE_INLINE UnsafeUTFIterator& operator++() U_LIFETIME_BOUND {
2182 }
else if (state_ == 0) {
2203 UnsafeUTFIterator result(*
this);
2206 }
else if (state_ == 0) {
2208 units_ = Impl::readAndInc(p0, p_);
2209 UnsafeUTFIterator result(*
this);
2214 UnsafeUTFIterator result(*
this);
2229 template <
typename Iter = UnitIter>
2231 std::enable_if_t<prv::bidirectional_iterator<Iter>, UnsafeUTFIterator&>
2232 operator--() U_LIFETIME_BOUND {
2235 p_ = units_.begin();
2237 units_ = Impl::decAndRead(p_);
2249 template<
typename Iter = UnitIter>
2251 std::enable_if_t<prv::bidirectional_iterator<Iter>, UnsafeUTFIterator>
2253 UnsafeUTFIterator result(*
this);
2259 friend class std::reverse_iterator<UnsafeUTFIterator<CP32, UnitIter>>;
2262 mutable UnitIter p_;
2265 mutable UnsafeCodeUnits<CP32, UnitIter> units_;
2270 mutable int8_t state_ = 0;
2275template<
typename CP32,
typename UnitIter>
2276class UnsafeUTFIterator<
2279 std::enable_if_t<!prv::forward_iterator<UnitIter>>> {
2280 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
2281 using Impl = UnsafeUTFImpl<CP32, UnitIter>;
2288 explicit Proxy(UnsafeCodeUnits<CP32, UnitIter> &units) : units_(units) {}
2289 UnsafeCodeUnits<CP32, UnitIter>& operator*() U_LIFETIME_BOUND {
return units_; }
2290 UnsafeCodeUnits<CP32, UnitIter>* operator->() U_LIFETIME_BOUND {
return &units_; }
2292 UnsafeCodeUnits<CP32, UnitIter> units_;
2296 using value_type = UnsafeCodeUnits<CP32, UnitIter>;
2297 using reference = value_type;
2298 using pointer = Proxy;
2299 using difference_type = prv::iter_difference_t<UnitIter>;
2300 using iterator_category = std::input_iterator_tag;
2302 U_FORCE_INLINE explicit UnsafeUTFIterator(UnitIter p) : p_(std::move(p)) {}
2304 U_FORCE_INLINE UnsafeUTFIterator(UnsafeUTFIterator &&src)
noexcept =
default;
2305 U_FORCE_INLINE UnsafeUTFIterator &operator=(UnsafeUTFIterator &&src)
noexcept =
default;
2307 U_FORCE_INLINE UnsafeUTFIterator(
const UnsafeUTFIterator &other) =
default;
2308 U_FORCE_INLINE UnsafeUTFIterator &operator=(
const UnsafeUTFIterator &other) =
default;
2311 return p_ == other.p_ && ahead_ == other.ahead_;
2319 !std::is_same_v<Sentinel, UnsafeUTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
2321 operator==(
const UnsafeUTFIterator &iter,
const Sentinel &s) {
2322 return !iter.ahead_ && iter.p_ == s;
2325#if U_CPLUSPLUS_VERSION < 20
2328 !std::is_same_v<Sentinel, UnsafeUTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
2330 operator==(
const Sentinel &s,
const UnsafeUTFIterator &iter) {
2331 return !iter.ahead_ && iter.p_ == s;
2336 !std::is_same_v<Sentinel, UnsafeUTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
2338 operator!=(
const UnsafeUTFIterator &iter,
const Sentinel &s) {
return !(iter == s); }
2342 !std::is_same_v<Sentinel, UnsafeUTFIterator> && !std::is_same_v<Sentinel, UnitIter>,
2344 operator!=(
const Sentinel &s,
const UnsafeUTFIterator &iter) {
return !(iter == s); }
2347 U_FORCE_INLINE UnsafeCodeUnits<CP32, UnitIter> operator*()
const {
2349 units_ = Impl::readAndInc(p_, p_);
2357 units_ = Impl::readAndInc(p_, p_);
2360 return Proxy(units_);
2363 U_FORCE_INLINE UnsafeUTFIterator& operator++() U_LIFETIME_BOUND {
2378 units_ = Impl::readAndInc(p_, p_);
2381 return Proxy(units_);
2386 mutable UnitIter p_;
2389 mutable UnsafeCodeUnits<CP32, UnitIter> units_ = {0, 0};
2393 mutable bool ahead_ =
false;
2403template<
typename CP32,
typename UnitIter>
2404class std::reverse_iterator<U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator<CP32, UnitIter>> {
2405 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
2406 using Impl = U_HEADER_ONLY_NAMESPACE::UnsafeUTFImpl<CP32, UnitIter>;
2413 explicit Proxy(UnsafeCodeUnits_ units) : units_(units) {}
2414 UnsafeCodeUnits_& operator*() U_LIFETIME_BOUND {
return units_; }
2415 UnsafeCodeUnits_* operator->() U_LIFETIME_BOUND {
return &units_; }
2417 UnsafeCodeUnits_ units_;
2421 using value_type = UnsafeCodeUnits_;
2422 using reference = value_type;
2423 using pointer = Proxy;
2425 using iterator_category = std::bidirectional_iterator_tag;
2427 U_FORCE_INLINE explicit reverse_iterator(U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator<CP32, UnitIter> iter) :
2428 p_(iter.base()), units_(0, 0, p_, p_) {}
2429 U_FORCE_INLINE reverse_iterator() : p_{}, units_(0, 0, p_, p_) {}
2431 U_FORCE_INLINE reverse_iterator(reverse_iterator &&src)
noexcept =
default;
2432 U_FORCE_INLINE reverse_iterator &operator=(reverse_iterator &&src)
noexcept =
default;
2434 U_FORCE_INLINE reverse_iterator(
const reverse_iterator &other) =
default;
2435 U_FORCE_INLINE reverse_iterator &operator=(
const reverse_iterator &other) =
default;
2438 return getLogicalPosition() == other.getLogicalPosition();
2442 U_FORCE_INLINE U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator<CP32, UnitIter> base()
const {
2443 return U_HEADER_ONLY_NAMESPACE::UnsafeUTFIterator<CP32, UnitIter>(
2444 getLogicalPosition());
2449 units_ = Impl::decAndRead(p_);
2457 units_ = Impl::decAndRead(p_);
2460 return Proxy(units_);
2467 }
else if (state_ == 0) {
2471 p_ = units_.begin();
2480 reverse_iterator result(*
this);
2483 }
else if (state_ == 0) {
2484 units_ = Impl::decAndRead(p_);
2485 reverse_iterator result(*
this);
2490 reverse_iterator result(*
this);
2492 p_ = units_.begin();
2504 units_ = Impl::readAndInc(p0, p_);
2510 reverse_iterator result(*
this);
2517 return state_ >= 0 ? p_ : units_.end();
2521 mutable UnitIter p_;
2524 mutable UnsafeCodeUnits_ units_;
2529 mutable int8_t state_ = 0;
2533namespace U_HEADER_ONLY_NAMESPACE {
2550template<
typename CP32,
typename UnitIter>
2551auto unsafeUTFIterator(UnitIter iter) {
2552 return UnsafeUTFIterator<CP32, UnitIter>(std::move(iter));
2582template<
typename CP32,
typename Range>
2583#if defined(__cpp_lib_ranges)
2584 requires std::ranges::range<Range>
2586class UnsafeUTFStringCodePoints {
2587 static_assert(
sizeof(CP32) == 4,
"CP32 must be a 32-bit type to hold a code point");
2593 UnsafeUTFStringCodePoints() =
default;
2600 template<
typename R = Range,
typename = std::enable_if_t<!std::is_reference_v<R>>>
2601 explicit UnsafeUTFStringCodePoints(Range unitRange) : unitRange(std::move(unitRange)) {}
2610 template<
typename R = Range,
typename = std::enable_if_t<std::is_reference_v<R>>,
typename =
void>
2611 explicit UnsafeUTFStringCodePoints(Range unitRange U_LIFETIME_BOUND) : unitRange(unitRange) {}
2614 UnsafeUTFStringCodePoints(
const UnsafeUTFStringCodePoints &other) =
default;
2617 UnsafeUTFStringCodePoints &operator=(
const UnsafeUTFStringCodePoints &other) =
default;
2624 return unsafeUTFIterator<CP32>(unitRange.begin());
2631 template<
typename R = Range,
typename = std::enable_if_t<prv::range<const R>>>
2632 auto begin()
const {
2633 return unsafeUTFIterator<CP32>(unitRange.begin());
2641 using UnitIter =
decltype(unitRange.begin());
2642 using LimitIter =
decltype(unitRange.end());
2643 if constexpr (!std::is_same_v<UnitIter, LimitIter>) {
2645 return unitRange.end();
2647 return unsafeUTFIterator<CP32>(unitRange.end());
2655 template<
typename R = Range,
typename = std::enable_if_t<prv::range<const R>>>
2657 using UnitIter =
decltype(unitRange.begin());
2658 using LimitIter =
decltype(unitRange.end());
2659 if constexpr (!std::is_same_v<UnitIter, LimitIter>) {
2661 return unitRange.end();
2663 return unsafeUTFIterator<CP32>(unitRange.end());
2671 auto rbegin()
const {
2672 return std::make_reverse_iterator(end());
2680 return std::make_reverse_iterator(begin());
2690 auto front()
const {
2710template<
typename CP32>
2711struct UnsafeUTFStringCodePointsAdaptor
2713 __cpp_lib_bind_back >= 2022'02
2714 : std::ranges::range_adaptor_closure<UnsafeUTFStringCodePointsAdaptor<CP32>>
2718 template<
typename Range>
2719 auto operator()(Range &&unitRange)
const {
2720#if defined(__cpp_lib_ranges) && __cpp_lib_ranges >= 2021'10
2721 return UnsafeUTFStringCodePoints<CP32, std::ranges::views::all_t<Range>>(std::forward<Range>(unitRange));
2723 if constexpr (prv::is_basic_string_view_v<std::decay_t<Range>>) {
2726 return UnsafeUTFStringCodePoints<CP32, std::decay_t<Range>>(std::forward<Range>(unitRange));
2728 return UnsafeUTFStringCodePoints<CP32, Range>(std::forward<Range>(unitRange));
2747template<
typename CP32>
2748constexpr UnsafeUTFStringCodePointsAdaptor<CP32> unsafeUTFStringCodePoints;
2753#if defined(__cpp_lib_ranges)
2754template <
typename CP32, UTFIllFormedBehavior behavior,
typename Range>
2755constexpr bool std::ranges::enable_borrowed_range<
2756 U_HEADER_ONLY_NAMESPACE::UTFStringCodePoints<CP32, behavior, Range>> =
2757 std::ranges::enable_borrowed_range<Range>;
2759template <
typename CP32,
typename Range>
2760constexpr bool std::ranges::enable_borrowed_range<
2761 U_HEADER_ONLY_NAMESPACE::UnsafeUTFStringCodePoints<CP32, Range>> =
2762 std::ranges::enable_borrowed_range<Range>;
U_COMMON_API UBool operator==(const StringPiece &x, const StringPiece &y)
Global operator == for StringPiece.
bool operator!=(const StringPiece &x, const StringPiece &y)
Global operator != for StringPiece.
#define U_SENTINEL
This value is intended for sentinel values for APIs that (take or) return single code points (UChar32...
#define U_FORCE_INLINE
Forces function inlining on compilers that are known to support it.
C API: 16-bit Unicode handling macros.
#define U16_IS_SURROGATE_TRAIL(c)
Assuming c is a surrogate code point (U16_IS_SURROGATE(c)), is it a trail surrogate?
#define U16_IS_SURROGATE_LEAD(c)
Assuming c is a surrogate code point (U16_IS_SURROGATE(c)), is it a lead surrogate?
#define U16_GET_SUPPLEMENTARY(lead, trail)
Get a supplementary code point value (U+10000..U+10ffff) from its lead and trail surrogates.
#define U16_IS_SURROGATE(c)
Is this code unit a surrogate (U+d800..U+dfff)?
#define U16_IS_LEAD(c)
Is this code unit a lead surrogate (U+d800..U+dbff)?
#define U16_IS_TRAIL(c)
Is this code unit a trail surrogate (U+dc00..U+dfff)?
C API: 8-bit Unicode handling macros.
#define U8_COUNT_TRAIL_BYTES_UNSAFE(leadByte)
Counts the trail bytes for a UTF-8 lead byte of a valid UTF-8 sequence.
#define U8_IS_VALID_LEAD3_AND_T1(lead, t1)
Internal 3-byte UTF-8 validity check.
#define U8_IS_VALID_LEAD4_AND_T1(lead, t1)
Internal 4-byte UTF-8 validity check.
#define U8_IS_SINGLE(c)
Does this code unit (byte) encode a code point by itself (US-ASCII 0..0x7f)?
#define U8_LEAD3_T1_BITS
Internal bit vector for 3-byte UTF-8 validity check, for use in U8_IS_VALID_LEAD3_AND_T1.
#define U8_LEAD4_T1_BITS
Internal bit vector for 4-byte UTF-8 validity check, for use in U8_IS_VALID_LEAD4_AND_T1.
#define U8_IS_LEAD(c)
Is this code unit (byte) a UTF-8 lead byte? (0xC2..0xF4)
#define U8_MASK_LEAD_BYTE(leadByte, countTrailBytes)
Mask a UTF-8 lead byte, leave only the lower bits that form part of the code point value.
#define U8_IS_TRAIL(c)
Is this code unit (byte) a UTF-8 trail byte? (0x80..0xBF)
typename std::iterator_traits< Iter >::difference_type iter_difference_t
constexpr bool is_basic_string_view_v
constexpr bool forward_iterator
typename std::iterator_traits< Iter >::value_type iter_value_t
constexpr bool bidirectional_iterator
UTFIllFormedBehavior
Some defined behaviors for handling ill-formed Unicode strings.
@ UTF_BEHAVIOR_FFFD
Returns U+FFFD Replacement Character.
@ UTF_BEHAVIOR_SURROGATE
UTF-8: Not allowed; UTF-16: returns the unpaired surrogate; UTF-32: returns the surrogate code point,...
@ UTF_BEHAVIOR_NEGATIVE
Returns a negative value (-1=U_SENTINEL) instead of a code point.
Basic definitions for ICU, for both C and C++ APIs.
C API: API for accessing ICU version numbers.