Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix regex tokenizing bug #131

Merged
merged 1 commit into from
Sep 24, 2019
+30 −37
Merged

Fix regex tokenizing bug #131

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions stl/inc/execution
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,6 @@ _STL_DISABLE_CLANG_WARNINGS
#pragma push_macro("new")
#undef new

// TRANSITION, Several names herein are declared as _Xxx_xxx2; in all such cases the corresponding
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For folks reading, these comments were transformed into tests that aren't yet present here in GitHub (I'm working on it right now, sorry!).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For the folks wondering why it matters: ABI.

When we change the relationship between an internal function _Fn and public function fn, we have to change the name of _Fn to avoid "old" fn in already-compiled code "in the wild" linking to freshly-compiled code and consequently calling the "new" _Fn with catastrophic results. Ditto "new" fn must not call "old" _Fn. When the change touches the function parameters, generally the mangled name of _Fn is different. If the change doesn't alter the function parameters, we'll change the name of _Fn directly, say by adding a 2.

When the name is changed directly the old name _Fn becomes available. We could then inadvertently reuse it for something else, with potentially the same catastrophic results. Rather than simply keeping a list of names to avoid - which people would have to check when writing new code - we have a test that crashes if such a name is reused.

// _Xxx_xxx name is an ABI zombie name

_EXTERN_C
// If on Windows XP, returns 1 (disabling parallelism); otherwise, returns the number of hardware threads available.
_NODISCARD unsigned int __stdcall __std_parallel_algorithms_hw_threads() noexcept;
Expand Down Expand Up @@ -2901,7 +2898,6 @@ struct _Bottom_up_tree_visitor {
}
};

// TRANSITION, _Static_partitioned_stable_sort2 is an ABI zombie name
template <class _BidIt, class _Pr>
struct _Static_partitioned_stable_sort3 {
using _Diff = _Iter_diff_t<_BidIt>;
Expand Down
1 change: 0 additions & 1 deletion stl/inc/list
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -576,7 +576,6 @@ public:
};

// STRUCT TEMPLATE _List_node_emplace_op2
// _List_node_emplace_op is an ABI zombie name
template <class _Alnode>
struct _List_node_emplace_op2 : _Alloc_construct_ptr<_Alnode> {
using _Alnode_traits = allocator_traits<_Alnode>;
Expand Down
7 changes: 0 additions & 7 deletions stl/inc/memory
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ _STL_DISABLE_CLANG_WARNINGS

_STD_BEGIN
// FUNCTION TEMPLATE uninitialized_copy_n
// TRANSITION: _Uninitialized_copy_n_unchecked and _Uninitialized_copy_n_unchecked1 are ABI zombie names
#if _HAS_IF_CONSTEXPR
template <class _InIt, class _Diff, class _NoThrowFwdIt>
_NoThrowFwdIt uninitialized_copy_n(const _InIt _First, const _Diff _Count_raw, _NoThrowFwdIt _Dest) {
Expand Down Expand Up @@ -142,7 +141,6 @@ _OutTy* uninitialized_move(const _InIt _First, const _InIt _Last, _OutTy (&_Dest
#endif // _ITERATOR_DEBUG_ARRAY_OVERLOADS

// FUNCTION TEMPLATE uninitialized_move_n
// TRANSITION, _Uninitialized_move_n_unchecked is an ABI zombie name
#if _HAS_IF_CONSTEXPR
template <class _InIt, class _Diff, class _NoThrowFwdIt>
pair<_InIt, _NoThrowFwdIt> uninitialized_move_n(_InIt _First, const _Diff _Count_raw, _NoThrowFwdIt _Dest) {
Expand Down Expand Up @@ -251,7 +249,6 @@ pair<_InTy*, _OutTy*> uninitialized_move_n(_InTy (&_First)[_InSize], const _Diff
#endif // _HAS_CXX17

// FUNCTION TEMPLATE uninitialized_fill_n
// TRANSITION: _Uninitialized_fill_n_unchecked is an ABI zombie name
#if _HAS_IF_CONSTEXPR
template <class _NoThrowFwdIt, class _Diff, class _Tval>
_NoThrowFwdIt uninitialized_fill_n(_NoThrowFwdIt _First, const _Diff _Count_raw, const _Tval& _Val) {
Expand Down Expand Up @@ -1480,8 +1477,6 @@ template <class _Dx, class _Ty>
_Dx* get_deleter(const shared_ptr<_Ty>&) noexcept = delete; // requires static RTTI
#endif // _HAS_STATIC_RTTI

// _Ref_count_obj is an ABI zombie name

// CLASS TEMPLATE _Ref_count_obj2
template <class _Ty>
class _Ref_count_obj2 : public _Ref_count_base { // handle reference counting for object in control block, no allocator
Expand Down Expand Up @@ -1549,8 +1544,6 @@ protected:
}
};

// _Ref_count_obj_alloc is an ABI zombie name

// CLASS TEMPLATE _Ref_count_obj_alloc2
template <class _Ty, class _Alloc>
class __declspec(empty_bases) _Ref_count_obj_alloc2 : public _Ebco_base<_Alloc>, public _Ref_count_base {
Expand Down
48 changes: 30 additions & 18 deletions stl/inc/regex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,8 @@ namespace regex_constants {
format_sed = 0x0400,
format_no_copy = 0x0800,
format_first_only = 0x1000,
_Match_not_null = 0x2000
_Match_not_null = 0x2000,
_Skip_zero_length = 0x4000,
};

_BITMASK_OPS(match_flag_type)
Expand Down Expand Up @@ -2213,17 +2214,21 @@ _NODISCARD bool regex_match(const basic_string<_Elem, _StTraits, _StAlloc>& _Str
_Re, _Flgs | regex_constants::match_any, true);
}

// FUNCTION TEMPLATE _Regex_search1
// FUNCTION TEMPLATE _Regex_search2
template <class _BidIt, class _Alloc, class _Elem, class _RxTraits, class _It>
bool _Regex_search1(_It _First, _It _Last, match_results<_BidIt, _Alloc>* _Matches,
const basic_regex<_Elem, _RxTraits>& _Re, regex_constants::match_flag_type _Flgs,
_It _Org) { // search for regular expression match in target text
bool _Regex_search2(_It _First, _It _Last, match_results<_BidIt, _Alloc>* _Matches,
const basic_regex<_Elem, _RxTraits>& _Re, regex_constants::match_flag_type _Flgs, _It _Org) {
// search for regular expression match in target text
if (_Re._Empty()) {
return false;
}

bool _Found = false;
_It _Begin = _First;
bool _Found = false;
const _It _Begin = _First;
if ((_Flgs & regex_constants::_Skip_zero_length) && _First != _Last) {
++_First;
}

_Matcher<_BidIt, _Elem, _RxTraits, _It> _Mx(
_First, _Last, _Re._Get_traits(), _Re._Get(), _Re.mark_count() + 1, _Re.flags(), _Flgs);

Expand Down Expand Up @@ -2258,15 +2263,15 @@ bool regex_search(_BidIt _First, _BidIt _Last, match_results<_BidIt, _Alloc>& _M
const basic_regex<_Elem, _RxTraits>& _Re, regex_constants::match_flag_type _Flgs = regex_constants::match_default) {
// search for regular expression match in target text
_Adl_verify_range(_First, _Last);
return _Regex_search1(_First, _Last, _STD addressof(_Matches), _Re, _Flgs, _First);
return _Regex_search2(_First, _Last, _STD addressof(_Matches), _Re, _Flgs, _First);
}

template <class _BidIt, class _Elem, class _RxTraits>
_NODISCARD bool regex_search(_BidIt _First, _BidIt _Last, const basic_regex<_Elem, _RxTraits>& _Re,
regex_constants::match_flag_type _Flgs = regex_constants::match_default) {
// search for regular expression match in target text
_Adl_verify_range(_First, _Last);
return _Regex_search1(_Get_unwrapped(_First), _Get_unwrapped(_Last),
return _Regex_search2(_Get_unwrapped(_First), _Get_unwrapped(_Last),
static_cast<match_results<_Unwrapped_t<_BidIt>>*>(nullptr), _Re, _Flgs | regex_constants::match_any,
_Get_unwrapped(_First));
}
Expand All @@ -2276,7 +2281,7 @@ _NODISCARD bool regex_search(_In_z_ const _Elem* _Str, const basic_regex<_Elem,
regex_constants::match_flag_type _Flgs = regex_constants::match_default) {
// search for regular expression match in target text
const _Elem* _Last = _Str + char_traits<_Elem>::length(_Str);
return _Regex_search1(
return _Regex_search2(
_Str, _Last, static_cast<match_results<const _Elem*>*>(nullptr), _Re, _Flgs | regex_constants::match_any, _Str);
}

Expand All @@ -2285,15 +2290,15 @@ bool regex_search(_In_z_ const _Elem* _Str, match_results<const _Elem*, _Alloc>&
const basic_regex<_Elem, _RxTraits>& _Re, regex_constants::match_flag_type _Flgs = regex_constants::match_default) {
// search for regular expression match in target text
const _Elem* _Last = _Str + char_traits<_Elem>::length(_Str);
return _Regex_search1(_Str, _Last, _STD addressof(_Matches), _Re, _Flgs, _Str);
return _Regex_search2(_Str, _Last, _STD addressof(_Matches), _Re, _Flgs, _Str);
}

template <class _StTraits, class _StAlloc, class _Alloc, class _Elem, class _RxTraits>
bool regex_search(const basic_string<_Elem, _StTraits, _StAlloc>& _Str,
match_results<typename basic_string<_Elem, _StTraits, _StAlloc>::const_iterator, _Alloc>& _Matches,
const basic_regex<_Elem, _RxTraits>& _Re, regex_constants::match_flag_type _Flgs = regex_constants::match_default) {
// search for regular expression match in target text
return _Regex_search1(_Str.begin(), _Str.end(), _STD addressof(_Matches), _Re, _Flgs, _Str.begin());
return _Regex_search2(_Str.begin(), _Str.end(), _STD addressof(_Matches), _Re, _Flgs, _Str.begin());
}

template <class _StTraits, class _StAlloc, class _Alloc, class _Elem, class _RxTraits>
Expand All @@ -2309,7 +2314,7 @@ _NODISCARD bool regex_search(const basic_string<_Elem, _StTraits, _StAlloc>& _St

_Iter _First = _Str.c_str();
_Iter _Last = _First + _Str.size();
return _Regex_search1(
return _Regex_search2(
_First, _Last, static_cast<match_results<_Iter>*>(nullptr), _Re, _Flgs | regex_constants::match_any, _First);
}

Expand All @@ -2324,7 +2329,7 @@ _OutIt _Regex_replace1(_OutIt _Result, _BidIt _First, _BidIt _Last, const basic_
regex_constants::match_flag_type _Not_null{};

while (
_Regex_search1(_Pos, _Last, _STD addressof(_Matches), _Re, _Flags | _Not_null, _Pos)) { // replace at each match
_Regex_search2(_Pos, _Last, _STD addressof(_Matches), _Re, _Flags | _Not_null, _Pos)) { // replace at each match
if (!(_Flgs & regex_constants::format_no_copy)) {
_Result = _STD copy(_Matches.prefix().first, _Matches.prefix().second, _Result);
}
Expand Down Expand Up @@ -2446,7 +2451,7 @@ public:
regex_constants::match_flag_type _Fl = regex_constants::match_default)
: _Begin(_First), _End(_Last), _MyRe(_STD addressof(_Re)), _Flags(_Fl) {
_Adl_verify_range(_Begin, _End);
if (!_Regex_search1(_Begin, _End, _STD addressof(_MyVal), *_MyRe, _Flags, _Begin)) {
if (!_Regex_search2(_Begin, _End, _STD addressof(_MyVal), *_MyRe, _Flags, _Begin)) {
_MyRe = nullptr;
} else {
this->_Adopt(_MyRe);
Expand Down Expand Up @@ -2498,6 +2503,7 @@ public:
_STL_VERIFY(this->_Getcont(), "regex_iterator orphaned");
#endif // _ITERATOR_DEBUG_LEVEL != 0

bool _Skip_empty_match = false;
if (_MyVal._At(0).first == _MyVal._At(0).second) { // handle zero-length match
if (_Start == _End) { // store end-of-sequence iterator
_MyRe = nullptr;
Expand All @@ -2510,17 +2516,23 @@ public:
}

// _Adl_verify_range(_Start, _End) checked in constructor
if (_Regex_search1(_Start, _End, _STD addressof(_MyVal), *_MyRe,
if (_Regex_search2(_Start, _End, _STD addressof(_MyVal), *_MyRe,
_Flags | regex_constants::match_not_null | regex_constants::match_continuous, _Begin)) {
return *this;
}

++_Start;
_Skip_empty_match = true;
}
_Flags = _Flags | regex_constants::match_prev_avail;

auto _Tmp_flags = _Flags;
if (_Skip_empty_match) {
_Tmp_flags |= regex_constants::_Skip_zero_length;
}

// _Adl_verify_range(_Start, _End) checked in constructor
if (!_Regex_search1(_Start, _End, _STD addressof(_MyVal), *_MyRe, _Flags, _Begin)) { // mark at end of sequence
if (!_Regex_search2(_Start, _End, _STD addressof(_MyVal), *_MyRe, _Tmp_flags, _Begin)) {
// mark at end of sequence
_MyRe = nullptr;
}

Expand Down
4 changes: 0 additions & 4 deletions stl/inc/vector
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -552,7 +552,6 @@ public:
}

private:
// _Move_from is an ABI zombie name
void _Move_construct(vector& _Right, true_type) noexcept { // move from _Right, stealing its contents
_Mypair._Myval2._Take_contents(_Right._Mypair._Myval2);
}
Expand Down Expand Up @@ -1574,8 +1573,6 @@ public:
}

private:
// _Udefault is an ABI zombie name

pointer _Ufill(pointer _Dest, const size_type _Count, const _Ty& _Val) {
// fill raw _Dest with _Count copies of _Val, using allocator
return _Uninitialized_fill_n(_Dest, _Count, _Val, _Getal());
Expand Down Expand Up @@ -1632,7 +1629,6 @@ private:
return _Geometric; // geometric growth is sufficient
}

// _Buy is an ABI zombie name
void _Buy_raw(const size_type _Newcapacity) {
// allocate array with _Newcapacity elements
auto& _My_data = _Mypair._Myval2;
Expand Down
1 change: 0 additions & 1 deletion stl/inc/xhash
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -999,7 +999,6 @@ public:
return _Vec.max_size() >> 1;
}

// _Hashval is an ABI zombie name
_NODISCARD size_type bucket(const key_type& _Keyval) const
noexcept(_Nothrow_hash<_Traits, key_type>) /* strengthened */ {
return _Traitsobj(_Keyval) & _Mask;
Expand Down
2 changes: 0 additions & 2 deletions stl/inc/xutility
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3397,7 +3397,6 @@ _DestTy* copy(_ExPo&&, _FwdIt1 _First, _FwdIt1 _Last, _DestTy (&_Dest)[_DestSize
#endif // _HAS_CXX17

// FUNCTION TEMPLATE copy_n
// TRANSITION: _Copy_n_unchecked, _Copy_n_unchecked1, _Copy_n_unchecked2, and _Copy_n_unchecked3 are ABI zombie names
#if _HAS_IF_CONSTEXPR
template <class _InIt, class _Diff, class _OutIt>
_OutIt copy_n(_InIt _First, _Diff _Count_raw, _OutIt _Dest) { // copy [_First, _First + _Count) to [_Dest, ...)
Expand Down Expand Up @@ -3849,7 +3848,6 @@ void fill(_ExPo&&, _FwdIt _First, _FwdIt _Last, const _Ty& _Val) noexcept /* ter
#endif // _HAS_CXX17

// FUNCTION TEMPLATE fill_n
// TRANSITION: _Fill_n_unchecked and _Fill_n_unchecked1 are ABI zombie names
#if _HAS_IF_CONSTEXPR
template <class _OutIt, class _Diff, class _Ty>
_OutIt fill_n(_OutIt _Dest, const _Diff _Count_raw, const _Ty& _Val) {
Expand Down