mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-06-28 15:28:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Upgrade to 2022-era LLVM LIBCXX

Browse source
This commit is contained in:
Justine Tunney 2024-05-27 02:12:27 -07:00
parent 2f4ca71f26
commit 8e68384e15
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
2078 changed files with 165657 additions and 65010 deletions
Download patch file Download diff file Expand all files Collapse all files

1768
third_party/libcxx/__pstl/internal/algorithm_fwd.h vendored Normal file
View file

File diff suppressed because it is too large Load diff

4174
third_party/libcxx/__pstl/internal/algorithm_impl.h vendored Normal file
View file

File diff suppressed because it is too large Load diff

76
third_party/libcxx/__pstl/internal/execution_defs.h vendored Normal file
View file

@ -0,0 +1,76 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_EXECUTION_POLICY_DEFS_H
#define _PSTL_EXECUTION_POLICY_DEFS_H
#include <__config>
#include <__type_traits/decay.h>
#include <__type_traits/enable_if.h>
#include <__type_traits/integral_constant.h>
#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
namespace __pstl {
namespace execution {
inline namespace v1 {
// 2.4, Sequential execution policy
class sequenced_policy {};
// 2.5, Parallel execution policy
class parallel_policy {};
// 2.6, Parallel+Vector execution policy
class parallel_unsequenced_policy {};
class unsequenced_policy {};
// 2.8, Execution policy objects
constexpr sequenced_policy seq{};
constexpr parallel_policy par{};
constexpr parallel_unsequenced_policy par_unseq{};
constexpr unsequenced_policy unseq{};
// 2.3, Execution policy type trait
template <class>
struct is_execution_policy : std::false_type {};
template <>
struct is_execution_policy<__pstl::execution::sequenced_policy> : std::true_type {};
template <>
struct is_execution_policy<__pstl::execution::parallel_policy> : std::true_type {};
template <>
struct is_execution_policy<__pstl::execution::parallel_unsequenced_policy> : std::true_type {};
template <>
struct is_execution_policy<__pstl::execution::unsequenced_policy> : std::true_type {};
template <class _Tp>
constexpr bool is_execution_policy_v = __pstl::execution::is_execution_policy<_Tp>::value;
} // namespace v1
} // namespace execution
namespace __internal {
template <class _ExecPolicy, class _Tp>
using __enable_if_execution_policy =
typename std::enable_if<__pstl::execution::is_execution_policy<typename std::decay<_ExecPolicy>::type>::value,
_Tp>::type;
template <class _IsVector>
struct __serial_tag;
template <class _IsVector>
struct __parallel_tag;
} // namespace __internal
} // namespace __pstl
#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
#endif /* _PSTL_EXECUTION_POLICY_DEFS_H */

97
third_party/libcxx/__pstl/internal/execution_impl.h vendored Normal file
View file

@ -0,0 +1,97 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_EXECUTION_IMPL_H
#define _PSTL_EXECUTION_IMPL_H
#include <__config>
#include <__iterator/iterator_traits.h>
#include <__type_traits/conditional.h>
#include <__type_traits/conjunction.h>
#include <__type_traits/decay.h>
#include <__type_traits/integral_constant.h>
#include <__type_traits/is_base_of.h>
#include <__pstl/internal/execution_defs.h>
#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
namespace __pstl {
namespace __internal {
template <typename _IteratorTag, typename... _IteratorTypes>
using __are_iterators_of = std::conjunction<
std::is_base_of<_IteratorTag, typename std::iterator_traits<std::decay_t<_IteratorTypes>>::iterator_category>...>;
template <typename... _IteratorTypes>
using __are_random_access_iterators = __are_iterators_of<std::random_access_iterator_tag, _IteratorTypes...>;
struct __serial_backend_tag {};
struct __tbb_backend_tag {};
struct __openmp_backend_tag {};
# if defined(_PSTL_PAR_BACKEND_TBB)
using __par_backend_tag = __tbb_backend_tag;
# elif defined(_PSTL_PAR_BACKEND_OPENMP)
using __par_backend_tag = __openmp_backend_tag;
# elif defined(_PSTL_PAR_BACKEND_SERIAL)
using __par_backend_tag = __serial_backend_tag;
# else
# error "A parallel backend must be specified";
# endif
template <class _IsVector>
struct __serial_tag {
using __is_vector = _IsVector;
};
template <class _IsVector>
struct __parallel_tag {
using __is_vector = _IsVector;
// backend tag can be change depending on
// TBB availability in the environment
using __backend_tag = __par_backend_tag;
};
template <class _IsVector, class... _IteratorTypes>
using __tag_type =
typename std::conditional<__internal::__are_random_access_iterators<_IteratorTypes...>::value,
__parallel_tag<_IsVector>,
__serial_tag<_IsVector>>::type;
template <class... _IteratorTypes>
_LIBCPP_HIDE_FROM_ABI __serial_tag</*_IsVector = */ std::false_type>
__select_backend(__pstl::execution::sequenced_policy, _IteratorTypes&&...) {
return {};
}
template <class... _IteratorTypes>
_LIBCPP_HIDE_FROM_ABI __serial_tag<__internal::__are_random_access_iterators<_IteratorTypes...>>
__select_backend(__pstl::execution::unsequenced_policy, _IteratorTypes&&...) {
return {};
}
template <class... _IteratorTypes>
_LIBCPP_HIDE_FROM_ABI __tag_type</*_IsVector = */ std::false_type, _IteratorTypes...>
__select_backend(__pstl::execution::parallel_policy, _IteratorTypes&&...) {
return {};
}
template <class... _IteratorTypes>
_LIBCPP_HIDE_FROM_ABI __tag_type<__internal::__are_random_access_iterators<_IteratorTypes...>, _IteratorTypes...>
__select_backend(__pstl::execution::parallel_unsequenced_policy, _IteratorTypes&&...) {
return {};
}
} // namespace __internal
} // namespace __pstl
#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
#endif /* _PSTL_EXECUTION_IMPL_H */

655
third_party/libcxx/__pstl/internal/glue_algorithm_defs.h vendored Normal file
View file

@ -0,0 +1,655 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_GLUE_ALGORITHM_DEFS_H
#define _PSTL_GLUE_ALGORITHM_DEFS_H
#include <__config>
#include <functional>
#include <iterator>
#include "execution_defs.h"
namespace std {
// [alg.find.end]
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
find_end(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __s_first,
_ForwardIterator2 __s_last,
_BinaryPredicate __pred);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
find_end(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __s_first,
_ForwardIterator2 __s_last);
// [alg.find_first_of]
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1> find_first_of(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __s_first,
_ForwardIterator2 __s_last,
_BinaryPredicate __pred);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
find_first_of(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __s_first,
_ForwardIterator2 __s_last);
// [alg.adjacent_find]
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
adjacent_find(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
adjacent_find(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred);
// [alg.count]
template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy,
typename iterator_traits<_ForwardIterator>::difference_type>
count(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value);
template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy,
typename iterator_traits<_ForwardIterator>::difference_type>
count_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred);
// [alg.search]
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
search(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __s_first,
_ForwardIterator2 __s_last,
_BinaryPredicate __pred);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
search(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __s_first,
_ForwardIterator2 __s_last);
template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
search_n(_ExecutionPolicy&& __exec,
_ForwardIterator __first,
_ForwardIterator __last,
_Size __count,
const _Tp& __value,
_BinaryPredicate __pred);
template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> search_n(
_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Size __count, const _Tp& __value);
// [alg.copy]
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
copy_if(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 result,
_Predicate __pred);
// [alg.swap]
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> swap_ranges(
_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2);
// [alg.replace]
template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
replace_if(_ExecutionPolicy&& __exec,
_ForwardIterator __first,
_ForwardIterator __last,
_UnaryPredicate __pred,
const _Tp& __new_value);
template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
replace(_ExecutionPolicy&& __exec,
_ForwardIterator __first,
_ForwardIterator __last,
const _Tp& __old_value,
const _Tp& __new_value);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _UnaryPredicate, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> replace_copy_if(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
_UnaryPredicate __pred,
const _Tp& __new_value);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> replace_copy(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
const _Tp& __old_value,
const _Tp& __new_value);
// [alg.generate]
template <class _ExecutionPolicy, class _ForwardIterator, class _Generator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
generate(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Generator __g);
template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Generator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
generate_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size count, _Generator __g);
// [alg.remove]
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> remove_copy_if(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
_Predicate __pred);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
remove_copy(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
const _Tp& __value);
template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
remove_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred);
template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
remove(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value);
// [alg.unique]
template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
unique(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred);
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
unique(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
unique_copy(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
_BinaryPredicate __pred);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
unique_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result);
// [alg.reverse]
template <class _ExecutionPolicy, class _BidirectionalIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
reverse(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last);
template <class _ExecutionPolicy, class _BidirectionalIterator, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
reverse_copy(_ExecutionPolicy&& __exec,
_BidirectionalIterator __first,
_BidirectionalIterator __last,
_ForwardIterator __d_first);
// [alg.rotate]
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
rotate(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
rotate_copy(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __middle,
_ForwardIterator1 __last,
_ForwardIterator2 __result);
// [alg.partitions]
template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
is_partitioned(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred);
template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
partition(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred);
template <class _ExecutionPolicy, class _BidirectionalIterator, class _UnaryPredicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _BidirectionalIterator> stable_partition(
_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last, _UnaryPredicate __pred);
template <class _ExecutionPolicy,
class _ForwardIterator,
class _ForwardIterator1,
class _ForwardIterator2,
class _UnaryPredicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
partition_copy(_ExecutionPolicy&& __exec,
_ForwardIterator __first,
_ForwardIterator __last,
_ForwardIterator1 __out_true,
_ForwardIterator2 __out_false,
_UnaryPredicate __pred);
// [alg.sort]
template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp);
template <class _ExecutionPolicy, class _RandomAccessIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last);
// [stable.sort]
template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
stable_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp);
template <class _ExecutionPolicy, class _RandomAccessIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
stable_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last);
// [mismatch]
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
mismatch(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_ForwardIterator2 __last2,
_BinaryPredicate __pred);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
mismatch(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_BinaryPredicate __pred);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
mismatch(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_ForwardIterator2 __last2);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
mismatch(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2);
// [alg.equal]
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
equal(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_BinaryPredicate __p);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
equal(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
equal(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_ForwardIterator2 __last2,
_BinaryPredicate __p);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
equal(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_ForwardIterator2 __last2);
// [alg.move]
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
move(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __d_first);
// [partial.sort]
template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
partial_sort(_ExecutionPolicy&& __exec,
_RandomAccessIterator __first,
_RandomAccessIterator __middle,
_RandomAccessIterator __last,
_Compare __comp);
template <class _ExecutionPolicy, class _RandomAccessIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
partial_sort(_ExecutionPolicy&& __exec,
_RandomAccessIterator __first,
_RandomAccessIterator __middle,
_RandomAccessIterator __last);
// [partial.sort.copy]
template <class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator, class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator> partial_sort_copy(
_ExecutionPolicy&& __exec,
_ForwardIterator __first,
_ForwardIterator __last,
_RandomAccessIterator __d_first,
_RandomAccessIterator __d_last,
_Compare __comp);
template <class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator> partial_sort_copy(
_ExecutionPolicy&& __exec,
_ForwardIterator __first,
_ForwardIterator __last,
_RandomAccessIterator __d_first,
_RandomAccessIterator __d_last);
// [is.sorted]
template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
is_sorted_until(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp);
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
is_sorted_until(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
is_sorted(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp);
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
is_sorted(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
// [alg.nth.element]
template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
nth_element(_ExecutionPolicy&& __exec,
_RandomAccessIterator __first,
_RandomAccessIterator __nth,
_RandomAccessIterator __last,
_Compare __comp);
template <class _ExecutionPolicy, class _RandomAccessIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
nth_element(_ExecutionPolicy&& __exec,
_RandomAccessIterator __first,
_RandomAccessIterator __nth,
_RandomAccessIterator __last);
// [alg.merge]
template <class _ExecutionPolicy,
class _ForwardIterator1,
class _ForwardIterator2,
class _ForwardIterator,
class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
merge(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_ForwardIterator2 __last2,
_ForwardIterator __d_first,
_Compare __comp);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
merge(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_ForwardIterator2 __last2,
_ForwardIterator __d_first);
template <class _ExecutionPolicy, class _BidirectionalIterator, class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
inplace_merge(_ExecutionPolicy&& __exec,
_BidirectionalIterator __first,
_BidirectionalIterator __middle,
_BidirectionalIterator __last,
_Compare __comp);
template <class _ExecutionPolicy, class _BidirectionalIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
inplace_merge(_ExecutionPolicy&& __exec,
_BidirectionalIterator __first,
_BidirectionalIterator __middle,
_BidirectionalIterator __last);
// [includes]
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
includes(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_ForwardIterator2 __last2,
_Compare __comp);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
includes(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_ForwardIterator2 __last2);
// [set.union]
template <class _ExecutionPolicy,
class _ForwardIterator1,
class _ForwardIterator2,
class _ForwardIterator,
class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
set_union(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_ForwardIterator2 __last2,
_ForwardIterator __result,
_Compare __comp);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
set_union(_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_ForwardIterator2 __last2,
_ForwardIterator __result);
// [set.intersection]
template <class _ExecutionPolicy,
class _ForwardIterator1,
class _ForwardIterator2,
class _ForwardIterator,
class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> set_intersection(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_ForwardIterator2 __last2,
_ForwardIterator __result,
_Compare __comp);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> set_intersection(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_ForwardIterator2 __last2,
_ForwardIterator __result);
// [set.difference]
template <class _ExecutionPolicy,
class _ForwardIterator1,
class _ForwardIterator2,
class _ForwardIterator,
class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> set_difference(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_ForwardIterator2 __last2,
_ForwardIterator __result,
_Compare __comp);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> set_difference(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_ForwardIterator2 __last2,
_ForwardIterator __result);
// [set.symmetric.difference]
template <class _ExecutionPolicy,
class _ForwardIterator1,
class _ForwardIterator2,
class _ForwardIterator,
class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> set_symmetric_difference(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_ForwardIterator2 __last2,
_ForwardIterator result,
_Compare __comp);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> set_symmetric_difference(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_ForwardIterator2 __last2,
_ForwardIterator __result);
// [is.heap]
template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator>
is_heap_until(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp);
template <class _ExecutionPolicy, class _RandomAccessIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator>
is_heap_until(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last);
template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
is_heap(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp);
template <class _ExecutionPolicy, class _RandomAccessIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
is_heap(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last);
// [alg.min.max]
template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
min_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp);
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
min_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
max_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp);
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
max_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator, _ForwardIterator>>
minmax_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp);
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator, _ForwardIterator>>
minmax_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
// [alg.lex.comparison]
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool> lexicographical_compare(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_ForwardIterator2 __last2,
_Compare __comp);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool> lexicographical_compare(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_ForwardIterator2 __last2);
} // namespace std
#endif /* _PSTL_GLUE_ALGORITHM_DEFS_H */

1081
third_party/libcxx/__pstl/internal/glue_algorithm_impl.h vendored Normal file
View file

File diff suppressed because it is too large Load diff

81
third_party/libcxx/__pstl/internal/glue_memory_defs.h vendored Normal file
View file

@ -0,0 +1,81 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_GLUE_MEMORY_DEFS_H
#define _PSTL_GLUE_MEMORY_DEFS_H
#include <__config>
#include "execution_defs.h"
namespace std {
// [uninitialized.copy]
template <class _ExecutionPolicy, class _InputIterator, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_copy(_ExecutionPolicy&& __exec, _InputIterator __first, _InputIterator __last, _ForwardIterator __result);
template <class _ExecutionPolicy, class _InputIterator, class _Size, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_copy_n(_ExecutionPolicy&& __exec, _InputIterator __first, _Size __n, _ForwardIterator __result);
// [uninitialized.move]
template <class _ExecutionPolicy, class _InputIterator, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_move(_ExecutionPolicy&& __exec, _InputIterator __first, _InputIterator __last, _ForwardIterator __result);
template <class _ExecutionPolicy, class _InputIterator, class _Size, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_move_n(_ExecutionPolicy&& __exec, _InputIterator __first, _Size __n, _ForwardIterator __result);
// [uninitialized.fill]
template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
uninitialized_fill(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value);
template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_fill_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n, const _Tp& __value);
// [specialized.destroy]
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
destroy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
destroy_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n);
// [uninitialized.construct.default]
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
uninitialized_default_construct(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_default_construct_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n);
// [uninitialized.construct.value]
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
uninitialized_value_construct(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_value_construct_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n);
} // namespace std
#endif /* _PSTL_GLUE_MEMORY_DEFS_H */

379
third_party/libcxx/__pstl/internal/glue_memory_impl.h vendored Normal file
View file

@ -0,0 +1,379 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_GLUE_MEMORY_IMPL_H
#define _PSTL_GLUE_MEMORY_IMPL_H
#include <__config>
#include "algorithm_fwd.h"
#include "execution_defs.h"
#include "utils.h"
#include "execution_impl.h"
namespace std {
// [uninitialized.copy]
template <class _ExecutionPolicy, class _InputIterator, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> uninitialized_copy(
_ExecutionPolicy&& __exec, _InputIterator __first, _InputIterator __last, _ForwardIterator __result) {
typedef typename iterator_traits<_InputIterator>::value_type _ValueType1;
typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType2;
typedef typename iterator_traits<_InputIterator>::reference _ReferenceType1;
typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType2;
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
return __pstl::__internal::__invoke_if_else(
std::integral_constant < bool,
std::is_trivial<_ValueType1>::value&& std::is_trivial<_ValueType2>::value > (),
[&]() {
return __pstl::__internal::__pattern_walk2_brick(
__dispatch_tag,
std::forward<_ExecutionPolicy>(__exec),
__first,
__last,
__result,
[](_InputIterator __begin, _InputIterator __end, _ForwardIterator __res) {
return __pstl::__internal::__brick_copy(__begin, __end, __res, __is_vector{});
});
},
[&]() {
return __pstl::__internal::__pattern_walk2(
__dispatch_tag,
std::forward<_ExecutionPolicy>(__exec),
__first,
__last,
__result,
[](_ReferenceType1 __val1, _ReferenceType2 __val2) { ::new (std::addressof(__val2)) _ValueType2(__val1); });
});
}
template <class _ExecutionPolicy, class _InputIterator, class _Size, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_copy_n(_ExecutionPolicy&& __exec, _InputIterator __first, _Size __n, _ForwardIterator __result) {
typedef typename iterator_traits<_InputIterator>::value_type _ValueType1;
typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType2;
typedef typename iterator_traits<_InputIterator>::reference _ReferenceType1;
typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType2;
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
return __pstl::__internal::__invoke_if_else(
std::integral_constant < bool,
std::is_trivial<_ValueType1>::value&& std::is_trivial<_ValueType2>::value > (),
[&]() {
return __pstl::__internal::__pattern_walk2_brick_n(
__dispatch_tag,
std::forward<_ExecutionPolicy>(__exec),
__first,
__n,
__result,
[](_InputIterator __begin, _Size __sz, _ForwardIterator __res) {
return __pstl::__internal::__brick_copy_n(__begin, __sz, __res, __is_vector{});
});
},
[&]() {
return __pstl::__internal::__pattern_walk2_n(
__dispatch_tag,
std::forward<_ExecutionPolicy>(__exec),
__first,
__n,
__result,
[](_ReferenceType1 __val1, _ReferenceType2 __val2) { ::new (std::addressof(__val2)) _ValueType2(__val1); });
});
}
// [uninitialized.move]
template <class _ExecutionPolicy, class _InputIterator, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> uninitialized_move(
_ExecutionPolicy&& __exec, _InputIterator __first, _InputIterator __last, _ForwardIterator __result) {
typedef typename iterator_traits<_InputIterator>::value_type _ValueType1;
typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType2;
typedef typename iterator_traits<_InputIterator>::reference _ReferenceType1;
typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType2;
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
return __pstl::__internal::__invoke_if_else(
std::integral_constant < bool,
std::is_trivial<_ValueType1>::value&& std::is_trivial<_ValueType2>::value > (),
[&]() {
return __pstl::__internal::__pattern_walk2_brick(
__dispatch_tag,
std::forward<_ExecutionPolicy>(__exec),
__first,
__last,
__result,
[](_InputIterator __begin, _InputIterator __end, _ForwardIterator __res) {
return __pstl::__internal::__brick_copy(__begin, __end, __res, __is_vector{});
});
},
[&]() {
return __pstl::__internal::__pattern_walk2(
__dispatch_tag,
std::forward<_ExecutionPolicy>(__exec),
__first,
__last,
__result,
[](_ReferenceType1 __val1, _ReferenceType2 __val2) {
::new (std::addressof(__val2)) _ValueType2(std::move(__val1));
});
});
}
template <class _ExecutionPolicy, class _InputIterator, class _Size, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_move_n(_ExecutionPolicy&& __exec, _InputIterator __first, _Size __n, _ForwardIterator __result) {
typedef typename iterator_traits<_InputIterator>::value_type _ValueType1;
typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType2;
typedef typename iterator_traits<_InputIterator>::reference _ReferenceType1;
typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType2;
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
return __pstl::__internal::__invoke_if_else(
std::integral_constant < bool,
std::is_trivial<_ValueType1>::value&& std::is_trivial<_ValueType2>::value > (),
[&]() {
return __pstl::__internal::__pattern_walk2_brick_n(
__dispatch_tag,
std::forward<_ExecutionPolicy>(__exec),
__first,
__n,
__result,
[](_InputIterator __begin, _Size __sz, _ForwardIterator __res) {
return __pstl::__internal::__brick_copy_n(__begin, __sz, __res, __is_vector{});
});
},
[&]() {
return __pstl::__internal::__pattern_walk2_n(
__dispatch_tag,
std::forward<_ExecutionPolicy>(__exec),
__first,
__n,
__result,
[](_ReferenceType1 __val1, _ReferenceType2 __val2) {
::new (std::addressof(__val2)) _ValueType2(std::move(__val1));
});
});
}
// [uninitialized.fill]
template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
uninitialized_fill(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
__pstl::__internal::__invoke_if_else(
std::is_arithmetic<_ValueType>(),
[&]() {
__pstl::__internal::__pattern_walk_brick(
__dispatch_tag,
std::forward<_ExecutionPolicy>(__exec),
__first,
__last,
[&__value](_ForwardIterator __begin, _ForwardIterator __end) {
__pstl::__internal::__brick_fill(__begin, __end, _ValueType(__value), __is_vector{});
});
},
[&]() {
__pstl::__internal::__pattern_walk1(
__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, [&__value](_ReferenceType __val) {
::new (std::addressof(__val)) _ValueType(__value);
});
});
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_fill_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n, const _Tp& __value) {
typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
return __pstl::__internal::__invoke_if_else(
std::is_arithmetic<_ValueType>(),
[&]() {
return __pstl::__internal::__pattern_walk_brick_n(
__dispatch_tag,
std::forward<_ExecutionPolicy>(__exec),
__first,
__n,
[&__value](_ForwardIterator __begin, _Size __count) {
return __pstl::__internal::__brick_fill_n(__begin, __count, _ValueType(__value), __is_vector{});
});
},
[&]() {
return __pstl::__internal::__pattern_walk1_n(
__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n, [&__value](_ReferenceType __val) {
::new (std::addressof(__val)) _ValueType(__value);
});
});
}
// [specialized.destroy]
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
destroy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last) {
typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
__pstl::__internal::__invoke_if_not(std::is_trivially_destructible<_ValueType>(), [&]() {
__pstl::__internal::__pattern_walk1(
__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, [](_ReferenceType __val) {
__val.~_ValueType();
});
});
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
destroy_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n) {
typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
return __pstl::__internal::__invoke_if_else(
std::is_trivially_destructible<_ValueType>(),
[&]() { return std::next(__first, __n); },
[&]() {
return __pstl::__internal::__pattern_walk1_n(
__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n, [](_ReferenceType __val) {
__val.~_ValueType();
});
});
}
// [uninitialized.construct.default]
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
uninitialized_default_construct(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last) {
typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
__pstl::__internal::__invoke_if_not(std::is_trivial<_ValueType>(), [&]() {
__pstl::__internal::__pattern_walk1(
__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, [](_ReferenceType __val) {
::new (std::addressof(__val)) _ValueType;
});
});
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_default_construct_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n) {
typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
return __pstl::__internal::__invoke_if_else(
std::is_trivial<_ValueType>(),
[&]() { return std::next(__first, __n); },
[&]() {
return __pstl::__internal::__pattern_walk1_n(
__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n, [](_ReferenceType __val) {
::new (std::addressof(__val)) _ValueType;
});
});
}
// [uninitialized.construct.value]
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
uninitialized_value_construct(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last) {
typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
__pstl::__internal::__invoke_if_else(
std::is_trivial<_ValueType>(),
[&]() {
__pstl::__internal::__pattern_walk_brick(
__dispatch_tag,
std::forward<_ExecutionPolicy>(__exec),
__first,
__last,
[](_ForwardIterator __begin, _ForwardIterator __end) {
__pstl::__internal::__brick_fill(__begin, __end, _ValueType(), __is_vector{});
});
},
[&]() {
__pstl::__internal::__pattern_walk1(
__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, [](_ReferenceType __val) {
::new (std::addressof(__val)) _ValueType();
});
});
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
uninitialized_value_construct_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n) {
typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
return __pstl::__internal::__invoke_if_else(
std::is_trivial<_ValueType>(),
[&]() {
return __pstl::__internal::__pattern_walk_brick_n(
__dispatch_tag,
std::forward<_ExecutionPolicy>(__exec),
__first,
__n,
[](_ForwardIterator __begin, _Size __count) {
return __pstl::__internal::__brick_fill_n(__begin, __count, _ValueType(), __is_vector{});
});
},
[&]() {
return __pstl::__internal::__pattern_walk1_n(
__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n, [](_ReferenceType __val) {
::new (std::addressof(__val)) _ValueType();
});
});
}
} // namespace std
#endif /* _PSTL_GLUE_MEMORY_IMPL_H */

175
third_party/libcxx/__pstl/internal/glue_numeric_defs.h vendored Normal file
View file

@ -0,0 +1,175 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_GLUE_NUMERIC_DEFS_H
#define _PSTL_GLUE_NUMERIC_DEFS_H
#include <__config>
#include <iterator>
#include "execution_defs.h"
namespace std {
// [reduce]
template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
reduce(_ExecutionPolicy&& __exec,
_ForwardIterator __first,
_ForwardIterator __last,
_Tp __init,
_BinaryOperation __binary_op);
template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Tp __init);
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy,
typename iterator_traits<_ForwardIterator>::value_type>
reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp> transform_reduce(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_Tp __init);
template <class _ExecutionPolicy,
class _ForwardIterator1,
class _ForwardIterator2,
class _Tp,
class _BinaryOperation1,
class _BinaryOperation2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp> transform_reduce(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_Tp __init,
_BinaryOperation1 __binary_op1,
_BinaryOperation2 __binary_op2);
template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp> transform_reduce(
_ExecutionPolicy&& __exec,
_ForwardIterator __first,
_ForwardIterator __last,
_Tp __init,
_BinaryOperation __binary_op,
_UnaryOperation __unary_op);
// [exclusive.scan]
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> exclusive_scan(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
_Tp __init);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> exclusive_scan(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
_Tp __init,
_BinaryOperation __binary_op);
// [inclusive.scan]
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> inclusive_scan(
_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> inclusive_scan(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
_BinaryOperation __binary_op);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> inclusive_scan(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
_BinaryOperation __binary_op,
_Tp __init);
// [transform.exclusive.scan]
template <class _ExecutionPolicy,
class _ForwardIterator1,
class _ForwardIterator2,
class _Tp,
class _BinaryOperation,
class _UnaryOperation>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> transform_exclusive_scan(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
_Tp __init,
_BinaryOperation __binary_op,
_UnaryOperation __unary_op);
// [transform.inclusive.scan]
template <class _ExecutionPolicy,
class _ForwardIterator1,
class _ForwardIterator2,
class _BinaryOperation,
class _UnaryOperation,
class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> transform_inclusive_scan(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
_BinaryOperation __binary_op,
_UnaryOperation __unary_op,
_Tp __init);
template <class _ExecutionPolicy,
class _ForwardIterator1,
class _ForwardIterator2,
class _UnaryOperation,
class _BinaryOperation>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> transform_inclusive_scan(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
_BinaryOperation __binary_op,
_UnaryOperation __unary_op);
// [adjacent.difference]
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> adjacent_difference(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __d_first,
_BinaryOperation op);
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> adjacent_difference(
_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __d_first);
} // namespace std
#endif /* _PSTL_GLUE_NUMERIC_DEFS_H */

319
third_party/libcxx/__pstl/internal/glue_numeric_impl.h vendored Normal file
View file

@ -0,0 +1,319 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_GLUE_NUMERIC_IMPL_H
#define _PSTL_GLUE_NUMERIC_IMPL_H
#include <__config>
#include <functional>
#include "execution_impl.h"
#include "numeric_fwd.h"
#include "utils.h"
namespace std {
// [reduce]
template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
reduce(_ExecutionPolicy&& __exec,
_ForwardIterator __first,
_ForwardIterator __last,
_Tp __init,
_BinaryOperation __binary_op) {
return transform_reduce(
std::forward<_ExecutionPolicy>(__exec), __first, __last, __init, __binary_op, __pstl::__internal::__no_op());
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Tp __init) {
return transform_reduce(
std::forward<_ExecutionPolicy>(__exec), __first, __last, __init, std::plus<_Tp>(), __pstl::__internal::__no_op());
}
template <class _ExecutionPolicy, class _ForwardIterator>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy,
typename iterator_traits<_ForwardIterator>::value_type>
reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last) {
typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
return transform_reduce(
std::forward<_ExecutionPolicy>(__exec),
__first,
__last,
_ValueType{},
std::plus<_ValueType>(),
__pstl::__internal::__no_op());
}
// [transform.reduce]
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp> transform_reduce(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_Tp __init) {
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
typedef typename iterator_traits<_ForwardIterator1>::value_type _InputType;
return __pstl::__internal::__pattern_transform_reduce(
__dispatch_tag,
std::forward<_ExecutionPolicy>(__exec),
__first1,
__last1,
__first2,
__init,
std::plus<_InputType>(),
std::multiplies<_InputType>());
}
template <class _ExecutionPolicy,
class _ForwardIterator1,
class _ForwardIterator2,
class _Tp,
class _BinaryOperation1,
class _BinaryOperation2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp> transform_reduce(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_Tp __init,
_BinaryOperation1 __binary_op1,
_BinaryOperation2 __binary_op2) {
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
return __pstl::__internal::__pattern_transform_reduce(
__dispatch_tag,
std::forward<_ExecutionPolicy>(__exec),
__first1,
__last1,
__first2,
__init,
__binary_op1,
__binary_op2);
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp> transform_reduce(
_ExecutionPolicy&& __exec,
_ForwardIterator __first,
_ForwardIterator __last,
_Tp __init,
_BinaryOperation __binary_op,
_UnaryOperation __unary_op) {
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
return __pstl::__internal::__pattern_transform_reduce(
__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __init, __binary_op, __unary_op);
}
// [exclusive.scan]
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> exclusive_scan(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
_Tp __init) {
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
using namespace __pstl;
return __internal::__pattern_transform_scan(
__dispatch_tag,
std::forward<_ExecutionPolicy>(__exec),
__first,
__last,
__result,
__pstl::__internal::__no_op(),
__init,
std::plus<_Tp>(),
/*inclusive=*/std::false_type());
}
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> exclusive_scan(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
_Tp __init,
_BinaryOperation __binary_op) {
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
using namespace __pstl;
return __internal::__pattern_transform_scan(
__dispatch_tag,
std::forward<_ExecutionPolicy>(__exec),
__first,
__last,
__result,
__pstl::__internal::__no_op(),
__init,
__binary_op,
/*inclusive=*/std::false_type());
}
// [inclusive.scan]
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> inclusive_scan(
_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result) {
typedef typename iterator_traits<_ForwardIterator1>::value_type _InputType;
return transform_inclusive_scan(
std::forward<_ExecutionPolicy>(__exec),
__first,
__last,
__result,
std::plus<_InputType>(),
__pstl::__internal::__no_op());
}
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> inclusive_scan(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
_BinaryOperation __binary_op) {
return transform_inclusive_scan(
std::forward<_ExecutionPolicy>(__exec), __first, __last, __result, __binary_op, __pstl::__internal::__no_op());
}
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> inclusive_scan(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
_BinaryOperation __binary_op,
_Tp __init) {
return transform_inclusive_scan(
std::forward<_ExecutionPolicy>(__exec),
__first,
__last,
__result,
__binary_op,
__pstl::__internal::__no_op(),
__init);
}
// [transform.exclusive.scan]
template <class _ExecutionPolicy,
class _ForwardIterator1,
class _ForwardIterator2,
class _Tp,
class _BinaryOperation,
class _UnaryOperation>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> transform_exclusive_scan(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
_Tp __init,
_BinaryOperation __binary_op,
_UnaryOperation __unary_op) {
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
return __pstl::__internal::__pattern_transform_scan(
__dispatch_tag,
std::forward<_ExecutionPolicy>(__exec),
__first,
__last,
__result,
__unary_op,
__init,
__binary_op,
/*inclusive=*/std::false_type());
}
// [transform.inclusive.scan]
template <class _ExecutionPolicy,
class _ForwardIterator1,
class _ForwardIterator2,
class _BinaryOperation,
class _UnaryOperation,
class _Tp>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> transform_inclusive_scan(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
_BinaryOperation __binary_op,
_UnaryOperation __unary_op,
_Tp __init) {
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
return __pstl::__internal::__pattern_transform_scan(
__dispatch_tag,
std::forward<_ExecutionPolicy>(__exec),
__first,
__last,
__result,
__unary_op,
__init,
__binary_op,
/*inclusive=*/std::true_type());
}
template <class _ExecutionPolicy,
class _ForwardIterator1,
class _ForwardIterator2,
class _UnaryOperation,
class _BinaryOperation>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> transform_inclusive_scan(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __result,
_BinaryOperation __binary_op,
_UnaryOperation __unary_op) {
if (__first != __last) {
auto __tmp = __unary_op(*__first);
*__result = __tmp;
return transform_inclusive_scan(
std::forward<_ExecutionPolicy>(__exec), ++__first, __last, ++__result, __binary_op, __unary_op, __tmp);
} else {
return __result;
}
}
// [adjacent.difference]
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> adjacent_difference(
_ExecutionPolicy&& __exec,
_ForwardIterator1 __first,
_ForwardIterator1 __last,
_ForwardIterator2 __d_first,
_BinaryOperation __op) {
if (__first == __last)
return __d_first;
auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __d_first);
return __pstl::__internal::__pattern_adjacent_difference(
__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __d_first, __op);
}
template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> adjacent_difference(
_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __d_first) {
typedef typename iterator_traits<_ForwardIterator1>::value_type _ValueType;
return adjacent_difference(
std::forward<_ExecutionPolicy>(__exec), __first, __last, __d_first, std::minus<_ValueType>());
}
} // namespace std
#endif /* _PSTL_GLUE_NUMERIC_IMPL_H_ */

106
third_party/libcxx/__pstl/internal/memory_impl.h vendored Normal file
View file

@ -0,0 +1,106 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_MEMORY_IMPL_H
#define _PSTL_MEMORY_IMPL_H
#include <__config>
#include <iterator>
#include "unseq_backend_simd.h"
namespace __pstl {
namespace __internal {
//------------------------------------------------------------------------
// uninitialized_move
//------------------------------------------------------------------------
template <typename _ForwardIterator, typename _OutputIterator>
_OutputIterator __brick_uninitialized_move(
_ForwardIterator __first,
_ForwardIterator __last,
_OutputIterator __result,
/*vector=*/std::false_type) noexcept {
using _ValueType = typename std::iterator_traits<_OutputIterator>::value_type;
for (; __first != __last; ++__first, ++__result) {
::new (std::addressof(*__result)) _ValueType(std::move(*__first));
}
return __result;
}
template <typename _RandomAccessIterator, typename _OutputIterator>
_OutputIterator __brick_uninitialized_move(
_RandomAccessIterator __first,
_RandomAccessIterator __last,
_OutputIterator __result,
/*vector=*/std::true_type) noexcept {
using __ValueType = typename std::iterator_traits<_OutputIterator>::value_type;
using _ReferenceType1 = typename std::iterator_traits<_RandomAccessIterator>::reference;
using _ReferenceType2 = typename std::iterator_traits<_OutputIterator>::reference;
return __unseq_backend::__simd_walk_2(
__first, __last - __first, __result, [](_ReferenceType1 __x, _ReferenceType2 __y) {
::new (std::addressof(__y)) __ValueType(std::move(__x));
});
}
template <typename _Iterator>
void __brick_destroy(_Iterator __first, _Iterator __last, /*vector*/ std::false_type) noexcept {
using _ValueType = typename std::iterator_traits<_Iterator>::value_type;
for (; __first != __last; ++__first)
__first->~_ValueType();
}
template <typename _RandomAccessIterator>
void __brick_destroy(_RandomAccessIterator __first, _RandomAccessIterator __last, /*vector*/ std::true_type) noexcept {
using _ValueType = typename std::iterator_traits<_RandomAccessIterator>::value_type;
using _ReferenceType = typename std::iterator_traits<_RandomAccessIterator>::reference;
__unseq_backend::__simd_walk_1(__first, __last - __first, [](_ReferenceType __x) { __x.~_ValueType(); });
}
//------------------------------------------------------------------------
// uninitialized copy
//------------------------------------------------------------------------
template <typename _ForwardIterator, typename _OutputIterator>
_OutputIterator __brick_uninitialized_copy(
_ForwardIterator __first,
_ForwardIterator __last,
_OutputIterator __result,
/*vector=*/std::false_type) noexcept {
using _ValueType = typename std::iterator_traits<_OutputIterator>::value_type;
for (; __first != __last; ++__first, ++__result) {
::new (std::addressof(*__result)) _ValueType(*__first);
}
return __result;
}
template <typename _RandomAccessIterator, typename _OutputIterator>
_OutputIterator __brick_uninitialized_copy(
_RandomAccessIterator __first,
_RandomAccessIterator __last,
_OutputIterator __result,
/*vector=*/std::true_type) noexcept {
using __ValueType = typename std::iterator_traits<_OutputIterator>::value_type;
using _ReferenceType1 = typename std::iterator_traits<_RandomAccessIterator>::reference;
using _ReferenceType2 = typename std::iterator_traits<_OutputIterator>::reference;
return __unseq_backend::__simd_walk_2(
__first, __last - __first, __result, [](_ReferenceType1 __x, _ReferenceType2 __y) {
::new (std::addressof(__y)) __ValueType(__x);
});
}
} // namespace __internal
} // namespace __pstl
#endif /* _PSTL_MEMORY_IMPL_H */

251
third_party/libcxx/__pstl/internal/numeric_fwd.h vendored Normal file
View file

@ -0,0 +1,251 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_NUMERIC_FWD_H
#define _PSTL_NUMERIC_FWD_H
#include <__config>
#include <type_traits>
#include <utility>
namespace __pstl {
namespace __internal {
//------------------------------------------------------------------------
// transform_reduce (version with two binary functions, according to draft N4659)
//------------------------------------------------------------------------
template <class _RandomAccessIterator1,
class _RandomAccessIterator2,
class _Tp,
class _BinaryOperation1,
class _BinaryOperation2>
_Tp __brick_transform_reduce(
_RandomAccessIterator1,
_RandomAccessIterator1,
_RandomAccessIterator2,
_Tp,
_BinaryOperation1,
_BinaryOperation2,
/*__is_vector=*/std::true_type) noexcept;
template <class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation1, class _BinaryOperation2>
_Tp __brick_transform_reduce(
_ForwardIterator1,
_ForwardIterator1,
_ForwardIterator2,
_Tp,
_BinaryOperation1,
_BinaryOperation2,
/*__is_vector=*/std::false_type) noexcept;
template <class _Tag,
class _ExecutionPolicy,
class _ForwardIterator1,
class _ForwardIterator2,
class _Tp,
class _BinaryOperation1,
class _BinaryOperation2>
_Tp __pattern_transform_reduce(
_Tag,
_ExecutionPolicy&&,
_ForwardIterator1,
_ForwardIterator1,
_ForwardIterator2,
_Tp,
_BinaryOperation1,
_BinaryOperation2) noexcept;
template <class _IsVector,
class _ExecutionPolicy,
class _RandomAccessIterator1,
class _RandomAccessIterator2,
class _Tp,
class _BinaryOperation1,
class _BinaryOperation2>
_Tp __pattern_transform_reduce(
__parallel_tag<_IsVector>,
_ExecutionPolicy&&,
_RandomAccessIterator1,
_RandomAccessIterator1,
_RandomAccessIterator2,
_Tp,
_BinaryOperation1,
_BinaryOperation2);
//------------------------------------------------------------------------
// transform_reduce (version with unary and binary functions)
//------------------------------------------------------------------------
template <class _RandomAccessIterator, class _Tp, class _UnaryOperation, class _BinaryOperation>
_Tp __brick_transform_reduce(
_RandomAccessIterator,
_RandomAccessIterator,
_Tp,
_BinaryOperation,
_UnaryOperation,
/*is_vector=*/std::true_type) noexcept;
template <class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
_Tp __brick_transform_reduce(
_ForwardIterator,
_ForwardIterator,
_Tp,
_BinaryOperation,
_UnaryOperation,
/*is_vector=*/std::false_type) noexcept;
template <class _Tag,
class _ExecutionPolicy,
class _ForwardIterator,
class _Tp,
class _BinaryOperation,
class _UnaryOperation>
_Tp __pattern_transform_reduce(
_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Tp, _BinaryOperation, _UnaryOperation) noexcept;
template <class _IsVector,
class _ExecutionPolicy,
class _RandomAccessIterator,
class _Tp,
class _BinaryOperation,
class _UnaryOperation>
_Tp __pattern_transform_reduce(
__parallel_tag<_IsVector>,
_ExecutionPolicy&&,
_RandomAccessIterator,
_RandomAccessIterator,
_Tp,
_BinaryOperation,
_UnaryOperation);
//------------------------------------------------------------------------
// transform_exclusive_scan
//
// walk3 evaluates f(x,y,z) for (x,y,z) drawn from [first1,last1), [first2,...), [first3,...)
//------------------------------------------------------------------------
template <class _ForwardIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation>
std::pair<_OutputIterator, _Tp> __brick_transform_scan(
_ForwardIterator,
_ForwardIterator,
_OutputIterator,
_UnaryOperation,
_Tp,
_BinaryOperation,
/*Inclusive*/ std::false_type) noexcept;
template <class _RandomAccessIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation>
std::pair<_OutputIterator, _Tp> __brick_transform_scan(
_RandomAccessIterator,
_RandomAccessIterator,
_OutputIterator,
_UnaryOperation,
_Tp,
_BinaryOperation,
/*Inclusive*/ std::true_type) noexcept;
template <class _Tag,
class _ExecutionPolicy,
class _ForwardIterator,
class _OutputIterator,
class _UnaryOperation,
class _Tp,
class _BinaryOperation,
class _Inclusive>
_OutputIterator __pattern_transform_scan(
_Tag,
_ExecutionPolicy&&,
_ForwardIterator,
_ForwardIterator,
_OutputIterator,
_UnaryOperation,
_Tp,
_BinaryOperation,
_Inclusive) noexcept;
template <class _IsVector,
class _ExecutionPolicy,
class _RandomAccessIterator,
class _OutputIterator,
class _UnaryOperation,
class _Tp,
class _BinaryOperation,
class _Inclusive>
typename std::enable_if<!std::is_floating_point<_Tp>::value, _OutputIterator>::type __pattern_transform_scan(
__parallel_tag<_IsVector> __tag,
_ExecutionPolicy&&,
_RandomAccessIterator,
_RandomAccessIterator,
_OutputIterator,
_UnaryOperation,
_Tp,
_BinaryOperation,
_Inclusive);
template <class _IsVector,
class _ExecutionPolicy,
class _RandomAccessIterator,
class _OutputIterator,
class _UnaryOperation,
class _Tp,
class _BinaryOperation,
class _Inclusive>
typename std::enable_if<std::is_floating_point<_Tp>::value, _OutputIterator>::type __pattern_transform_scan(
__parallel_tag<_IsVector>,
_ExecutionPolicy&&,
_RandomAccessIterator,
_RandomAccessIterator,
_OutputIterator,
_UnaryOperation,
_Tp,
_BinaryOperation,
_Inclusive);
//------------------------------------------------------------------------
// adjacent_difference
//------------------------------------------------------------------------
template <class _ForwardIterator, class _OutputIterator, class _BinaryOperation>
_OutputIterator __brick_adjacent_difference(
_ForwardIterator,
_ForwardIterator,
_OutputIterator,
_BinaryOperation,
/*is_vector*/ std::false_type) noexcept;
template <class _RandomAccessIterator, class _OutputIterator, class _BinaryOperation>
_OutputIterator __brick_adjacent_difference(
_RandomAccessIterator,
_RandomAccessIterator,
_OutputIterator,
_BinaryOperation,
/*is_vector*/ std::true_type) noexcept;
template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _BinaryOperation>
_OutputIterator __pattern_adjacent_difference(
_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _OutputIterator, _BinaryOperation) noexcept;
template <class _IsVector,
class _ExecutionPolicy,
class _RandomAccessIterator,
class _OutputIterator,
class _BinaryOperation>
_OutputIterator __pattern_adjacent_difference(
__parallel_tag<_IsVector>,
_ExecutionPolicy&&,
_RandomAccessIterator,
_RandomAccessIterator,
_OutputIterator,
_BinaryOperation);
} // namespace __internal
} // namespace __pstl
#endif /* _PSTL_NUMERIC_FWD_H */

536
third_party/libcxx/__pstl/internal/numeric_impl.h vendored Normal file
View file

@ -0,0 +1,536 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_NUMERIC_IMPL_H
#define _PSTL_NUMERIC_IMPL_H
#include <__assert>
#include <__config>
#include <iterator>
#include <type_traits>
#include <numeric>
#include "parallel_backend.h"
#include "execution_impl.h"
#include "unseq_backend_simd.h"
#include "algorithm_fwd.h"
namespace __pstl {
namespace __internal {
//------------------------------------------------------------------------
// transform_reduce (version with two binary functions, according to draft N4659)
//------------------------------------------------------------------------
template <class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation1, class _BinaryOperation2>
_Tp __brick_transform_reduce(
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_Tp __init,
_BinaryOperation1 __binary_op1,
_BinaryOperation2 __binary_op2,
/*is_vector=*/std::false_type) noexcept {
return std::inner_product(__first1, __last1, __first2, __init, __binary_op1, __binary_op2);
}
template <class _RandomAccessIterator1,
class _RandomAccessIterator2,
class _Tp,
class _BinaryOperation1,
class _BinaryOperation2>
_Tp __brick_transform_reduce(
_RandomAccessIterator1 __first1,
_RandomAccessIterator1 __last1,
_RandomAccessIterator2 __first2,
_Tp __init,
_BinaryOperation1 __binary_op1,
_BinaryOperation2 __binary_op2,
/*is_vector=*/std::true_type) noexcept {
typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
return __unseq_backend::__simd_transform_reduce(
__last1 - __first1, __init, __binary_op1, [=, &__binary_op2](_DifferenceType __i) {
return __binary_op2(__first1[__i], __first2[__i]);
});
}
template <class _Tag,
class _ExecutionPolicy,
class _ForwardIterator1,
class _ForwardIterator2,
class _Tp,
class _BinaryOperation1,
class _BinaryOperation2>
_Tp __pattern_transform_reduce(
_Tag,
_ExecutionPolicy&&,
_ForwardIterator1 __first1,
_ForwardIterator1 __last1,
_ForwardIterator2 __first2,
_Tp __init,
_BinaryOperation1 __binary_op1,
_BinaryOperation2 __binary_op2) noexcept {
return __brick_transform_reduce(
__first1, __last1, __first2, __init, __binary_op1, __binary_op2, typename _Tag::__is_vector{});
}
template <class _IsVector,
class _ExecutionPolicy,
class _RandomAccessIterator1,
class _RandomAccessIterator2,
class _Tp,
class _BinaryOperation1,
class _BinaryOperation2>
_Tp __pattern_transform_reduce(
__parallel_tag<_IsVector> __tag,
_ExecutionPolicy&& __exec,
_RandomAccessIterator1 __first1,
_RandomAccessIterator1 __last1,
_RandomAccessIterator2 __first2,
_Tp __init,
_BinaryOperation1 __binary_op1,
_BinaryOperation2 __binary_op2) {
using __backend_tag = typename decltype(__tag)::__backend_tag;
return __internal::__except_handler([&]() {
return __par_backend::__parallel_transform_reduce(
__backend_tag{},
std::forward<_ExecutionPolicy>(__exec),
__first1,
__last1,
[__first1, __first2, __binary_op2](_RandomAccessIterator1 __i) mutable {
return __binary_op2(*__i, *(__first2 + (__i - __first1)));
},
__init,
__binary_op1, // Combine
[__first1, __first2, __binary_op1, __binary_op2](
_RandomAccessIterator1 __i, _RandomAccessIterator1 __j, _Tp __init) -> _Tp {
return __internal::__brick_transform_reduce(
__i, __j, __first2 + (__i - __first1), __init, __binary_op1, __binary_op2, _IsVector{});
});
});
}
//------------------------------------------------------------------------
// transform_reduce (version with unary and binary functions)
//------------------------------------------------------------------------
template <class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
_Tp __brick_transform_reduce(
_ForwardIterator __first,
_ForwardIterator __last,
_Tp __init,
_BinaryOperation __binary_op,
_UnaryOperation __unary_op,
/*is_vector=*/std::false_type) noexcept {
return std::transform_reduce(__first, __last, __init, __binary_op, __unary_op);
}
template <class _RandomAccessIterator, class _Tp, class _UnaryOperation, class _BinaryOperation>
_Tp __brick_transform_reduce(
_RandomAccessIterator __first,
_RandomAccessIterator __last,
_Tp __init,
_BinaryOperation __binary_op,
_UnaryOperation __unary_op,
/*is_vector=*/std::true_type) noexcept {
typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
return __unseq_backend::__simd_transform_reduce(
__last - __first, __init, __binary_op, [=, &__unary_op](_DifferenceType __i) {
return __unary_op(__first[__i]);
});
}
template <class _Tag,
class _ExecutionPolicy,
class _ForwardIterator,
class _Tp,
class _BinaryOperation,
class _UnaryOperation>
_Tp __pattern_transform_reduce(
_Tag,
_ExecutionPolicy&&,
_ForwardIterator __first,
_ForwardIterator __last,
_Tp __init,
_BinaryOperation __binary_op,
_UnaryOperation __unary_op) noexcept {
return __internal::__brick_transform_reduce(
__first, __last, __init, __binary_op, __unary_op, typename _Tag::__is_vector{});
}
template <class _IsVector,
class _ExecutionPolicy,
class _RandomAccessIterator,
class _Tp,
class _BinaryOperation,
class _UnaryOperation>
_Tp __pattern_transform_reduce(
__parallel_tag<_IsVector> __tag,
_ExecutionPolicy&& __exec,
_RandomAccessIterator __first,
_RandomAccessIterator __last,
_Tp __init,
_BinaryOperation __binary_op,
_UnaryOperation __unary_op) {
using __backend_tag = typename decltype(__tag)::__backend_tag;
return __internal::__except_handler([&]() {
return __par_backend::__parallel_transform_reduce(
__backend_tag{},
std::forward<_ExecutionPolicy>(__exec),
__first,
__last,
[__unary_op](_RandomAccessIterator __i) mutable { return __unary_op(*__i); },
__init,
__binary_op,
[__unary_op, __binary_op](_RandomAccessIterator __i, _RandomAccessIterator __j, _Tp __init) {
return __internal::__brick_transform_reduce(__i, __j, __init, __binary_op, __unary_op, _IsVector{});
});
});
}
//------------------------------------------------------------------------
// transform_exclusive_scan
//
// walk3 evaluates f(x,y,z) for (x,y,z) drawn from [first1,last1), [first2,...), [first3,...)
//------------------------------------------------------------------------
// Exclusive form
template <class _ForwardIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation>
std::pair<_OutputIterator, _Tp> __brick_transform_scan(
_ForwardIterator __first,
_ForwardIterator __last,
_OutputIterator __result,
_UnaryOperation __unary_op,
_Tp __init,
_BinaryOperation __binary_op,
/*Inclusive*/ std::false_type,
/*is_vector=*/std::false_type) noexcept {
for (; __first != __last; ++__first, ++__result) {
*__result = __init;
__init = __binary_op(__init, __unary_op(*__first));
}
return std::make_pair(__result, __init);
}
// Inclusive form
template <class _RandomAccessIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation>
std::pair<_OutputIterator, _Tp> __brick_transform_scan(
_RandomAccessIterator __first,
_RandomAccessIterator __last,
_OutputIterator __result,
_UnaryOperation __unary_op,
_Tp __init,
_BinaryOperation __binary_op,
/*Inclusive*/ std::true_type,
/*is_vector=*/std::false_type) noexcept {
for (; __first != __last; ++__first, ++__result) {
__init = __binary_op(__init, __unary_op(*__first));
*__result = __init;
}
return std::make_pair(__result, __init);
}
// type is arithmetic and binary operation is a user defined operation.
template <typename _Tp, typename _BinaryOperation>
using is_arithmetic_udop =
std::integral_constant<bool,
std::is_arithmetic<_Tp>::value && !std::is_same<_BinaryOperation, std::plus<_Tp>>::value>;
// [restriction] - T shall be DefaultConstructible.
// [violation] - default ctor of T shall set the identity value for binary_op.
template <class _RandomAccessIterator,
class _OutputIterator,
class _UnaryOperation,
class _Tp,
class _BinaryOperation,
class _Inclusive>
typename std::enable_if<!is_arithmetic_udop<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
__brick_transform_scan(
_RandomAccessIterator __first,
_RandomAccessIterator __last,
_OutputIterator __result,
_UnaryOperation __unary_op,
_Tp __init,
_BinaryOperation __binary_op,
_Inclusive,
/*is_vector=*/std::true_type) noexcept {
#if defined(_PSTL_UDS_PRESENT)
return __unseq_backend::__simd_scan(
__first, __last - __first, __result, __unary_op, __init, __binary_op, _Inclusive());
#else
// We need to call serial brick here to call function for inclusive and exclusive scan that depends on _Inclusive()
// value
return __internal::__brick_transform_scan(
__first,
__last,
__result,
__unary_op,
__init,
__binary_op,
_Inclusive(),
/*is_vector=*/std::false_type());
#endif
}
template <class _RandomAccessIterator,
class _OutputIterator,
class _UnaryOperation,
class _Tp,
class _BinaryOperation,
class _Inclusive>
typename std::enable_if<is_arithmetic_udop<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
__brick_transform_scan(
_RandomAccessIterator __first,
_RandomAccessIterator __last,
_OutputIterator __result,
_UnaryOperation __unary_op,
_Tp __init,
_BinaryOperation __binary_op,
_Inclusive,
/*is_vector=*/std::true_type) noexcept {
return __internal::__brick_transform_scan(
__first,
__last,
__result,
__unary_op,
__init,
__binary_op,
_Inclusive(),
/*is_vector=*/std::false_type());
}
template <class _Tag,
class _ExecutionPolicy,
class _ForwardIterator,
class _OutputIterator,
class _UnaryOperation,
class _Tp,
class _BinaryOperation,
class _Inclusive>
_OutputIterator __pattern_transform_scan(
_Tag,
_ExecutionPolicy&&,
_ForwardIterator __first,
_ForwardIterator __last,
_OutputIterator __result,
_UnaryOperation __unary_op,
_Tp __init,
_BinaryOperation __binary_op,
_Inclusive) noexcept {
return __internal::__brick_transform_scan(
__first, __last, __result, __unary_op, __init, __binary_op, _Inclusive(), typename _Tag::__is_vector{})
.first;
}
template <class _IsVector,
class _ExecutionPolicy,
class _RandomAccessIterator,
class _OutputIterator,
class _UnaryOperation,
class _Tp,
class _BinaryOperation,
class _Inclusive>
typename std::enable_if<!std::is_floating_point<_Tp>::value, _OutputIterator>::type __pattern_transform_scan(
__parallel_tag<_IsVector> __tag,
_ExecutionPolicy&& __exec,
_RandomAccessIterator __first,
_RandomAccessIterator __last,
_OutputIterator __result,
_UnaryOperation __unary_op,
_Tp __init,
_BinaryOperation __binary_op,
_Inclusive) {
using __backend_tag = typename decltype(__tag)::__backend_tag;
typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
return __internal::__except_handler([&]() {
__par_backend::__parallel_transform_scan(
__backend_tag{},
std::forward<_ExecutionPolicy>(__exec),
__last - __first,
[__first, __unary_op](_DifferenceType __i) mutable { return __unary_op(__first[__i]); },
__init,
__binary_op,
[__first, __unary_op, __binary_op](_DifferenceType __i, _DifferenceType __j, _Tp __init) {
// Execute serial __brick_transform_reduce, due to the explicit SIMD vectorization (reduction) requires a
// commutative operation for the guarantee of correct scan.
return __internal::__brick_transform_reduce(
__first + __i,
__first + __j,
__init,
__binary_op,
__unary_op,
/*__is_vector*/ std::false_type());
},
[__first, __unary_op, __binary_op, __result](_DifferenceType __i, _DifferenceType __j, _Tp __init) {
return __internal::__brick_transform_scan(
__first + __i,
__first + __j,
__result + __i,
__unary_op,
__init,
__binary_op,
_Inclusive(),
_IsVector{})
.second;
});
return __result + (__last - __first);
});
}
template <class _IsVector,
class _ExecutionPolicy,
class _RandomAccessIterator,
class _OutputIterator,
class _UnaryOperation,
class _Tp,
class _BinaryOperation,
class _Inclusive>
typename std::enable_if<std::is_floating_point<_Tp>::value, _OutputIterator>::type __pattern_transform_scan(
__parallel_tag<_IsVector> __tag,
_ExecutionPolicy&& __exec,
_RandomAccessIterator __first,
_RandomAccessIterator __last,
_OutputIterator __result,
_UnaryOperation __unary_op,
_Tp __init,
_BinaryOperation __binary_op,
_Inclusive) {
using __backend_tag = typename decltype(__tag)::__backend_tag;
typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
_DifferenceType __n = __last - __first;
if (__n <= 0) {
return __result;
}
return __internal::__except_handler([&]() {
__par_backend::__parallel_strict_scan(
__backend_tag{},
std::forward<_ExecutionPolicy>(__exec),
__n,
__init,
[__first, __unary_op, __binary_op, __result](_DifferenceType __i, _DifferenceType __len) {
return __internal::__brick_transform_scan(
__first + __i,
__first + (__i + __len),
__result + __i,
__unary_op,
_Tp{},
__binary_op,
_Inclusive(),
_IsVector{})
.second;
},
__binary_op,
[__result, &__binary_op](_DifferenceType __i, _DifferenceType __len, _Tp __initial) {
return *(std::transform(__result + __i,
__result + __i + __len,
__result + __i,
[&__initial, &__binary_op](const _Tp& __x) {
return __binary_op(__initial, __x);
}) -
1);
},
[](_Tp) {});
return __result + (__last - __first);
});
}
//------------------------------------------------------------------------
// adjacent_difference
//------------------------------------------------------------------------
template <class _ForwardIterator, class _OutputIterator, class _BinaryOperation>
_OutputIterator __brick_adjacent_difference(
_ForwardIterator __first,
_ForwardIterator __last,
_OutputIterator __d_first,
_BinaryOperation __op,
/*is_vector*/ std::false_type) noexcept {
return std::adjacent_difference(__first, __last, __d_first, __op);
}
template <class _RandomAccessIterator1, class _RandomAccessIterator2, class BinaryOperation>
_RandomAccessIterator2 __brick_adjacent_difference(
_RandomAccessIterator1 __first,
_RandomAccessIterator1 __last,
_RandomAccessIterator2 __d_first,
BinaryOperation __op,
/*is_vector=*/std::true_type) noexcept {
_LIBCPP_ASSERT(__first != __last, "Range cannot be empty");
typedef typename std::iterator_traits<_RandomAccessIterator1>::reference _ReferenceType1;
typedef typename std::iterator_traits<_RandomAccessIterator2>::reference _ReferenceType2;
auto __n = __last - __first;
*__d_first = *__first;
return __unseq_backend::__simd_walk_3(
__first + 1,
__n - 1,
__first,
__d_first + 1,
[&__op](_ReferenceType1 __x, _ReferenceType1 __y, _ReferenceType2 __z) { __z = __op(__x, __y); });
}
template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _BinaryOperation>
_OutputIterator __pattern_adjacent_difference(
_Tag,
_ExecutionPolicy&&,
_ForwardIterator __first,
_ForwardIterator __last,
_OutputIterator __d_first,
_BinaryOperation __op) noexcept {
return __internal::__brick_adjacent_difference(__first, __last, __d_first, __op, typename _Tag::__is_vector{});
}
template <class _IsVector,
class _ExecutionPolicy,
class _RandomAccessIterator1,
class _RandomAccessIterator2,
class _BinaryOperation>
_RandomAccessIterator2 __pattern_adjacent_difference(
__parallel_tag<_IsVector> __tag,
_ExecutionPolicy&& __exec,
_RandomAccessIterator1 __first,
_RandomAccessIterator1 __last,
_RandomAccessIterator2 __d_first,
_BinaryOperation __op) {
_LIBCPP_ASSERT(__first != __last, "range cannot be empty");
typedef typename std::iterator_traits<_RandomAccessIterator1>::reference _ReferenceType1;
typedef typename std::iterator_traits<_RandomAccessIterator2>::reference _ReferenceType2;
using __backend_tag = typename decltype(__tag)::__backend_tag;
*__d_first = *__first;
__par_backend::__parallel_for(
__backend_tag{},
std::forward<_ExecutionPolicy>(__exec),
__first,
__last - 1,
[&__op, __d_first, __first](_RandomAccessIterator1 __b, _RandomAccessIterator1 __e) {
_RandomAccessIterator2 __d_b = __d_first + (__b - __first);
__internal::__brick_walk3(
__b,
__e,
__b + 1,
__d_b + 1,
[&__op](_ReferenceType1 __x, _ReferenceType1 __y, _ReferenceType2 __z) { __z = __op(__y, __x); },
_IsVector{});
});
return __d_first + (__last - __first);
}
} // namespace __internal
} // namespace __pstl
#endif /* _PSTL_NUMERIC_IMPL_H */

64
third_party/libcxx/__pstl/internal/omp/parallel_for.h vendored Normal file
View file

@ -0,0 +1,64 @@
// -*- C++ -*-
// -*-===----------------------------------------------------------------------===//
//
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_INTERNAL_OMP_PARALLEL_FOR_H
#define _PSTL_INTERNAL_OMP_PARALLEL_FOR_H
#include <cstddef>
#include "util.h"
namespace __pstl
{
namespace __omp_backend
{
template <class _Index, class _Fp>
void
__parallel_for_body(_Index __first, _Index __last, _Fp __f)
{
// initial partition of the iteration space into chunks
auto __policy = __omp_backend::__chunk_partitioner(__first, __last);
// To avoid over-subscription we use taskloop for the nested parallelism
_PSTL_PRAGMA(omp taskloop untied mergeable)
for (std::size_t __chunk = 0; __chunk < __policy.__n_chunks; ++__chunk)
{
__pstl::__omp_backend::__process_chunk(__policy, __first, __chunk, __f);
}
}
//------------------------------------------------------------------------
// Notation:
// Evaluation of brick f[i,j) for each subrange [i,j) of [first, last)
//------------------------------------------------------------------------
template <class _ExecutionPolicy, class _Index, class _Fp>
void
__parallel_for(__pstl::__internal::__openmp_backend_tag, _ExecutionPolicy&&, _Index __first, _Index __last, _Fp __f)
{
if (omp_in_parallel())
{
// we don't create a nested parallel region in an existing parallel
// region: just create tasks
__pstl::__omp_backend::__parallel_for_body(__first, __last, __f);
}
else
{
// in any case (nested or non-nested) one parallel region is created and
// only one thread creates a set of tasks
_PSTL_PRAGMA(omp parallel)
_PSTL_PRAGMA(omp single nowait) { __pstl::__omp_backend::__parallel_for_body(__first, __last, __f); }
}
}
} // namespace __omp_backend
} // namespace __pstl
#endif // _PSTL_INTERNAL_OMP_PARALLEL_FOR_H

59
third_party/libcxx/__pstl/internal/omp/parallel_for_each.h vendored Normal file
View file

@ -0,0 +1,59 @@
// -*- C++ -*-
// -*-===----------------------------------------------------------------------===//
//
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_INTERNAL_OMP_PARALLEL_FOR_EACH_H
#define _PSTL_INTERNAL_OMP_PARALLEL_FOR_EACH_H
#include "util.h"
namespace __pstl
{
namespace __omp_backend
{
template <class _ForwardIterator, class _Fp>
void
__parallel_for_each_body(_ForwardIterator __first, _ForwardIterator __last, _Fp __f)
{
using DifferenceType = typename std::iterator_traits<_ForwardIterator>::difference_type;
// TODO: Think of an approach to remove the std::distance call
auto __size = std::distance(__first, __last);
_PSTL_PRAGMA(omp taskloop untied mergeable)
for (DifferenceType __index = 0; __index < __size; ++__index)
{
// TODO: Think of an approach to remove the increment here each time.
auto __iter = std::next(__first, __index);
__f(*__iter);
}
}
template <class _ExecutionPolicy, class _ForwardIterator, class _Fp>
void
__parallel_for_each(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Fp __f)
{
if (omp_in_parallel())
{
// we don't create a nested parallel region in an existing parallel
// region: just create tasks
__pstl::__omp_backend::__parallel_for_each_body(__first, __last, __f);
}
else
{
// in any case (nested or non-nested) one parallel region is created and
// only one thread creates a set of tasks
_PSTL_PRAGMA(omp parallel)
_PSTL_PRAGMA(omp single nowait) { __pstl::__omp_backend::__parallel_for_each_body(__first, __last, __f); }
}
}
} // namespace __omp_backend
} // namespace __pstl
#endif // _PSTL_INTERNAL_OMP_PARALLEL_FOR_EACH_H

50
third_party/libcxx/__pstl/internal/omp/parallel_invoke.h vendored Normal file
View file

@ -0,0 +1,50 @@
// -*- C++ -*-
// -*-===----------------------------------------------------------------------===//
//
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_INTERNAL_OMP_PARALLEL_INVOKE_H
#define _PSTL_INTERNAL_OMP_PARALLEL_INVOKE_H
#include "util.h"
namespace __pstl
{
namespace __omp_backend
{
template <typename _F1, typename _F2>
void
__parallel_invoke_body(_F1&& __f1, _F2&& __f2)
{
_PSTL_PRAGMA(omp taskgroup)
{
_PSTL_PRAGMA(omp task untied mergeable) { std::forward<_F1>(__f1)(); }
_PSTL_PRAGMA(omp task untied mergeable) { std::forward<_F2>(__f2)(); }
}
}
template <class _ExecutionPolicy, typename _F1, typename _F2>
void
__parallel_invoke(__pstl::__internal::__openmp_backend_tag, _ExecutionPolicy&&, _F1&& __f1, _F2&& __f2)
{
if (omp_in_parallel())
{
__pstl::__omp_backend::__parallel_invoke_body(std::forward<_F1>(__f1), std::forward<_F2>(__f2));
}
else
{
_PSTL_PRAGMA(omp parallel)
_PSTL_PRAGMA(omp single nowait)
__pstl::__omp_backend::__parallel_invoke_body(std::forward<_F1>(__f1), std::forward<_F2>(__f2));
}
}
} // namespace __omp_backend
} // namespace __pstl
#endif // _PSTL_INTERNAL_OMP_PARALLEL_INVOKE_H

98
third_party/libcxx/__pstl/internal/omp/parallel_merge.h vendored Normal file
View file

@ -0,0 +1,98 @@
// -*- C++ -*-
// -*-===----------------------------------------------------------------------===//
//
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_INTERNAL_OMP_PARALLEL_MERGE_H
#define _PSTL_INTERNAL_OMP_PARALLEL_MERGE_H
#include "util.h"
namespace __pstl
{
namespace __omp_backend
{
template <typename _RandomAccessIterator1, typename _RandomAccessIterator2, typename _RandomAccessIterator3,
typename _Compare, typename _LeafMerge>
void
__parallel_merge_body(std::size_t __size_x, std::size_t __size_y, _RandomAccessIterator1 __xs,
_RandomAccessIterator1 __xe, _RandomAccessIterator2 __ys, _RandomAccessIterator2 __ye,
_RandomAccessIterator3 __zs, _Compare __comp, _LeafMerge __leaf_merge)
{
if (__size_x + __size_y <= __omp_backend::__default_chunk_size)
{
__leaf_merge(__xs, __xe, __ys, __ye, __zs, __comp);
return;
}
_RandomAccessIterator1 __xm;
_RandomAccessIterator2 __ym;
if (__size_x < __size_y)
{
__ym = __ys + (__size_y / 2);
__xm = std::upper_bound(__xs, __xe, *__ym, __comp);
}
else
{
__xm = __xs + (__size_x / 2);
__ym = std::lower_bound(__ys, __ye, *__xm, __comp);
}
auto __zm = __zs + (__xm - __xs) + (__ym - __ys);
_PSTL_PRAGMA(omp task untied mergeable default(none)
firstprivate(__xs, __xm, __ys, __ym, __zs, __comp, __leaf_merge))
__pstl::__omp_backend::__parallel_merge_body(__xm - __xs, __ym - __ys, __xs, __xm, __ys, __ym, __zs, __comp,
__leaf_merge);
_PSTL_PRAGMA(omp task untied mergeable default(none)
firstprivate(__xm, __xe, __ym, __ye, __zm, __comp, __leaf_merge))
__pstl::__omp_backend::__parallel_merge_body(__xe - __xm, __ye - __ym, __xm, __xe, __ym, __ye, __zm, __comp,
__leaf_merge);
_PSTL_PRAGMA(omp taskwait)
}
template <class _ExecutionPolicy, typename _RandomAccessIterator1, typename _RandomAccessIterator2,
typename _RandomAccessIterator3, typename _Compare, typename _LeafMerge>
void
__parallel_merge(__pstl::__internal::__openmp_backend_tag, _ExecutionPolicy&& /*__exec*/, _RandomAccessIterator1 __xs,
_RandomAccessIterator1 __xe, _RandomAccessIterator2 __ys, _RandomAccessIterator2 __ye,
_RandomAccessIterator3 __zs, _Compare __comp, _LeafMerge __leaf_merge)
{
std::size_t __size_x = __xe - __xs;
std::size_t __size_y = __ye - __ys;
/*
* Run the merge in parallel by chunking it up. Use the smaller range (if any) as the iteration range, and the
* larger range as the search range.
*/
if (omp_in_parallel())
{
__pstl::__omp_backend::__parallel_merge_body(__size_x, __size_y, __xs, __xe, __ys, __ye, __zs, __comp,
__leaf_merge);
}
else
{
_PSTL_PRAGMA(omp parallel)
{
_PSTL_PRAGMA(omp single nowait)
__pstl::__omp_backend::__parallel_merge_body(__size_x, __size_y, __xs, __xe, __ys, __ye, __zs, __comp,
__leaf_merge);
}
}
}
} // namespace __omp_backend
} // namespace __pstl
#endif // _PSTL_INTERNAL_OMP_PARALLEL_MERGE_H

73
third_party/libcxx/__pstl/internal/omp/parallel_reduce.h vendored Normal file
View file

@ -0,0 +1,73 @@
// -*- C++ -*-
// -*-===----------------------------------------------------------------------===//
//
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_INTERNAL_OMP_PARALLEL_REDUCE_H
#define _PSTL_INTERNAL_OMP_PARALLEL_REDUCE_H
#include "util.h"
namespace __pstl
{
namespace __omp_backend
{
template <class _RandomAccessIterator, class _Value, typename _RealBody, typename _Reduction>
_Value
__parallel_reduce_body(_RandomAccessIterator __first, _RandomAccessIterator __last, _Value __identity,
_RealBody __real_body, _Reduction __reduce)
{
if (__should_run_serial(__first, __last))
{
return __real_body(__first, __last, __identity);
}
auto __middle = __first + ((__last - __first) / 2);
_Value __v1(__identity), __v2(__identity);
__parallel_invoke_body(
[&]() { __v1 = __parallel_reduce_body(__first, __middle, __identity, __real_body, __reduce); },
[&]() { __v2 = __parallel_reduce_body(__middle, __last, __identity, __real_body, __reduce); });
return __reduce(__v1, __v2);
}
//------------------------------------------------------------------------
// Notation:
// r(i,j,init) returns reduction of init with reduction over [i,j)
// c(x,y) combines values x and y that were the result of r
//------------------------------------------------------------------------
template <class _ExecutionPolicy, class _RandomAccessIterator, class _Value, typename _RealBody, typename _Reduction>
_Value
__parallel_reduce(__pstl::__internal::__openmp_backend_tag, _ExecutionPolicy&&, _RandomAccessIterator __first,
_RandomAccessIterator __last, _Value __identity, _RealBody __real_body, _Reduction __reduction)
{
// We don't create a nested parallel region in an existing parallel region:
// just create tasks.
if (omp_in_parallel())
{
return __pstl::__omp_backend::__parallel_reduce_body(__first, __last, __identity, __real_body, __reduction);
}
// In any case (nested or non-nested) one parallel region is created and only
// one thread creates a set of tasks.
_Value __res = __identity;
_PSTL_PRAGMA(omp parallel)
_PSTL_PRAGMA(omp single nowait)
{
__res = __pstl::__omp_backend::__parallel_reduce_body(__first, __last, __identity, __real_body, __reduction);
}
return __res;
}
} // namespace __omp_backend
} // namespace __pstl
#endif // _PSTL_INTERNAL_OMP_PARALLEL_REDUCE_H

136
third_party/libcxx/__pstl/internal/omp/parallel_scan.h vendored Normal file
View file

@ -0,0 +1,136 @@
// -*- C++ -*-
// -*-===----------------------------------------------------------------------===//
//
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_INTERNAL_OMP_PARALLEL_SCAN_H
#define _PSTL_INTERNAL_OMP_PARALLEL_SCAN_H
#include "parallel_invoke.h"
namespace __pstl
{
namespace __omp_backend
{
template <typename _Index>
_Index
__split(_Index __m)
{
_Index __k = 1;
while (2 * __k < __m)
__k *= 2;
return __k;
}
template <typename _Index, typename _Tp, typename _Rp, typename _Cp>
void
__upsweep(_Index __i, _Index __m, _Index __tilesize, _Tp* __r, _Index __lastsize, _Rp __reduce, _Cp __combine)
{
if (__m == 1)
__r[0] = __reduce(__i * __tilesize, __lastsize);
else
{
_Index __k = __split(__m);
__omp_backend::__parallel_invoke_body(
[=] { __omp_backend::__upsweep(__i, __k, __tilesize, __r, __tilesize, __reduce, __combine); },
[=] {
__omp_backend::__upsweep(__i + __k, __m - __k, __tilesize, __r + __k, __lastsize, __reduce, __combine);
});
if (__m == 2 * __k)
__r[__m - 1] = __combine(__r[__k - 1], __r[__m - 1]);
}
}
template <typename _Index, typename _Tp, typename _Cp, typename _Sp>
void
__downsweep(_Index __i, _Index __m, _Index __tilesize, _Tp* __r, _Index __lastsize, _Tp __initial, _Cp __combine,
_Sp __scan)
{
if (__m == 1)
__scan(__i * __tilesize, __lastsize, __initial);
else
{
const _Index __k = __split(__m);
__omp_backend::__parallel_invoke_body(
[=] { __omp_backend::__downsweep(__i, __k, __tilesize, __r, __tilesize, __initial, __combine, __scan); },
// Assumes that __combine never throws.
// TODO: Consider adding a requirement for user functors to be constant.
[=, &__combine]
{
__omp_backend::__downsweep(__i + __k, __m - __k, __tilesize, __r + __k, __lastsize,
__combine(__initial, __r[__k - 1]), __combine, __scan);
});
}
}
template <typename _ExecutionPolicy, typename _Index, typename _Tp, typename _Rp, typename _Cp, typename _Sp,
typename _Ap>
void
__parallel_strict_scan_body(_Index __n, _Tp __initial, _Rp __reduce, _Cp __combine, _Sp __scan, _Ap __apex)
{
_Index __p = omp_get_num_threads();
const _Index __slack = 4;
_Index __tilesize = (__n - 1) / (__slack * __p) + 1;
_Index __m = (__n - 1) / __tilesize;
__buffer<_Tp> __buf(__m + 1);
_Tp* __r = __buf.get();
__omp_backend::__upsweep(_Index(0), _Index(__m + 1), __tilesize, __r, __n - __m * __tilesize, __reduce, __combine);
std::size_t __k = __m + 1;
_Tp __t = __r[__k - 1];
while ((__k &= __k - 1))
{
__t = __combine(__r[__k - 1], __t);
}
__apex(__combine(__initial, __t));
__omp_backend::__downsweep(_Index(0), _Index(__m + 1), __tilesize, __r, __n - __m * __tilesize, __initial,
__combine, __scan);
}
template <class _ExecutionPolicy, typename _Index, typename _Tp, typename _Rp, typename _Cp, typename _Sp, typename _Ap>
void
__parallel_strict_scan(__pstl::__internal::__openmp_backend_tag, _ExecutionPolicy&&, _Index __n, _Tp __initial,
_Rp __reduce, _Cp __combine, _Sp __scan, _Ap __apex)
{
if (__n <= __default_chunk_size)
{
_Tp __sum = __initial;
if (__n)
{
__sum = __combine(__sum, __reduce(_Index(0), __n));
}
__apex(__sum);
if (__n)
{
__scan(_Index(0), __n, __initial);
}
return;
}
if (omp_in_parallel())
{
__pstl::__omp_backend::__parallel_strict_scan_body<_ExecutionPolicy>(__n, __initial, __reduce, __combine,
__scan, __apex);
}
else
{
_PSTL_PRAGMA(omp parallel)
_PSTL_PRAGMA(omp single nowait)
{
__pstl::__omp_backend::__parallel_strict_scan_body<_ExecutionPolicy>(__n, __initial, __reduce, __combine,
__scan, __apex);
}
}
}
} // namespace __omp_backend
} // namespace __pstl
#endif // _PSTL_INTERNAL_OMP_PARALLEL_SCAN_H

33
third_party/libcxx/__pstl/internal/omp/parallel_stable_partial_sort.h vendored Normal file
View file

@ -0,0 +1,33 @@
// -*- C++ -*-
// -*-===----------------------------------------------------------------------===//
//
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_INTERNAL_OMP_PARALLEL_STABLE_PARTIAL_SORT_H
#define _PSTL_INTERNAL_OMP_PARALLEL_STABLE_PARTIAL_SORT_H
#include "util.h"
namespace __pstl
{
namespace __omp_backend
{
template <typename _RandomAccessIterator, typename _Compare, typename _LeafSort>
void
__parallel_stable_partial_sort(__pstl::__internal::__openmp_backend_tag, _RandomAccessIterator __xs,
_RandomAccessIterator __xe, _Compare __comp, _LeafSort __leaf_sort,
std::size_t /* __nsort */)
{
// TODO: "Parallel partial sort needs to be implemented.");
__leaf_sort(__xs, __xe, __comp);
}
} // namespace __omp_backend
} // namespace __pstl
#endif // _PSTL_INTERNAL_OMP_PARALLEL_STABLE_PARTIAL_SORT_H

160
third_party/libcxx/__pstl/internal/omp/parallel_stable_sort.h vendored Normal file
View file

@ -0,0 +1,160 @@
// -*- C++ -*-
// -*-===----------------------------------------------------------------------===//
//
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_INTERNAL_OMP_PARALLEL_STABLE_SORT_H
#define _PSTL_INTERNAL_OMP_PARALLEL_STABLE_SORT_H
#include "util.h"
#include "parallel_merge.h"
namespace __pstl
{
namespace __omp_backend
{
namespace __sort_details
{
struct __move_value
{
template <typename _Iterator, typename _OutputIterator>
void
operator()(_Iterator __x, _OutputIterator __z) const
{
*__z = std::move(*__x);
}
};
template <typename _RandomAccessIterator, typename _OutputIterator>
_OutputIterator
__parallel_move_range(_RandomAccessIterator __first1, _RandomAccessIterator __last1, _OutputIterator __d_first)
{
std::size_t __size = __last1 - __first1;
// Perform serial moving of small chunks
if (__size <= __default_chunk_size)
{
return std::move(__first1, __last1, __d_first);
}
// Perform parallel moving of larger chunks
auto __policy = __pstl::__omp_backend::__chunk_partitioner(__first1, __last1);
_PSTL_PRAGMA(omp taskloop)
for (std::size_t __chunk = 0; __chunk < __policy.__n_chunks; ++__chunk)
{
__pstl::__omp_backend::__process_chunk(__policy, __first1, __chunk,
[&](auto __chunk_first, auto __chunk_last)
{
auto __chunk_offset = __chunk_first - __first1;
auto __output_it = __d_first + __chunk_offset;
std::move(__chunk_first, __chunk_last, __output_it);
});
}
return __d_first + __size;
}
struct __move_range
{
template <typename _RandomAccessIterator, typename _OutputIterator>
_OutputIterator
operator()(_RandomAccessIterator __first1, _RandomAccessIterator __last1, _OutputIterator __d_first) const
{
return __pstl::__omp_backend::__sort_details::__parallel_move_range(__first1, __last1, __d_first);
}
};
} // namespace __sort_details
template <typename _RandomAccessIterator, typename _Compare, typename _LeafSort>
void
__parallel_stable_sort_body(_RandomAccessIterator __xs, _RandomAccessIterator __xe, _Compare __comp,
_LeafSort __leaf_sort)
{
using _ValueType = typename std::iterator_traits<_RandomAccessIterator>::value_type;
using _VecType = typename std::vector<_ValueType>;
using _OutputIterator = typename _VecType::iterator;
using _MoveValue = typename __omp_backend::__sort_details::__move_value;
using _MoveRange = __omp_backend::__sort_details::__move_range;
if (__should_run_serial(__xs, __xe))
{
__leaf_sort(__xs, __xe, __comp);
}
else
{
std::size_t __size = __xe - __xs;
auto __mid = __xs + (__size / 2);
__pstl::__omp_backend::__parallel_invoke_body(
[&]() { __parallel_stable_sort_body(__xs, __mid, __comp, __leaf_sort); },
[&]() { __parallel_stable_sort_body(__mid, __xe, __comp, __leaf_sort); });
// Perform a parallel merge of the sorted ranges into __output_data.
_VecType __output_data(__size);
_MoveValue __move_value;
_MoveRange __move_range;
__utils::__serial_move_merge __merge(__size);
__pstl::__omp_backend::__parallel_merge_body(
__mid - __xs, __xe - __mid, __xs, __mid, __mid, __xe, __output_data.begin(), __comp,
[&__merge, &__move_value, &__move_range](_RandomAccessIterator __as, _RandomAccessIterator __ae,
_RandomAccessIterator __bs, _RandomAccessIterator __be,
_OutputIterator __cs, _Compare __comp)
{ __merge(__as, __ae, __bs, __be, __cs, __comp, __move_value, __move_value, __move_range, __move_range); });
// Move the values from __output_data back in the original source range.
__pstl::__omp_backend::__sort_details::__parallel_move_range(__output_data.begin(), __output_data.end(), __xs);
}
}
template <class _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare, typename _LeafSort>
void
__parallel_stable_sort(__pstl::__internal::__openmp_backend_tag __tag, _ExecutionPolicy&& /*__exec*/,
_RandomAccessIterator __xs, _RandomAccessIterator __xe, _Compare __comp, _LeafSort __leaf_sort,
std::size_t __nsort = 0)
{
auto __count = static_cast<std::size_t>(__xe - __xs);
if (__count <= __default_chunk_size || __nsort < __count)
{
__leaf_sort(__xs, __xe, __comp);
return;
}
// TODO: the partial sort implementation should
// be shared with the other backends.
if (omp_in_parallel())
{
if (__count <= __nsort)
{
__pstl::__omp_backend::__parallel_stable_sort_body(__xs, __xe, __comp, __leaf_sort);
}
else
{
__pstl::__omp_backend::__parallel_stable_partial_sort(__tag, __xs, __xe, __comp, __leaf_sort, __nsort);
}
}
else
{
_PSTL_PRAGMA(omp parallel)
_PSTL_PRAGMA(omp single nowait)
if (__count <= __nsort)
{
__pstl::__omp_backend::__parallel_stable_sort_body(__xs, __xe, __comp, __leaf_sort);
}
else
{
__pstl::__omp_backend::__parallel_stable_partial_sort(__tag, __xs, __xe, __comp, __leaf_sort, __nsort);
}
}
}
} // namespace __omp_backend
} // namespace __pstl
#endif // _PSTL_INTERNAL_OMP_PARALLEL_STABLE_SORT_H

113
third_party/libcxx/__pstl/internal/omp/parallel_transform_reduce.h vendored Normal file
View file

@ -0,0 +1,113 @@
// -*- C++ -*-
// -*-===----------------------------------------------------------------------===//
//
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_INTERNAL_OMP_PARALLEL_TRANSFORM_REDUCE_H
#define _PSTL_INTERNAL_OMP_PARALLEL_TRANSFORM_REDUCE_H
#include "util.h"
namespace __pstl
{
namespace __omp_backend
{
//------------------------------------------------------------------------
// parallel_transform_reduce
//
// Notation:
// r(i,j,init) returns reduction of init with reduction over [i,j)
// u(i) returns f(i,i+1,identity) for a hypothetical left identity element
// of r c(x,y) combines values x and y that were the result of r or u
//------------------------------------------------------------------------
template <class _RandomAccessIterator, class _UnaryOp, class _Value, class _Combiner, class _Reduction>
auto
__transform_reduce_body(_RandomAccessIterator __first, _RandomAccessIterator __last, _UnaryOp __unary_op, _Value __init,
_Combiner __combiner, _Reduction __reduction)
{
const std::size_t __num_threads = omp_get_num_threads();
const std::size_t __size = __last - __first;
// Initial partition of the iteration space into chunks. If the range is too small,
// this will result in a nonsense policy, so we check on the size as well below.
auto __policy = __omp_backend::__chunk_partitioner(__first + __num_threads, __last);
if (__size <= __num_threads || __policy.__n_chunks < 2)
{
return __reduction(__first, __last, __init);
}
// Here, we cannot use OpenMP UDR because we must store the init value in
// the combiner and it will be used several times. Although there should be
// the only one; we manually generate the identity elements for each thread.
std::vector<_Value> __accums;
__accums.reserve(__num_threads);
// initialize accumulators for all threads
for (std::size_t __i = 0; __i < __num_threads; ++__i)
{
__accums.emplace_back(__unary_op(__first + __i));
}
// main loop
_PSTL_PRAGMA(omp taskloop shared(__accums))
for (std::size_t __chunk = 0; __chunk < __policy.__n_chunks; ++__chunk)
{
__pstl::__omp_backend::__process_chunk(__policy, __first + __num_threads, __chunk,
[&](auto __chunk_first, auto __chunk_last)
{
auto __thread_num = omp_get_thread_num();
__accums[__thread_num] =
__reduction(__chunk_first, __chunk_last, __accums[__thread_num]);
});
}
// combine by accumulators
for (std::size_t __i = 0; __i < __num_threads; ++__i)
{
__init = __combiner(__init, __accums[__i]);
}
return __init;
}
template <class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryOp, class _Value, class _Combiner,
class _Reduction>
_Value
__parallel_transform_reduce(__pstl::__internal::__openmp_backend_tag, _ExecutionPolicy&&, _RandomAccessIterator __first,
_RandomAccessIterator __last, _UnaryOp __unary_op, _Value __init, _Combiner __combiner,
_Reduction __reduction)
{
_Value __result = __init;
if (omp_in_parallel())
{
// We don't create a nested parallel region in an existing parallel
// region: just create tasks
__result = __pstl::__omp_backend::__transform_reduce_body(__first, __last, __unary_op, __init, __combiner,
__reduction);
}
else
{
// Create a parallel region, and a single thread will create tasks
// for the region.
_PSTL_PRAGMA(omp parallel)
_PSTL_PRAGMA(omp single nowait)
{
__result = __pstl::__omp_backend::__transform_reduce_body(__first, __last, __unary_op, __init, __combiner,
__reduction);
}
}
return __result;
}
} // namespace __omp_backend
} // namespace __pstl
#endif // _PSTL_INTERNAL_OMP_PARALLEL_TRANSFORM_REDUCE_H

32
third_party/libcxx/__pstl/internal/omp/parallel_transform_scan.h vendored Normal file
View file

@ -0,0 +1,32 @@
// -*- C++ -*-
// -*-===----------------------------------------------------------------------===//
//
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_INTERNAL_OMP_PARALLEL_TRANSFORM_SCAN_H
#define _PSTL_INTERNAL_OMP_PARALLEL_TRANSFORM_SCAN_H
#include "util.h"
namespace __pstl
{
namespace __omp_backend
{
template <class _ExecutionPolicy, class _Index, class _Up, class _Tp, class _Cp, class _Rp, class _Sp>
_Tp
__parallel_transform_scan(__pstl::__internal::__openmp_backend_tag, _ExecutionPolicy&&, _Index __n, _Up /* __u */,
_Tp __init, _Cp /* __combine */, _Rp /* __brick_reduce */, _Sp __scan)
{
// TODO: parallelize this function.
return __scan(_Index(0), __n, __init);
}
} // namespace __omp_backend
} // namespace __pstl
#endif // _PSTL_INTERNAL_OMP_PARALLEL_TRANSFORM_SCAN_H

171
third_party/libcxx/__pstl/internal/omp/util.h vendored Normal file
View file

@ -0,0 +1,171 @@
// -*- C++ -*-
// -*-===----------------------------------------------------------------------===//
//
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_INTERNAL_OMP_UTIL_H
#define _PSTL_INTERNAL_OMP_UTIL_H
#include <algorithm>
#include <atomic>
#include <iterator>
#include <cstddef>
#include <cstdio>
#include <memory>
#include <vector>
#include <omp.h>
#include "third_party/libcxx/__pstl/internal/parallel_backend_utils.h"
#include "third_party/libcxx/__pstl/internal/unseq_backend_simd.h"
#include "third_party/libcxx/__pstl/internal/utils.h"
// Portability "#pragma" definition
#ifdef _MSC_VER
# define _PSTL_PRAGMA(x) __pragma(x)
#else
# define _PSTL_PRAGMA(x) _Pragma(# x)
#endif
namespace __pstl
{
namespace __omp_backend
{
//------------------------------------------------------------------------
// use to cancel execution
//------------------------------------------------------------------------
inline void
__cancel_execution()
{
// TODO: Figure out how to make cancelation work.
}
//------------------------------------------------------------------------
// raw buffer
//------------------------------------------------------------------------
template <typename _Tp>
class __buffer
{
std::allocator<_Tp> __allocator_;
_Tp* __ptr_;
const std::size_t __buf_size_;
__buffer(const __buffer&) = delete;
void
operator=(const __buffer&) = delete;
public:
__buffer(std::size_t __n) : __allocator_(), __ptr_(__allocator_.allocate(__n)), __buf_size_(__n) {}
operator bool() const { return __ptr_ != nullptr; }
_Tp*
get() const
{
return __ptr_;
}
~__buffer() { __allocator_.deallocate(__ptr_, __buf_size_); }
};
// Preliminary size of each chunk: requires further discussion
inline constexpr std::size_t __default_chunk_size = 2048;
// Convenience function to determine when we should run serial.
template <typename _Iterator, std::enable_if_t<!std::is_integral<_Iterator>::value, bool> = true>
constexpr auto
__should_run_serial(_Iterator __first, _Iterator __last) -> bool
{
using _difference_type = typename std::iterator_traits<_Iterator>::difference_type;
auto __size = std::distance(__first, __last);
return __size <= static_cast<_difference_type>(__default_chunk_size);
}
template <typename _Index, std::enable_if_t<std::is_integral<_Index>::value, bool> = true>
constexpr auto
__should_run_serial(_Index __first, _Index __last) -> bool
{
using _difference_type = _Index;
auto __size = __last - __first;
return __size <= static_cast<_difference_type>(__default_chunk_size);
}
struct __chunk_metrics
{
std::size_t __n_chunks;
std::size_t __chunk_size;
std::size_t __first_chunk_size;
};
// The iteration space partitioner according to __requested_chunk_size
template <class _RandomAccessIterator, class _Size = std::size_t>
auto
__chunk_partitioner(_RandomAccessIterator __first, _RandomAccessIterator __last,
_Size __requested_chunk_size = __default_chunk_size) -> __chunk_metrics
{
/*
* This algorithm improves distribution of elements in chunks by avoiding
* small tail chunks. The leftover elements that do not fit neatly into
* the chunk size are redistributed to early chunks. This improves
* utilization of the processor's prefetch and reduces the number of
* tasks needed by 1.
*/
const _Size __n = __last - __first;
_Size __n_chunks = 0;
_Size __chunk_size = 0;
_Size __first_chunk_size = 0;
if (__n < __requested_chunk_size)
{
__chunk_size = __n;
__first_chunk_size = __n;
__n_chunks = 1;
return __chunk_metrics{__n_chunks, __chunk_size, __first_chunk_size};
}
__n_chunks = (__n / __requested_chunk_size) + 1;
__chunk_size = __n / __n_chunks;
__first_chunk_size = __chunk_size;
const _Size __n_leftover_items = __n - (__n_chunks * __chunk_size);
if (__n_leftover_items == __chunk_size)
{
__n_chunks += 1;
return __chunk_metrics{__n_chunks, __chunk_size, __first_chunk_size};
}
else if (__n_leftover_items == 0)
{
__first_chunk_size = __chunk_size;
return __chunk_metrics{__n_chunks, __chunk_size, __first_chunk_size};
}
const _Size __n_extra_items_per_chunk = __n_leftover_items / __n_chunks;
const _Size __n_final_leftover_items = __n_leftover_items - (__n_extra_items_per_chunk * __n_chunks);
__chunk_size += __n_extra_items_per_chunk;
__first_chunk_size = __chunk_size + __n_final_leftover_items;
return __chunk_metrics{__n_chunks, __chunk_size, __first_chunk_size};
}
template <typename _Iterator, typename _Index, typename _Func>
void
__process_chunk(const __chunk_metrics& __metrics, _Iterator __base, _Index __chunk_index, _Func __f)
{
auto __this_chunk_size = __chunk_index == 0 ? __metrics.__first_chunk_size : __metrics.__chunk_size;
auto __index = __chunk_index == 0 ? 0
: (__chunk_index * __metrics.__chunk_size) +
(__metrics.__first_chunk_size - __metrics.__chunk_size);
auto __first = __base + __index;
auto __last = __first + __this_chunk_size;
__f(__first, __last);
}
} // namespace __omp_backend
} // namespace __pstl
#endif // _PSTL_INTERNAL_OMP_UTIL_H

39
third_party/libcxx/__pstl/internal/parallel_backend.h vendored Normal file
View file

@ -0,0 +1,39 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_PARALLEL_BACKEND_H
#define _PSTL_PARALLEL_BACKEND_H
#include <__config>
#if defined(_PSTL_PAR_BACKEND_SERIAL)
# include "parallel_backend_serial.h"
# if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
namespace __pstl
{
namespace __par_backend = __serial_backend;
} // namespace __pstl
# endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
#elif defined(_PSTL_PAR_BACKEND_TBB)
# include "parallel_backend_tbb.h"
namespace __pstl
{
namespace __par_backend = __tbb_backend;
}
#elif defined(_PSTL_PAR_BACKEND_OPENMP)
# include "parallel_backend_omp.h"
namespace __pstl
{
namespace __par_backend = __omp_backend;
}
#else
# error "No backend set"
#endif
#endif /* _PSTL_PARALLEL_BACKEND_H */

58
third_party/libcxx/__pstl/internal/parallel_backend_omp.h vendored Normal file
View file

@ -0,0 +1,58 @@
// -*- C++ -*-
// -*-===----------------------------------------------------------------------===//
//
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_PARALLEL_BACKEND_OMP_H
#define _PSTL_PARALLEL_BACKEND_OMP_H
//------------------------------------------------------------------------
// parallel_invoke
//------------------------------------------------------------------------
#include "third_party/libcxx/__pstl/internal/omp/parallel_invoke.h"
//------------------------------------------------------------------------
// parallel_for
//------------------------------------------------------------------------
#include "third_party/libcxx/__pstl/internal/omp/parallel_for.h"
//------------------------------------------------------------------------
// parallel_for_each
//------------------------------------------------------------------------
#include "third_party/libcxx/__pstl/internal/omp/parallel_for_each.h"
//------------------------------------------------------------------------
// parallel_reduce
//------------------------------------------------------------------------
#include "third_party/libcxx/__pstl/internal/omp/parallel_reduce.h"
#include "third_party/libcxx/__pstl/internal/omp/parallel_transform_reduce.h"
//------------------------------------------------------------------------
// parallel_scan
//------------------------------------------------------------------------
#include "third_party/libcxx/__pstl/internal/omp/parallel_scan.h"
#include "third_party/libcxx/__pstl/internal/omp/parallel_transform_scan.h"
//------------------------------------------------------------------------
// parallel_stable_sort
//------------------------------------------------------------------------
#include "third_party/libcxx/__pstl/internal/omp/parallel_stable_partial_sort.h"
#include "third_party/libcxx/__pstl/internal/omp/parallel_stable_sort.h"
//------------------------------------------------------------------------
// parallel_merge
//------------------------------------------------------------------------
#include "third_party/libcxx/__pstl/internal/omp/parallel_merge.h"
#endif //_PSTL_PARALLEL_BACKEND_OMP_H

114
third_party/libcxx/__pstl/internal/parallel_backend_serial.h vendored Normal file
View file

@ -0,0 +1,114 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_PARALLEL_BACKEND_SERIAL_H
#define _PSTL_PARALLEL_BACKEND_SERIAL_H
#include <__config>
#include <__memory/allocator.h>
#include <__pstl/internal/execution_impl.h>
#include <__utility/forward.h>
#include <cstddef>
#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
namespace __pstl
{
namespace __serial_backend
{
template <typename _Tp>
class __buffer
{
std::allocator<_Tp> __allocator_;
_Tp* __ptr_;
const std::size_t __buf_size_;
__buffer(const __buffer&) = delete;
void
operator=(const __buffer&) = delete;
public:
_LIBCPP_HIDE_FROM_ABI
__buffer(std::size_t __n) : __allocator_(), __ptr_(__allocator_.allocate(__n)), __buf_size_(__n) {}
_LIBCPP_HIDE_FROM_ABI operator bool() const { return __ptr_ != nullptr; }
_LIBCPP_HIDE_FROM_ABI _Tp*
get() const
{
return __ptr_;
}
_LIBCPP_HIDE_FROM_ABI ~__buffer() { __allocator_.deallocate(__ptr_, __buf_size_); }
};
template <class _ExecutionPolicy, class _Value, class _Index, typename _RealBody, typename _Reduction>
_LIBCPP_HIDE_FROM_ABI _Value
__parallel_reduce(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _Index __first, _Index __last,
const _Value& __identity, const _RealBody& __real_body, const _Reduction&)
{
if (__first == __last)
{
return __identity;
}
else
{
return __real_body(__first, __last, __identity);
}
}
template <class _ExecutionPolicy, class _Index, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduce>
_LIBCPP_HIDE_FROM_ABI _Tp
__parallel_transform_reduce(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _Index __first, _Index __last,
_UnaryOp, _Tp __init, _BinaryOp, _Reduce __reduce)
{
return __reduce(__first, __last, __init);
}
template <class _ExecutionPolicy, typename _Index, typename _Tp, typename _Rp, typename _Cp, typename _Sp, typename _Ap>
_LIBCPP_HIDE_FROM_ABI void
__parallel_strict_scan(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _Index __n, _Tp __initial,
_Rp __reduce, _Cp __combine, _Sp __scan, _Ap __apex)
{
_Tp __sum = __initial;
if (__n)
__sum = __combine(__sum, __reduce(_Index(0), __n));
__apex(__sum);
if (__n)
__scan(_Index(0), __n, __initial);
}
template <class _ExecutionPolicy, class _Index, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduce, class _Scan>
_LIBCPP_HIDE_FROM_ABI _Tp
__parallel_transform_scan(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _Index __n, _UnaryOp,
_Tp __init, _BinaryOp, _Reduce, _Scan __scan)
{
return __scan(_Index(0), __n, __init);
}
template <class _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare, typename _LeafSort>
_LIBCPP_HIDE_FROM_ABI void
__parallel_stable_sort(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _RandomAccessIterator __first,
_RandomAccessIterator __last, _Compare __comp, _LeafSort __leaf_sort, std::size_t = 0)
{
__leaf_sort(__first, __last, __comp);
}
template <class _ExecutionPolicy, typename _F1, typename _F2>
_LIBCPP_HIDE_FROM_ABI void
__parallel_invoke(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _F1&& __f1, _F2&& __f2)
{
std::forward<_F1>(__f1)();
std::forward<_F2>(__f2)();
}
} // namespace __serial_backend
} // namespace __pstl
#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
#endif /* _PSTL_PARALLEL_BACKEND_SERIAL_H */

1293
third_party/libcxx/__pstl/internal/parallel_backend_tbb.h vendored Normal file
View file

File diff suppressed because it is too large Load diff

260
third_party/libcxx/__pstl/internal/parallel_backend_utils.h vendored Normal file
View file

@ -0,0 +1,260 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_PARALLEL_BACKEND_UTILS_H
#define _PSTL_PARALLEL_BACKEND_UTILS_H
#include <__assert>
#include <__config>
#include <__iterator/iterator_traits.h>
#include <__memory/addressof.h>
#include "utils.h"
namespace __pstl
{
namespace __utils
{
//! Destroy sequence [xs,xe)
struct __serial_destroy
{
template <typename _RandomAccessIterator>
void
operator()(_RandomAccessIterator __zs, _RandomAccessIterator __ze)
{
typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _ValueType;
while (__zs != __ze)
{
--__ze;
(*__ze).~_ValueType();
}
}
};
//! Merge sequences [__xs,__xe) and [__ys,__ye) to output sequence [__zs,(__xe-__xs)+(__ye-__ys)), using std::move
struct __serial_move_merge
{
const std::size_t _M_nmerge;
explicit __serial_move_merge(std::size_t __nmerge) : _M_nmerge(__nmerge) {}
template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Compare,
class _MoveValueX, class _MoveValueY, class _MoveSequenceX, class _MoveSequenceY>
void
operator()(_RandomAccessIterator1 __xs, _RandomAccessIterator1 __xe, _RandomAccessIterator2 __ys,
_RandomAccessIterator2 __ye, _RandomAccessIterator3 __zs, _Compare __comp, _MoveValueX __move_value_x,
_MoveValueY __move_value_y, _MoveSequenceX __move_sequence_x, _MoveSequenceY __move_sequence_y)
{
constexpr bool __same_move_val = std::is_same<_MoveValueX, _MoveValueY>::value;
constexpr bool __same_move_seq = std::is_same<_MoveSequenceX, _MoveSequenceY>::value;
auto __n = _M_nmerge;
_LIBCPP_ASSERT(__n > 0, "");
auto __nx = __xe - __xs;
//auto __ny = __ye - __ys;
_RandomAccessIterator3 __zs_beg = __zs;
if (__xs != __xe)
{
if (__ys != __ye)
{
for (;;)
{
if (__comp(*__ys, *__xs))
{
const auto __i = __zs - __zs_beg;
if (__i < __nx)
__move_value_x(__ys, __zs);
else
__move_value_y(__ys, __zs);
++__zs, --__n;
if (++__ys == __ye)
{
break;
}
else if (__n == 0)
{
const auto __j = __zs - __zs_beg;
if (__same_move_seq || __j < __nx)
__zs = __move_sequence_x(__ys, __ye, __zs);
else
__zs = __move_sequence_y(__ys, __ye, __zs);
break;
}
}
else
{
const auto __i = __zs - __zs_beg;
if (__same_move_val || __i < __nx)
__move_value_x(__xs, __zs);
else
__move_value_y(__xs, __zs);
++__zs, --__n;
if (++__xs == __xe)
{
const auto __j = __zs - __zs_beg;
if (__same_move_seq || __j < __nx)
__move_sequence_x(__ys, __ye, __zs);
else
__move_sequence_y(__ys, __ye, __zs);
return;
}
else if (__n == 0)
{
const auto __j = __zs - __zs_beg;
if (__same_move_seq || __j < __nx)
{
__zs = __move_sequence_x(__xs, __xe, __zs);
__move_sequence_x(__ys, __ye, __zs);
}
else
{
__zs = __move_sequence_y(__xs, __xe, __zs);
__move_sequence_y(__ys, __ye, __zs);
}
return;
}
}
}
}
__ys = __xs;
__ye = __xe;
}
const auto __i = __zs - __zs_beg;
if (__same_move_seq || __i < __nx)
__move_sequence_x(__ys, __ye, __zs);
else
__move_sequence_y(__ys, __ye, __zs);
}
};
template <typename _ForwardIterator1, typename _ForwardIterator2, typename _OutputIterator, typename _Compare,
typename _CopyConstructRange>
_OutputIterator
__set_union_construct(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
_ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
_CopyConstructRange __cc_range)
{
using _Tp = typename std::iterator_traits<_OutputIterator>::value_type;
for (; __first1 != __last1; ++__result)
{
if (__first2 == __last2)
return __cc_range(__first1, __last1, __result);
if (__comp(*__first2, *__first1))
{
::new (std::addressof(*__result)) _Tp(*__first2);
++__first2;
}
else
{
::new (std::addressof(*__result)) _Tp(*__first1);
if (!__comp(*__first1, *__first2))
++__first2;
++__first1;
}
}
return __cc_range(__first2, __last2, __result);
}
template <typename _ForwardIterator1, typename _ForwardIterator2, typename _OutputIterator, typename _Compare>
_OutputIterator
__set_intersection_construct(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
_ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp)
{
using _Tp = typename std::iterator_traits<_OutputIterator>::value_type;
for (; __first1 != __last1 && __first2 != __last2;)
{
if (__comp(*__first1, *__first2))
++__first1;
else
{
if (!__comp(*__first2, *__first1))
{
::new (std::addressof(*__result)) _Tp(*__first1);
++__result;
++__first1;
}
++__first2;
}
}
return __result;
}
template <typename _ForwardIterator1, typename _ForwardIterator2, typename _OutputIterator, typename _Compare,
typename _CopyConstructRange>
_OutputIterator
__set_difference_construct(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
_ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
_CopyConstructRange __cc_range)
{
using _Tp = typename std::iterator_traits<_OutputIterator>::value_type;
for (; __first1 != __last1;)
{
if (__first2 == __last2)
return __cc_range(__first1, __last1, __result);
if (__comp(*__first1, *__first2))
{
::new (std::addressof(*__result)) _Tp(*__first1);
++__result;
++__first1;
}
else
{
if (!__comp(*__first2, *__first1))
++__first1;
++__first2;
}
}
return __result;
}
template <typename _ForwardIterator1, typename _ForwardIterator2, typename _OutputIterator, typename _Compare,
typename _CopyConstructRange>
_OutputIterator
__set_symmetric_difference_construct(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
_ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
_CopyConstructRange __cc_range)
{
using _Tp = typename std::iterator_traits<_OutputIterator>::value_type;
for (; __first1 != __last1;)
{
if (__first2 == __last2)
return __cc_range(__first1, __last1, __result);
if (__comp(*__first1, *__first2))
{
::new (std::addressof(*__result)) _Tp(*__first1);
++__result;
++__first1;
}
else
{
if (__comp(*__first2, *__first1))
{
::new (std::addressof(*__result)) _Tp(*__first2);
++__result;
}
else
++__first1;
++__first2;
}
}
return __cc_range(__first2, __last2, __result);
}
} // namespace __utils
} // namespace __pstl
#endif /* _PSTL_PARALLEL_BACKEND_UTILS_H */

762
third_party/libcxx/__pstl/internal/unseq_backend_simd.h vendored Normal file
View file

@ -0,0 +1,762 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_UNSEQ_BACKEND_SIMD_H
#define _PSTL_UNSEQ_BACKEND_SIMD_H
#include <__config>
#include <__functional/operations.h>
#include <__iterator/iterator_traits.h>
#include <__type_traits/is_arithmetic.h>
#include <__type_traits/is_same.h>
#include <__utility/move.h>
#include <__utility/pair.h>
#include <cstddef>
#include <cstdint>
#include <__pstl/internal/utils.h>
// This header defines the minimum set of vector routines required
// to support parallel STL.
#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
namespace __pstl
{
namespace __unseq_backend
{
// Expect vector width up to 64 (or 512 bit)
const std::size_t __lane_size = 64;
template <class _Iterator, class _DifferenceType, class _Function>
_LIBCPP_HIDE_FROM_ABI _Iterator
__simd_walk_1(_Iterator __first, _DifferenceType __n, _Function __f) noexcept
{
_PSTL_PRAGMA_SIMD
for (_DifferenceType __i = 0; __i < __n; ++__i)
__f(__first[__i]);
return __first + __n;
}
template <class _Iterator1, class _DifferenceType, class _Iterator2, class _Function>
_LIBCPP_HIDE_FROM_ABI _Iterator2
__simd_walk_2(_Iterator1 __first1, _DifferenceType __n, _Iterator2 __first2, _Function __f) noexcept
{
_PSTL_PRAGMA_SIMD
for (_DifferenceType __i = 0; __i < __n; ++__i)
__f(__first1[__i], __first2[__i]);
return __first2 + __n;
}
template <class _Iterator1, class _DifferenceType, class _Iterator2, class _Iterator3, class _Function>
_LIBCPP_HIDE_FROM_ABI _Iterator3
__simd_walk_3(_Iterator1 __first1, _DifferenceType __n, _Iterator2 __first2, _Iterator3 __first3,
_Function __f) noexcept
{
_PSTL_PRAGMA_SIMD
for (_DifferenceType __i = 0; __i < __n; ++__i)
__f(__first1[__i], __first2[__i], __first3[__i]);
return __first3 + __n;
}
// TODO: check whether __simd_first() can be used here
template <class _Index, class _DifferenceType, class _Pred>
_LIBCPP_HIDE_FROM_ABI bool
__simd_or(_Index __first, _DifferenceType __n, _Pred __pred) noexcept
{
_DifferenceType __block_size = 4 < __n ? 4 : __n;
const _Index __last = __first + __n;
while (__last != __first)
{
int32_t __flag = 1;
_PSTL_PRAGMA_SIMD_REDUCTION(& : __flag)
for (_DifferenceType __i = 0; __i < __block_size; ++__i)
if (__pred(*(__first + __i)))
__flag = 0;
if (!__flag)
return true;
__first += __block_size;
if (__last - __first >= __block_size << 1)
{
// Double the block _Size. Any unnecessary iterations can be amortized against work done so far.
__block_size <<= 1;
}
else
{
__block_size = __last - __first;
}
}
return false;
}
template <class _Index1, class _DifferenceType, class _Index2, class _Pred>
_LIBCPP_HIDE_FROM_ABI std::pair<_Index1, _Index2>
__simd_first(_Index1 __first1, _DifferenceType __n, _Index2 __first2, _Pred __pred) noexcept
{
const _Index1 __last1 = __first1 + __n;
const _Index2 __last2 = __first2 + __n;
// Experiments show good block sizes like this
const _DifferenceType __block_size = 8;
alignas(__lane_size) _DifferenceType __lane[__block_size] = {0};
while (__last1 - __first1 >= __block_size)
{
_DifferenceType __found = 0;
_DifferenceType __i;
_PSTL_PRAGMA_SIMD_REDUCTION(|
: __found) for (__i = 0; __i < __block_size; ++__i)
{
const _DifferenceType __t = __pred(__first1[__i], __first2[__i]);
__lane[__i] = __t;
__found |= __t;
}
if (__found)
{
_DifferenceType __i2;
// This will vectorize
for (__i2 = 0; __i2 < __block_size; ++__i2)
{
if (__lane[__i2])
break;
}
return std::make_pair(__first1 + __i2, __first2 + __i2);
}
__first1 += __block_size;
__first2 += __block_size;
}
//Keep remainder scalar
for (; __last1 != __first1; ++__first1, ++__first2)
if (__pred(*(__first1), *(__first2)))
return std::make_pair(__first1, __first2);
return std::make_pair(__last1, __last2);
}
template <class _Index, class _DifferenceType, class _Pred>
_LIBCPP_HIDE_FROM_ABI _DifferenceType
__simd_count(_Index __index, _DifferenceType __n, _Pred __pred) noexcept
{
_DifferenceType __count = 0;
_PSTL_PRAGMA_SIMD_REDUCTION(+ : __count)
for (_DifferenceType __i = 0; __i < __n; ++__i)
if (__pred(*(__index + __i)))
++__count;
return __count;
}
template <class _InputIterator, class _DifferenceType, class _OutputIterator, class _BinaryPredicate>
_LIBCPP_HIDE_FROM_ABI _OutputIterator
__simd_unique_copy(_InputIterator __first, _DifferenceType __n, _OutputIterator __result,
_BinaryPredicate __pred) noexcept
{
if (__n == 0)
return __result;
_DifferenceType __cnt = 1;
__result[0] = __first[0];
_PSTL_PRAGMA_SIMD
for (_DifferenceType __i = 1; __i < __n; ++__i)
{
if (!__pred(__first[__i], __first[__i - 1]))
{
__result[__cnt] = __first[__i];
++__cnt;
}
}
return __result + __cnt;
}
template <class _InputIterator, class _DifferenceType, class _OutputIterator, class _Assigner>
_LIBCPP_HIDE_FROM_ABI _OutputIterator
__simd_assign(_InputIterator __first, _DifferenceType __n, _OutputIterator __result, _Assigner __assigner) noexcept
{
_PSTL_USE_NONTEMPORAL_STORES_IF_ALLOWED
_PSTL_PRAGMA_SIMD
for (_DifferenceType __i = 0; __i < __n; ++__i)
__assigner(__first + __i, __result + __i);
return __result + __n;
}
template <class _InputIterator, class _DifferenceType, class _OutputIterator, class _UnaryPredicate>
_LIBCPP_HIDE_FROM_ABI _OutputIterator
__simd_copy_if(_InputIterator __first, _DifferenceType __n, _OutputIterator __result, _UnaryPredicate __pred) noexcept
{
_DifferenceType __cnt = 0;
_PSTL_PRAGMA_SIMD
for (_DifferenceType __i = 0; __i < __n; ++__i)
{
if (__pred(__first[__i]))
{
__result[__cnt] = __first[__i];
++__cnt;
}
}
return __result + __cnt;
}
template <class _InputIterator, class _DifferenceType, class _BinaryPredicate>
_LIBCPP_HIDE_FROM_ABI _DifferenceType
__simd_calc_mask_2(_InputIterator __first, _DifferenceType __n, bool* __mask, _BinaryPredicate __pred) noexcept
{
_DifferenceType __count = 0;
_PSTL_PRAGMA_SIMD_REDUCTION(+ : __count)
for (_DifferenceType __i = 0; __i < __n; ++__i)
{
__mask[__i] = !__pred(__first[__i], __first[__i - 1]);
__count += __mask[__i];
}
return __count;
}
template <class _InputIterator, class _DifferenceType, class _UnaryPredicate>
_LIBCPP_HIDE_FROM_ABI _DifferenceType
__simd_calc_mask_1(_InputIterator __first, _DifferenceType __n, bool* __mask, _UnaryPredicate __pred) noexcept
{
_DifferenceType __count = 0;
_PSTL_PRAGMA_SIMD_REDUCTION(+ : __count)
for (_DifferenceType __i = 0; __i < __n; ++__i)
{
__mask[__i] = __pred(__first[__i]);
__count += __mask[__i];
}
return __count;
}
template <class _InputIterator, class _DifferenceType, class _OutputIterator, class _Assigner>
_LIBCPP_HIDE_FROM_ABI void
__simd_copy_by_mask(_InputIterator __first, _DifferenceType __n, _OutputIterator __result, bool* __mask,
_Assigner __assigner) noexcept
{
_DifferenceType __cnt = 0;
_PSTL_PRAGMA_SIMD
for (_DifferenceType __i = 0; __i < __n; ++__i)
{
if (__mask[__i])
{
{
__assigner(__first + __i, __result + __cnt);
++__cnt;
}
}
}
}
template <class _InputIterator, class _DifferenceType, class _OutputIterator1, class _OutputIterator2>
_LIBCPP_HIDE_FROM_ABI void
__simd_partition_by_mask(_InputIterator __first, _DifferenceType __n, _OutputIterator1 __out_true,
_OutputIterator2 __out_false, bool* __mask) noexcept
{
_DifferenceType __cnt_true = 0, __cnt_false = 0;
_PSTL_PRAGMA_SIMD
for (_DifferenceType __i = 0; __i < __n; ++__i)
{
if (__mask[__i])
{
__out_true[__cnt_true] = __first[__i];
++__cnt_true;
}
else
{
__out_false[__cnt_false] = __first[__i];
++__cnt_false;
}
}
}
template <class _Index, class _DifferenceType, class _Generator>
_LIBCPP_HIDE_FROM_ABI _Index
__simd_generate_n(_Index __first, _DifferenceType __size, _Generator __g) noexcept
{
_PSTL_USE_NONTEMPORAL_STORES_IF_ALLOWED
_PSTL_PRAGMA_SIMD
for (_DifferenceType __i = 0; __i < __size; ++__i)
__first[__i] = __g();
return __first + __size;
}
template <class _Index, class _BinaryPredicate>
_LIBCPP_HIDE_FROM_ABI _Index
__simd_adjacent_find(_Index __first, _Index __last, _BinaryPredicate __pred, bool __or_semantic) noexcept
{
if (__last - __first < 2)
return __last;
typedef typename std::iterator_traits<_Index>::difference_type _DifferenceType;
_DifferenceType __i = 0;
// Experiments show good block sizes like this
//TODO: to consider tuning block_size for various data types
const _DifferenceType __block_size = 8;
alignas(__lane_size) _DifferenceType __lane[__block_size] = {0};
while (__last - __first >= __block_size)
{
_DifferenceType __found = 0;
_PSTL_PRAGMA_SIMD_REDUCTION(|
: __found) for (__i = 0; __i < __block_size - 1; ++__i)
{
//TODO: to improve SIMD vectorization
const _DifferenceType __t = __pred(*(__first + __i), *(__first + __i + 1));
__lane[__i] = __t;
__found |= __t;
}
//Process a pair of elements on a boundary of a data block
if (__first + __block_size < __last && __pred(*(__first + __i), *(__first + __i + 1)))
__lane[__i] = __found = 1;
if (__found)
{
if (__or_semantic)
return __first;
// This will vectorize
for (__i = 0; __i < __block_size; ++__i)
if (__lane[__i])
break;
return __first + __i; //As far as found is true a __result (__lane[__i] is true) is guaranteed
}
__first += __block_size;
}
//Process the rest elements
for (; __last - __first > 1; ++__first)
if (__pred(*__first, *(__first + 1)))
return __first;
return __last;
}
// It was created to reduce the code inside std::enable_if
template <typename _Tp, typename _BinaryOperation>
using is_arithmetic_plus = std::integral_constant<bool, std::is_arithmetic<_Tp>::value &&
std::is_same<_BinaryOperation, std::plus<_Tp>>::value>;
template <typename _DifferenceType, typename _Tp, typename _BinaryOperation, typename _UnaryOperation>
_LIBCPP_HIDE_FROM_ABI typename std::enable_if<is_arithmetic_plus<_Tp, _BinaryOperation>::value, _Tp>::type
__simd_transform_reduce(_DifferenceType __n, _Tp __init, _BinaryOperation, _UnaryOperation __f) noexcept
{
_PSTL_PRAGMA_SIMD_REDUCTION(+ : __init)
for (_DifferenceType __i = 0; __i < __n; ++__i)
__init += __f(__i);
return __init;
}
template <typename _Size, typename _Tp, typename _BinaryOperation, typename _UnaryOperation>
_LIBCPP_HIDE_FROM_ABI typename std::enable_if<!is_arithmetic_plus<_Tp, _BinaryOperation>::value, _Tp>::type
__simd_transform_reduce(_Size __n, _Tp __init, _BinaryOperation __binary_op, _UnaryOperation __f) noexcept
{
const _Size __block_size = __lane_size / sizeof(_Tp);
if (__n > 2 * __block_size && __block_size > 1)
{
alignas(__lane_size) char __lane_buffer[__lane_size];
_Tp* __lane = reinterpret_cast<_Tp*>(__lane_buffer);
// initializer
_PSTL_PRAGMA_SIMD
for (_Size __i = 0; __i < __block_size; ++__i)
{
::new (__lane + __i) _Tp(__binary_op(__f(__i), __f(__block_size + __i)));
}
// main loop
_Size __i = 2 * __block_size;
const _Size __last_iteration = __block_size * (__n / __block_size);
for (; __i < __last_iteration; __i += __block_size)
{
_PSTL_PRAGMA_SIMD
for (_Size __j = 0; __j < __block_size; ++__j)
{
__lane[__j] = __binary_op(__lane[__j], __f(__i + __j));
}
}
// remainder
_PSTL_PRAGMA_SIMD
for (_Size __j = 0; __j < __n - __last_iteration; ++__j)
{
__lane[__j] = __binary_op(__lane[__j], __f(__last_iteration + __j));
}
// combiner
for (_Size __j = 0; __j < __block_size; ++__j)
{
__init = __binary_op(__init, __lane[__j]);
}
// destroyer
_PSTL_PRAGMA_SIMD
for (_Size __j = 0; __j < __block_size; ++__j)
{
__lane[__j].~_Tp();
}
}
else
{
for (_Size __i = 0; __i < __n; ++__i)
{
__init = __binary_op(__init, __f(__i));
}
}
return __init;
}
// Exclusive scan for "+" and arithmetic types
template <class _InputIterator, class _Size, class _OutputIterator, class _UnaryOperation, class _Tp,
class _BinaryOperation>
_LIBCPP_HIDE_FROM_ABI
typename std::enable_if<is_arithmetic_plus<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
__simd_scan(_InputIterator __first, _Size __n, _OutputIterator __result, _UnaryOperation __unary_op, _Tp __init,
_BinaryOperation, /*Inclusive*/ std::false_type)
{
_PSTL_PRAGMA_SIMD_SCAN(+ : __init)
for (_Size __i = 0; __i < __n; ++__i)
{
__result[__i] = __init;
_PSTL_PRAGMA_SIMD_EXCLUSIVE_SCAN(__init)
__init += __unary_op(__first[__i]);
}
return std::make_pair(__result + __n, __init);
}
// As soon as we cannot call __binary_op in "combiner" we create a wrapper over _Tp to encapsulate __binary_op
template <typename _Tp, typename _BinaryOp>
struct _Combiner
{
_Tp __value_;
_BinaryOp* __bin_op_; // Here is a pointer to function because of default ctor
_LIBCPP_HIDE_FROM_ABI _Combiner() : __value_{}, __bin_op_(nullptr) {}
_LIBCPP_HIDE_FROM_ABI
_Combiner(const _Tp& __value, const _BinaryOp* __bin_op)
: __value_(__value), __bin_op_(const_cast<_BinaryOp*>(__bin_op)) {}
_LIBCPP_HIDE_FROM_ABI _Combiner(const _Combiner& __obj) : __value_{}, __bin_op_(__obj.__bin_op) {}
_LIBCPP_HIDE_FROM_ABI void
operator()(const _Combiner& __obj)
{
__value_ = (*__bin_op_)(__value_, __obj.__value_);
}
};
// Exclusive scan for other binary operations and types
template <class _InputIterator, class _Size, class _OutputIterator, class _UnaryOperation, class _Tp,
class _BinaryOperation>
_LIBCPP_HIDE_FROM_ABI
typename std::enable_if<!is_arithmetic_plus<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
__simd_scan(_InputIterator __first, _Size __n, _OutputIterator __result, _UnaryOperation __unary_op, _Tp __init,
_BinaryOperation __binary_op, /*Inclusive*/ std::false_type)
{
typedef _Combiner<_Tp, _BinaryOperation> _CombinerType;
_CombinerType __combined_init{__init, &__binary_op};
_PSTL_PRAGMA_DECLARE_REDUCTION(__bin_op, _CombinerType)
_PSTL_PRAGMA_SIMD_SCAN(__bin_op : __combined_init)
for (_Size __i = 0; __i < __n; ++__i)
{
__result[__i] = __combined_init.__value_;
_PSTL_PRAGMA_SIMD_EXCLUSIVE_SCAN(__combined_init)
__combined_init.__value_ = __binary_op(__combined_init.__value_, __unary_op(__first[__i]));
}
return std::make_pair(__result + __n, __combined_init.__value_);
}
// Inclusive scan for "+" and arithmetic types
template <class _InputIterator, class _Size, class _OutputIterator, class _UnaryOperation, class _Tp,
class _BinaryOperation>
_LIBCPP_HIDE_FROM_ABI
typename std::enable_if<is_arithmetic_plus<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
__simd_scan(_InputIterator __first, _Size __n, _OutputIterator __result, _UnaryOperation __unary_op, _Tp __init,
_BinaryOperation, /*Inclusive*/ std::true_type)
{
_PSTL_PRAGMA_SIMD_SCAN(+ : __init)
for (_Size __i = 0; __i < __n; ++__i)
{
__init += __unary_op(__first[__i]);
_PSTL_PRAGMA_SIMD_INCLUSIVE_SCAN(__init)
__result[__i] = __init;
}
return std::make_pair(__result + __n, __init);
}
// Inclusive scan for other binary operations and types
template <class _InputIterator, class _Size, class _OutputIterator, class _UnaryOperation, class _Tp,
class _BinaryOperation>
_LIBCPP_HIDE_FROM_ABI
typename std::enable_if<!is_arithmetic_plus<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
__simd_scan(_InputIterator __first, _Size __n, _OutputIterator __result, _UnaryOperation __unary_op, _Tp __init,
_BinaryOperation __binary_op, std::true_type)
{
typedef _Combiner<_Tp, _BinaryOperation> _CombinerType;
_CombinerType __combined_init{__init, &__binary_op};
_PSTL_PRAGMA_DECLARE_REDUCTION(__bin_op, _CombinerType)
_PSTL_PRAGMA_SIMD_SCAN(__bin_op : __combined_init)
for (_Size __i = 0; __i < __n; ++__i)
{
__combined_init.__value_ = __binary_op(__combined_init.__value_, __unary_op(__first[__i]));
_PSTL_PRAGMA_SIMD_INCLUSIVE_SCAN(__combined_init)
__result[__i] = __combined_init.__value_;
}
return std::make_pair(__result + __n, __combined_init.__value_);
}
// [restriction] - std::iterator_traits<_ForwardIterator>::value_type should be DefaultConstructible.
// complexity [violation] - We will have at most (__n-1 + number_of_lanes) comparisons instead of at most __n-1.
template <typename _ForwardIterator, typename _Size, typename _Compare>
_LIBCPP_HIDE_FROM_ABI _ForwardIterator
__simd_min_element(_ForwardIterator __first, _Size __n, _Compare __comp) noexcept
{
if (__n == 0)
{
return __first;
}
typedef typename std::iterator_traits<_ForwardIterator>::value_type _ValueType;
struct _ComplexType
{
_ValueType __min_val_;
_Size __min_ind_;
_Compare* __min_comp_;
_LIBCPP_HIDE_FROM_ABI _ComplexType() : __min_val_{}, __min_ind_{}, __min_comp_(nullptr) {}
_LIBCPP_HIDE_FROM_ABI _ComplexType(const _ValueType& __val, const _Compare* __comp)
: __min_val_(__val), __min_ind_(0), __min_comp_(const_cast<_Compare*>(__comp))
{
}
_LIBCPP_HIDE_FROM_ABI _ComplexType(const _ComplexType& __obj)
: __min_val_(__obj.__min_val_), __min_ind_(__obj.__min_ind_), __min_comp_(__obj.__min_comp_)
{
}
_PSTL_PRAGMA_DECLARE_SIMD
_LIBCPP_HIDE_FROM_ABI void
operator()(const _ComplexType& __obj)
{
if (!(*__min_comp_)(__min_val_, __obj.__min_val_) &&
((*__min_comp_)(__obj.__min_val_, __min_val_) || __obj.__min_ind_ - __min_ind_ < 0))
{
__min_val_ = __obj.__min_val_;
__min_ind_ = __obj.__min_ind_;
}
}
};
_ComplexType __init{*__first, &__comp};
_PSTL_PRAGMA_DECLARE_REDUCTION(__min_func, _ComplexType)
_PSTL_PRAGMA_SIMD_REDUCTION(__min_func : __init)
for (_Size __i = 1; __i < __n; ++__i)
{
const _ValueType __min_val = __init.__min_val_;
const _ValueType __current = __first[__i];
if (__comp(__current, __min_val))
{
__init.__min_val_ = __current;
__init.__min_ind_ = __i;
}
}
return __first + __init.__min_ind_;
}
// [restriction] - std::iterator_traits<_ForwardIterator>::value_type should be DefaultConstructible.
// complexity [violation] - We will have at most (2*(__n-1) + 4*number_of_lanes) comparisons instead of at most [1.5*(__n-1)].
template <typename _ForwardIterator, typename _Size, typename _Compare>
_LIBCPP_HIDE_FROM_ABI std::pair<_ForwardIterator, _ForwardIterator>
__simd_minmax_element(_ForwardIterator __first, _Size __n, _Compare __comp) noexcept
{
if (__n == 0)
{
return std::make_pair(__first, __first);
}
typedef typename std::iterator_traits<_ForwardIterator>::value_type _ValueType;
struct _ComplexType
{
_ValueType __min_val_;
_ValueType __max_val_;
_Size __min_ind_;
_Size __max_ind_;
_Compare* __minmax_comp;
_LIBCPP_HIDE_FROM_ABI _ComplexType()
: __min_val_{}, __max_val_{}, __min_ind_{}, __max_ind_{}, __minmax_comp(nullptr) {}
_LIBCPP_HIDE_FROM_ABI _ComplexType(
const _ValueType& __min_val, const _ValueType& __max_val, const _Compare* __comp)
: __min_val_(__min_val), __max_val_(__max_val), __min_ind_(0), __max_ind_(0),
__minmax_comp(const_cast<_Compare*>(__comp))
{
}
_LIBCPP_HIDE_FROM_ABI _ComplexType(const _ComplexType& __obj)
: __min_val_(__obj.__min_val_), __max_val_(__obj.__max_val_), __min_ind_(__obj.__min_ind_),
__max_ind_(__obj.__max_ind_), __minmax_comp(__obj.__minmax_comp)
{
}
_LIBCPP_HIDE_FROM_ABI void
operator()(const _ComplexType& __obj)
{
// min
if ((*__minmax_comp)(__obj.__min_val_, __min_val_))
{
__min_val_ = __obj.__min_val_;
__min_ind_ = __obj.__min_ind_;
}
else if (!(*__minmax_comp)(__min_val_, __obj.__min_val_))
{
__min_val_ = __obj.__min_val_;
__min_ind_ = (__min_ind_ - __obj.__min_ind_ < 0) ? __min_ind_ : __obj.__min_ind_;
}
// max
if ((*__minmax_comp)(__max_val_, __obj.__max_val_))
{
__max_val_ = __obj.__max_val_;
__max_ind_ = __obj.__max_ind_;
}
else if (!(*__minmax_comp)(__obj.__max_val_, __max_val_))
{
__max_val_ = __obj.__max_val_;
__max_ind_ = (__max_ind_ - __obj.__max_ind_ < 0) ? __obj.__max_ind_ : __max_ind_;
}
}
};
_ComplexType __init{*__first, *__first, &__comp};
_PSTL_PRAGMA_DECLARE_REDUCTION(__min_func, _ComplexType);
_PSTL_PRAGMA_SIMD_REDUCTION(__min_func : __init)
for (_Size __i = 1; __i < __n; ++__i)
{
auto __min_val = __init.__min_val_;
auto __max_val = __init.__max_val_;
auto __current = __first + __i;
if (__comp(*__current, __min_val))
{
__init.__min_val_ = *__current;
__init.__min_ind_ = __i;
}
else if (!__comp(*__current, __max_val))
{
__init.__max_val_ = *__current;
__init.__max_ind_ = __i;
}
}
return std::make_pair(__first + __init.__min_ind_, __first + __init.__max_ind_);
}
template <class _InputIterator, class _DifferenceType, class _OutputIterator1, class _OutputIterator2,
class _UnaryPredicate>
_LIBCPP_HIDE_FROM_ABI std::pair<_OutputIterator1, _OutputIterator2>
__simd_partition_copy(_InputIterator __first, _DifferenceType __n, _OutputIterator1 __out_true,
_OutputIterator2 __out_false, _UnaryPredicate __pred) noexcept
{
_DifferenceType __cnt_true = 0, __cnt_false = 0;
_PSTL_PRAGMA_SIMD
for (_DifferenceType __i = 0; __i < __n; ++__i)
{
if (__pred(__first[__i]))
{
__out_true[__cnt_true] = __first[__i];
++__cnt_true;
}
else
{
__out_false[__cnt_false] = __first[__i];
++__cnt_false;
}
}
return std::make_pair(__out_true + __cnt_true, __out_false + __cnt_false);
}
template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
_LIBCPP_HIDE_FROM_ABI _ForwardIterator1
__simd_find_first_of(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
_ForwardIterator2 __s_last, _BinaryPredicate __pred) noexcept
{
typedef typename std::iterator_traits<_ForwardIterator1>::difference_type _DifferencType;
const _DifferencType __n1 = __last - __first;
const _DifferencType __n2 = __s_last - __s_first;
if (__n1 == 0 || __n2 == 0)
{
return __last; // according to the standard
}
// Common case
// If first sequence larger than second then we'll run simd_first with parameters of first sequence.
// Otherwise, vice versa.
if (__n1 < __n2)
{
for (; __first != __last; ++__first)
{
if (__unseq_backend::__simd_or(
__s_first, __n2,
__internal::__equal_value_by_pred<decltype(*__first), _BinaryPredicate>(*__first, __pred)))
{
return __first;
}
}
}
else
{
for (; __s_first != __s_last; ++__s_first)
{
const auto __result = __unseq_backend::__simd_first(
__first, _DifferencType(0), __n1, [__s_first, &__pred](_ForwardIterator1 __it, _DifferencType __i) {
return __pred(__it[__i], *__s_first);
});
if (__result != __last)
{
return __result;
}
}
}
return __last;
}
template <class _RandomAccessIterator, class _DifferenceType, class _UnaryPredicate>
_LIBCPP_HIDE_FROM_ABI _RandomAccessIterator
__simd_remove_if(_RandomAccessIterator __first, _DifferenceType __n, _UnaryPredicate __pred) noexcept
{
// find first element we need to remove
auto __current = __unseq_backend::__simd_first(
__first, _DifferenceType(0), __n,
[&__pred](_RandomAccessIterator __it, _DifferenceType __i) { return __pred(__it[__i]); });
__n -= __current - __first;
// if we have in sequence only one element that pred(__current[1]) != false we can exit the function
if (__n < 2)
{
return __current;
}
_DifferenceType __cnt = 0;
_PSTL_PRAGMA_SIMD
for (_DifferenceType __i = 1; __i < __n; ++__i)
{
if (!__pred(__current[__i]))
{
__current[__cnt] = std::move(__current[__i]);
++__cnt;
}
}
return __current + __cnt;
}
} // namespace __unseq_backend
} // namespace __pstl
#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
#endif /* _PSTL_UNSEQ_BACKEND_SIMD_H */

144
third_party/libcxx/__pstl/internal/utils.h vendored Normal file
View file

@ -0,0 +1,144 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _PSTL_UTILS_H
#define _PSTL_UTILS_H
#include <__config>
#include <__exception/terminate.h>
#include <__utility/forward.h>
#include <new>
#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
namespace __pstl {
namespace __internal {
template <typename _Fp>
_LIBCPP_HIDE_FROM_ABI auto __except_handler(_Fp __f) -> decltype(__f()) {
#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
try {
#endif // _LIBCPP_HAS_NO_EXCEPTIONS
return __f();
#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
} catch (const std::bad_alloc&) {
throw; // re-throw bad_alloc according to the standard [algorithms.parallel.exceptions]
} catch (...) {
std::terminate(); // Good bye according to the standard [algorithms.parallel.exceptions]
}
#endif // _LIBCPP_HAS_NO_EXCEPTIONS
}
template <typename _Fp>
_LIBCPP_HIDE_FROM_ABI void __invoke_if(std::true_type, _Fp __f) {
__f();
}
template <typename _Fp>
_LIBCPP_HIDE_FROM_ABI void __invoke_if(std::false_type, _Fp) {}
template <typename _Fp>
_LIBCPP_HIDE_FROM_ABI void __invoke_if_not(std::false_type, _Fp __f) {
__f();
}
template <typename _Fp>
_LIBCPP_HIDE_FROM_ABI void __invoke_if_not(std::true_type, _Fp) {}
template <typename _F1, typename _F2>
_LIBCPP_HIDE_FROM_ABI auto __invoke_if_else(std::true_type, _F1 __f1, _F2) -> decltype(__f1()) {
return __f1();
}
template <typename _F1, typename _F2>
_LIBCPP_HIDE_FROM_ABI auto __invoke_if_else(std::false_type, _F1, _F2 __f2) -> decltype(__f2()) {
return __f2();
}
//! Unary operator that returns reference to its argument.
struct __no_op {
template <typename _Tp>
_LIBCPP_HIDE_FROM_ABI _Tp&& operator()(_Tp&& __a) const {
return std::forward<_Tp>(__a);
}
};
template <typename _Pred>
class __reorder_pred {
_Pred __pred_;
public:
_LIBCPP_HIDE_FROM_ABI explicit __reorder_pred(_Pred __pred) : __pred_(__pred) {}
template <typename _FTp, typename _STp>
_LIBCPP_HIDE_FROM_ABI bool operator()(_FTp&& __a, _STp&& __b) {
return __pred_(std::forward<_STp>(__b), std::forward<_FTp>(__a));
}
};
//! Like a polymorphic lambda for pred(...,value)
template <typename _Tp, typename _Predicate>
class __equal_value_by_pred {
const _Tp& __value_;
_Predicate __pred_;
public:
_LIBCPP_HIDE_FROM_ABI __equal_value_by_pred(const _Tp& __value, _Predicate __pred)
: __value_(__value), __pred_(__pred) {}
template <typename _Arg>
_LIBCPP_HIDE_FROM_ABI bool operator()(_Arg&& __arg) {
return __pred_(std::forward<_Arg>(__arg), __value_);
}
};
//! Like a polymorphic lambda for ==value
template <typename _Tp>
class __equal_value {
const _Tp& __value_;
public:
_LIBCPP_HIDE_FROM_ABI explicit __equal_value(const _Tp& __value) : __value_(__value) {}
template <typename _Arg>
_LIBCPP_HIDE_FROM_ABI bool operator()(_Arg&& __arg) const {
return std::forward<_Arg>(__arg) == __value_;
}
};
//! Logical negation of ==value
template <typename _Tp>
class __not_equal_value {
const _Tp& __value_;
public:
_LIBCPP_HIDE_FROM_ABI explicit __not_equal_value(const _Tp& __value) : __value_(__value) {}
template <typename _Arg>
_LIBCPP_HIDE_FROM_ABI bool operator()(_Arg&& __arg) const {
return !(std::forward<_Arg>(__arg) == __value_);
}
};
template <typename _ForwardIterator, typename _Compare>
_LIBCPP_HIDE_FROM_ABI _ForwardIterator
__cmp_iterators_by_values(_ForwardIterator __a, _ForwardIterator __b, _Compare __comp) {
if (__a < __b) { // we should return closer iterator
return __comp(*__b, *__a) ? __b : __a;
} else {
return __comp(*__a, *__b) ? __a : __b;
}
}
} // namespace __internal
} // namespace __pstl
#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
#endif /* _PSTL_UTILS_H */