// Simd fixed_size ABI specific implementations -*- C++ -*-

// Copyright (C) 2020-2021 Free Software Foundation, Inc.
//
// This file is part of the GNU ISO C++ Library.  This library is free
// software; you can redistribute it and/or modify it under the
// terms of the GNU General Public License as published by the
// Free Software Foundation; either version 3, or (at your option)
// any later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// Under Section 7 of GPL version 3, you are granted additional
// permissions described in the GCC Runtime Library Exception, version
// 3.1, as published by the Free Software Foundation.

// You should have received a copy of the GNU General Public License and
// a copy of the GCC Runtime Library Exception along with this program;
// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
// <http://www.gnu.org/licenses/>.

/*
 * The fixed_size ABI gives the following guarantees:
 *  - simd objects are passed via the stack
 *  - memory layout of `simd<_Tp, _Np>` is equivalent to `array<_Tp, _Np>`
 *  - alignment of `simd<_Tp, _Np>` is `_Np * sizeof(_Tp)` if _Np is __a
 *    power-of-2 value, otherwise `std::__bit_ceil(_Np * sizeof(_Tp))` (Note:
 *    if the alignment were to exceed the system/compiler maximum, it is bounded
 *    to that maximum)
 *  - simd_mask objects are passed like bitset<_Np>
 *  - memory layout of `simd_mask<_Tp, _Np>` is equivalent to `bitset<_Np>`
 *  - alignment of `simd_mask<_Tp, _Np>` is equal to the alignment of
 *    `bitset<_Np>`
 */

#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_FIXED_SIZE_H_
#define _GLIBCXX_EXPERIMENTAL_SIMD_FIXED_SIZE_H_

#if __cplusplus >= 201703L

#include <array>

_GLIBCXX_SIMD_BEGIN_NAMESPACE

// __simd_tuple_element {{{
template <size_t _I, typename _Tp>
  struct __simd_tuple_element;

template <typename _Tp, typename _A0, typename... _As>
  struct __simd_tuple_element<0, _SimdTuple<_Tp, _A0, _As...>>
  { using type = simd<_Tp, _A0>; };

template <size_t _I, typename _Tp, typename _A0, typename... _As>
  struct __simd_tuple_element<_I, _SimdTuple<_Tp, _A0, _As...>>
  {
    using type =
      typename __simd_tuple_element<_I - 1, _SimdTuple<_Tp, _As...>>::type;
  };

template <size_t _I, typename _Tp>
  using __simd_tuple_element_t = typename __simd_tuple_element<_I, _Tp>::type;

// }}}
// __simd_tuple_concat {{{

template <typename _Tp, typename... _A0s, typename... _A1s>
  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp, _A0s..., _A1s...>
  __simd_tuple_concat(const _SimdTuple<_Tp, _A0s...>& __left,
		      const _SimdTuple<_Tp, _A1s...>& __right)
  {
    if constexpr (sizeof...(_A0s) == 0)
      return __right;
    else if constexpr (sizeof...(_A1s) == 0)
      return __left;
    else
      return {__left.first, __simd_tuple_concat(__left.second, __right)};
  }

template <typename _Tp, typename _A10, typename... _A1s>
  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp, simd_abi::scalar, _A10,
					       _A1s...>
  __simd_tuple_concat(const _Tp& __left,
		      const _SimdTuple<_Tp, _A10, _A1s...>& __right)
  { return {__left, __right}; }

// }}}
// __simd_tuple_pop_front {{{
// Returns the next _SimdTuple in __x that has _Np elements less.
// Precondition: _Np must match the number of elements in __first (recursively)
template <size_t _Np, typename _Tp>
  _GLIBCXX_SIMD_INTRINSIC constexpr decltype(auto)
  __simd_tuple_pop_front(_Tp&& __x)
  {
    if constexpr (_Np == 0)
      return static_cast<_Tp&&>(__x);
    else
      {
	using _Up = __remove_cvref_t<_Tp>;
	static_assert(_Np >= _Up::_S_first_size);
	return __simd_tuple_pop_front<_Np - _Up::_S_first_size>(__x.second);
      }
  }

// }}}
// __get_simd_at<_Np> {{{1
struct __as_simd {};

struct __as_simd_tuple {};

template <typename _Tp, typename _A0, typename... _Abis>
  _GLIBCXX_SIMD_INTRINSIC constexpr simd<_Tp, _A0>
  __simd_tuple_get_impl(__as_simd, const _SimdTuple<_Tp, _A0, _Abis...>& __t,
			_SizeConstant<0>)
  { return {__private_init, __t.first}; }

template <typename _Tp, typename _A0, typename... _Abis>
  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
  __simd_tuple_get_impl(__as_simd_tuple,
			const _SimdTuple<_Tp, _A0, _Abis...>& __t,
			_SizeConstant<0>)
  { return __t.first; }

template <typename _Tp, typename _A0, typename... _Abis>
  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
  __simd_tuple_get_impl(__as_simd_tuple, _SimdTuple<_Tp, _A0, _Abis...>& __t,
			_SizeConstant<0>)
  { return __t.first; }

template <typename _R, size_t _Np, typename _Tp, typename... _Abis>
  _GLIBCXX_SIMD_INTRINSIC constexpr auto
  __simd_tuple_get_impl(_R, const _SimdTuple<_Tp, _Abis...>& __t,
			_SizeConstant<_Np>)
  { return __simd_tuple_get_impl(_R(), __t.second, _SizeConstant<_Np - 1>()); }

template <size_t _Np, typename _Tp, typename... _Abis>
  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
  __simd_tuple_get_impl(__as_simd_tuple, _SimdTuple<_Tp, _Abis...>& __t,
			_SizeConstant<_Np>)
  {
    return __simd_tuple_get_impl(__as_simd_tuple(), __t.second,
				 _SizeConstant<_Np - 1>());
  }

template <size_t _Np, typename _Tp, typename... _Abis>
  _GLIBCXX_SIMD_INTRINSIC constexpr auto
  __get_simd_at(const _SimdTuple<_Tp, _Abis...>& __t)
  { return __simd_tuple_get_impl(__as_simd(), __t, _SizeConstant<_Np>()); }

// }}}
// __get_tuple_at<_Np> {{{
template <size_t _Np, typename _Tp, typename... _Abis>
  _GLIBCXX_SIMD_INTRINSIC constexpr auto
  __get_tuple_at(const _SimdTuple<_Tp, _Abis...>& __t)
  {
    return __simd_tuple_get_impl(__as_simd_tuple(), __t, _SizeConstant<_Np>());
  }

template <size_t _Np, typename _Tp, typename... _Abis>
  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
  __get_tuple_at(_SimdTuple<_Tp, _Abis...>& __t)
  {
    return __simd_tuple_get_impl(__as_simd_tuple(), __t, _SizeConstant<_Np>());
  }

// __tuple_element_meta {{{1
template <typename _Tp, typename _Abi, size_t _Offset>
  struct __tuple_element_meta : public _Abi::_SimdImpl
  {
    static_assert(is_same_v<typename _Abi::_SimdImpl::abi_type,
			    _Abi>); // this fails e.g. when _SimdImpl is an
				    // alias for _SimdImplBuiltin<_DifferentAbi>
    using value_type = _Tp;
    using abi_type = _Abi;
    using _Traits = _SimdTraits<_Tp, _Abi>;
    using _MaskImpl = typename _Abi::_MaskImpl;
    using _MaskMember = typename _Traits::_MaskMember;
    using simd_type = simd<_Tp, _Abi>;
    static constexpr size_t _S_offset = _Offset;
    static constexpr size_t _S_size() { return simd_size<_Tp, _Abi>::value; }
    static constexpr _MaskImpl _S_mask_impl = {};

    template <size_t _Np, bool _Sanitized>
      _GLIBCXX_SIMD_INTRINSIC static auto
      _S_submask(_BitMask<_Np, _Sanitized> __bits)
      { return __bits.template _M_extract<_Offset, _S_size()>(); }

    template <size_t _Np, bool _Sanitized>
      _GLIBCXX_SIMD_INTRINSIC static _MaskMember
      _S_make_mask(_BitMask<_Np, _Sanitized> __bits)
      {
	return _MaskImpl::template _S_convert<_Tp>(
	  __bits.template _M_extract<_Offset, _S_size()>()._M_sanitized());
      }

    _GLIBCXX_SIMD_INTRINSIC static _ULLong
    _S_mask_to_shifted_ullong(_MaskMember __k)
    { return _MaskImpl::_S_to_bits(__k).to_ullong() << _Offset; }
  };

template <size_t _Offset, typename _Tp, typename _Abi, typename... _As>
  __tuple_element_meta<_Tp, _Abi, _Offset>
  __make_meta(const _SimdTuple<_Tp, _Abi, _As...>&)
  { return {}; }

// }}}1
// _WithOffset wrapper class {{{
template <size_t _Offset, typename _Base>
  struct _WithOffset : public _Base
  {
    static inline constexpr size_t _S_offset = _Offset;

    _GLIBCXX_SIMD_INTRINSIC char* _M_as_charptr()
    {
      return reinterpret_cast<char*>(this)
	     + _S_offset * sizeof(typename _Base::value_type);
    }

    _GLIBCXX_SIMD_INTRINSIC const char* _M_as_charptr() const
    {
      return reinterpret_cast<const char*>(this)
	     + _S_offset * sizeof(typename _Base::value_type);
    }
  };

// make _WithOffset<_WithOffset> ill-formed to use:
template <size_t _O0, size_t _O1, typename _Base>
  struct _WithOffset<_O0, _WithOffset<_O1, _Base>> {};

template <size_t _Offset, typename _Tp>
  decltype(auto)
  __add_offset(_Tp& __base)
  { return static_cast<_WithOffset<_Offset, __remove_cvref_t<_Tp>>&>(__base); }

template <size_t _Offset, typename _Tp>
  decltype(auto)
  __add_offset(const _Tp& __base)
  {
    return static_cast<const _WithOffset<_Offset, __remove_cvref_t<_Tp>>&>(
      __base);
  }

template <size_t _Offset, size_t _ExistingOffset, typename _Tp>
  decltype(auto)
  __add_offset(_WithOffset<_ExistingOffset, _Tp>& __base)
  {
    return static_cast<_WithOffset<_Offset + _ExistingOffset, _Tp>&>(
      static_cast<_Tp&>(__base));
  }

template <size_t _Offset, size_t _ExistingOffset, typename _Tp>
  decltype(auto)
  __add_offset(const _WithOffset<_ExistingOffset, _Tp>& __base)
  {
    return static_cast<const _WithOffset<_Offset + _ExistingOffset, _Tp>&>(
      static_cast<const _Tp&>(__base));
  }

template <typename _Tp>
  constexpr inline size_t __offset = 0;

template <size_t _Offset, typename _Tp>
  constexpr inline size_t __offset<_WithOffset<_Offset, _Tp>>
    = _WithOffset<_Offset, _Tp>::_S_offset;

template <typename _Tp>
  constexpr inline size_t __offset<const _Tp> = __offset<_Tp>;

template <typename _Tp>
  constexpr inline size_t __offset<_Tp&> = __offset<_Tp>;

template <typename _Tp>
  constexpr inline size_t __offset<_Tp&&> = __offset<_Tp>;

// }}}
// _SimdTuple specializations {{{1
// empty {{{2
template <typename _Tp>
  struct _SimdTuple<_Tp>
  {
    using value_type = _Tp;
    static constexpr size_t _S_tuple_size = 0;
    static constexpr size_t _S_size() { return 0; }
  };

// _SimdTupleData {{{2
template <typename _FirstType, typename _SecondType>
  struct _SimdTupleData
  {
    _FirstType first;
    _SecondType second;

    _GLIBCXX_SIMD_INTRINSIC
    constexpr bool _M_is_constprop() const
    {
      if constexpr (is_class_v<_FirstType>)
	return first._M_is_constprop() && second._M_is_constprop();
      else
	return __builtin_constant_p(first) && second._M_is_constprop();
    }
  };

template <typename _FirstType, typename _Tp>
  struct _SimdTupleData<_FirstType, _SimdTuple<_Tp>>
  {
    _FirstType first;
    static constexpr _SimdTuple<_Tp> second = {};

    _GLIBCXX_SIMD_INTRINSIC
    constexpr bool _M_is_constprop() const
    {
      if constexpr (is_class_v<_FirstType>)
	return first._M_is_constprop();
      else
	return __builtin_constant_p(first);
    }
  };

// 1 or more {{{2
template <typename _Tp, typename _Abi0, typename... _Abis>
  struct _SimdTuple<_Tp, _Abi0, _Abis...>
    : _SimdTupleData<typename _SimdTraits<_Tp, _Abi0>::_SimdMember,
		     _SimdTuple<_Tp, _Abis...>>
  {
    static_assert(!__is_fixed_size_abi_v<_Abi0>);
    using value_type = _Tp;
    using _FirstType = typename _SimdTraits<_Tp, _Abi0>::_SimdMember;
    using _FirstAbi = _Abi0;
    using _SecondType = _SimdTuple<_Tp, _Abis...>;
    static constexpr size_t _S_tuple_size = sizeof...(_Abis) + 1;

    static constexpr size_t _S_size()
    { return simd_size_v<_Tp, _Abi0> + _SecondType::_S_size(); }

    static constexpr size_t _S_first_size = simd_size_v<_Tp, _Abi0>;
    static constexpr bool _S_is_homogeneous = (is_same_v<_Abi0, _Abis> && ...);

    using _Base = _SimdTupleData<typename _SimdTraits<_Tp, _Abi0>::_SimdMember,
				 _SimdTuple<_Tp, _Abis...>>;
    using _Base::first;
    using _Base::second;

    _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple() = default;
    _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple(const _SimdTuple&) = default;
    _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple& operator=(const _SimdTuple&)
      = default;

    template <typename _Up>
      _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple(_Up&& __x)
      : _Base{static_cast<_Up&&>(__x)} {}

    template <typename _Up, typename _Up2>
      _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple(_Up&& __x, _Up2&& __y)
      : _Base{static_cast<_Up&&>(__x), static_cast<_Up2&&>(__y)} {}

    template <typename _Up>
      _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple(_Up&& __x, _SimdTuple<_Tp>)
      : _Base{static_cast<_Up&&>(__x)} {}

    _GLIBCXX_SIMD_INTRINSIC char* _M_as_charptr()
    { return reinterpret_cast<char*>(this); }

    _GLIBCXX_SIMD_INTRINSIC const char* _M_as_charptr() const
    { return reinterpret_cast<const char*>(this); }

    template <size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC constexpr auto& _M_at()
      {
	if constexpr (_Np == 0)
	  return first;
	else
	  return second.template _M_at<_Np - 1>();
      }

    template <size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC constexpr const auto& _M_at() const
      {
	if constexpr (_Np == 0)
	  return first;
	else
	  return second.template _M_at<_Np - 1>();
      }

    template <size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC constexpr auto _M_simd_at() const
      {
	if constexpr (_Np == 0)
	  return simd<_Tp, _Abi0>(__private_init, first);
	else
	  return second.template _M_simd_at<_Np - 1>();
      }

    template <size_t _Offset = 0, typename _Fp>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple
      _S_generate(_Fp&& __gen, _SizeConstant<_Offset> = {})
      {
	auto&& __first = __gen(__tuple_element_meta<_Tp, _Abi0, _Offset>());
	if constexpr (_S_tuple_size == 1)
	  return {__first};
	else
	  return {__first,
		  _SecondType::_S_generate(
		    static_cast<_Fp&&>(__gen),
		    _SizeConstant<_Offset + simd_size_v<_Tp, _Abi0>>())};
      }

    template <size_t _Offset = 0, typename _Fp, typename... _More>
      _GLIBCXX_SIMD_INTRINSIC _SimdTuple
      _M_apply_wrapped(_Fp&& __fun, const _More&... __more) const
      {
	auto&& __first
	  = __fun(__make_meta<_Offset>(*this), first, __more.first...);
	if constexpr (_S_tuple_size == 1)
	  return {__first};
	else
	  return {
	    __first,
	    second.template _M_apply_wrapped<_Offset + simd_size_v<_Tp, _Abi0>>(
	      static_cast<_Fp&&>(__fun), __more.second...)};
      }

    template <typename _Tup>
      _GLIBCXX_SIMD_INTRINSIC constexpr decltype(auto)
      _M_extract_argument(_Tup&& __tup) const
      {
	using _TupT = typename __remove_cvref_t<_Tup>::value_type;
	if constexpr (is_same_v<_SimdTuple, __remove_cvref_t<_Tup>>)
	  return __tup.first;
	else if (__builtin_is_constant_evaluated())
	  return __fixed_size_storage_t<_TupT, _S_first_size>::_S_generate([&](
	    auto __meta) constexpr {
	    return __meta._S_generator(
	      [&](auto __i) constexpr { return __tup[__i]; },
	      static_cast<_TupT*>(nullptr));
	  });
	else
	  return [&]() {
	    __fixed_size_storage_t<_TupT, _S_first_size> __r;
	    __builtin_memcpy(__r._M_as_charptr(), __tup._M_as_charptr(),
			     sizeof(__r));
	    return __r;
	  }();
      }

    template <typename _Tup>
      _GLIBCXX_SIMD_INTRINSIC constexpr auto&
      _M_skip_argument(_Tup&& __tup) const
      {
	static_assert(_S_tuple_size > 1);
	using _Up = __remove_cvref_t<_Tup>;
	constexpr size_t __off = __offset<_Up>;
	if constexpr (_S_first_size == _Up::_S_first_size && __off == 0)
	  return __tup.second;
	else if constexpr (_S_first_size > _Up::_S_first_size
			   && _S_first_size % _Up::_S_first_size == 0
			   && __off == 0)
	  return __simd_tuple_pop_front<_S_first_size>(__tup);
	else if constexpr (_S_first_size + __off < _Up::_S_first_size)
	  return __add_offset<_S_first_size>(__tup);
	else if constexpr (_S_first_size + __off == _Up::_S_first_size)
	  return __tup.second;
	else
	  __assert_unreachable<_Tup>();
      }

    template <size_t _Offset, typename... _More>
      _GLIBCXX_SIMD_INTRINSIC constexpr void
      _M_assign_front(const _SimdTuple<_Tp, _Abi0, _More...>& __x) &
      {
	static_assert(_Offset == 0);
	first = __x.first;
	if constexpr (sizeof...(_More) > 0)
	  {
	    static_assert(sizeof...(_Abis) >= sizeof...(_More));
	    second.template _M_assign_front<0>(__x.second);
	  }
      }

    template <size_t _Offset>
      _GLIBCXX_SIMD_INTRINSIC constexpr void
      _M_assign_front(const _FirstType& __x) &
      {
	static_assert(_Offset == 0);
	first = __x;
      }

    template <size_t _Offset, typename... _As>
      _GLIBCXX_SIMD_INTRINSIC constexpr void
      _M_assign_front(const _SimdTuple<_Tp, _As...>& __x) &
      {
	__builtin_memcpy(_M_as_charptr() + _Offset * sizeof(value_type),
			 __x._M_as_charptr(),
			 sizeof(_Tp) * _SimdTuple<_Tp, _As...>::_S_size());
      }

    /*
     * Iterate over the first objects in this _SimdTuple and call __fun for each
     * of them. If additional arguments are passed via __more, chunk them into
     * _SimdTuple or __vector_type_t objects of the same number of values.
     */
    template <typename _Fp, typename... _More>
      _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple
      _M_apply_per_chunk(_Fp&& __fun, _More&&... __more) const
      {
	if constexpr ((...
		       || conjunction_v<
			 is_lvalue_reference<_More>,
			 negation<is_const<remove_reference_t<_More>>>>) )
	  {
	    // need to write back at least one of __more after calling __fun
	    auto&& __first = [&](auto... __args) constexpr
	    {
	      auto __r = __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), first,
			       __args...);
	      [[maybe_unused]] auto&& __ignore_me = {(
		[](auto&& __dst, const auto& __src) {
		  if constexpr (is_assignable_v<decltype(__dst),
						decltype(__dst)>)
		    {
		      __dst.template _M_assign_front<__offset<decltype(__dst)>>(
			__src);
		    }
		}(static_cast<_More&&>(__more), __args),
		0)...};
	      return __r;
	    }
	    (_M_extract_argument(__more)...);
	    if constexpr (_S_tuple_size == 1)
	      return {__first};
	    else
	      return {__first,
		      second._M_apply_per_chunk(static_cast<_Fp&&>(__fun),
						_M_skip_argument(__more)...)};
	  }
	else
	  {
	    auto&& __first = __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), first,
				   _M_extract_argument(__more)...);
	    if constexpr (_S_tuple_size == 1)
	      return {__first};
	    else
	      return {__first,
		      second._M_apply_per_chunk(static_cast<_Fp&&>(__fun),
						_M_skip_argument(__more)...)};
	  }
      }

    template <typename _R = _Tp, typename _Fp, typename... _More>
      _GLIBCXX_SIMD_INTRINSIC auto _M_apply_r(_Fp&& __fun,
					      const _More&... __more) const
      {
	auto&& __first = __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), first,
			       __more.first...);
	if constexpr (_S_tuple_size == 1)
	  return __first;
	else
	  return __simd_tuple_concat<_R>(
	    __first, second.template _M_apply_r<_R>(static_cast<_Fp&&>(__fun),
						    __more.second...));
      }

    template <typename _Fp, typename... _More>
      _GLIBCXX_SIMD_INTRINSIC constexpr friend _SanitizedBitMask<_S_size()>
      _M_test(const _Fp& __fun, const _SimdTuple& __x, const _More&... __more)
      {
	const _SanitizedBitMask<_S_first_size> __first
	  = _Abi0::_MaskImpl::_S_to_bits(
	    __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), __x.first,
		  __more.first...));
	if constexpr (_S_tuple_size == 1)
	  return __first;
	else
	  return _M_test(__fun, __x.second, __more.second...)
	    ._M_prepend(__first);
      }

    template <typename _Up, _Up _I>
      _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
      operator[](integral_constant<_Up, _I>) const noexcept
      {
	if constexpr (_I < simd_size_v<_Tp, _Abi0>)
	  return _M_subscript_read(_I);
	else
	  return second[integral_constant<_Up, _I - simd_size_v<_Tp, _Abi0>>()];
      }

    _Tp operator[](size_t __i) const noexcept
    {
      if constexpr (_S_tuple_size == 1)
	return _M_subscript_read(__i);
      else
	{
#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
	  return reinterpret_cast<const __may_alias<_Tp>*>(this)[__i];
#else
	  if constexpr (__is_scalar_abi<_Abi0>())
	    {
	      const _Tp* ptr = &first;
	      return ptr[__i];
	    }
	  else
	    return __i < simd_size_v<_Tp, _Abi0>
		     ? _M_subscript_read(__i)
		     : second[__i - simd_size_v<_Tp, _Abi0>];
#endif
	}
    }

    void _M_set(size_t __i, _Tp __val) noexcept
    {
      if constexpr (_S_tuple_size == 1)
	return _M_subscript_write(__i, __val);
      else
	{
#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
	  reinterpret_cast<__may_alias<_Tp>*>(this)[__i] = __val;
#else
	  if (__i < simd_size_v<_Tp, _Abi0>)
	    _M_subscript_write(__i, __val);
	  else
	    second._M_set(__i - simd_size_v<_Tp, _Abi0>, __val);
#endif
	}
    }

  private:
    // _M_subscript_read/_write {{{
    _Tp _M_subscript_read([[maybe_unused]] size_t __i) const noexcept
    {
      if constexpr (__is_vectorizable_v<_FirstType>)
	return first;
      else
	return first[__i];
    }

    void _M_subscript_write([[maybe_unused]] size_t __i, _Tp __y) noexcept
    {
      if constexpr (__is_vectorizable_v<_FirstType>)
	first = __y;
      else
	first._M_set(__i, __y);
    }

    // }}}
  };

// __make_simd_tuple {{{1
template <typename _Tp, typename _A0>
  _GLIBCXX_SIMD_INTRINSIC _SimdTuple<_Tp, _A0>
  __make_simd_tuple(simd<_Tp, _A0> __x0)
  { return {__data(__x0)}; }

template <typename _Tp, typename _A0, typename... _As>
  _GLIBCXX_SIMD_INTRINSIC _SimdTuple<_Tp, _A0, _As...>
  __make_simd_tuple(const simd<_Tp, _A0>& __x0, const simd<_Tp, _As>&... __xs)
  { return {__data(__x0), __make_simd_tuple(__xs...)}; }

template <typename _Tp, typename _A0>
  _GLIBCXX_SIMD_INTRINSIC _SimdTuple<_Tp, _A0>
  __make_simd_tuple(const typename _SimdTraits<_Tp, _A0>::_SimdMember& __arg0)
  { return {__arg0}; }

template <typename _Tp, typename _A0, typename _A1, typename... _Abis>
  _GLIBCXX_SIMD_INTRINSIC _SimdTuple<_Tp, _A0, _A1, _Abis...>
  __make_simd_tuple(
    const typename _SimdTraits<_Tp, _A0>::_SimdMember& __arg0,
    const typename _SimdTraits<_Tp, _A1>::_SimdMember& __arg1,
    const typename _SimdTraits<_Tp, _Abis>::_SimdMember&... __args)
  { return {__arg0, __make_simd_tuple<_Tp, _A1, _Abis...>(__arg1, __args...)}; }

// __to_simd_tuple {{{1
template <typename _Tp, size_t _Np, typename _V, size_t _NV, typename... _VX>
  _GLIBCXX_SIMD_INTRINSIC constexpr __fixed_size_storage_t<_Tp, _Np>
  __to_simd_tuple(const array<_V, _NV>& __from, const _VX... __fromX);

template <typename _Tp, size_t _Np,
	  size_t _Offset = 0, // skip this many elements in __from0
	  typename _R = __fixed_size_storage_t<_Tp, _Np>, typename _V0,
	  typename _V0VT = _VectorTraits<_V0>, typename... _VX>
  _GLIBCXX_SIMD_INTRINSIC _R constexpr __to_simd_tuple(const _V0 __from0,
						       const _VX... __fromX)
  {
    static_assert(is_same_v<typename _V0VT::value_type, _Tp>);
    static_assert(_Offset < _V0VT::_S_full_size);
    using _R0 = __vector_type_t<_Tp, _R::_S_first_size>;
    if constexpr (_R::_S_tuple_size == 1)
      {
	if constexpr (_Np == 1)
	  return _R{__from0[_Offset]};
	else if constexpr (_Offset == 0 && _V0VT::_S_full_size >= _Np)
	  return _R{__intrin_bitcast<_R0>(__from0)};
	else if constexpr (_Offset * 2 == _V0VT::_S_full_size
			   && _V0VT::_S_full_size / 2 >= _Np)
	  return _R{__intrin_bitcast<_R0>(__extract_part<1, 2>(__from0))};
	else if constexpr (_Offset * 4 == _V0VT::_S_full_size
			   && _V0VT::_S_full_size / 4 >= _Np)
	  return _R{__intrin_bitcast<_R0>(__extract_part<1, 4>(__from0))};
	else
	  __assert_unreachable<_Tp>();
      }
    else
      {
	if constexpr (1 == _R::_S_first_size)
	  { // extract one scalar and recurse
	    if constexpr (_Offset + 1 < _V0VT::_S_full_size)
	      return _R{__from0[_Offset],
			__to_simd_tuple<_Tp, _Np - 1, _Offset + 1>(__from0,
								   __fromX...)};
	    else
	      return _R{__from0[_Offset],
			__to_simd_tuple<_Tp, _Np - 1, 0>(__fromX...)};
	  }

	// place __from0 into _R::first and recurse for __fromX -> _R::second
	else if constexpr (_V0VT::_S_full_size == _R::_S_first_size
			   && _Offset == 0)
	  return _R{__from0,
		    __to_simd_tuple<_Tp, _Np - _R::_S_first_size>(__fromX...)};

	// place lower part of __from0 into _R::first and recurse with _Offset
	else if constexpr (_V0VT::_S_full_size > _R::_S_first_size
			   && _Offset == 0)
	  return _R{__intrin_bitcast<_R0>(__from0),
		    __to_simd_tuple<_Tp, _Np - _R::_S_first_size,
				    _R::_S_first_size>(__from0, __fromX...)};

	// place lower part of second quarter of __from0 into _R::first and
	// recurse with _Offset
	else if constexpr (_Offset * 4 == _V0VT::_S_full_size
			   && _V0VT::_S_full_size >= 4 * _R::_S_first_size)
	  return _R{__intrin_bitcast<_R0>(__extract_part<2, 4>(__from0)),
		    __to_simd_tuple<_Tp, _Np - _R::_S_first_size,
				    _Offset + _R::_S_first_size>(__from0,
								 __fromX...)};

	// place lower half of high half of __from0 into _R::first and recurse
	// with _Offset
	else if constexpr (_Offset * 2 == _V0VT::_S_full_size
			   && _V0VT::_S_full_size >= 4 * _R::_S_first_size)
	  return _R{__intrin_bitcast<_R0>(__extract_part<2, 4>(__from0)),
		    __to_simd_tuple<_Tp, _Np - _R::_S_first_size,
				    _Offset + _R::_S_first_size>(__from0,
								 __fromX...)};

	// place high half of __from0 into _R::first and recurse with __fromX
	else if constexpr (_Offset * 2 == _V0VT::_S_full_size
			   && _V0VT::_S_full_size / 2 >= _R::_S_first_size)
	  return _R{__intrin_bitcast<_R0>(__extract_part<1, 2>(__from0)),
		    __to_simd_tuple<_Tp, _Np - _R::_S_first_size, 0>(
		      __fromX...)};

	// ill-formed if some unforseen pattern is needed
	else
	  __assert_unreachable<_Tp>();
      }
  }

template <typename _Tp, size_t _Np, typename _V, size_t _NV, typename... _VX>
  _GLIBCXX_SIMD_INTRINSIC constexpr __fixed_size_storage_t<_Tp, _Np>
  __to_simd_tuple(const array<_V, _NV>& __from, const _VX... __fromX)
  {
    if constexpr (is_same_v<_Tp, _V>)
      {
	static_assert(
	  sizeof...(_VX) == 0,
	  "An array of scalars must be the last argument to __to_simd_tuple");
	return __call_with_subscripts(
	  __from,
	  make_index_sequence<_NV>(), [&](const auto... __args) constexpr {
	    return __simd_tuple_concat(
	      _SimdTuple<_Tp, simd_abi::scalar>{__args}..., _SimdTuple<_Tp>());
	  });
      }
    else
      return __call_with_subscripts(
	__from,
	make_index_sequence<_NV>(), [&](const auto... __args) constexpr {
	  return __to_simd_tuple<_Tp, _Np>(__args..., __fromX...);
	});
  }

template <size_t, typename _Tp>
  using __to_tuple_helper = _Tp;

template <typename _Tp, typename _A0, size_t _NOut, size_t _Np,
	  size_t... _Indexes>
  _GLIBCXX_SIMD_INTRINSIC __fixed_size_storage_t<_Tp, _NOut>
  __to_simd_tuple_impl(index_sequence<_Indexes...>,
      const array<__vector_type_t<_Tp, simd_size_v<_Tp, _A0>>, _Np>& __args)
  {
    return __make_simd_tuple<_Tp, __to_tuple_helper<_Indexes, _A0>...>(
      __args[_Indexes]...);
  }

template <typename _Tp, typename _A0, size_t _NOut, size_t _Np,
	  typename _R = __fixed_size_storage_t<_Tp, _NOut>>
  _GLIBCXX_SIMD_INTRINSIC _R
  __to_simd_tuple_sized(
    const array<__vector_type_t<_Tp, simd_size_v<_Tp, _A0>>, _Np>& __args)
  {
    static_assert(_Np * simd_size_v<_Tp, _A0> >= _NOut);
    return __to_simd_tuple_impl<_Tp, _A0, _NOut>(
      make_index_sequence<_R::_S_tuple_size>(), __args);
  }

// __optimize_simd_tuple {{{1
template <typename _Tp>
  _GLIBCXX_SIMD_INTRINSIC _SimdTuple<_Tp>
  __optimize_simd_tuple(const _SimdTuple<_Tp>)
  { return {}; }

template <typename _Tp, typename _Ap>
  _GLIBCXX_SIMD_INTRINSIC const _SimdTuple<_Tp, _Ap>&
  __optimize_simd_tuple(const _SimdTuple<_Tp, _Ap>& __x)
  { return __x; }

template <typename _Tp, typename _A0, typename _A1, typename... _Abis,
	  typename _R = __fixed_size_storage_t<
	    _Tp, _SimdTuple<_Tp, _A0, _A1, _Abis...>::_S_size()>>
  _GLIBCXX_SIMD_INTRINSIC _R
  __optimize_simd_tuple(const _SimdTuple<_Tp, _A0, _A1, _Abis...>& __x)
  {
    using _Tup = _SimdTuple<_Tp, _A0, _A1, _Abis...>;
    if constexpr (is_same_v<_R, _Tup>)
      return __x;
    else if constexpr (is_same_v<typename _R::_FirstType,
				 typename _Tup::_FirstType>)
      return {__x.first, __optimize_simd_tuple(__x.second)};
    else if constexpr (__is_scalar_abi<_A0>()
		       || _A0::template _S_is_partial<_Tp>)
      return {__generate_from_n_evaluations<_R::_S_first_size,
					    typename _R::_FirstType>(
		[&](auto __i) { return __x[__i]; }),
	      __optimize_simd_tuple(
		__simd_tuple_pop_front<_R::_S_first_size>(__x))};
    else if constexpr (is_same_v<_A0, _A1>
	&& _R::_S_first_size == simd_size_v<_Tp, _A0> + simd_size_v<_Tp, _A1>)
      return {__concat(__x.template _M_at<0>(), __x.template _M_at<1>()),
	      __optimize_simd_tuple(__x.second.second)};
    else if constexpr (sizeof...(_Abis) >= 2
	&& _R::_S_first_size == (4 * simd_size_v<_Tp, _A0>)
	&& simd_size_v<_Tp, _A0> == __simd_tuple_element_t<
	    (sizeof...(_Abis) >= 2 ? 3 : 0), _Tup>::size())
      return {
	__concat(__concat(__x.template _M_at<0>(), __x.template _M_at<1>()),
		 __concat(__x.template _M_at<2>(), __x.template _M_at<3>())),
	__optimize_simd_tuple(__x.second.second.second.second)};
    else
      {
	static_assert(sizeof(_R) == sizeof(__x));
	_R __r;
	__builtin_memcpy(__r._M_as_charptr(), __x._M_as_charptr(),
			 sizeof(_Tp) * _R::_S_size());
	return __r;
      }
  }

// __for_each(const _SimdTuple &, Fun) {{{1
template <size_t _Offset = 0, typename _Tp, typename _A0, typename _Fp>
  _GLIBCXX_SIMD_INTRINSIC constexpr void
  __for_each(const _SimdTuple<_Tp, _A0>& __t, _Fp&& __fun)
  { static_cast<_Fp&&>(__fun)(__make_meta<_Offset>(__t), __t.first); }

template <size_t _Offset = 0, typename _Tp, typename _A0, typename _A1,
	  typename... _As, typename _Fp>
  _GLIBCXX_SIMD_INTRINSIC constexpr void
  __for_each(const _SimdTuple<_Tp, _A0, _A1, _As...>& __t, _Fp&& __fun)
  {
    __fun(__make_meta<_Offset>(__t), __t.first);
    __for_each<_Offset + simd_size<_Tp, _A0>::value>(__t.second,
						     static_cast<_Fp&&>(__fun));
  }

// __for_each(_SimdTuple &, Fun) {{{1
template <size_t _Offset = 0, typename _Tp, typename _A0, typename _Fp>
  _GLIBCXX_SIMD_INTRINSIC constexpr void
  __for_each(_SimdTuple<_Tp, _A0>& __t, _Fp&& __fun)
  { static_cast<_Fp&&>(__fun)(__make_meta<_Offset>(__t), __t.first); }

template <size_t _Offset = 0, typename _Tp, typename _A0, typename _A1,
	  typename... _As, typename _Fp>
  _GLIBCXX_SIMD_INTRINSIC constexpr void
  __for_each(_SimdTuple<_Tp, _A0, _A1, _As...>& __t, _Fp&& __fun)
  {
    __fun(__make_meta<_Offset>(__t), __t.first);
    __for_each<_Offset + simd_size<_Tp, _A0>::value>(__t.second,
						     static_cast<_Fp&&>(__fun));
  }

// __for_each(_SimdTuple &, const _SimdTuple &, Fun) {{{1
template <size_t _Offset = 0, typename _Tp, typename _A0, typename _Fp>
  _GLIBCXX_SIMD_INTRINSIC constexpr void
  __for_each(_SimdTuple<_Tp, _A0>& __a, const _SimdTuple<_Tp, _A0>& __b,
	     _Fp&& __fun)
  {
    static_cast<_Fp&&>(__fun)(__make_meta<_Offset>(__a), __a.first, __b.first);
  }

template <size_t _Offset = 0, typename _Tp, typename _A0, typename _A1,
	  typename... _As, typename _Fp>
  _GLIBCXX_SIMD_INTRINSIC constexpr void
  __for_each(_SimdTuple<_Tp, _A0, _A1, _As...>& __a,
	     const _SimdTuple<_Tp, _A0, _A1, _As...>& __b, _Fp&& __fun)
  {
    __fun(__make_meta<_Offset>(__a), __a.first, __b.first);
    __for_each<_Offset + simd_size<_Tp, _A0>::value>(__a.second, __b.second,
						     static_cast<_Fp&&>(__fun));
  }

// __for_each(const _SimdTuple &, const _SimdTuple &, Fun) {{{1
template <size_t _Offset = 0, typename _Tp, typename _A0, typename _Fp>
  _GLIBCXX_SIMD_INTRINSIC constexpr void
  __for_each(const _SimdTuple<_Tp, _A0>& __a, const _SimdTuple<_Tp, _A0>& __b,
	     _Fp&& __fun)
  {
    static_cast<_Fp&&>(__fun)(__make_meta<_Offset>(__a), __a.first, __b.first);
  }

template <size_t _Offset = 0, typename _Tp, typename _A0, typename _A1,
	  typename... _As, typename _Fp>
  _GLIBCXX_SIMD_INTRINSIC constexpr void
  __for_each(const _SimdTuple<_Tp, _A0, _A1, _As...>& __a,
	     const _SimdTuple<_Tp, _A0, _A1, _As...>& __b, _Fp&& __fun)
  {
    __fun(__make_meta<_Offset>(__a), __a.first, __b.first);
    __for_each<_Offset + simd_size<_Tp, _A0>::value>(__a.second, __b.second,
						     static_cast<_Fp&&>(__fun));
  }

// }}}1
// __extract_part(_SimdTuple) {{{
template <int _Index, int _Total, int _Combine, typename _Tp, typename _A0,
	  typename... _As>
  _GLIBCXX_SIMD_INTRINSIC auto // __vector_type_t or _SimdTuple
  __extract_part(const _SimdTuple<_Tp, _A0, _As...>& __x)
  {
    // worst cases:
    // (a) 4, 4, 4 => 3, 3, 3, 3 (_Total = 4)
    // (b) 2, 2, 2 => 3, 3       (_Total = 2)
    // (c) 4, 2 => 2, 2, 2       (_Total = 3)
    using _Tuple = _SimdTuple<_Tp, _A0, _As...>;
    static_assert(_Index + _Combine <= _Total && _Index >= 0 && _Total >= 1);
    constexpr size_t _Np = _Tuple::_S_size();
    static_assert(_Np >= _Total && _Np % _Total == 0);
    constexpr size_t __values_per_part = _Np / _Total;
    [[maybe_unused]] constexpr size_t __values_to_skip
      = _Index * __values_per_part;
    constexpr size_t __return_size = __values_per_part * _Combine;
    using _RetAbi = simd_abi::deduce_t<_Tp, __return_size>;

    // handle (optimize) the simple cases
    if constexpr (_Index == 0 && _Tuple::_S_first_size == __return_size)
      return __x.first._M_data;
    else if constexpr (_Index == 0 && _Total == _Combine)
      return __x;
    else if constexpr (_Index == 0 && _Tuple::_S_first_size >= __return_size)
      return __intrin_bitcast<__vector_type_t<_Tp, __return_size>>(
	__as_vector(__x.first));

    // recurse to skip unused data members at the beginning of _SimdTuple
    else if constexpr (__values_to_skip >= _Tuple::_S_first_size)
      { // recurse
	if constexpr (_Tuple::_S_first_size % __values_per_part == 0)
	  {
	    constexpr int __parts_in_first
	      = _Tuple::_S_first_size / __values_per_part;
	    return __extract_part<_Index - __parts_in_first,
				  _Total - __parts_in_first, _Combine>(
	      __x.second);
	  }
	else
	  return __extract_part<__values_to_skip - _Tuple::_S_first_size,
				_Np - _Tuple::_S_first_size, __return_size>(
	    __x.second);
      }

    // extract from multiple _SimdTuple data members
    else if constexpr (__return_size > _Tuple::_S_first_size - __values_to_skip)
      {
#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
	const __may_alias<_Tp>* const element_ptr
	  = reinterpret_cast<const __may_alias<_Tp>*>(&__x) + __values_to_skip;
	return __as_vector(simd<_Tp, _RetAbi>(element_ptr, element_aligned));
#else
	[[maybe_unused]] constexpr size_t __offset = __values_to_skip;
	return __as_vector(simd<_Tp, _RetAbi>([&](auto __i) constexpr {
	  constexpr _SizeConstant<__i + __offset> __k;
	  return __x[__k];
	}));
#endif
      }

    // all of the return values are in __x.first
    else if constexpr (_Tuple::_S_first_size % __values_per_part == 0)
      return __extract_part<_Index, _Tuple::_S_first_size / __values_per_part,
			    _Combine>(__x.first);
    else
      return __extract_part<__values_to_skip, _Tuple::_S_first_size,
			    _Combine * __values_per_part>(__x.first);
  }

// }}}
// __fixed_size_storage_t<_Tp, _Np>{{{
template <typename _Tp, int _Np, typename _Tuple,
	  typename _Next = simd<_Tp, _AllNativeAbis::_BestAbi<_Tp, _Np>>,
	  int _Remain = _Np - int(_Next::size())>
  struct __fixed_size_storage_builder;

template <typename _Tp, int _Np>
  struct __fixed_size_storage
  : public __fixed_size_storage_builder<_Tp, _Np, _SimdTuple<_Tp>> {};

template <typename _Tp, int _Np, typename... _As, typename _Next>
  struct __fixed_size_storage_builder<_Tp, _Np, _SimdTuple<_Tp, _As...>, _Next,
				      0>
  { using type = _SimdTuple<_Tp, _As..., typename _Next::abi_type>; };

template <typename _Tp, int _Np, typename... _As, typename _Next, int _Remain>
  struct __fixed_size_storage_builder<_Tp, _Np, _SimdTuple<_Tp, _As...>, _Next,
				      _Remain>
  {
    using type = typename __fixed_size_storage_builder<
      _Tp, _Remain, _SimdTuple<_Tp, _As..., typename _Next::abi_type>>::type;
  };

// }}}
// _AbisInSimdTuple {{{
template <typename _Tp>
  struct _SeqOp;

template <size_t _I0, size_t... _Is>
  struct _SeqOp<index_sequence<_I0, _Is...>>
  {
    using _FirstPlusOne = index_sequence<_I0 + 1, _Is...>;
    using _NotFirstPlusOne = index_sequence<_I0, (_Is + 1)...>;
    template <size_t _First, size_t _Add>
    using _Prepend = index_sequence<_First, _I0 + _Add, (_Is + _Add)...>;
  };

template <typename _Tp>
  struct _AbisInSimdTuple;

template <typename _Tp>
  struct _AbisInSimdTuple<_SimdTuple<_Tp>>
  {
    using _Counts = index_sequence<0>;
    using _Begins = index_sequence<0>;
  };

template <typename _Tp, typename _Ap>
  struct _AbisInSimdTuple<_SimdTuple<_Tp, _Ap>>
  {
    using _Counts = index_sequence<1>;
    using _Begins = index_sequence<0>;
  };

template <typename _Tp, typename _A0, typename... _As>
  struct _AbisInSimdTuple<_SimdTuple<_Tp, _A0, _A0, _As...>>
  {
    using _Counts = typename _SeqOp<typename _AbisInSimdTuple<
      _SimdTuple<_Tp, _A0, _As...>>::_Counts>::_FirstPlusOne;
    using _Begins = typename _SeqOp<typename _AbisInSimdTuple<
      _SimdTuple<_Tp, _A0, _As...>>::_Begins>::_NotFirstPlusOne;
  };

template <typename _Tp, typename _A0, typename _A1, typename... _As>
  struct _AbisInSimdTuple<_SimdTuple<_Tp, _A0, _A1, _As...>>
  {
    using _Counts = typename _SeqOp<typename _AbisInSimdTuple<
      _SimdTuple<_Tp, _A1, _As...>>::_Counts>::template _Prepend<1, 0>;
    using _Begins = typename _SeqOp<typename _AbisInSimdTuple<
      _SimdTuple<_Tp, _A1, _As...>>::_Begins>::template _Prepend<0, 1>;
  };

// }}}
// __autocvt_to_simd {{{
template <typename _Tp, bool = is_arithmetic_v<__remove_cvref_t<_Tp>>>
  struct __autocvt_to_simd
  {
    _Tp _M_data;
    using _TT = __remove_cvref_t<_Tp>;

    operator _TT()
    { return _M_data; }

    operator _TT&()
    {
      static_assert(is_lvalue_reference<_Tp>::value, "");
      static_assert(!is_const<_Tp>::value, "");
      return _M_data;
    }

    operator _TT*()
    {
      static_assert(is_lvalue_reference<_Tp>::value, "");
      static_assert(!is_const<_Tp>::value, "");
      return &_M_data;
    }

    constexpr inline __autocvt_to_simd(_Tp dd) : _M_data(dd) {}

    template <typename _Abi>
      operator simd<typename _TT::value_type, _Abi>()
      { return {__private_init, _M_data}; }

    template <typename _Abi>
      operator simd<typename _TT::value_type, _Abi>&()
      {
	return *reinterpret_cast<simd<typename _TT::value_type, _Abi>*>(
	  &_M_data);
      }

    template <typename _Abi>
      operator simd<typename _TT::value_type, _Abi>*()
      {
	return reinterpret_cast<simd<typename _TT::value_type, _Abi>*>(
	  &_M_data);
      }
  };

template <typename _Tp>
  __autocvt_to_simd(_Tp &&) -> __autocvt_to_simd<_Tp>;

template <typename _Tp>
  struct __autocvt_to_simd<_Tp, true>
  {
    using _TT = __remove_cvref_t<_Tp>;
    _Tp _M_data;
    fixed_size_simd<_TT, 1> _M_fd;

    constexpr inline __autocvt_to_simd(_Tp dd) : _M_data(dd), _M_fd(_M_data) {}

    ~__autocvt_to_simd()
    { _M_data = __data(_M_fd).first; }

    operator fixed_size_simd<_TT, 1>()
    { return _M_fd; }

    operator fixed_size_simd<_TT, 1> &()
    {
      static_assert(is_lvalue_reference<_Tp>::value, "");
      static_assert(!is_const<_Tp>::value, "");
      return _M_fd;
    }

    operator fixed_size_simd<_TT, 1> *()
    {
      static_assert(is_lvalue_reference<_Tp>::value, "");
      static_assert(!is_const<_Tp>::value, "");
      return &_M_fd;
    }
  };

// }}}

struct _CommonImplFixedSize;
template <int _Np> struct _SimdImplFixedSize;
template <int _Np> struct _MaskImplFixedSize;
// simd_abi::_Fixed {{{
template <int _Np>
  struct simd_abi::_Fixed
  {
    template <typename _Tp> static constexpr size_t _S_size = _Np;
    template <typename _Tp> static constexpr size_t _S_full_size = _Np;
    // validity traits {{{
    struct _IsValidAbiTag : public __bool_constant<(_Np > 0)> {};

    template <typename _Tp>
      struct _IsValidSizeFor
      : __bool_constant<(_Np <= simd_abi::max_fixed_size<_Tp>)> {};

    template <typename _Tp>
      struct _IsValid : conjunction<_IsValidAbiTag, __is_vectorizable<_Tp>,
				    _IsValidSizeFor<_Tp>> {};

    template <typename _Tp>
      static constexpr bool _S_is_valid_v = _IsValid<_Tp>::value;

    // }}}
    // _S_masked {{{
    _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np>
    _S_masked(_BitMask<_Np> __x)
    { return __x._M_sanitized(); }

    _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np>
    _S_masked(_SanitizedBitMask<_Np> __x)
    { return __x; }

    // }}}
    // _*Impl {{{
    using _CommonImpl = _CommonImplFixedSize;
    using _SimdImpl = _SimdImplFixedSize<_Np>;
    using _MaskImpl = _MaskImplFixedSize<_Np>;

    // }}}
    // __traits {{{
    template <typename _Tp, bool = _S_is_valid_v<_Tp>>
      struct __traits : _InvalidTraits {};

    template <typename _Tp>
      struct __traits<_Tp, true>
      {
	using _IsValid = true_type;
	using _SimdImpl = _SimdImplFixedSize<_Np>;
	using _MaskImpl = _MaskImplFixedSize<_Np>;

	// simd and simd_mask member types {{{
	using _SimdMember = __fixed_size_storage_t<_Tp, _Np>;
	using _MaskMember = _SanitizedBitMask<_Np>;

	static constexpr size_t _S_simd_align
	  = std::__bit_ceil(_Np * sizeof(_Tp));

	static constexpr size_t _S_mask_align = alignof(_MaskMember);

	// }}}
	// _SimdBase / base class for simd, providing extra conversions {{{
	struct _SimdBase
	{
	  // The following ensures, function arguments are passed via the stack.
	  // This is important for ABI compatibility across TU boundaries
	  _SimdBase(const _SimdBase&) {}
	  _SimdBase() = default;

	  explicit operator const _SimdMember &() const
	  { return static_cast<const simd<_Tp, _Fixed>*>(this)->_M_data; }

	  explicit operator array<_Tp, _Np>() const
	  {
	    array<_Tp, _Np> __r;
	    // _SimdMember can be larger because of higher alignment
	    static_assert(sizeof(__r) <= sizeof(_SimdMember), "");
	    __builtin_memcpy(__r.data(), &static_cast<const _SimdMember&>(*this),
			     sizeof(__r));
	    return __r;
	  }
	};

	// }}}
	// _MaskBase {{{
	// empty. The bitset interface suffices
	struct _MaskBase {};

	// }}}
	// _SimdCastType {{{
	struct _SimdCastType
	{
	  _SimdCastType(const array<_Tp, _Np>&);
	  _SimdCastType(const _SimdMember& dd) : _M_data(dd) {}
	  explicit operator const _SimdMember &() const { return _M_data; }

	private:
	  const _SimdMember& _M_data;
	};

	// }}}
	// _MaskCastType {{{
	class _MaskCastType
	{
	  _MaskCastType() = delete;
	};
	// }}}
      };
    // }}}
  };

// }}}
// _CommonImplFixedSize {{{
struct _CommonImplFixedSize
{
  // _S_store {{{
  template <typename _Tp, typename... _As>
    _GLIBCXX_SIMD_INTRINSIC static void
    _S_store(const _SimdTuple<_Tp, _As...>& __x, void* __addr)
    {
      constexpr size_t _Np = _SimdTuple<_Tp, _As...>::_S_size();
      __builtin_memcpy(__addr, &__x, _Np * sizeof(_Tp));
    }

  // }}}
};

// }}}
// _SimdImplFixedSize {{{1
// fixed_size should not inherit from _SimdMathFallback in order for
// specializations in the used _SimdTuple Abis to get used
template <int _Np>
  struct _SimdImplFixedSize
  {
    // member types {{{2
    using _MaskMember = _SanitizedBitMask<_Np>;

    template <typename _Tp>
      using _SimdMember = __fixed_size_storage_t<_Tp, _Np>;

    template <typename _Tp>
      static constexpr size_t _S_tuple_size = _SimdMember<_Tp>::_S_tuple_size;

    template <typename _Tp>
      using _Simd = simd<_Tp, simd_abi::fixed_size<_Np>>;

    template <typename _Tp>
      using _TypeTag = _Tp*;

    // broadcast {{{2
    template <typename _Tp>
      static constexpr inline _SimdMember<_Tp> _S_broadcast(_Tp __x) noexcept
      {
	return _SimdMember<_Tp>::_S_generate([&](auto __meta) constexpr {
	  return __meta._S_broadcast(__x);
	});
      }

    // _S_generator {{{2
    template <typename _Fp, typename _Tp>
      static constexpr inline _SimdMember<_Tp> _S_generator(_Fp&& __gen,
							    _TypeTag<_Tp>)
      {
	return _SimdMember<_Tp>::_S_generate([&__gen](auto __meta) constexpr {
	  return __meta._S_generator(
	    [&](auto __i) constexpr {
	      return __i < _Np ? __gen(_SizeConstant<__meta._S_offset + __i>())
			       : 0;
	    },
	    _TypeTag<_Tp>());
	});
      }

    // _S_load {{{2
    template <typename _Tp, typename _Up>
      static inline _SimdMember<_Tp> _S_load(const _Up* __mem,
					     _TypeTag<_Tp>) noexcept
      {
	return _SimdMember<_Tp>::_S_generate([&](auto __meta) {
	  return __meta._S_load(&__mem[__meta._S_offset], _TypeTag<_Tp>());
	});
      }

    // _S_masked_load {{{2
    template <typename _Tp, typename... _As, typename _Up>
      static inline _SimdTuple<_Tp, _As...>
      _S_masked_load(const _SimdTuple<_Tp, _As...>& __old,
		     const _MaskMember __bits, const _Up* __mem) noexcept
      {
	auto __merge = __old;
	__for_each(__merge, [&](auto __meta, auto& __native) {
	  if (__meta._S_submask(__bits).any())
#pragma GCC diagnostic push
	  // __mem + __mem._S_offset could be UB ([expr.add]/4.3, but it punts
	  // the responsibility for avoiding UB to the caller of the masked load
	  // via the mask. Consequently, the compiler may assume this branch is
	  // unreachable, if the pointer arithmetic is UB.
#pragma GCC diagnostic ignored "-Warray-bounds"
	    __native
	      = __meta._S_masked_load(__native, __meta._S_make_mask(__bits),
				      __mem + __meta._S_offset);
#pragma GCC diagnostic pop
	});
	return __merge;
      }

    // _S_store {{{2
    template <typename _Tp, typename _Up>
      static inline void _S_store(const _SimdMember<_Tp>& __v, _Up* __mem,
				  _TypeTag<_Tp>) noexcept
      {
	__for_each(__v, [&](auto __meta, auto __native) {
	  __meta._S_store(__native, &__mem[__meta._S_offset], _TypeTag<_Tp>());
	});
      }

    // _S_masked_store {{{2
    template <typename _Tp, typename... _As, typename _Up>
      static inline void _S_masked_store(const _SimdTuple<_Tp, _As...>& __v,
					 _Up* __mem,
					 const _MaskMember __bits) noexcept
      {
	__for_each(__v, [&](auto __meta, auto __native) {
	  if (__meta._S_submask(__bits).any())
#pragma GCC diagnostic push
	  // __mem + __mem._S_offset could be UB ([expr.add]/4.3, but it punts
	  // the responsibility for avoiding UB to the caller of the masked
	  // store via the mask. Consequently, the compiler may assume this
	  // branch is unreachable, if the pointer arithmetic is UB.
#pragma GCC diagnostic ignored "-Warray-bounds"
	    __meta._S_masked_store(__native, __mem + __meta._S_offset,
				   __meta._S_make_mask(__bits));
#pragma GCC diagnostic pop
	});
      }

    // negation {{{2
    template <typename _Tp, typename... _As>
      static inline _MaskMember
      _S_negate(const _SimdTuple<_Tp, _As...>& __x) noexcept
      {
	_MaskMember __bits = 0;
	__for_each(
	  __x, [&__bits](auto __meta, auto __native) constexpr {
	    __bits
	      |= __meta._S_mask_to_shifted_ullong(__meta._S_negate(__native));
	  });
	return __bits;
      }

    // reductions {{{2
    template <typename _Tp, typename _BinaryOperation>
      static constexpr inline _Tp _S_reduce(const _Simd<_Tp>& __x,
					    const _BinaryOperation& __binary_op)
      {
	using _Tup = _SimdMember<_Tp>;
	const _Tup& __tup = __data(__x);
	if constexpr (_Tup::_S_tuple_size == 1)
	  return _Tup::_FirstAbi::_SimdImpl::_S_reduce(
	    __tup.template _M_simd_at<0>(), __binary_op);
	else if constexpr (_Tup::_S_tuple_size == 2 && _Tup::_S_size() > 2
			   && _Tup::_SecondType::_S_size() == 1)
	  {
	    return __binary_op(simd<_Tp, simd_abi::scalar>(
				 reduce(__tup.template _M_simd_at<0>(),
					__binary_op)),
			       __tup.template _M_simd_at<1>())[0];
	  }
	else if constexpr (_Tup::_S_tuple_size == 2 && _Tup::_S_size() > 4
			   && _Tup::_SecondType::_S_size() == 2)
	  {
	    return __binary_op(
	      simd<_Tp, simd_abi::scalar>(
		reduce(__tup.template _M_simd_at<0>(), __binary_op)),
	      simd<_Tp, simd_abi::scalar>(
		reduce(__tup.template _M_simd_at<1>(), __binary_op)))[0];
	  }
	else
	  {
	    const auto& __x2 = __call_with_n_evaluations<
	      __div_roundup(_Tup::_S_tuple_size, 2)>(
	      [](auto __first_simd, auto... __remaining) {
		if constexpr (sizeof...(__remaining) == 0)
		  return __first_simd;
		else
		  {
		    using _Tup2
		      = _SimdTuple<_Tp,
				   typename decltype(__first_simd)::abi_type,
				   typename decltype(__remaining)::abi_type...>;
		    return fixed_size_simd<_Tp, _Tup2::_S_size()>(
		      __private_init,
		      __make_simd_tuple(__first_simd, __remaining...));
		  }
	      },
	      [&](auto __i) {
		auto __left = __tup.template _M_simd_at<2 * __i>();
		if constexpr (2 * __i + 1 == _Tup::_S_tuple_size)
		  return __left;
		else
		  {
		    auto __right = __tup.template _M_simd_at<2 * __i + 1>();
		    using _LT = decltype(__left);
		    using _RT = decltype(__right);
		    if constexpr (_LT::size() == _RT::size())
		      return __binary_op(__left, __right);
		    else
		      {
			_GLIBCXX_SIMD_USE_CONSTEXPR_API
			typename _LT::mask_type __k(
			  __private_init,
			  [](auto __j) constexpr { return __j < _RT::size(); });
			_LT __ext_right = __left;
			where(__k, __ext_right)
			  = __proposed::resizing_simd_cast<_LT>(__right);
			where(__k, __left) = __binary_op(__left, __ext_right);
			return __left;
		      }
		  }
	      });
	    return reduce(__x2, __binary_op);
	  }
      }

    // _S_min, _S_max {{{2
    template <typename _Tp, typename... _As>
      static inline constexpr _SimdTuple<_Tp, _As...>
      _S_min(const _SimdTuple<_Tp, _As...>& __a,
	     const _SimdTuple<_Tp, _As...>& __b)
      {
	return __a._M_apply_per_chunk(
	  [](auto __impl, auto __aa, auto __bb) constexpr {
	    return __impl._S_min(__aa, __bb);
	  },
	  __b);
      }

    template <typename _Tp, typename... _As>
      static inline constexpr _SimdTuple<_Tp, _As...>
      _S_max(const _SimdTuple<_Tp, _As...>& __a,
	     const _SimdTuple<_Tp, _As...>& __b)
      {
	return __a._M_apply_per_chunk(
	  [](auto __impl, auto __aa, auto __bb) constexpr {
	    return __impl._S_max(__aa, __bb);
	  },
	  __b);
      }

    // _S_complement {{{2
    template <typename _Tp, typename... _As>
      static inline constexpr _SimdTuple<_Tp, _As...>
      _S_complement(const _SimdTuple<_Tp, _As...>& __x) noexcept
      {
	return __x._M_apply_per_chunk([](auto __impl, auto __xx) constexpr {
	  return __impl._S_complement(__xx);
	});
      }

    // _S_unary_minus {{{2
    template <typename _Tp, typename... _As>
      static inline constexpr _SimdTuple<_Tp, _As...>
      _S_unary_minus(const _SimdTuple<_Tp, _As...>& __x) noexcept
      {
	return __x._M_apply_per_chunk([](auto __impl, auto __xx) constexpr {
	  return __impl._S_unary_minus(__xx);
	});
      }

    // arithmetic operators {{{2

#define _GLIBCXX_SIMD_FIXED_OP(name_, op_)                                     \
    template <typename _Tp, typename... _As>                                   \
      static inline constexpr _SimdTuple<_Tp, _As...> name_(                   \
	const _SimdTuple<_Tp, _As...> __x, const _SimdTuple<_Tp, _As...> __y)  \
      {                                                                        \
	return __x._M_apply_per_chunk(                                         \
	  [](auto __impl, auto __xx, auto __yy) constexpr {                    \
	    return __impl.name_(__xx, __yy);                                   \
	  },                                                                   \
	  __y);                                                                \
      }

    _GLIBCXX_SIMD_FIXED_OP(_S_plus, +)
    _GLIBCXX_SIMD_FIXED_OP(_S_minus, -)
    _GLIBCXX_SIMD_FIXED_OP(_S_multiplies, *)
    _GLIBCXX_SIMD_FIXED_OP(_S_divides, /)
    _GLIBCXX_SIMD_FIXED_OP(_S_modulus, %)
    _GLIBCXX_SIMD_FIXED_OP(_S_bit_and, &)
    _GLIBCXX_SIMD_FIXED_OP(_S_bit_or, |)
    _GLIBCXX_SIMD_FIXED_OP(_S_bit_xor, ^)
    _GLIBCXX_SIMD_FIXED_OP(_S_bit_shift_left, <<)
    _GLIBCXX_SIMD_FIXED_OP(_S_bit_shift_right, >>)
#undef _GLIBCXX_SIMD_FIXED_OP

    template <typename _Tp, typename... _As>
      static inline constexpr _SimdTuple<_Tp, _As...>
      _S_bit_shift_left(const _SimdTuple<_Tp, _As...>& __x, int __y)
      {
	return __x._M_apply_per_chunk([__y](auto __impl, auto __xx) constexpr {
	  return __impl._S_bit_shift_left(__xx, __y);
	});
      }

    template <typename _Tp, typename... _As>
      static inline constexpr _SimdTuple<_Tp, _As...>
      _S_bit_shift_right(const _SimdTuple<_Tp, _As...>& __x, int __y)
      {
	return __x._M_apply_per_chunk([__y](auto __impl, auto __xx) constexpr {
	  return __impl._S_bit_shift_right(__xx, __y);
	});
      }

  // math {{{2
#define _GLIBCXX_SIMD_APPLY_ON_TUPLE(_RetTp, __name)                           \
    template <typename _Tp, typename... _As, typename... _More>                \
      static inline __fixed_size_storage_t<_RetTp, _Np>                        \
	_S_##__name(const _SimdTuple<_Tp, _As...>& __x,                        \
		    const _More&... __more)                                    \
      {                                                                        \
	if constexpr (sizeof...(_More) == 0)                                   \
	  {                                                                    \
	    if constexpr (is_same_v<_Tp, _RetTp>)                              \
	      return __x._M_apply_per_chunk(                                   \
		[](auto __impl, auto __xx) constexpr {                         \
		  using _V = typename decltype(__impl)::simd_type;             \
		  return __data(__name(_V(__private_init, __xx)));             \
		});                                                            \
	    else                                                               \
	      return __optimize_simd_tuple(                                    \
		__x.template _M_apply_r<_RetTp>([](auto __impl, auto __xx) {   \
		  return __impl._S_##__name(__xx);                             \
		}));                                                           \
	  }                                                                    \
	else if constexpr (                                                    \
	  is_same_v<                                                           \
	    _Tp,                                                               \
	    _RetTp> && (... && is_same_v<_SimdTuple<_Tp, _As...>, _More>) )    \
	  return __x._M_apply_per_chunk(                                       \
	    [](auto __impl, auto __xx, auto... __pack) constexpr {             \
	      using _V = typename decltype(__impl)::simd_type;                 \
	      return __data(__name(_V(__private_init, __xx),                   \
				   _V(__private_init, __pack)...));            \
	    },                                                                 \
	    __more...);                                                        \
	else if constexpr (is_same_v<_Tp, _RetTp>)                             \
	  return __x._M_apply_per_chunk(                                       \
	    [](auto __impl, auto __xx, auto... __pack) constexpr {             \
	      using _V = typename decltype(__impl)::simd_type;                 \
	      return __data(__name(_V(__private_init, __xx),                   \
				   __autocvt_to_simd(__pack)...));             \
	    },                                                                 \
	    __more...);                                                        \
	else                                                                   \
	  __assert_unreachable<_Tp>();                                         \
      }

    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, acos)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, asin)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, atan)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, atan2)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, cos)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, sin)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, tan)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, acosh)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, asinh)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, atanh)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, cosh)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, sinh)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, tanh)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, exp)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, exp2)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, expm1)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(int, ilogb)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log10)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log1p)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log2)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, logb)
    // modf implemented in simd_math.h
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp,
				 scalbn) // double scalbn(double x, int exp);
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, scalbln)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, cbrt)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, abs)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fabs)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, pow)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, sqrt)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, erf)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, erfc)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, lgamma)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, tgamma)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, trunc)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, ceil)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, floor)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, nearbyint)

    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, rint)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(long, lrint)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(long long, llrint)

    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, round)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(long, lround)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(long long, llround)

    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, ldexp)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fmod)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, remainder)
    // copysign in simd_math.h
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, nextafter)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fdim)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fmax)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fmin)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fma)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(int, fpclassify)
#undef _GLIBCXX_SIMD_APPLY_ON_TUPLE

    template <typename _Tp, typename... _Abis>
      static _SimdTuple<_Tp, _Abis...> _S_remquo(
	const _SimdTuple<_Tp, _Abis...>& __x,
	const _SimdTuple<_Tp, _Abis...>& __y,
	__fixed_size_storage_t<int, _SimdTuple<_Tp, _Abis...>::_S_size()>* __z)
      {
	return __x._M_apply_per_chunk(
	  [](auto __impl, const auto __xx, const auto __yy, auto& __zz) {
	    return __impl._S_remquo(__xx, __yy, &__zz);
	  },
	  __y, *__z);
      }

    template <typename _Tp, typename... _As>
      static inline _SimdTuple<_Tp, _As...>
      _S_frexp(const _SimdTuple<_Tp, _As...>& __x,
	       __fixed_size_storage_t<int, _Np>& __exp) noexcept
      {
	return __x._M_apply_per_chunk(
	  [](auto __impl, const auto& __a, auto& __b) {
	    return __data(
	      frexp(typename decltype(__impl)::simd_type(__private_init, __a),
		    __autocvt_to_simd(__b)));
	  },
	  __exp);
      }

#define _GLIBCXX_SIMD_TEST_ON_TUPLE_(name_)                                    \
    template <typename _Tp, typename... _As>                                   \
      static inline _MaskMember                                                \
	_S_##name_(const _SimdTuple<_Tp, _As...>& __x) noexcept                \
      {                                                                        \
	return _M_test([](auto __impl,                                         \
			  auto __xx) { return __impl._S_##name_(__xx); },      \
		       __x);                                                   \
      }

    _GLIBCXX_SIMD_TEST_ON_TUPLE_(isinf)
    _GLIBCXX_SIMD_TEST_ON_TUPLE_(isfinite)
    _GLIBCXX_SIMD_TEST_ON_TUPLE_(isnan)
    _GLIBCXX_SIMD_TEST_ON_TUPLE_(isnormal)
    _GLIBCXX_SIMD_TEST_ON_TUPLE_(signbit)
#undef _GLIBCXX_SIMD_TEST_ON_TUPLE_

    // _S_increment & _S_decrement{{{2
    template <typename... _Ts>
      _GLIBCXX_SIMD_INTRINSIC static constexpr void
      _S_increment(_SimdTuple<_Ts...>& __x)
      {
	__for_each(
	  __x, [](auto __meta, auto& native) constexpr {
	    __meta._S_increment(native);
	  });
      }

    template <typename... _Ts>
      _GLIBCXX_SIMD_INTRINSIC static constexpr void
      _S_decrement(_SimdTuple<_Ts...>& __x)
      {
	__for_each(
	  __x, [](auto __meta, auto& native) constexpr {
	    __meta._S_decrement(native);
	  });
      }

    // compares {{{2
#define _GLIBCXX_SIMD_CMP_OPERATIONS(__cmp)                                    \
    template <typename _Tp, typename... _As>                                   \
      _GLIBCXX_SIMD_INTRINSIC constexpr static _MaskMember                     \
      __cmp(const _SimdTuple<_Tp, _As...>& __x,                                \
	    const _SimdTuple<_Tp, _As...>& __y)                                \
      {                                                                        \
	return _M_test(                                                        \
	  [](auto __impl, auto __xx, auto __yy) constexpr {                    \
	    return __impl.__cmp(__xx, __yy);                                   \
	  },                                                                   \
	  __x, __y);                                                           \
      }

    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_equal_to)
    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_not_equal_to)
    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_less)
    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_less_equal)
    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isless)
    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_islessequal)
    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isgreater)
    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isgreaterequal)
    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_islessgreater)
    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isunordered)
#undef _GLIBCXX_SIMD_CMP_OPERATIONS

    // smart_reference access {{{2
    template <typename _Tp, typename... _As, typename _Up>
      _GLIBCXX_SIMD_INTRINSIC static void _S_set(_SimdTuple<_Tp, _As...>& __v,
						 int __i, _Up&& __x) noexcept
      { __v._M_set(__i, static_cast<_Up&&>(__x)); }

    // _S_masked_assign {{{2
    template <typename _Tp, typename... _As>
      _GLIBCXX_SIMD_INTRINSIC static void
      _S_masked_assign(const _MaskMember __bits, _SimdTuple<_Tp, _As...>& __lhs,
		       const __type_identity_t<_SimdTuple<_Tp, _As...>>& __rhs)
      {
	__for_each(
	  __lhs, __rhs,
	  [&](auto __meta, auto& __native_lhs, auto __native_rhs) constexpr {
	    __meta._S_masked_assign(__meta._S_make_mask(__bits), __native_lhs,
				    __native_rhs);
	  });
      }

    // Optimization for the case where the RHS is a scalar. No need to broadcast
    // the scalar to a simd first.
    template <typename _Tp, typename... _As>
      _GLIBCXX_SIMD_INTRINSIC static void
      _S_masked_assign(const _MaskMember __bits, _SimdTuple<_Tp, _As...>& __lhs,
		       const __type_identity_t<_Tp> __rhs)
      {
	__for_each(
	  __lhs, [&](auto __meta, auto& __native_lhs) constexpr {
	    __meta._S_masked_assign(__meta._S_make_mask(__bits), __native_lhs,
				    __rhs);
	  });
      }

    // _S_masked_cassign {{{2
    template <typename _Op, typename _Tp, typename... _As>
      static inline void _S_masked_cassign(const _MaskMember __bits,
					   _SimdTuple<_Tp, _As...>& __lhs,
					   const _SimdTuple<_Tp, _As...>& __rhs,
					   _Op __op)
      {
	__for_each(
	  __lhs, __rhs,
	  [&](auto __meta, auto& __native_lhs, auto __native_rhs) constexpr {
	    __meta.template _S_masked_cassign(__meta._S_make_mask(__bits),
					      __native_lhs, __native_rhs, __op);
	  });
      }

    // Optimization for the case where the RHS is a scalar. No need to broadcast
    // the scalar to a simd first.
    template <typename _Op, typename _Tp, typename... _As>
      static inline void _S_masked_cassign(const _MaskMember __bits,
					   _SimdTuple<_Tp, _As...>& __lhs,
					   const _Tp& __rhs, _Op __op)
      {
	__for_each(
	  __lhs, [&](auto __meta, auto& __native_lhs) constexpr {
	    __meta.template _S_masked_cassign(__meta._S_make_mask(__bits),
					      __native_lhs, __rhs, __op);
	  });
      }

    // _S_masked_unary {{{2
    template <template <typename> class _Op, typename _Tp, typename... _As>
      static inline _SimdTuple<_Tp, _As...>
      _S_masked_unary(const _MaskMember __bits,
		      const _SimdTuple<_Tp, _As...> __v) // TODO: const-ref __v?
      {
	return __v._M_apply_wrapped([&__bits](auto __meta,
					      auto __native) constexpr {
	  return __meta.template _S_masked_unary<_Op>(__meta._S_make_mask(
							__bits),
						      __native);
	});
      }

    // }}}2
  };

// _MaskImplFixedSize {{{1
template <int _Np>
  struct _MaskImplFixedSize
  {
    static_assert(
      sizeof(_ULLong) * __CHAR_BIT__ >= _Np,
      "The fixed_size implementation relies on one _ULLong being able to store "
      "all boolean elements."); // required in load & store

    // member types {{{
    using _Abi = simd_abi::fixed_size<_Np>;

    using _MaskMember = _SanitizedBitMask<_Np>;

    template <typename _Tp>
      using _FirstAbi = typename __fixed_size_storage_t<_Tp, _Np>::_FirstAbi;

    template <typename _Tp>
      using _TypeTag = _Tp*;

    // }}}
    // _S_broadcast {{{
    template <typename>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
      _S_broadcast(bool __x)
      { return __x ? ~_MaskMember() : _MaskMember(); }

    // }}}
    // _S_load {{{
    template <typename>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
      _S_load(const bool* __mem)
      {
	using _Ip = __int_for_sizeof_t<bool>;
	// the following load uses element_aligned and relies on __mem already
	// carrying alignment information from when this load function was
	// called.
	const simd<_Ip, _Abi> __bools(reinterpret_cast<const __may_alias<_Ip>*>(
					__mem),
				      element_aligned);
	return __data(__bools != 0);
      }

    // }}}
    // _S_to_bits {{{
    template <bool _Sanitized>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np>
      _S_to_bits(_BitMask<_Np, _Sanitized> __x)
      {
	if constexpr (_Sanitized)
	  return __x;
	else
	  return __x._M_sanitized();
      }

    // }}}
    // _S_convert {{{
    template <typename _Tp, typename _Up, typename _UAbi>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
      _S_convert(simd_mask<_Up, _UAbi> __x)
      {
	return _UAbi::_MaskImpl::_S_to_bits(__data(__x))
	  .template _M_extract<0, _Np>();
      }

    // }}}
    // _S_from_bitmask {{{2
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static _MaskMember
      _S_from_bitmask(_MaskMember __bits, _TypeTag<_Tp>) noexcept
      { return __bits; }

    // _S_load {{{2
    static inline _MaskMember _S_load(const bool* __mem) noexcept
    {
      // TODO: _UChar is not necessarily the best type to use here. For smaller
      // _Np _UShort, _UInt, _ULLong, float, and double can be more efficient.
      _ULLong __r = 0;
      using _Vs = __fixed_size_storage_t<_UChar, _Np>;
      __for_each(_Vs{}, [&](auto __meta, auto) {
	__r |= __meta._S_mask_to_shifted_ullong(
	  __meta._S_mask_impl._S_load(&__mem[__meta._S_offset],
				      _SizeConstant<__meta._S_size()>()));
      });
      return __r;
    }

    // _S_masked_load {{{2
    static inline _MaskMember _S_masked_load(_MaskMember __merge,
					     _MaskMember __mask,
					     const bool* __mem) noexcept
    {
      _BitOps::_S_bit_iteration(__mask.to_ullong(), [&](auto __i) {
	__merge.set(__i, __mem[__i]);
      });
      return __merge;
    }

    // _S_store {{{2
    static inline void _S_store(const _MaskMember __bitmask,
				bool* __mem) noexcept
    {
      if constexpr (_Np == 1)
	__mem[0] = __bitmask[0];
      else
	_FirstAbi<_UChar>::_CommonImpl::_S_store_bool_array(__bitmask, __mem);
    }

    // _S_masked_store {{{2
    static inline void _S_masked_store(const _MaskMember __v, bool* __mem,
				       const _MaskMember __k) noexcept
    {
      _BitOps::_S_bit_iteration(__k, [&](auto __i) { __mem[__i] = __v[__i]; });
    }

    // logical and bitwise operators {{{2
    _GLIBCXX_SIMD_INTRINSIC static _MaskMember
    _S_logical_and(const _MaskMember& __x, const _MaskMember& __y) noexcept
    { return __x & __y; }

    _GLIBCXX_SIMD_INTRINSIC static _MaskMember
    _S_logical_or(const _MaskMember& __x, const _MaskMember& __y) noexcept
    { return __x | __y; }

    _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
    _S_bit_not(const _MaskMember& __x) noexcept
    { return ~__x; }

    _GLIBCXX_SIMD_INTRINSIC static _MaskMember
    _S_bit_and(const _MaskMember& __x, const _MaskMember& __y) noexcept
    { return __x & __y; }

    _GLIBCXX_SIMD_INTRINSIC static _MaskMember
    _S_bit_or(const _MaskMember& __x, const _MaskMember& __y) noexcept
    { return __x | __y; }

    _GLIBCXX_SIMD_INTRINSIC static _MaskMember
    _S_bit_xor(const _MaskMember& __x, const _MaskMember& __y) noexcept
    { return __x ^ __y; }

    // smart_reference access {{{2
    _GLIBCXX_SIMD_INTRINSIC static void _S_set(_MaskMember& __k, int __i,
					       bool __x) noexcept
    { __k.set(__i, __x); }

    // _S_masked_assign {{{2
    _GLIBCXX_SIMD_INTRINSIC static void
    _S_masked_assign(const _MaskMember __k, _MaskMember& __lhs,
		     const _MaskMember __rhs)
    { __lhs = (__lhs & ~__k) | (__rhs & __k); }

    // Optimization for the case where the RHS is a scalar.
    _GLIBCXX_SIMD_INTRINSIC static void _S_masked_assign(const _MaskMember __k,
							 _MaskMember& __lhs,
							 const bool __rhs)
    {
      if (__rhs)
	__lhs |= __k;
      else
	__lhs &= ~__k;
    }

    // }}}2
    // _S_all_of {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static bool _S_all_of(simd_mask<_Tp, _Abi> __k)
      { return __data(__k).all(); }

    // }}}
    // _S_any_of {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static bool _S_any_of(simd_mask<_Tp, _Abi> __k)
      { return __data(__k).any(); }

    // }}}
    // _S_none_of {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static bool _S_none_of(simd_mask<_Tp, _Abi> __k)
      { return __data(__k).none(); }

    // }}}
    // _S_some_of {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static bool
      _S_some_of([[maybe_unused]] simd_mask<_Tp, _Abi> __k)
      {
	if constexpr (_Np == 1)
	  return false;
	else
	  return __data(__k).any() && !__data(__k).all();
      }

    // }}}
    // _S_popcount {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static int _S_popcount(simd_mask<_Tp, _Abi> __k)
      { return __data(__k).count(); }

    // }}}
    // _S_find_first_set {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static int
      _S_find_first_set(simd_mask<_Tp, _Abi> __k)
      { return std::__countr_zero(__data(__k).to_ullong()); }

    // }}}
    // _S_find_last_set {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static int
      _S_find_last_set(simd_mask<_Tp, _Abi> __k)
      { return std::__bit_width(__data(__k).to_ullong()) - 1; }

    // }}}
  };
// }}}1

_GLIBCXX_SIMD_END_NAMESPACE
#endif // __cplusplus >= 201703L
#endif // _GLIBCXX_EXPERIMENTAL_SIMD_FIXED_SIZE_H_

// vim: foldmethod=marker sw=2 noet ts=8 sts=2 tw=80