/*
 * Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.  Oracle designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Oracle in the LICENSE file that accompanied this code.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 */
package jdk.incubator.vector;

import java.lang.foreign.MemorySegment;
import java.nio.ByteOrder;
import java.util.function.IntUnaryOperator;

import jdk.internal.vm.annotation.ForceInline;
import jdk.internal.vm.vector.VectorSupport;

import static jdk.incubator.vector.VectorOperators.*;

@SuppressWarnings("cast")
abstract sealed class AbstractVector<E> extends Vector<E>
        permits ByteVector, DoubleVector, FloatVector, IntVector, LongVector, ShortVector {
    /**
     * The order of vector bytes when stored in natural,
     * array elements of the same lane type.
     * This is also the behavior of the
     * VectorSupport load/store instructions.
     * If these instructions gain the capability to do
     * byte swapping on the fly, add a bit to those
     * instructions, but let this polarity be the
     * "neutral" or "default" setting of the bit.
     */
    /*package-private*/
    static final ByteOrder NATIVE_ENDIAN = ByteOrder.nativeOrder();

    /**
     * The order of vector bytes as stored in the register
     * file.  This becomes visible with the asRaw[Type]Vector
     * operations, which convert between the internal byte-wise
     * representation and the typed lane-wise representation.
     * It is very possible for a platform to have big-endian
     * memory layout and little-endian register layout,
     * so this is a different setting from NATIVE_ENDIAN.
     * In fact, both Intel and ARM use LE conventions here.
     * Future work may be needed for resolutely BE platforms.
     */
    /*package-private*/
    static final ByteOrder REGISTER_ENDIAN = ByteOrder.LITTLE_ENDIAN;

    /*package-private*/
    static final int OFFSET_IN_RANGE = 1;

    /*package-private*/
    static final int OFFSET_OUT_OF_RANGE = 0;

    /*package-private*/
    AbstractVector(Object bits) {
        super(bits);
    }

    // Extractors

    /*package-private*/
    abstract AbstractSpecies<E> vspecies();

    /*package-private*/
    abstract int laneTypeOrdinal();

    @Override
    @ForceInline
    public final VectorSpecies<E> species() {
        return vspecies();
    }

    // Something to make types match up better:

    @Override
    @ForceInline
    public final
    <F> Vector<F> check(VectorSpecies<F> species) {
        return check0(species);
    }

    @ForceInline
    @SuppressWarnings("unchecked")
    /*package-private*/ final
    <F> AbstractVector<F> check0(VectorSpecies<F> species) {
        if (!sameSpecies(species)) {
            throw AbstractSpecies.checkFailed(this, species);
        }
        return (AbstractVector<F>) this;
    }

    /**
     * {@inheritDoc} <!--workaround-->
     */
    @Override
    @ForceInline
    public final
    <F> Vector<F> check(Class<F> elementType) {
        return check0(elementType);
    }

    @ForceInline
    @SuppressWarnings("unchecked")
    /*package-private*/ final
    <F> AbstractVector<F> check0(Class<F> elementType) {
        if (this.elementType() != elementType) {
            throw AbstractSpecies.checkFailed(this, elementType);
        }
        return (AbstractVector<F>) this;
    }

    @ForceInline
    @SuppressWarnings("unchecked")
    /*package-private*/ final
    <F> AbstractVector<F> check(Vector<F> other) {
        if (!sameSpecies(other)) {
            throw AbstractSpecies.checkFailed(this, other);
        }
        return (AbstractVector<F>) this;
    }

    @ForceInline
    private boolean sameSpecies(Vector<?> other) {
        // It's simpler and faster to do a class check.
        boolean same = (this.getClass() == other.getClass());
        // Make sure it works, too!
        assert(same == (this.species() == other.species())) : same;
        return same;
    }

    @ForceInline
    private boolean sameSpecies(VectorSpecies<?> species) {
        // It's simpler and faster to do a class check,
        // even if you have to load a dummy vector.
        AbstractVector<?> other = ((AbstractSpecies<?>)species).dummyVector();
        boolean same = (this.getClass() == other.getClass());
        // Make sure it works, too!
        assert(same == (this.species() == species)) : same;
        return same;
    }

    /**
     * {@inheritDoc} <!--workaround-->
     */
    @Override
    @ForceInline
    public final VectorMask<E> maskAll(boolean bit) {
        return species().maskAll(bit);
    }

    // Make myself into a vector of the same shape
    // and same information content but different lane type
    /*package-private*/
    abstract AbstractVector<?> asVectorRaw(LaneType laneType);

    // Make myself into a byte vector of the same shape
    /*package-private*/
    abstract ByteVector asByteVectorRaw();

    /*package-private*/
    @ForceInline
    final AbstractVector<?> asVectorRawTemplate(LaneType laneType) {
        // NOTE:  This assumes that convert0('X')
        // respects REGISTER_ENDIAN order.
        return convert0('X', vspecies().withLanes(laneType)).swapIfNeeded(vspecies());
    }

    @ForceInline
    static <T> VectorShuffle<T> normalizeSubLanesForSpecies(AbstractSpecies<T> targetSpecies, int subLanesPerSrc) {
        final int lanes = targetSpecies.laneCount();

        if ((lanes % subLanesPerSrc) != 0) {
            throw new IllegalArgumentException("laneCount " + lanes + " not divisible by subLanesPerSrc " + subLanesPerSrc);
        }

        // Each group corresponds to one source lane.
        // For each group, reverse the lanes inside that group.
        final int groups = lanes / subLanesPerSrc;
        int[] map = new int[lanes];
        for (int g = 0; g < groups; ++g) {
            int base = g * subLanesPerSrc;
            for (int j = 0; j < subLanesPerSrc; ++j) {
                 map[base + j] = base + (subLanesPerSrc - 1 - j);
            }
        }
        return VectorShuffle.fromArray(targetSpecies, map, 0);
    }

    @ForceInline
    final int subLanesToSwap(AbstractSpecies<?> srcSpecies) {
        if (java.nio.ByteOrder.nativeOrder() != ByteOrder.BIG_ENDIAN) {
            return -1;
        }
        int sBytes = srcSpecies.elementSize();
        int tBytes = vspecies().elementSize();

        // No lane reordering needed for same size or widening reinterprets
        if (sBytes == tBytes || (sBytes % tBytes) != 0) {
            return -1;
        }
        int subLanesPerSrc = sBytes / tBytes;
        return subLanesPerSrc;
    }

    /*package-private*/
    @ForceInline
    ByteVector asByteVectorRawTemplate() {
        return (ByteVector) asVectorRawTemplate(LaneType.BYTE);
    }


    abstract AbstractMask<E> maskFromArray(boolean[] bits);

    abstract <F> VectorShuffle<F> bitsToShuffle(AbstractSpecies<F> dsp);

    /*package-private*/
    @ForceInline
    final <F> VectorShuffle<F> bitsToShuffleTemplate(AbstractSpecies<F> dsp) {
        int etype = vspecies().laneTypeOrdinal();
        Class<?> dvtype = dsp.shuffleType();
        int dtype = dsp.asIntegral().laneTypeOrdinal();
        int dlength = dsp.dummyVector().length();
        return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
                                     getClass(), etype, length(),
                                     dvtype, dtype, dlength,
                                     this, dsp,
                                     AbstractVector::bitsToShuffle0);
    }

    abstract <F> VectorShuffle<F> bitsToShuffle0(AbstractSpecies<F> dsp);

    abstract <F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp, boolean wrap);

    abstract AbstractShuffle<E> iotaShuffle();

    abstract AbstractShuffle<E> iotaShuffle(int start, int step, boolean wrap);

    @ForceInline
    final VectorShuffle<E> iotaShuffleTemplate(int start, int step, boolean wrap) {
        if ((length() & (length() - 1)) != 0) {
            // Uncommon path, the length is not a power of 2
            return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, length())))
                        : shuffleFromOp(i -> i * step + start);
        }

        AbstractVector<?> iota = vspecies().asIntegral().iota();
        iota = (AbstractVector<?>) iota.lanewise(VectorOperators.MUL, step)
                .lanewise(VectorOperators.ADD, start);
        return iota.toShuffle(vspecies(), wrap);
    }

    abstract AbstractShuffle<E> shuffleFromArray(int[] indexes, int i);

    abstract AbstractShuffle<E> shuffleFromOp(IntUnaryOperator fn);

    /*package-private*/
    abstract AbstractVector<E> fromMemorySegment0(MemorySegment ms, long offset);

    /*package-private*/
    abstract AbstractVector<E> maybeSwap(ByteOrder bo);

    /*package-private*/
    abstract AbstractVector<?> swapIfNeeded(AbstractSpecies<?> srcSpecies);

    /*package-private*/
    @ForceInline
    VectorShuffle<Byte> swapBytesShuffle() {
        return vspecies().swapBytesShuffle();
    }

    /**
     * {@inheritDoc} <!--workaround-->
     */
    @Override
    @ForceInline
    public ShortVector reinterpretAsShorts() {
        return (ShortVector) asVectorRaw(LaneType.SHORT);
    }

    /**
     * {@inheritDoc} <!--workaround-->
     */
    @Override
    @ForceInline
    public IntVector reinterpretAsInts() {
        return (IntVector) asVectorRaw(LaneType.INT);
    }

    /**
     * {@inheritDoc} <!--workaround-->
     */
    @Override
    @ForceInline
    public LongVector reinterpretAsLongs() {
        return (LongVector) asVectorRaw(LaneType.LONG);
    }

    /**
     * {@inheritDoc} <!--workaround-->
     */
    @Override
    @ForceInline
    public FloatVector reinterpretAsFloats() {
        return (FloatVector) asVectorRaw(LaneType.FLOAT);
    }

    /**
     * {@inheritDoc} <!--workaround-->
     */
    @Override
    @ForceInline
    public DoubleVector reinterpretAsDoubles() {
        return (DoubleVector) asVectorRaw(LaneType.DOUBLE);
    }

    /**
     * {@inheritDoc} <!--workaround-->
     */
    @Override
    @ForceInline
    public final <F>
    Vector<F> convert(Conversion<E,F> conv, int part) {
        // Shape invariance is simple to implement.
        // It's part of the API because shape invariance
        // is the default mode of operation, and
        // shape-shifting operations must advertise
        // themselves.
        ConversionImpl<E,F> c = (ConversionImpl<E,F>) conv;
        @SuppressWarnings("unchecked")
        VectorSpecies<F> rsp = (VectorSpecies<F>)
            vspecies().withLanes(c.range());
        return convertShape(conv, rsp, part);
    }

    /**
     * {@inheritDoc} <!--workaround-->
     */
    @Override
    @ForceInline
    public final <F>
    Vector<F> castShape(VectorSpecies<F> toSpecies, int part) {
        // This is an odd mix of shape conversion plus
        // lanewise conversions.  It seems to be useful
        // sometimes as a shorthand, though maybe we
        // can drop it.
        AbstractSpecies<E> vsp = vspecies();
        AbstractSpecies<F> rsp = (AbstractSpecies<F>) toSpecies;
        @SuppressWarnings("unchecked")
        ConversionImpl<E,F> c = (ConversionImpl<E,F>)
            ConversionImpl.ofCast(vsp.laneType, rsp.laneType);
        return convertShape(c, rsp, part);
    }

    /**
     * {@inheritDoc} <!--workaround-->
     */
    @Override
    @ForceInline
    public abstract <F>
    Vector<F> convertShape(Conversion<E,F> conv, VectorSpecies<F> rsp, int part);

    /**
     * This is the template for Vector::reinterpretShape, to be
     * specialized by each distinct vector class.
     */
    /*package-private*/
    @ForceInline
    final <F>
    AbstractVector<F> reinterpretShapeTemplate(VectorSpecies<F> toSpecies, int part) {
        AbstractSpecies<F> rsp = (AbstractSpecies<F>) toSpecies;
        AbstractSpecies<E> vsp = vspecies();
        if (part == 0) {
            // Works the same for in-place, selection (truncation) and insertion (padding)
            return convert0('X', rsp);
        } else {
            int origin = shapeChangeOrigin(vsp, rsp, false, part);
            //System.out.println("*** origin = "+origin+", part = "+part+", reinterpret");
            if (part > 0) {  // Selection (truncation): slice first then cast.
                return slice(origin).convert0('X', rsp);
            } else {  // Insertion (padding): cast first then unslice.
                return rsp.zero().slice(rsp.laneCount() - origin,
                                        convert0('X', rsp));
            }
        }
    }

    @Override
    public abstract AbstractVector<E> slice(int origin, Vector<E> v1);

    @Override
    public abstract AbstractVector<E> slice(int origin);

    /**
     * This is the template for Vector::convertShape, to be
     * specialized by each distinct vector class.
     */
    /*package-private*/
    @ForceInline
    final <F>
    AbstractVector<F> convertShapeTemplate(Conversion<E,F> conv, VectorSpecies<F> toSpecies, int part) {
        ConversionImpl<E,F> c = (ConversionImpl<E,F>) conv;
        AbstractSpecies<F> rsp = (AbstractSpecies<F>) toSpecies;
        AbstractSpecies<E> vsp = vspecies();
        char kind = c.kind();
        switch (kind) {
        case 'C': // Regular cast conversion.
            break;
        case 'I':  // Identity conversion => reinterpret.
            assert(c.sizeChangeLog2() == 0);
            kind = 'X';
            break;
        case 'Z':  // Lane-wise expansion with zero padding.
            assert(c.sizeChangeLog2() > 0);
            assert(c.range().elementKind == 'I');
            break;
        case 'R':  // Lane-wise reinterpret conversion.
            if (c.sizeChangeLog2() != 0) {
                kind = 'Z';  // some goofy stuff here
                break;
            }
            kind = 'X';  // No size change => reinterpret whole vector
            break;
        default:
            throw new AssertionError(c);
        }
        vsp.check(c.domain());  // apply dynamic check to conv
        rsp.check(c.range());   // apply dynamic check to conv
        if (part == 0) {
            // Works the same for in-place, selection (truncation) and insertion (padding)
            return convert0(kind, rsp);
        } else {
            int origin = shapeChangeOrigin(vsp, rsp, true, part);
            //System.out.println("*** origin = "+origin+", part = "+part+", lanewise");
            if (part > 0) {  // Selection (truncation): slice first then cast.
                return slice(origin).convert0(kind, rsp);
            } else {  // Insertion (padding): cast first then unslice.
                return rsp.zero().slice(rsp.laneCount() - origin,
                                        convert0(kind, rsp));
            }
        }
    }

    /**
     * Check a part number and return it multiplied by the appropriate
     * block factor to yield the origin of the operand block, as a
     * lane number.  For selection (truncation) the origin is reckoned
     * in the domain vector, since the domain vector has too much
     * information and must be sliced.  For insertion (padding) the
     * origin is reckoned in the range vector, since the range vector
     * has too many lanes and the result must be unsliced at the same
     * position as the inverse selection.  If the conversion is lanewise,
     * then lane sizes may be changing as well.  This affects the logical
     * size of the result, and so the domain size is multiplied or
     * divided by the lane size change.
     */
    /*package-private*/
    @ForceInline
    static
    int shapeChangeOrigin(AbstractSpecies<?> dsp,
                          AbstractSpecies<?> rsp,
                          boolean lanewise,
                          int part) {
        // domain / input:          X        -> ETYPE
        // logical result:          f(X)     -> FTYPE
        // range / physical output: Y        -> FTYPE
        int domSizeLog2 = dsp.vectorShape.vectorBitSizeLog2;
        int phySizeLog2 = rsp.vectorShape.vectorBitSizeLog2;
        // Logical expansion ratio = ML
        //   lanewise=true  (logical lanewise conversion)
        //   -> ML = phyElementSize / domElementSize
        //   lanewise=false (reinterpret)
        //   -> ML = 1
        int laneChangeLog2 = 0;
        if (lanewise) {
            laneChangeLog2 = (rsp.laneType.elementSizeLog2 -
                              dsp.laneType.elementSizeLog2);
        }
        // ML = |f(X)| / |X|
        // -> |f(X)| = |X| * ML
        int resSizeLog2 = domSizeLog2 + laneChangeLog2;
        // resSizeLog2 = 0 => logical result only has a single byte
        //                    e.g. cast L64 (1-element long) to B8 (1-element byte)
        // resSizeLog2 < 0 => logical result has less than a byte -> impossible!
        //                    currently, all vectors start with at least 8 bytes,
        //                    and we can contract by at most a factor of 8.
        assert(resSizeLog2 >= 0);
        if (!partInRange(resSizeLog2, phySizeLog2, part)) {
            // fall through...
        } else if (resSizeLog2 > phySizeLog2) {
            // Selection (truncation).
            // Output selection ratio:
            //   MS = |f(X)| / |Y|
            //
            // We must slice one of MS blocks from the domain.
            // The block has L lanes:
            //   L = X.length        / MS
            //     = (|X| / |ETYPE|) / MS
            //     = (|X| / |ETYPE|) / (|f(X)| / |Y|)
            //     = |Y| / (|f(X)| / |X|) / |ETYPE|
            //     = |Y| / ML             / |ETYPE|
            //
            //   origin = part * L
            //
            int s = phySizeLog2 - laneChangeLog2 - dsp.laneType.elementSizeLog2;
            // Scale the part number by the input block size, in input lanes.
            if ((s & 31) == s)  // sanity check
                return part << s;
        } else {
            // Insertion (padding).
            // Output expansion ratio:
            //   MO = |Y| / |f(X)|
            //
            // We must drop the logical result into one of MO blocks of the range.
            // The block has L lanes:
            //   L = Y.length        / MO
            //     = (|Y| / |FTYPE|) / MO
            //     = (|Y| / |FTYPE|) / (|Y| / |f(X)|)
            //     = |f(X)| / |FTYPE|
            //
            //   origin = -part * L
            //
            int s = resSizeLog2 - rsp.laneType.elementSizeLog2;
            // Scale the part number by the output block size, in output lanes.
            if ((s & 31) == s)  // sanity check
                return -part << s;
        }
        throw wrongPart(dsp, rsp, lanewise, part);
    }

    @ForceInline
    private static boolean partInRange(int resSizeLog2, int phySizeLog2, int part) {
        // Let's try a branch-free version of this.
        int diff = (resSizeLog2 - phySizeLog2);
        int sign = (diff >> -1);
        //d = Math.abs(diff);
        //d = (sign == 0 ? diff : sign == -1 ? 1 + ~diff);
        int d = (diff ^ sign) - sign;
        assert(d == Math.abs(diff) && d <= 16);  // let's not go crazy here
        //p = part * sign;
        int p = (part ^ sign) - sign;
        // z = sign == 0  ? 0<=part<(1<<d),  == (part & (-1 << d)) == 0
        // z = sign == -1 ? 0<=-part<(1<<d), == (-part & (-1 << d)) == 0
        boolean z = (p & (-1 << d)) == 0;
        assert(z == partInRangeSlow(resSizeLog2, phySizeLog2, part)) : z;
        return z;
    }

    private static boolean partInRangeSlow(int resSizeLog2, int phySizeLog2, int part) {
        int limit = partLimit(resSizeLog2, phySizeLog2);
        if (limit > 0) {  // selection (output is truncation of logical result)
            return 0 <= part && part < limit;
        } else if (limit < 0) {  // insertion (output is logical result with padding)
            return limit < part && part <= 0;
        } else { // in-place
            return (part == 0);
        }
    }

    private static int partLimit(int resSizeLog2, int phySizeLog2) {
        if (resSizeLog2 > phySizeLog2) {  // selection (truncation)
            return 1 << (resSizeLog2 - phySizeLog2);
        } else if (resSizeLog2 < phySizeLog2) {  // insertion (padding)
            return -1 << (phySizeLog2 - resSizeLog2);
        } else { // in-place
            return 0;
        }
    }

    private static
    ArrayIndexOutOfBoundsException
    wrongPart(AbstractSpecies<?> dsp,
              AbstractSpecies<?> rsp,
              boolean lanewise,
              int part) {
        // domain / input:          X        -> ETYPE
        // logical result:          f(X)     -> FTYPE
        // range / physical output: Y        -> FTYPE
        int domSizeLog2 = dsp.vectorShape.vectorBitSizeLog2;
        int phySizeLog2 = rsp.vectorShape.vectorBitSizeLog2;

        // ------------------- Logical Expansion ----------------
        int laneChangeLog2 = 0;
        String logicalOp = null;
        if (lanewise) {
            // Logical lanewise conversion
            //   ML = |FTYPE| / |ETYPE| = phyElementSize / domElementSize
            laneChangeLog2 = (rsp.laneType.elementSizeLog2 -
                              dsp.laneType.elementSizeLog2);
            if (laneChangeLog2 > 0) {
                String ML = String.format("%d", 1 << laneChangeLog2);
                logicalOp = String.format("conversion lanewise expanding by ML=%s", ML);
            } else if (laneChangeLog2 < 0) {
                String ML = String.format("1/%d", 1 << -laneChangeLog2);
                logicalOp = String.format("conversion lanewise contracting by ML=%s", ML);
            } else {
                logicalOp = "conversion lanewise in-place (ML=1)";
            }
        } else {
            // Logical reinterpret
            laneChangeLog2 = 0;
            logicalOp = "reinterpreting (ML=1)";
        }

        // ------------------- Physical Expansion ----------------
        String physicalOp = null;
        if (domSizeLog2 < phySizeLog2) {
            String MP = String.format("%d", 1 << (phySizeLog2 - domSizeLog2));
            physicalOp = String.format("expansion by MP=%s", MP);
        } else if (phySizeLog2 < domSizeLog2) {
            String MP = String.format("1/%d", 1 << (domSizeLog2 - phySizeLog2));
            physicalOp = String.format("contraction by MP=%s", MP);
        } else {
            physicalOp = "shape-invariant (MP=1)";
        }

        // ---------- Output Expansion (Select/Insert) ------------
        int limit = partLimit(domSizeLog2 + laneChangeLog2, phySizeLog2);
        String outputOp = null;
        String partRange = null;
        if (limit < -1) {
            outputOp = String.format("output insertion with MO=%d", -limit);
            partRange = String.format("in [%d..0]", limit+1);
        } else if (limit > 1) {
            outputOp = String.format("output selection with MS=%d", limit);
            partRange = String.format("in [0..%d]", limit-1);
        } else {
            outputOp = "output in-place (MO=MS=1)";
            partRange = "0";
        }
        String msg = String.format("bad part number %d should be %s, %s; logical: %s; physical: %s; %s -> %s.",
                                   part, partRange, outputOp, logicalOp, physicalOp, dsp, rsp);
        return new ArrayIndexOutOfBoundsException(msg);
    }

    /*package-private*/
    ArithmeticException divZeroException() {
        throw new ArithmeticException("zero vector lane in dividend "+this);
    }

    /**
     * Helper function for all sorts of byte-wise reinterpretation casts.
     * This function kicks in after intrinsic failure.
     */
    /*package-private*/
    @ForceInline
    final <F>
    AbstractVector<F> defaultReinterpret(AbstractSpecies<F> rsp) {
        int blen = Math.max(this.bitSize(), rsp.vectorBitSize()) / Byte.SIZE;
        ByteOrder bo = ByteOrder.nativeOrder();
        MemorySegment ms = MemorySegment.ofArray(new byte[blen]);
        this.intoMemorySegment(ms, 0, bo);
        VectorMask<F> m = rsp.maskAll(true);
        // enum-switches don't optimize properly JDK-8161245
        switch (rsp.laneType.switchKey) {
        case LaneType.SK_BYTE:
            return ByteVector.fromMemorySegment(rsp.check(byte.class), ms, 0, bo, m.check(byte.class)).check0(rsp);
        case LaneType.SK_SHORT:
            return ShortVector.fromMemorySegment(rsp.check(short.class), ms, 0, bo, m.check(short.class)).check0(rsp);
        case LaneType.SK_INT:
            return IntVector.fromMemorySegment(rsp.check(int.class), ms, 0, bo, m.check(int.class)).check0(rsp);
        case LaneType.SK_LONG:
            return LongVector.fromMemorySegment(rsp.check(long.class), ms, 0, bo, m.check(long.class)).check0(rsp);
        case LaneType.SK_FLOAT:
            return FloatVector.fromMemorySegment(rsp.check(float.class), ms, 0, bo, m.check(float.class)).check0(rsp);
        case LaneType.SK_DOUBLE:
            return DoubleVector.fromMemorySegment(rsp.check(double.class), ms, 0, bo, m.check(double.class)).check0(rsp);
        default:
            throw new AssertionError(rsp.toString());
        }
    }

    /**
     * Helper function for all sorts of lane-wise conversions.
     * This function kicks in after intrinsic failure.
     */
    /*package-private*/
    @ForceInline
    final <F>
    AbstractVector<F> defaultCast(AbstractSpecies<F> dsp) {
        int rlength = dsp.laneCount;
        if (vspecies().laneType.elementKind == 'F') {
            // Buffer input values in a double array.
            double[] lanes = toDoubleArray();
            int limit = Math.min(lanes.length, rlength);
            // enum-switches don't optimize properly JDK-8161245
            switch (dsp.laneType.switchKey) {
            case LaneType.SK_BYTE: {
                byte[] a = new byte[rlength];
                for (int i = 0; i < limit; i++) {
                    a[i] = (byte) lanes[i];
                }
                return ByteVector.fromArray(dsp.check(byte.class), a, 0).check0(dsp);
            }
            case LaneType.SK_SHORT: {
                short[] a = new short[rlength];
                for (int i = 0; i < limit; i++) {
                    a[i] = (short) lanes[i];
                }
                return ShortVector.fromArray(dsp.check(short.class), a, 0).check0(dsp);
            }
            case LaneType.SK_INT: {
                int[] a = new int[rlength];
                for (int i = 0; i < limit; i++) {
                    a[i] = (int) lanes[i];
                }
                return IntVector.fromArray(dsp.check(int.class), a, 0).check0(dsp);
            }
            case LaneType.SK_LONG: {
                long[] a = new long[rlength];
                for (int i = 0; i < limit; i++) {
                    a[i] = (long) lanes[i];
                }
                return LongVector.fromArray(dsp.check(long.class), a, 0).check0(dsp);
            }
            case LaneType.SK_FLOAT: {
                float[] a = new float[rlength];
                for (int i = 0; i < limit; i++) {
                    a[i] = (float) lanes[i];
                }
                return FloatVector.fromArray(dsp.check(float.class), a, 0).check0(dsp);
            }
            case LaneType.SK_DOUBLE: {
                double[] a = new double[rlength];
                for (int i = 0; i < limit; i++) {
                    a[i] = (double) lanes[i];
                }
                return DoubleVector.fromArray(dsp.check(double.class), a, 0).check0(dsp);
            }
            default: break;
            }
        } else {
            // Buffer input values in a long array.
            long[] lanes = toLongArray();
            int limit = Math.min(lanes.length, rlength);
            // enum-switches don't optimize properly JDK-8161245
            switch (dsp.laneType.switchKey) {
            case LaneType.SK_BYTE: {
                byte[] a = new byte[rlength];
                for (int i = 0; i < limit; i++) {
                    a[i] = (byte) lanes[i];
                }
                return ByteVector.fromArray(dsp.check(byte.class), a, 0).check0(dsp);
            }
            case LaneType.SK_SHORT: {
                short[] a = new short[rlength];
                for (int i = 0; i < limit; i++) {
                    a[i] = (short) lanes[i];
                }
                return ShortVector.fromArray(dsp.check(short.class), a, 0).check0(dsp);
            }
            case LaneType.SK_INT: {
                int[] a = new int[rlength];
                for (int i = 0; i < limit; i++) {
                    a[i] = (int) lanes[i];
                }
                return IntVector.fromArray(dsp.check(int.class), a, 0).check0(dsp);
            }
            case LaneType.SK_LONG: {
                long[] a = new long[rlength];
                for (int i = 0; i < limit; i++) {
                    a[i] = (long) lanes[i];
                }
                return LongVector.fromArray(dsp.check(long.class), a, 0).check0(dsp);
            }
            case LaneType.SK_FLOAT: {
                float[] a = new float[rlength];
                for (int i = 0; i < limit; i++) {
                    a[i] = (float) lanes[i];
                }
                return FloatVector.fromArray(dsp.check(float.class), a, 0).check0(dsp);
            }
            case LaneType.SK_DOUBLE: {
                double[] a = new double[rlength];
                for (int i = 0; i < limit; i++) {
                    a[i] = (double) lanes[i];
                }
                return DoubleVector.fromArray(dsp.check(double.class), a, 0).check0(dsp);
            }
            default: break;
            }
        }
        throw new AssertionError();
    }

    /**
     * Helper function for all sorts of lane-wise unsigned conversions.
     * This function kicks in after intrinsic failure.
     */
    /*package-private*/
    @ForceInline
    final <F>
    AbstractVector<F> defaultUCast(AbstractSpecies<F> dsp) {
        AbstractSpecies<?> vsp = this.vspecies();
        if (vsp.elementSize() >= dsp.elementSize()) {
            // clip in place
            return this.convert0('C', dsp);
        } else {
            // extend in place, but remove unwanted sign extension
            long mask = -1L >>> -vsp.elementSize();
            return (AbstractVector<F>) this.convert0('C', dsp).lanewise(AND, dsp.broadcast(mask));
        }
    }

    // Constant-folded access to conversion intrinsics:

    /**
     * Dispatch on conversion kind and target species.
     * The code of this is arranged to fold up if the
     * vector class is constant and the target species
     * is also constant.  This is often the case.
     * Residual non-folded code may also perform acceptably
     * in some cases due to type profiling, especially
     * of rvtype.  If only one shape is being used,
     * the profiling of rvtype should help speculatively
     * fold the code even when the target species is
     * not a constant.
     */
    /*package-private*/
    @ForceInline
    final <F>
    AbstractVector<F> convert0(char kind, AbstractSpecies<F> rsp) {
        // Derive some JIT-time constants:
        Class<?> vtype;
        int etype;        // fill in after switch (constant)
        int vlength;      // fill in after switch (mark type profile?)
        Class<?> rvtype;  // fill in after switch (mark type profile)
        int rtype;
        int rlength;
        switch (kind) {
        case 'Z':  // lane-wise size change, maybe with sign clip
            AbstractSpecies<?> rspi = rsp.asIntegral();
            AbstractSpecies<?> vsp = this.vspecies();
            AbstractSpecies<?> vspi = vsp.asIntegral();
            AbstractVector<?> biti = vspi == vsp ? this : this.convert0('X', vspi);
            rtype = rspi.laneTypeOrdinal();
            rlength = rspi.laneCount();
            etype = vspi.laneTypeOrdinal();
            vlength = vspi.laneCount();
            rvtype = rspi.dummyVector().getClass();
            vtype = vspi.dummyVector().getClass();
            int opc = vspi.elementSize() < rspi.elementSize() ? VectorSupport.VECTOR_OP_UCAST : VectorSupport.VECTOR_OP_CAST;
            AbstractVector<?> bitv = VectorSupport.convert(opc,
                    vtype, etype, vlength,
                    rvtype, rtype, rlength,
                    biti, rspi,
                    AbstractVector::defaultUCast);
            return (rspi == rsp ? bitv.check0(rsp) : bitv.convert0('X', rsp));
        case 'C':  // lane-wise cast (but not identity)
            rtype = rsp.laneTypeOrdinal();
            rlength = rsp.laneCount();
            etype = this.vspecies().laneTypeOrdinal(); // (profile)
            vlength = this.length();  // (profile)
            rvtype = rsp.dummyVector().getClass();  // (profile)
            vtype = this.getClass();
            return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
                    vtype, etype, vlength,
                    rvtype, rtype, rlength,
                    this, rsp,
                    AbstractVector::defaultCast);
        case 'X':  // reinterpret cast, not lane-wise if lane sizes differ
            rtype = rsp.laneTypeOrdinal();
            rlength = rsp.laneCount();
            etype = this.vspecies().laneTypeOrdinal(); // (profile)
            vlength = this.length();  // (profile)
            rvtype = rsp.dummyVector().getClass();  // (profile)
            vtype = this.getClass();
            return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET,
                    vtype, etype, vlength,
                    rvtype, rtype, rlength,
                    this, rsp,
                    AbstractVector::defaultReinterpret);
        }
        throw new AssertionError();
    }

    static {
        // Recode uses of VectorSupport.reinterpret if this assertion fails:
        assert(REGISTER_ENDIAN == ByteOrder.LITTLE_ENDIAN);
    }
}
