    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.io.Serializable;
    016import java.lang.reflect.Array;
    017import java.text.Format;
    018import java.util.ArrayList;
    019import java.util.Arrays;
    020import java.util.List;
    021import java.util.concurrent.ConcurrentHashMap;
    022import java.util.concurrent.ConcurrentMap;
    023
    024import org.eclipse.january.DatasetException;
    025import org.eclipse.january.IMonitor;
    026import org.eclipse.january.MetadataException;
    027import org.eclipse.january.metadata.ErrorMetadata;
    028import org.eclipse.january.metadata.MetadataFactory;
    029import org.eclipse.january.metadata.MetadataType;
    030import org.eclipse.january.metadata.StatisticsMetadata;
    031import org.eclipse.january.metadata.internal.ErrorMetadataImpl;
    032import org.eclipse.january.metadata.internal.StatisticsMetadataImpl;
    033
    034/**
    035 * Generic container class for data
    036 * <p/>
    037 * Each subclass has an array of primitive types, elements of this array are grouped or
    038 * compounded to make items
    039 * <p/>
    040 * Data items can be boolean, integer, float, complex float, vector float, etc
    041 */
    042public abstract class AbstractDataset extends LazyDatasetBase implements Dataset {
    043        // pin UID to base class
    044        private static final long serialVersionUID = Dataset.serialVersionUID;
    045
    046        protected int size; // number of items
    047
    048        transient protected AbstractDataset base; // is null when not a view
    049        protected int[] stride; // can be null for row-major, contiguous datasets
    050        protected int offset;
    051
    052        /**
    053         * The data itself, held in a 1D array, but the object will wrap it to appear as possessing as many dimensions as
    054         * wanted
    055         */
    056        protected Serializable odata = null;
    057
    058        /**
    059         * Set aliased data as base data
    060         */
    061        abstract protected void setData();
    062
    063        /**
    064         * Constructor required for serialisation.
    065         */
    066        public AbstractDataset() {
    067        }
    068
    069        @Override
    070        public synchronized Dataset synchronizedCopy() {
    071                return clone();
    072        }
    073
    074        @Override
    075        public boolean equals(Object obj) {
    076                if (this == obj) {
    077                        return true;
    078                }
    079                if (obj == null) {
    080                        return false;
    081                }
    082                if (!getClass().equals(obj.getClass())) {
    083                        if (getRank() == 0) // for zero-rank datasets
    084                                return obj.equals(getObjectAbs(0));
    085                        return false;
    086                }
    087
    088                Dataset other = (Dataset) obj;
    089                if (getElementsPerItem() != other.getElementsPerItem())
    090                        return false;
    091                if (size != other.getSize())
    092                        return false;
    093                if (!Arrays.equals(shape, other.getShapeRef())) {
    094                        return false;
    095                }
    096
    097                return true;
    098        }
    099
    100        @Override
    101        public int hashCode() {
    102                return getStats().getHash(shape);
    103        }
    104
    105        @Override
    106        abstract public AbstractDataset clone();
    107
    108        protected Format stringFormat = null;
    109
    110        @Override
    111        public void setStringFormat(Format format) {
    112                stringFormat = format;
    113        }
    114
    115        @Override
    116        public Dataset copy(final int dtype) {
    117                if (getDType() == dtype) {
    118                        return clone();
    119                }
    120                return DatasetUtils.copy(this, dtype);
    121        }
    122
    123        @Override
    124        public <T extends Dataset> T copy(Class<T> clazz) {
    125                return DatasetUtils.copy(clazz, this);
    126        }
    127
    128        @Override
    129        public Dataset cast(final int dtype) {
    130                if (getDType() == dtype) {
    131                        return this;
    132                }
    133                return DatasetUtils.cast(this, dtype);
    134        }
    135
    136        @Override
    137        public <T extends Dataset> T cast(Class<T> clazz) {
    138                return DatasetUtils.cast(clazz, this);
    139        }
    140
    141        @Override
    142        public Dataset cast(final boolean repeat, final int dtype, final int isize) {
    143                if (getDType() == dtype && getElementsPerItem() == isize) {
    144                        return this;
    145                }
    146                return DatasetUtils.cast(this, repeat, dtype, isize);
    147        }
    148
    149        @Override
    150        abstract public AbstractDataset getView(boolean deepCopyMetadata);
    151
    152        /**
    153         * Copy fields from original to view
    154         * @param orig
    155         * @param view
    156         * @param clone if true, then clone everything but bulk data
    157         * @param cloneMetadata if true, clone metadata
    158         */
    159        protected static void copyToView(Dataset orig, AbstractDataset view, boolean clone, boolean cloneMetadata) {
    160                view.name = orig.getName();
    161                view.size = orig.getSize();
    162                view.odata = orig.getBuffer();
    163                view.offset = orig.getOffset();
    164                view.base = orig instanceof AbstractDataset ? ((AbstractDataset) orig).base : null;
    165
    166                if (clone) {
    167                        view.shape = orig.getShape();
    168                        view.stride = orig instanceof AbstractDataset && ((AbstractDataset) orig).stride != null ?
    169                                        ((AbstractDataset) orig).stride.clone() : null;
    170                } else {
    171                        view.shape = orig.getShapeRef();
    172                        view.stride = orig instanceof AbstractDataset ? ((AbstractDataset) orig).stride : null;
    173                }
    174
    175                view.metadata = getMetadataMap(orig, cloneMetadata);
    176                int odtype = orig.getDType();
    177                int vdtype = view.getDType();
    178                if (odtype != vdtype) {
    179                        view.setDirty();
    180                }
    181        }
    182
    183        /**
    184         * @since 2.0
    185         */
    186        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> getMetadataMap(Dataset a, boolean clone) {
    187                if (a == null)
    188                        return null;
    189
    190                List<MetadataType> all = null;
    191                try {
    192                        all = a.getMetadata(null);
    193                } catch (Exception e) {
    194                }
    195                if (all == null)
    196                        return null;
    197
    198                ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
    199
    200                for (MetadataType m : all) {
    201                        if (m == null) {
    202                                continue;
    203                        }
    204                        Class<? extends MetadataType> c = findMetadataTypeSubInterfaces(m.getClass());
    205                        List<MetadataType> l = map.get(c);
    206                        if (l == null) {
    207                                l = new ArrayList<MetadataType>();
    208                                map.put(c, l);
    209                        }
    210                        if (clone)
    211                                m = m.clone();
    212                        l.add(m);
    213                }
    214                return map;
    215        }
    216
    217        @Override
    218        public IntegerDataset getIndices() {
    219                final IntegerDataset ret = DatasetUtils.indices(shape);
    220                if (getName() != null) {
    221                        ret.setName("Indices of " + getName());
    222                }
    223                return ret;
    224        }
    225
    226        @Override
    227        public Dataset getTransposedView(int... axes) {
    228                axes = checkPermutatedAxes(shape, axes);
    229
    230                AbstractDataset t = getView(true);
    231                if (axes == null || getRank() == 1)
    232                        return t;
    233
    234                int rank = shape.length;
    235                int[] tstride = new int[rank];
    236                int[] toffset = new int[1];
    237                int[] nshape = createStrides(new SliceND(shape), this, tstride, toffset);
    238                int[] nstride = new int[rank];
    239                for (int i = 0; i < rank; i++) {
    240                        final int ax = axes[i];
    241                        nstride[i] = tstride[ax];
    242                        nshape[i] = shape[ax];
    243                }
    244                t.shape = nshape;
    245                t.stride = nstride;
    246                t.offset = toffset[0];
    247                t.base = this;
    248                t.setDirty();
    249                t.transposeMetadata(axes);
    250                return t;
    251        }
    252
    253        @Override
    254        public Dataset transpose(int... axes) {
    255                Dataset t = getTransposedView(axes);
    256                return t == null ? clone() : t.clone();
    257        }
    258
    259        @Override
    260        public Dataset swapAxes(int axis1, int axis2) {
    261                int rank = shape.length;
    262                if (axis1 < 0)
    263                        axis1 += rank;
    264                if (axis2 < 0)
    265                        axis2 += rank;
    266
    267                if (axis1 < 0 || axis2 < 0 || axis1 >= rank || axis2 >= rank) {
    268                        logger.error("Axis value invalid - out of range");
    269                        throw new IllegalArgumentException("Axis value invalid - out of range");
    270                }
    271
    272                if (rank == 1 || axis1 == axis2) {
    273                        return this;
    274                }
    275
    276                int[] axes = new int[rank];
    277                for (int i = 0; i < rank; i++) {
    278                        axes[i] = i;
    279                }
    280
    281                axes[axis1] = axis2;
    282                axes[axis2] = axis1;
    283                return getTransposedView(axes);
    284        }
    285
    286        private boolean isContiguous() {
    287                if (stride == null)
    288                        return true;
    289
    290                if (offset != 0)
    291                        return false;
    292
    293                int s = getElementsPerItem();
    294                for (int j = getRank() - 1; j >= 0; j--) {
    295                        if (stride[j] != s) {
    296                                return false;
    297                        }
    298                        s *= shape[j];
    299                }
    300
    301                return true;
    302        }
    303
    304        @Override
    305        public Dataset flatten() {
    306                if (!isContiguous()) { // need to make a copy if not contiguous
    307                        return clone().flatten();
    308                }
    309                return reshape(size);
    310        }
    311
    312        /**
    313         * Fill dataset from object at depth dimension
    314         * @param obj
    315         * @param depth
    316         * @param pos position
    317         */
    318        protected void fillData(Object obj, final int depth, final int[] pos) {
    319                if (obj == null) {
    320                        int dtype = getDType();
    321                        if (dtype == FLOAT32)
    322                                set(Float.NaN, pos);
    323                        else if (dtype == FLOAT64)
    324                                set(Double.NaN, pos);
    325                        return;
    326                }
    327
    328                Class<?> clazz = obj.getClass();
    329                if (obj instanceof List<?>) {
    330                        List<?> jl = (List<?>) obj;
    331                        int l = jl.size();
    332                        for (int i = 0; i < l; i++) {
    333                                Object lo = jl.get(i);
    334                                fillData(lo, depth + 1, pos);
    335                                pos[depth]++;
    336                        }
    337                        pos[depth] = 0;
    338                } else if (clazz.isArray()) {
    339                        int l = Array.getLength(obj);
    340                        if (clazz.equals(odata.getClass())) {
    341                                System.arraycopy(obj, 0, odata, get1DIndex(pos), l);
    342                        } else if (clazz.getComponentType().isPrimitive()) {
    343                                for (int i = 0; i < l; i++) {
    344                                        set(Array.get(obj, i), pos);
    345                                        pos[depth]++;
    346                                }
    347                                pos[depth] = 0;
    348                        } else {
    349                                for (int i = 0; i < l; i++) {
    350                                        fillData(Array.get(obj, i), depth + 1, pos);
    351                                        pos[depth]++;
    352                                }
    353                                pos[depth] = 0;
    354                        }
    355                } else if (obj instanceof IDataset) {
    356                        boolean[] a = new boolean[shape.length];
    357                        for (int i = depth; i < a.length; i++)
    358                                a[i] = true;
    359                        setSlice(obj, getSliceIteratorFromAxes(pos, a));
    360                } else {
    361                        set(obj, pos);
    362                }
    363        }
    364
    365        @Override
    366        public IndexIterator getIterator(final boolean withPosition) {
    367                if (stride != null) {
    368                        return base.getSize() == 1  ? (withPosition ? new PositionIterator(offset, shape) :
    369                                new SingleItemIterator(offset, size)) : new StrideIterator(shape, stride, offset);
    370                }
    371                return withPosition ? new ContiguousIteratorWithPosition(shape, size) : new ContiguousIterator(size);
    372        }
    373
    374        @Override
    375        public IndexIterator getIterator() {
    376                return getIterator(false);
    377        }
    378
    379        @Override
    380        public PositionIterator getPositionIterator(final int... axes) {
    381                return new PositionIterator(shape, axes);
    382        }
    383
    384        @Override
    385        public IndexIterator getSliceIterator(final int[] start, final int[] stop, final int[] step) {
    386                return getSliceIterator(new SliceND(shape, start, stop, step));
    387        }
    388
    389        /**
    390         * @param slice
    391         * @return an slice iterator that operates like an IndexIterator
    392         */
    393        public IndexIterator getSliceIterator(SliceND slice) {
    394                if (ShapeUtils.calcLongSize(slice.getShape()) == 0) {
    395                        return new NullIterator(shape, slice.getShape());
    396                }
    397                if (stride != null)
    398                        return new StrideIterator(getElementsPerItem(), shape, stride, offset, slice);
    399
    400                return new SliceIterator(shape, size, slice);
    401        }
    402
    403        @Override
    404        public SliceIterator getSliceIteratorFromAxes(final int[] pos, boolean[] axes) {
    405                int rank = shape.length;
    406                int[] start;
    407                int[] stop = new int[rank];
    408                int[] step = new int[rank];
    409
    410                if (pos == null) {
    411                        start = new int[rank];
    412                } else if (pos.length == rank) {
    413                        start = pos.clone();
    414                } else {
    415                        throw new IllegalArgumentException("pos array length is not equal to rank of dataset");
    416                }
    417                if (axes == null) {
    418                        axes = new boolean[rank];
    419                        Arrays.fill(axes, true);
    420                } else if (axes.length != rank) {
    421                        throw new IllegalArgumentException("axes array length is not equal to rank of dataset");
    422                }
    423
    424                for (int i = 0; i < rank; i++) {
    425                        if (axes[i]) {
    426                                stop[i] = shape[i];
    427                        } else {
    428                                stop[i] = start[i] + 1;
    429                        }
    430                        step[i] = 1;
    431                }
    432                return (SliceIterator) getSliceIterator(start, stop, step);
    433        }
    434
    435        @Override
    436        public BooleanIterator getBooleanIterator(Dataset choice) {
    437                return getBooleanIterator(choice, true);
    438        }
    439
    440        @Override
    441        public BooleanIterator getBooleanIterator(Dataset choice, boolean value) {
    442                return new BooleanIterator(getIterator(), choice, value);
    443        }
    444
    445        @Override
    446        public Dataset getByBoolean(Dataset selection) {
    447                checkCompatibility(selection);
    448
    449                final int length = ((Number) selection.sum()).intValue();
    450                final int is = getElementsPerItem();
    451                @SuppressWarnings("deprecation")
    452                Dataset r = DatasetFactory.zeros(is, new int[] { length }, getDType());
    453                BooleanIterator biter = getBooleanIterator(selection);
    454
    455                int i = 0;
    456                while (biter.hasNext()) {
    457                        r.setObjectAbs(i, getObjectAbs(biter.index));
    458                        i += is;
    459                }
    460                return r;
    461        }
    462
    463        @Override
    464        public Dataset getBy1DIndex(IntegerDataset index) {
    465                final int is = getElementsPerItem();
    466                @SuppressWarnings("deprecation")
    467                final Dataset r = DatasetFactory.zeros(is, index.getShape(), getDType());
    468                final IntegerIterator iter = new IntegerIterator(index, size, is);
    469
    470                int i = 0;
    471                while (iter.hasNext()) {
    472                        r.setObjectAbs(i, getObjectAbs(iter.index));
    473                        i += is;
    474                }
    475                return r;
    476        }
    477
    478        @Override
    479        public Dataset getByIndexes(final Object... indexes) {
    480                final IntegersIterator iter = new IntegersIterator(shape, indexes);
    481                final int is = getElementsPerItem();
    482                @SuppressWarnings("deprecation")
    483                final Dataset r = DatasetFactory.zeros(is, iter.getShape(), getDType());
    484
    485                final int[] pos = iter.getPos();
    486                int i = 0;
    487                while (iter.hasNext()) {
    488                        r.setObjectAbs(i, getObject(pos));
    489                        i += is;
    490                }
    491                return r;
    492        }
    493
    494        @Override
    495        public Class<?> getElementClass() {
    496                return DTypeUtils.getElementClass(getDType());
    497        }
    498
    499        @Override
    500        public boolean hasFloatingPointElements() {
    501                Class<?> cls = getElementClass();
    502                return cls == Float.class || cls == Double.class;
    503        }
    504
    505        @Override
    506        public int getElementsPerItem() {
    507                return DTypeUtils.getElementsPerItem(getDType());
    508        }
    509
    510        @Override
    511        public int getItemBytes() {
    512                return DTypeUtils.getItemBytes(getDType(), getElementsPerItem());
    513        }
    514
    515        @Override
    516        public String getName() {
    517                return name;
    518        }
    519
    520        @Override
    521        public void setName(final String name) {
    522                this.name = name;
    523        }
    524
    525        @Override
    526        public int getSize() {
    527                return size;
    528        }
    529
    530        @Override
    531        public int[] getShape() {
    532                // make a copy of the dimensions data, and put that out
    533                if (shape == null) {
    534                        logger.warn("Shape is null!!!");
    535                        return new int[] {};
    536                }
    537                return shape.clone();
    538        }
    539
    540        @Override
    541        public int getRank() {
    542                return shape == null ? 0 : shape.length;
    543        }
    544
    545        @Override
    546        public int getNbytes() {
    547                return getSize() * getItemBytes();
    548        }
    549
    550        /**
    551         * Check for -1 placeholder in shape and replace if necessary
    552         * @param shape
    553         * @param size
    554         */
    555        private void checkShape(int[] shape, int size) {
    556                int rank = shape.length;
    557                int found = -1;
    558                int nsize = 1;
    559                for (int i = 0; i < rank; i++) {
    560                        int d = shape[i];
    561                        if (d == -1) {
    562                                if (found == -1) {
    563                                        found = i;
    564                                } else {
    565                                        logger.error("Can only have one -1 placeholder in shape");
    566                                        throw new IllegalArgumentException("Can only have one -1 placeholder in shape");
    567                                }
    568                        } else {
    569                                nsize *= d;
    570                        }
    571                }
    572                if (found >= 0) {
    573                        shape[found] = size/nsize;
    574                } else if (nsize != size && !(rank == 0 && size == 0)) {
    575                        logger.error("New shape is not same size as old shape");
    576                        throw new IllegalArgumentException("New size is not same as the old size. Old size is "+size+" new size is "+nsize+" and shape is "+Arrays.toString(shape));
    577                }
    578        }
    579
    580        @Override
    581        public void setShape(final int... shape) {
    582                int[] nshape = shape.clone();
    583                checkShape(nshape, size);
    584                if (Arrays.equals(this.shape, nshape)) {
    585                        return;
    586                }
    587
    588                if (stride != null) {
    589                        // the only compatible shapes are ones where new dimensions are factors of old dimensions
    590                        // or are combined adjacent old dimensions
    591                        int[] oshape = this.shape;
    592                        int orank = oshape.length;
    593                        int nrank = nshape.length;
    594                        int diff = nrank - orank;
    595                        int[] nstride = new int[nrank];
    596                        boolean ones = true;
    597                        // work forwards for broadcasting cases
    598                        for (int i = 0, j = 0; i < orank || j < nrank;) {
    599                                if (j >= diff && i < orank && j < nrank && oshape[i] == nshape[j]) {
    600                                        nstride[j++] = stride[i++];
    601                                } else if (j < nrank && nshape[j] == 1) {
    602                                        nstride[j++] = 0;
    603                                } else if (i < orank && oshape[i] == 1) {
    604                                        i++;
    605                                } else {
    606                                        if (j < nrank)
    607                                                j++;
    608                                        if (i < orank)
    609                                                i++;
    610                                        ones = false;
    611                                }
    612                        }
    613                        if (!ones) { // not just ones differ in shapes
    614                                int[] ostride = stride;
    615                                int ob = 0;
    616                                int oe = 1;
    617                                int nb = 0;
    618                                int ne = 1;
    619                                while (ob < orank && nb < nrank) {
    620                                        int ol = oshape[ob];
    621                                        int nl = nshape[nb];
    622
    623                                        if (nl < ol) { // find group of shape dimensions that form common size
    624                                                do { // case where new shape spreads single dimension over several dimensions
    625                                                        if (ne == nrank) {
    626                                                                break;
    627                                                        }
    628                                                        nl *= nshape[ne++];
    629                                                } while (nl < ol);
    630                                                if (nl != ol) {
    631                                                        logger.error("Subshape is incompatible with single dimension");
    632                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
    633                                                }
    634                                                int on = ne - 1;
    635                                                while (nshape[on] == 1) {
    636                                                        on--;
    637                                                }
    638
    639                                                nstride[on] = ostride[ob];
    640                                                for (int n = on - 1; n >= nb; n--) {
    641                                                        if (nshape[n] == 1)
    642                                                                continue;
    643
    644                                                        nstride[n] = nshape[on] * nstride[on];
    645                                                        on = n;
    646                                                }
    647                                        } else if (ol < nl) {
    648                                                do { // case where new shape combines several dimensions into one dimension
    649                                                        if (oe == orank) {
    650                                                                break;
    651                                                        }
    652                                                        ol *= oshape[oe++];
    653                                                } while (ol < nl);
    654                                                if (nl != ol) {
    655                                                        logger.error("Single dimension is incompatible with subshape");
    656                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
    657                                                }
    658
    659                                                int oo = oe - 1;
    660                                                while (oshape[oo] == 1) {
    661                                                        oo--;
    662                                                }
    663                                                int os = ostride[oo];
    664                                                for (int o = oo - 1; o >= ob; o--) {
    665                                                        if (oshape[o] == 1)
    666                                                                continue;
    667                                                        if (ostride[o] != oshape[oo] * ostride[oo]) {
    668                                                                logger.error("Subshape cannot be a non-contiguous view");
    669                                                                throw new IllegalArgumentException("Subshape cannot be a non-contiguous view");
    670                                                        }
    671                                                        oo = o;
    672                                                }
    673                                                nstride[nb] = os;
    674                                        } else {
    675                                                nstride[nb] = ostride[ob];
    676                                        }
    677
    678                                        ob = oe++;
    679                                        nb = ne++;
    680                                }
    681                        }
    682
    683                        stride = nstride;
    684                }
    685
    686                setDirty();
    687                if (this.shape != null) {
    688                        reshapeMetadata(this.shape, nshape);
    689                        this.shape = nshape;
    690                }
    691        }
    692
    693        @Override
    694        public int[] getShapeRef() {
    695                return shape;
    696        }
    697
    698        @Override
    699        public int getOffset() {
    700                return offset;
    701        }
    702
    703        @Override
    704        public int[] getStrides() {
    705                return stride;
    706        }
    707
    708        @Override
    709        public Serializable getBuffer() {
    710                return odata;
    711        }
    712
    713        @Override
    714        public void overrideInternal(Serializable buffer, int... shape) {
    715                if (buffer != null) {
    716                        odata = buffer;
    717                        setData();
    718                        setDirty();
    719                }
    720
    721                if (shape != null) {
    722                        this.shape = shape.clone();
    723                        size = ShapeUtils.calcSize(this.shape);
    724                }
    725        }
    726
    727        /**
    728         * Create a stride array from dataset
    729         * @param a dataset
    730         * @param offset output offset
    731         * @return new strides
    732         */
    733        public static int[] createStrides(Dataset a, final int[] offset) {
    734                return createStrides(a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), offset);
    735        }
    736
    737        /**
    738         * Create a stride array from dataset
    739         * @param isize
    740         * @param shape
    741         * @param oStride original stride
    742         * @param oOffset original offset (only used if there is an original stride)
    743         * @param offset output offset
    744         * @return new strides
    745         */
    746        public static int[] createStrides(final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] offset) {
    747                int rank = shape.length;
    748                final int[] stride;
    749                if (oStride == null) {
    750                        offset[0] = 0;
    751                        stride = new int[rank];
    752                        int s = isize;
    753                        for (int j = rank - 1; j >= 0; j--) {
    754                                stride[j] = s;
    755                                s *= shape[j];
    756                        }
    757                } else {
    758                        offset[0] = oOffset;
    759                        stride = oStride.clone();
    760                }
    761                return stride;
    762        }
    763
    764        /**
    765         * Create a stride array from slice information and a dataset
    766         * @param slice
    767         * @param a dataset
    768         * @param stride output stride
    769         * @param offset output offset
    770         * @return new shape
    771         */
    772        public static int[] createStrides(final SliceND slice, final Dataset a, final int[] stride, final int[] offset) {
    773                return createStrides(slice, a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), stride, offset);
    774        }
    775
    776        /**
    777         * Create a stride array from slice and dataset information
    778         * @param slice
    779         * @param isize
    780         * @param shape
    781         * @param oStride original stride
    782         * @param oOffset original offset (only used if there is an original stride)
    783         * @param stride output stride
    784         * @param offset output offset
    785         * @return new shape
    786         */
    787        public static int[] createStrides(final SliceND slice, final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] stride, final int[] offset) {
    788                int[] lstart = slice.getStart();
    789                int[] lstep = slice.getStep();
    790                int[] newShape = slice.getShape();
    791                int rank = shape.length;
    792
    793                if (oStride == null) {
    794                        int s = isize;
    795                        offset[0] = 0;
    796                        for (int j = rank - 1; j >= 0; j--) {
    797                                stride[j] = s * lstep[j];
    798                                offset[0] += s * lstart[j];
    799                                s *= shape[j];
    800                        }
    801                } else {
    802                        offset[0] = oOffset;
    803                        for (int j = 0; j < rank; j++) {
    804                                int s = oStride[j];
    805                                stride[j] = lstep[j] * s;
    806                                offset[0] += lstart[j] * s;
    807                        }
    808                }
    809
    810                return newShape;
    811        }
    812
    813        @Override
    814        public Dataset getBroadcastView(int... broadcastShape) {
    815                AbstractDataset view = getView(true);
    816
    817                if (!Arrays.equals(shape, broadcastShape)) {
    818                        List<int[]> nShapes = BroadcastUtils.broadcastShapesToMax(broadcastShape, shape);
    819                        view.setShape(nShapes.get(0));
    820                        view.stride = BroadcastUtils.createBroadcastStrides(view, broadcastShape);
    821                        view.base = this;
    822                        view.shape = broadcastShape.clone();
    823                        view.size = ShapeUtils.calcSize(broadcastShape);
    824                        if (view.name == null || view.name.isEmpty()) {
    825                                view.name = "Broadcast from " + Arrays.toString(shape);
    826                        } else {
    827                                view.name = "Broadcast of " + view.name + " from " + Arrays.toString(shape);
    828                        }
    829                }
    830                return view;
    831        }
    832
    833        @Override
    834        public Dataset getSliceView(final int[] start, final int[] stop, final int[] step) {
    835                return getSliceView(new SliceND(shape, start, stop, step));
    836        }
    837
    838        @Override
    839        public Dataset getSliceView(Slice... slice) {
    840                if (slice == null || slice.length == 0) {
    841                        return getView(true);
    842                }
    843
    844                return getSliceView(new SliceND(shape, slice));
    845        }
    846
    847        /**
    848         * Get a slice of the dataset. The returned dataset is a view on a selection of items
    849         * @param slice
    850         * @return slice view
    851         */
    852        @Override
    853        public Dataset getSliceView(SliceND slice) {
    854                if (slice.isAll()) {
    855                        return getView(true);
    856                }
    857
    858                final int rank = shape.length;
    859                int[] sStride = new int[rank];
    860                int[] sOffset = new int[1];
    861
    862                int[] sShape = createStrides(slice, this, sStride, sOffset);
    863
    864                AbstractDataset s = getView(false);
    865                s.shape = sShape;
    866                s.size = ShapeUtils.calcSize(sShape);
    867                s.stride = sStride;
    868                s.offset = sOffset[0];
    869                s.base = this;
    870
    871                s.metadata = copyMetadata();
    872                s.sliceMetadata(true, slice);
    873
    874                s.setDirty();
    875                s.setName(name + BLOCK_OPEN + slice + BLOCK_CLOSE);
    876
    877                return s;
    878        }
    879
    880        /**
    881         * Get flattened view index of given position
    882         * @param pos
    883         *            the integer array specifying the n-D position
    884         * @return the index on the flattened dataset
    885         */
    886        private int getFlat1DIndex(final int[] pos) {
    887                final int imax = pos.length;
    888                if (imax == 0) {
    889                        return 0;
    890                }
    891
    892                return get1DIndexFromShape(pos);
    893        }
    894
    895        /**
    896         * @since 2.0
    897         */
    898        protected int getFirst1DIndex() {
    899                if (shape == null) {
    900                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    901                }
    902                return stride == null ? 0 : offset;
    903        }
    904
    905        @Override
    906        public int get1DIndex(final int... n) {
    907                if (n.length == 0 && shape.length == 0)
    908                        return offset;
    909
    910                return stride == null ? get1DIndexFromShape(n) : get1DIndexFromStrides(n);
    911        }
    912
    913        private static void throwAIOOBException(int i, int s, int d) {
    914                throw new ArrayIndexOutOfBoundsException("Index (" + i + ") out of range [-" + s + "," + s
    915                                + "] in dimension " + d);
    916        }
    917
    918        /**
    919         * @param i
    920         * @return the index on the data array corresponding to that location
    921         */
    922        protected int get1DIndex(int i) {
    923                if (shape == null) {
    924                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    925                }
    926                if (shape.length > 1) {
    927                        logger.error("This dataset is not 1D but was addressed as such");
    928                        throw new UnsupportedOperationException("This dataset is not 1D but was addressed as such");
    929                }
    930                if (i < 0) {
    931                        i += shape[0];
    932                }
    933                if (i < 0 || i >= shape[0]) {
    934                        throwAIOOBException(i, shape[0], 0);
    935                }
    936                return stride == null ? i : i*stride[0] + offset;
    937        }
    938
    939        /**
    940         * @param i
    941         * @param j
    942         * @return the index on the data array corresponding to that location
    943         */
    944        protected int get1DIndex(int i, int j) {
    945                if (shape == null) {
    946                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    947                }
    948                if (shape.length != 2) {
    949                        logger.error("This dataset is not 2D but was addressed as such");
    950                        throw new UnsupportedOperationException("This dataset is not 2D but was addressed as such");
    951                }
    952                if (i < 0) {
    953                        i += shape[0];
    954                }
    955                if (i < 0 || i >= shape[0]) {
    956                        throwAIOOBException(i, shape[0], 0);
    957                }
    958                if (j < 0) {
    959                        j += shape[1];
    960                }
    961                if (j < 0 || j >= shape[1]) {
    962                        throwAIOOBException(i, shape[1], 1);
    963                }
    964                return stride == null ? i*shape[1] + j : i*stride[0] + j*stride[1] + offset;
    965        }
    966
    967        protected int get1DIndexFromShape(final int[] n) {
    968                return get1DIndexFromShape(shape, n);
    969        }
    970
    971        protected static int get1DIndexFromShape(final int[] shape, final int[] n) {
    972                if (shape == null) {
    973                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    974                }
    975                final int rank = shape.length;
    976                if (rank != n.length) {
    977                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
    978                        logger.error(errMsg);
    979                        throw new IllegalArgumentException(errMsg);
    980                }
    981                int index = 0;
    982                for (int i = 0; i < rank; i++) {
    983                        final int si = shape[i];
    984                        int ni = n[i];
    985                        if (ni < 0) {
    986                                ni += si;
    987                        }
    988                        if (ni < 0 || ni >= si) {
    989                                throwAIOOBException(ni, si, i);
    990                        }
    991                        index = index * si + ni;
    992                }
    993
    994                return index;
    995        }
    996
    997        private int get1DIndexFromStrides(final int[] n) {
    998                return get1DIndexFromStrides(shape, stride, offset, n);
    999        }
    1000
    1001        private static int get1DIndexFromStrides(final int[] shape, final int[] stride, final int offset, final int[] n) {
    1002                if (shape == null) {
    1003                        throw new IllegalArgumentException("Cannot find an index from a null shape");
    1004                }
    1005                final int rank = shape.length;
    1006                if (rank != n.length) {
    1007                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
    1008                        logger.error(errMsg);
    1009                        throw new IllegalArgumentException(errMsg);
    1010                }
    1011                int index = offset;
    1012                for (int i = 0; i < rank; i++) {
    1013                        final int st = stride[i];
    1014                        if (st != 0) { // not broadcasted
    1015                                final int si = shape[i];
    1016                                int ni = n[i];
    1017                                if (ni < 0) {
    1018                                        ni += si;
    1019                                }
    1020                                if (ni < 0 || ni >= si) {
    1021                                        throwAIOOBException(ni, si, i);
    1022                                }
    1023                                index += st * ni;
    1024                        }
    1025                }
    1026                return index;
    1027        }
    1028
    1029        @Override
    1030        public int[] getNDPosition(final int n) {
    1031                if (isIndexInRange(n)) {
    1032                        throw new IllegalArgumentException("Index provided " + n
    1033                                        + "is larger then the size of the containing array");
    1034                }
    1035
    1036                return stride == null ? ShapeUtils.getNDPositionFromShape(n, shape) : getNDPositionFromStrides(n);
    1037        }
    1038
    1039        private boolean isIndexInRange(final int n) {
    1040                if (stride == null) {
    1041                        return n >= size;
    1042                }
    1043                return n >= getBufferLength();
    1044        }
    1045
    1046        /**
    1047         * @return entire buffer length
    1048         */
    1049        abstract protected int getBufferLength();
    1050
    1051        private int[] getNDPositionFromStrides(int n) {
    1052                n -= offset;
    1053                int rank = shape.length;
    1054                if (rank == 1) {
    1055                        return new int[] { n / stride[0] };
    1056                }
    1057
    1058                int[] output = new int[rank];
    1059                int i = 0;
    1060                while (i != n) { // TODO find more efficient way than this exhaustive search
    1061                        int j = rank - 1;
    1062                        for (; j >= 0; j--) {
    1063                                output[j]++;
    1064                                i += stride[j];
    1065                                if (output[j] >= shape[j]) {
    1066                                        output[j] = 0;
    1067                                        i -= shape[j] * stride[j];
    1068                                } else {
    1069                                        break;
    1070                                }
    1071                        }
    1072                        if (j == -1) {
    1073                                logger.error("Index was not found in this strided dataset");
    1074                                throw new IllegalArgumentException("Index was not found in this strided dataset");
    1075                        }
    1076                }
    1077
    1078                return output;
    1079        }
    1080
    1081        @Override
    1082        public int checkAxis(int axis) {
    1083                return checkAxis(shape.length, axis);
    1084        }
    1085
    1086        /**
    1087         * Check that axis is in range [-rank,rank)
    1088         *
    1089         * @param rank
    1090         * @param axis
    1091         * @return sanitized axis in range [0, rank)
    1092         */
    1093        protected static int checkAxis(int rank, int axis) {
    1094                if (axis < 0) {
    1095                        axis += rank;
    1096                }
    1097
    1098                if (axis < 0 || axis >= rank) {
    1099                        throw new IndexOutOfBoundsException("Axis " + axis + " given is out of range [0, " + rank + ")");
    1100                }
    1101                return axis;
    1102        }
    1103
    1104        protected static final char BLOCK_OPEN = '[';
    1105        protected static final char BLOCK_CLOSE = ']';
    1106
    1107        @Override
    1108        public String toString() {
    1109                return toString(false);
    1110        }
    1111
    1112        @Override
    1113        public String toString(boolean showData) {
    1114                final int rank = shape == null ? 0 : shape.length;
    1115                final StringBuilder out = new StringBuilder();
    1116                if (DTypeUtils.isDTypeElemental(getDType())) {
    1117                        out.append("Dataset ");
    1118                } else {
    1119                        out.append("Compound dataset (");
    1120                        out.append(getElementsPerItem());
    1121                        out.append(") ");
    1122                }
    1123
    1124                if (!showData) {
    1125                        if (name != null && name.length() > 0) {
    1126                                out.append("'");
    1127                                out.append(name);
    1128                                out.append("' has shape ");
    1129                        } else {
    1130                                out.append("shape is ");
    1131                        }
    1132
    1133                        out.append(BLOCK_OPEN);
    1134                        if (rank > 0) {
    1135                                out.append(shape[0]);
    1136                        }
    1137                        for (int i = 1; i < rank; i++) {
    1138                                out.append(", " + shape[i]);
    1139                        }
    1140                        out.append(BLOCK_CLOSE);
    1141                        return out.toString();
    1142                }
    1143
    1144                if (size == 0) {
    1145                        return out.toString();
    1146                }
    1147
    1148                if (rank > 0) {
    1149                        int[] pos = new int[rank];
    1150                        final StringBuilder lead = new StringBuilder();
    1151                        printBlocks(out, lead, 0, pos);
    1152                } else {
    1153                        out.append(getString());
    1154                }
    1155                return out.toString();
    1156        }
    1157
    1158        /**
    1159         * Limit to strings output via the toString() method
    1160         */
    1161        private static int maxStringLength = 120;
    1162
    1163        /**
    1164         * Set maximum line length for toString() method
    1165         * @param maxLineLength
    1166         */
    1167        public static void setMaxLineLength(int maxLineLength) {
    1168                maxStringLength = maxLineLength;
    1169        }
    1170
    1171        /**
    1172         * @return maximum line length for toString() method
    1173         */
    1174        public static int getMaxLineLength() {
    1175                return maxStringLength;
    1176        }
    1177
    1178        /**
    1179         * Limit to number of sub-blocks output via the toString() method
    1180         */
    1181        private static final int MAX_SUBBLOCKS = 6;
    1182
    1183        private final static String SEPARATOR = ",";
    1184        private final static String SPACE = " ";
    1185        private final static String ELLIPSIS = "...";
    1186        private final static String NEWLINE = "\n";
    1187
    1188        /**
    1189         * Make a line of output for last dimension of dataset
    1190         *
    1191         * @param start
    1192         * @return line
    1193         */
    1194        private StringBuilder makeLine(final int end, final int... start) {
    1195                StringBuilder line = new StringBuilder();
    1196                final int[] pos;
    1197                if (end >= start.length) {
    1198                        pos = Arrays.copyOf(start, end + 1);
    1199                } else {
    1200                        pos = start;
    1201                }
    1202                pos[end] = 0;
    1203                line.append(BLOCK_OPEN);
    1204                line.append(getString(pos));
    1205
    1206                final int length = shape[end];
    1207
    1208                // trim elements printed if length exceed estimate of maximum elements
    1209                int excess = length - maxStringLength / 3; // space + number + separator
    1210                if (excess > 0) {
    1211                        int index = (length - excess) / 2;
    1212                        for (int y = 1; y < index; y++) {
    1213                                line.append(SEPARATOR + SPACE);
    1214                                pos[end] = y;
    1215                                line.append(getString(pos));
    1216                        }
    1217                        index = (length + excess) / 2;
    1218                        for (int y = index; y < length; y++) {
    1219                                line.append(SEPARATOR + SPACE);
    1220                                pos[end] = y;
    1221                                line.append(getString(pos));
    1222                        }
    1223                } else {
    1224                        for (int y = 1; y < length; y++) {
    1225                                line.append(SEPARATOR + SPACE);
    1226                                pos[end] = y;
    1227                                line.append(getString(pos));
    1228                        }
    1229                }
    1230                line.append(BLOCK_CLOSE);
    1231
    1232                // trim string down to limit
    1233                excess = line.length() - maxStringLength - ELLIPSIS.length() - 1;
    1234                if (excess > 0) {
    1235                        int index = line.substring(0, (line.length() - excess) / 2).lastIndexOf(SEPARATOR) + 2;
    1236                        StringBuilder out = new StringBuilder(line.subSequence(0, index));
    1237                        out.append(ELLIPSIS + SEPARATOR);
    1238                        index = line.substring((line.length() + excess) / 2).indexOf(SEPARATOR) + (line.length() + excess) / 2 + 1;
    1239                        out.append(line.subSequence(index, line.length()));
    1240                        return out;
    1241                }
    1242
    1243                return line;
    1244        }
    1245
    1246        /**
    1247         * recursive method to print blocks
    1248         */
    1249        private void printBlocks(final StringBuilder out, final StringBuilder lead, final int level, final int[] pos) {
    1250                if (out.length() > 0) {
    1251                        char last = out.charAt(out.length() - 1);
    1252                        if (last != BLOCK_OPEN) {
    1253                                out.append(lead);
    1254                        }
    1255                }
    1256                final int end = getRank() - 1;
    1257                if (level != end) {
    1258                        out.append(BLOCK_OPEN);
    1259                        int length = shape[level];
    1260
    1261                        // first sub-block
    1262                        pos[level] = 0;
    1263                        StringBuilder newlead = new StringBuilder(lead);
    1264                        newlead.append(SPACE);
    1265                        printBlocks(out, newlead, level + 1, pos);
    1266                        if (length < 2) { // escape
    1267                                out.append(BLOCK_CLOSE);
    1268                                return;
    1269                        }
    1270
    1271                        out.append(SEPARATOR + NEWLINE);
    1272                        for (int i = level + 1; i < end; i++) {
    1273                                out.append(NEWLINE);
    1274                        }
    1275
    1276                        // middle sub-blocks
    1277                        if (length < MAX_SUBBLOCKS) {
    1278                                for (int x = 1; x < length - 1; x++) {
    1279                                        pos[level] = x;
    1280                                        printBlocks(out, newlead, level + 1, pos);
    1281                                        if (end <= level + 1) {
    1282                                                out.append(SEPARATOR + NEWLINE);
    1283                                        } else {
    1284                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
    1285                                        }
    1286                                }
    1287                        } else {
    1288                                final int excess = length - MAX_SUBBLOCKS;
    1289                                int xmax = (length - excess) / 2;
    1290                                for (int x = 1; x < xmax; x++) {
    1291                                        pos[level] = x;
    1292                                        printBlocks(out, newlead, level + 1, pos);
    1293                                        if (end <= level + 1) {
    1294                                                out.append(SEPARATOR + NEWLINE);
    1295                                        } else {
    1296                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
    1297                                        }
    1298                                }
    1299                                out.append(newlead);
    1300                                out.append(ELLIPSIS + SEPARATOR + NEWLINE);
    1301                                xmax = (length + excess) / 2;
    1302                                for (int x = xmax; x < length - 1; x++) {
    1303                                        pos[level] = x;
    1304                                        printBlocks(out, newlead, level + 1, pos);
    1305                                        if (end <= level + 1) {
    1306                                                out.append(SEPARATOR + NEWLINE);
    1307                                        } else {
    1308                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
    1309                                        }
    1310                                }
    1311                        }
    1312
    1313                        // last sub-block
    1314                        pos[level] = length - 1;
    1315                        printBlocks(out, newlead, level + 1, pos);
    1316                        out.append(BLOCK_CLOSE);
    1317                } else {
    1318                        out.append(makeLine(end, pos));
    1319                }
    1320        }
    1321
    1322        @Override
    1323        public Dataset squeezeEnds() {
    1324                return squeeze(true);
    1325        }
    1326
    1327        @Override
    1328        public Dataset squeeze() {
    1329                return squeeze(false);
    1330        }
    1331
    1332        @Override
    1333        public Dataset squeeze(boolean onlyFromEnds) {
    1334                final int[] tshape = ShapeUtils.squeezeShape(shape, onlyFromEnds);
    1335                final int[] oshape = shape;
    1336                if (stride == null) {
    1337                        shape = tshape;
    1338                } else {
    1339                        int rank = shape.length;
    1340                        int trank = tshape.length;
    1341                        if (trank < rank) {
    1342                                int[] tstride = new int[tshape.length];
    1343                                if (onlyFromEnds) {
    1344                                        for (int i = 0; i < rank; i++) {
    1345                                                if (shape[i] != 1) {
    1346                                                        for (int k = 0; k < trank; k++) {
    1347                                                                tstride[k] = stride[i++];
    1348                                                        }
    1349                                                        break;
    1350                                                }
    1351                                        }
    1352                                } else {
    1353                                        int t = 0;
    1354                                        for (int i = 0; i < rank; i++) {
    1355                                                if (shape[i] != 1) {
    1356                                                        tstride[t++] = stride[i];
    1357                                                }
    1358                                        }
    1359                                }
    1360                                shape = tshape;
    1361                                stride = tstride;
    1362                        }
    1363                }
    1364
    1365                setDirty();
    1366                reshapeMetadata(oshape, shape);
    1367                return this;
    1368        }
    1369
    1370        @Override
    1371        public boolean isCompatibleWith(final ILazyDataset g) {
    1372                return ShapeUtils.areShapesCompatible(shape, g.getShape());
    1373        }
    1374
    1375        @Override
    1376        public void checkCompatibility(final ILazyDataset g) throws IllegalArgumentException {
    1377                ShapeUtils.checkCompatibility(this, g);
    1378        }
    1379
    1380        @Override
    1381        public Dataset reshape(final int... shape) {
    1382                Dataset a;
    1383                try {
    1384                        a = getView(true);
    1385                        a.setShape(shape);
    1386                } catch (IllegalArgumentException e) {
    1387                        a = clone();
    1388                        a.setShape(shape);
    1389                }
    1390                return a;
    1391        }
    1392
    1393        /**
    1394         * @param start
    1395         * @param stop
    1396         * @param step
    1397         * @return number of steps to take
    1398         */
    1399        protected static int calcSteps(final double start, final double stop, final double step) {
    1400                return Math.max(0, (int) Math.ceil((stop - start) / step));
    1401        }
    1402
    1403        @Override
    1404        public boolean isComplex() {
    1405                int type = getDType();
    1406                return type == COMPLEX64 || type == COMPLEX128;
    1407        }
    1408
    1409        @Override
    1410        public Dataset getRealPart() {
    1411                return this;
    1412        }
    1413
    1414        @Override
    1415        public Dataset getRealView() {
    1416                return getView(true);
    1417        }
    1418
    1419        @Override
    1420        public Dataset getSlice(final int[] start, final int[] stop, final int[] step) {
    1421                return getSlice(new SliceND(shape, start, stop, step));
    1422        }
    1423
    1424        @Override
    1425        public Dataset getSlice(Slice... slice) {
    1426                return getSlice(new SliceND(shape, slice));
    1427        }
    1428
    1429        @Override
    1430        public Dataset getSlice(IMonitor monitor, Slice... slice) {
    1431                return getSlice(slice);
    1432        }
    1433
    1434        @Override
    1435        public Dataset getSlice(IMonitor monitor, SliceND slice) {
    1436                return getSlice(slice);
    1437        }
    1438
    1439        @Override
    1440        public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) {
    1441                return getSlice(start, stop, step);
    1442        }
    1443
    1444        /**
    1445         * Get a slice of the dataset. The returned dataset is a copied selection of items
    1446         * @param slice
    1447         * @return The dataset of the sliced data
    1448         */
    1449        @Override
    1450        public Dataset getSlice(final SliceND slice) {
    1451                SliceIterator it = (SliceIterator) getSliceIterator(slice);
    1452                AbstractDataset s = getSlice(it);
    1453                s.metadata = copyMetadata();
    1454                s.setDirty();
    1455                s.sliceMetadata(true, slice);
    1456                return s;
    1457        }
    1458
    1459        /**
    1460         * Get a slice of the dataset. The returned dataset is a copied selection of items
    1461         *
    1462         * @param iterator Slice iterator
    1463         * @return The dataset of the sliced data
    1464         */
    1465        abstract public AbstractDataset getSlice(final SliceIterator iterator);
    1466
    1467        @SuppressWarnings("deprecation")
    1468        @Override
    1469        public Dataset setSlice(final Object obj, final SliceND slice) {
    1470                Dataset ds;
    1471                if (obj instanceof Dataset) {
    1472                        ds = (Dataset) obj;
    1473                } else if (obj instanceof IDataset) {
    1474                        ds = DatasetUtils.convertToDataset((IDataset) obj);
    1475                } else {
    1476                        int dtype = getDType();
    1477                        if (dtype != Dataset.BOOL) {
    1478                                dtype = DTypeUtils.getLargestDType(dtype);
    1479                        }
    1480                        ds = DatasetFactory.createFromObject(getElementsPerItem(), dtype, obj);
    1481                }
    1482
    1483                return setSlicedView(getSliceView(slice), ds);
    1484        }
    1485
    1486        @Override
    1487        public Dataset setSlice(final Object obj, final int[] start, final int[] stop, final int[] step) {
    1488                return setSlice(obj, new SliceND(shape, start, stop, step));
    1489        }
    1490
    1491        /**
    1492         * Set a view of current dataset to given dataset with broadcasting
    1493         * @param view
    1494         * @param d
    1495         * @return this dataset
    1496         */
    1497        abstract Dataset setSlicedView(Dataset view, Dataset d);
    1498
    1499        @Override
    1500        public Dataset setSlice(Object obj, Slice... slice) {
    1501                if (slice == null || slice.length == 0) {
    1502                        return setSlice(obj, new SliceND(shape));
    1503                }
    1504                return setSlice(obj, new SliceND(shape, slice));
    1505        }
    1506
    1507        @Override
    1508        public boolean all() {
    1509                return Comparisons.allTrue(this);
    1510        }
    1511
    1512        @Override
    1513        public BooleanDataset all(final int axis) {
    1514                return Comparisons.allTrue(this, axis);
    1515        }
    1516
    1517        @Override
    1518        public boolean any() {
    1519                return Comparisons.anyTrue(this);
    1520        }
    1521
    1522        @Override
    1523        public BooleanDataset any(final int axis) {
    1524                return Comparisons.anyTrue(this, axis);
    1525        }
    1526
    1527        @Override
    1528        public Dataset ifloorDivide(final Object o) {
    1529                return idivide(o).ifloor();
    1530        }
    1531
    1532        @Override
    1533        public double residual(final Object o) {
    1534                return residual(o, null, false);
    1535        }
    1536
    1537        @Override
    1538        public double residual(final Object o, boolean ignoreNaNs) {
    1539                return residual(o, null, ignoreNaNs);
    1540        }
    1541
    1542        /**
    1543         * @since 2.0
    1544         */
    1545        @SuppressWarnings("unchecked")
    1546        protected StatisticsMetadata<Number> getStats() {
    1547                StatisticsMetadata<Number> md = getFirstMetadata(StatisticsMetadata.class);
    1548                if (md == null || md.isDirty()) {
    1549                        md = new StatisticsMetadataImpl<Number>();
    1550                        md.initialize(this);
    1551                        setMetadata(md);
    1552                }
    1553                return md;
    1554        }
    1555
    1556        /**
    1557         * @since 2.0
    1558         */
    1559        @SuppressWarnings("unchecked")
    1560        protected StatisticsMetadata<String> getStringStats() {
    1561                StatisticsMetadata<String> md = getFirstMetadata(StatisticsMetadata.class);
    1562                if (md == null || md.isDirty()) {
    1563                        md = new StatisticsMetadataImpl<String>();
    1564                        md.initialize(this);
    1565                        setMetadata(md);
    1566                }
    1567                return md;
    1568        }
    1569
    1570        @Override
    1571        public Number max(boolean... ignoreInvalids) {
    1572                return getStats().getMaximum(ignoreInvalids);
    1573        }
    1574
    1575        @Override
    1576        public Dataset max(int axis, boolean... ignoreInvalids) {
    1577                return getStats().getMaximum(axis, ignoreInvalids);
    1578        }
    1579
    1580        @Override
    1581        public Number min(boolean... ignoreInvalids) {
    1582                return getStats().getMinimum(ignoreInvalids);
    1583        }
    1584
    1585        @Override
    1586        public Dataset min(int axis, boolean... ignoreInvalids) {
    1587                return getStats().getMinimum(axis, ignoreInvalids);
    1588        }
    1589
    1590        @Override
    1591        public int argMax(boolean... ignoreInvalids) {
    1592                return getFlat1DIndex(maxPos(ignoreInvalids));
    1593        }
    1594
    1595        /**
    1596         * @since 2.0
    1597         */
    1598        @Override
    1599        public IntegerDataset argMax(int axis, boolean... ignoreInvalids) {
    1600                return (IntegerDataset) getStats().getArgMaximum(axis, ignoreInvalids);
    1601        }
    1602
    1603        @Override
    1604        public int argMin(boolean... ignoreInvalids) {
    1605                return getFlat1DIndex(minPos(ignoreInvalids));
    1606        }
    1607
    1608        /**
    1609         * @since 2.0
    1610         */
    1611        @Override
    1612        public IntegerDataset argMin(int axis, boolean... ignoreInvalids) {
    1613                return (IntegerDataset) getStats().getArgMinimum(axis, ignoreInvalids);
    1614        }
    1615
    1616        @Override
    1617        public Number peakToPeak(boolean... ignoreInvalids) {
    1618                return DTypeUtils.fromDoubleToBiggestNumber(max(ignoreInvalids).doubleValue() - min(ignoreInvalids).doubleValue(), getDType());
    1619        }
    1620
    1621        @Override
    1622        public Dataset peakToPeak(int axis,  boolean... ignoreInvalids) {
    1623                return Maths.subtract(max(axis, ignoreInvalids), min(axis, ignoreInvalids));
    1624        }
    1625
    1626
    1627        @Override
    1628        public long count(boolean... ignoreInvalids) {
    1629                return getStats().getCount(ignoreInvalids);
    1630        }
    1631
    1632        @Override
    1633        public Dataset count(int axis, boolean... ignoreInvalids) {
    1634                return getStats().getCount(axis, ignoreInvalids);
    1635        }
    1636
    1637        @Override
    1638        public Object sum(boolean... ignoreInvalids) {
    1639                return getStats().getSum(ignoreInvalids);
    1640        }
    1641
    1642        @Override
    1643        public Dataset sum(int axis, boolean... ignoreInvalids) {
    1644                return getStats().getSum(axis, ignoreInvalids);
    1645        }
    1646
    1647        @Override
    1648        public Object product(boolean... ignoreInvalids) {
    1649                return Stats.product(this, ignoreInvalids);
    1650        }
    1651
    1652        @Override
    1653        public Dataset product(int axis, boolean... ignoreInvalids) {
    1654                return Stats.product(this, axis, ignoreInvalids);
    1655        }
    1656
    1657        @Override
    1658        public Object mean(boolean... ignoreInvalids) {
    1659                return getStats().getMean(ignoreInvalids);
    1660        }
    1661
    1662        @Override
    1663        public Dataset mean(int axis, boolean... ignoreInvalids) {
    1664                return getStats().getMean(axis, ignoreInvalids);
    1665        }
    1666
    1667        @Override
    1668        public double variance() {
    1669                return variance(false);
    1670        }
    1671
    1672        @Override
    1673        public double variance(boolean isWholePopulation, boolean... ignoreInvalids) {
    1674                return getStats().getVariance(isWholePopulation, ignoreInvalids);
    1675        }
    1676
    1677        @Override
    1678        public Dataset variance(int axis) {
    1679                return getStats().getVariance(axis, false);
    1680        }
    1681
    1682        @Override
    1683        public Dataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
    1684                return getStats().getVariance(axis, isWholePopulation, ignoreInvalids);
    1685        }
    1686
    1687        @Override
    1688        public double stdDeviation() {
    1689                return Math.sqrt(variance());
    1690        }
    1691
    1692        @Override
    1693        public double stdDeviation(boolean isWholePopulation, boolean... ignoreInvalids) {
    1694                return Math.sqrt(variance(isWholePopulation, ignoreInvalids));
    1695        }
    1696
    1697        @Override
    1698        public Dataset stdDeviation(int axis) {
    1699                return Maths.sqrt(variance(axis, false));
    1700        }
    1701
    1702        @Override
    1703        public Dataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
    1704                return Maths.sqrt(variance(axis, isWholePopulation, ignoreInvalids));
    1705        }
    1706
    1707        @Override
    1708        public double rootMeanSquare(boolean... ignoreInvalids) {
    1709                StatisticsMetadata<Number> stats = getStats();
    1710                final double mean = stats.getMean(ignoreInvalids).doubleValue();
    1711                final double var = stats.getVariance(true, ignoreInvalids);
    1712                return Math.sqrt(var + mean * mean);
    1713        }
    1714
    1715        @Override
    1716        public Dataset rootMeanSquare(int axis, boolean... ignoreInvalids) {
    1717                StatisticsMetadata<Number> stats = getStats();
    1718                Dataset v = stats.getVariance(axis, true, ignoreInvalids);
    1719                Dataset m = stats.getMean(axis, ignoreInvalids);
    1720                Dataset result = Maths.multiply(m, m);
    1721                return Maths.sqrt(result.iadd(v));
    1722        }
    1723
    1724        /**
    1725         * Set item from compatible dataset in a direct and speedy way. Remember to setDirty afterwards.
    1726         *
    1727         * @param dindex
    1728         * @param sindex
    1729         * @param src
    1730         *            is the source data buffer
    1731         */
    1732        protected abstract void setItemDirect(final int dindex, final int sindex, final Object src);
    1733
    1734        /**
    1735         * @return error broadcasted to current shape
    1736         */
    1737        private Dataset getBroadcastedInternalError() {
    1738                if (shape == null) {
    1739                        throw new IllegalArgumentException("Cannot get error for null dataset");
    1740                }
    1741                ILazyDataset led = super.getErrors();
    1742                if (led == null)
    1743                        return null;
    1744
    1745                Dataset ed = null;
    1746                try {
    1747                        ed = DatasetUtils.sliceAndConvertLazyDataset(led);
    1748                } catch (DatasetException e) {
    1749                        logger.error("Could not get data from lazy dataset", e);
    1750                }
    1751                if (led != ed) {
    1752                        setErrors(ed); // set back
    1753                }
    1754
    1755                return ed.getBroadcastView(shape);
    1756        }
    1757
    1758        @Override
    1759        public Dataset getErrors() {
    1760                Dataset ed = getBroadcastedInternalError();
    1761                if (ed == null)
    1762                        return null;
    1763
    1764                return ed;
    1765        }
    1766
    1767        @Override
    1768        public double getError() {
    1769                Dataset ed = getBroadcastedInternalError();
    1770                if (ed == null)
    1771                        return 0;
    1772
    1773                return ed.getDouble();
    1774        }
    1775
    1776        @Override
    1777        public double getError(final int i) {
    1778                Dataset ed = getBroadcastedInternalError();
    1779                if (ed == null)
    1780                        return 0;
    1781
    1782                return ed.getDouble(i);
    1783        }
    1784
    1785        @Override
    1786        public double getError(final int i, final int j) {
    1787                Dataset ed = getBroadcastedInternalError();
    1788                if (ed == null)
    1789                        return 0;
    1790
    1791                return ed.getDouble(i, j);
    1792        }
    1793
    1794        @Override
    1795        public double getError(int... pos) {
    1796                Dataset ed = getBroadcastedInternalError();
    1797                if (ed == null)
    1798                        return 0;
    1799
    1800                return ed.getDouble(pos);
    1801        }
    1802
    1803        @Override
    1804        public double[] getErrorArray(final int i) {
    1805                Dataset ed = getBroadcastedInternalError();
    1806                if (ed == null)
    1807                        return null;
    1808
    1809                return new double[] {getError(i)};
    1810        }
    1811
    1812        @Override
    1813        public double[] getErrorArray(final int i, final int j) {
    1814                Dataset ed = getBroadcastedInternalError();
    1815                if (ed == null)
    1816                        return null;
    1817
    1818                return new double[] {getError(i, j)};
    1819        }
    1820
    1821        @Override
    1822        public double[] getErrorArray(int... pos) {
    1823                Dataset ed = getBroadcastedInternalError();
    1824                if (ed == null)
    1825                        return null;
    1826
    1827                return new double[] {getError(pos)};
    1828        }
    1829
    1830        protected Dataset getInternalSquaredError() {
    1831                Dataset sed = getErrorBuffer().getBroadcastView(shape);
    1832                return sed;
    1833        }
    1834
    1835        @Override
    1836        public Dataset getErrorBuffer() {
    1837                ErrorMetadata emd = getErrorMetadata();
    1838                if (emd == null)
    1839                        return null;
    1840
    1841                if (!(emd instanceof ErrorMetadataImpl)) {
    1842                        ILazyDataset led = emd.getError();
    1843                        Dataset ed;
    1844                        try {
    1845                                ed = DatasetUtils.sliceAndConvertLazyDataset(led);
    1846                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
    1847                                setMetadata(emd);
    1848                                emd.setError(ed);
    1849                        } catch (MetadataException me) {
    1850                                logger.error("Could not create metadata", me);
    1851                        } catch (DatasetException e) {
    1852                                logger.error("Could not get data from lazy dataset", e);
    1853                        }
    1854                }
    1855
    1856                return ((ErrorMetadataImpl) emd).getSquaredError();
    1857        }
    1858
    1859        /**
    1860         * Set a copy of the buffer that backs the (squared) error data
    1861         * @param buffer can be null, anything that can be used to create a DoubleDataset or CompoundDoubleDataset
    1862         */
    1863        @Override
    1864        public void setErrorBuffer(Serializable buffer) {
    1865                if (shape == null) {
    1866                        throw new IllegalArgumentException("Cannot set error buffer for null dataset");
    1867                }
    1868                if (buffer == null) {
    1869                        clearMetadata(ErrorMetadata.class);
    1870                        return;
    1871                }
    1872
    1873                IDataset d = (IDataset) createFromSerializable(buffer, false);
    1874                ErrorMetadata emd = getErrorMetadata();
    1875                if (!(emd instanceof ErrorMetadataImpl)) {
    1876                        try {
    1877                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
    1878                                setMetadata(emd);
    1879                        } catch (MetadataException me) {
    1880                                logger.error("Could not create metadata", me);
    1881                        }
    1882                }
    1883                ((ErrorMetadataImpl) emd).setSquaredError(d);
    1884        }
    1885}