    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.util.Arrays;
    016
    017import org.eclipse.january.DatasetException;
    018import org.eclipse.january.IMonitor;
    019import org.eclipse.january.metadata.StatisticsMetadata;
    020import org.eclipse.january.metadata.internal.StatisticsMetadataImpl;
    021
    022/**
    023 * Generic container class for data that is compound in nature
    024 *
    025 * Each subclass has an array of compound types, items of this array are composed of primitive types
    026 *
    027 * Data items can be Complex, Vector, etc
    028 *
    029 */
    030public abstract class AbstractCompoundDataset extends AbstractDataset implements CompoundDataset {
    031        // pin UID to base class
    032        private static final long serialVersionUID = Dataset.serialVersionUID;
    033
    034        protected int isize; // number of elements per item
    035
    036        @Override
    037        public int getElementsPerItem() {
    038                return isize;
    039        }
    040
    041        @Override
    042        protected int get1DIndex(final int i) {
    043                int n = super.get1DIndex(i);
    044                return stride == null ? isize * n : n;
    045        }
    046
    047        @Override
    048        protected int get1DIndex(final int i, final int j) {
    049                int n = super.get1DIndex(i, j);
    050                return stride == null ? isize * n : n;
    051        }
    052
    053        @Override
    054        protected int get1DIndexFromShape(final int[] n) {
    055                return isize * super.get1DIndexFromShape(n);
    056        }
    057
    058        @Override
    059        public Dataset getUniqueItems() {
    060                throw new UnsupportedOperationException("Cannot sort compound datasets");
    061        }
    062
    063        @Override
    064        public IndexIterator getIterator(final boolean withPosition) {
    065                if (stride != null) {
    066                        return base.getSize() == 1 ?
    067                                        (withPosition ? new PositionIterator(offset, shape) :  new SingleItemIterator(offset, size)) : new StrideIterator(isize, shape, stride, offset);
    068                }
    069                return withPosition ? getSliceIterator(null, null, null) :
    070                        new ContiguousIterator(size, isize);
    071        }
    072
    073        /**
    074         * Get an iterator that picks out the chosen element from all items
    075         * @param element
    076         * @return an iterator
    077         */
    078        public IndexIterator getIterator(int element) {
    079                if (element < 0)
    080                        element += isize;
    081                if (element < 0 || element > isize) {
    082                        logger.error("Invalid choice of element: {}/{}", element, isize);
    083                        throw new IllegalArgumentException("Invalid choice of element: " + element + "/" + isize);
    084                }
    085
    086                final IndexIterator it;
    087                if (stride != null) {
    088                        it = base.getSize() == 1 ? new SingleItemIterator(offset + element, size) : new StrideIterator(isize, shape, stride, offset, element);
    089                } else {
    090                        it = new ContiguousIterator(size, isize, element);
    091                }
    092
    093                return it;
    094        }
    095
    096        @Override
    097        public IndexIterator getSliceIterator(SliceND slice) {
    098                if (stride != null)
    099                        return new StrideIterator(isize, shape, stride, offset, slice);
    100
    101                return new SliceIterator(shape, size, isize, slice);
    102        }
    103
    104        /**
    105         * Constructor required for serialisation.
    106         */
    107        public AbstractCompoundDataset() {
    108        }
    109
    110        @Override
    111        public boolean equals(Object obj) {
    112                if (!super.equals(obj)) {
    113                        return false;
    114                }
    115
    116                CompoundDataset other = (CompoundDataset) obj;
    117                return isize == other.getElementsPerItem();
    118        }
    119
    120        @Override
    121        public int hashCode() {
    122                return getCompoundStats().getHash(shape);
    123        }
    124
    125        @Override
    126        public CompoundDataset cast(boolean repeat, int dtype, int isize) {
    127                return (CompoundDataset) super.cast(repeat, dtype, isize);
    128        }
    129
    130        @Override
    131        public CompoundDataset cast(int dtype) {
    132                return (CompoundDataset) super.cast(dtype);
    133        }
    134
    135        @Override
    136        abstract public AbstractCompoundDataset clone();
    137
    138        @Override
    139        public CompoundDataset flatten() {
    140                return (CompoundDataset) super.flatten();
    141        }
    142
    143        @Override
    144        public CompoundDataset getBy1DIndex(IntegerDataset index) {
    145                return (CompoundDataset) super.getBy1DIndex(index);
    146        }
    147
    148        @Override
    149        public CompoundDataset getByBoolean(Dataset selection) {
    150                return (CompoundDataset) super.getByBoolean(selection);
    151        }
    152
    153        @Override
    154        public CompoundDataset getByIndexes(Object... indexes) {
    155                return (CompoundDataset) super.getByIndexes(indexes);
    156        }
    157
    158        @Override
    159        public CompoundDataset getSlice(IMonitor mon, int[] start, int[] stop, int[] step) {
    160                return (CompoundDataset) super.getSlice(mon, start, stop, step);
    161        }
    162
    163        @Override
    164        public CompoundDataset getSlice(IMonitor mon, Slice... slice) {
    165                return (CompoundDataset) super.getSlice(mon, slice);
    166        }
    167
    168        @Override
    169        public CompoundDataset getSlice(IMonitor mon, SliceND slice) {
    170                return (CompoundDataset) super.getSlice(mon, slice);
    171        }
    172
    173        @Override
    174        public CompoundDataset getSlice(int[] start, int[] stop, int[] step) {
    175                return (CompoundDataset) super.getSlice(start, stop, step);
    176        }
    177
    178        @Override
    179        public CompoundDataset getSlice(Slice... slice) {
    180                return (CompoundDataset) super.getSlice(slice);
    181        }
    182
    183        @Override
    184        public CompoundDataset getSlice(SliceND slice) {
    185                return (CompoundDataset) super.getSlice(slice);
    186        }
    187
    188        @Override
    189        abstract public AbstractCompoundDataset getSlice(SliceIterator iterator);
    190
    191        @Override
    192        public CompoundDataset getSliceView(int[] start, int[] stop, int[] step) {
    193                return (CompoundDataset) super.getSliceView(start, stop, step);
    194        }
    195
    196        @Override
    197        public CompoundDataset getSliceView(Slice... slice) {
    198                return (CompoundDataset) super.getSliceView(slice);
    199        }
    200
    201        @Override
    202        public CompoundDataset getSliceView(SliceND slice) {
    203                return (CompoundDataset) super.getSliceView(slice);
    204        }
    205
    206        @Override
    207        public CompoundDataset getTransposedView(int... axes) {
    208                return (CompoundDataset) super.getTransposedView(axes);
    209        }
    210
    211        @Override
    212        abstract public AbstractCompoundDataset getView(boolean deepCopyMetadata);
    213
    214        @Override
    215        public CompoundDataset getBroadcastView(int... broadcastShape) {
    216                return (CompoundDataset) super.getBroadcastView(broadcastShape);
    217        }
    218
    219        @Override
    220        public CompoundDataset ifloorDivide(Object o) {
    221                return (CompoundDataset) super.ifloorDivide(o);
    222        }
    223
    224        @Override
    225        public CompoundDataset reshape(int... shape) {
    226                return (CompoundDataset) super.reshape(shape);
    227        }
    228
    229        @Override
    230        public CompoundDataset setSlice(Object obj, int[] start, int[] stop, int[] step) {
    231                return (CompoundDataset) super.setSlice(obj, start, stop, step);
    232        }
    233
    234        @Override
    235        public CompoundDataset setSlice(Object object, Slice... slice) {
    236                return (CompoundDataset) super.setSlice(object, slice);
    237        }
    238
    239        @Override
    240        public CompoundDataset sort(Integer axis) {
    241                throw new UnsupportedOperationException("Cannot sort dataset");
    242        }
    243
    244        @Override
    245        public CompoundDataset squeezeEnds() {
    246                return (CompoundDataset) super.squeezeEnds();
    247        }
    248
    249        @Override
    250        public CompoundDataset squeeze() {
    251                return (CompoundDataset) super.squeeze();
    252        }
    253
    254        @Override
    255        public CompoundDataset squeeze(boolean onlyFromEnd) {
    256                return (CompoundDataset) super.squeeze(onlyFromEnd);
    257        }
    258
    259        @Override
    260        public CompoundDataset swapAxes(int axis1, int axis2) {
    261                return (CompoundDataset) super.swapAxes(axis1, axis2);
    262        }
    263
    264        @Override
    265        public synchronized CompoundDataset synchronizedCopy() {
    266                return clone();
    267        }
    268
    269        @Override
    270        public CompoundDataset transpose(int... axes) {
    271                return (CompoundDataset) super.transpose(axes);
    272        }
    273
    274        /**
    275         * @since 2.0
    276         */
    277        abstract protected double getFirstValue();
    278
    279        abstract protected double getFirstValue(final int i);
    280
    281        abstract protected double getFirstValue(final int i, final int j);
    282
    283        abstract protected double getFirstValue(final int...pos);
    284
    285        @Override
    286        public boolean getBoolean() {
    287                return getFirstValue() != 0;
    288        }
    289
    290        @Override
    291        public boolean getBoolean(final int i) {
    292                return getFirstValue(i) != 0;
    293        }
    294
    295        @Override
    296        public boolean getBoolean(final int i, final int j) {
    297                return getFirstValue(i, j) != 0;
    298        }
    299
    300        @Override
    301        public boolean getBoolean(final int... pos) {
    302                return getFirstValue(pos) != 0;
    303        }
    304
    305        @Override
    306        public byte getByte() {
    307                return (byte) getFirstValue();
    308        }
    309
    310        @Override
    311        public byte getByte(final int i) {
    312                return (byte) getFirstValue(i);
    313        }
    314
    315        @Override
    316        public byte getByte(final int i, final int j) {
    317                return (byte) getFirstValue(i, j);
    318        }
    319
    320        @Override
    321        public byte getByte(final int... pos) {
    322                return (byte) getFirstValue(pos);
    323        }
    324
    325        @Override
    326        public short getShort() {
    327                return (short) getFirstValue();
    328        }
    329
    330        @Override
    331        public short getShort(final int i) {
    332                return (short) getFirstValue(i);
    333        }
    334
    335        @Override
    336        public short getShort(final int i, final int j) {
    337                return (short) getFirstValue(i, j);
    338        }
    339
    340        @Override
    341        public short getShort(final int... pos) {
    342                return (short) getFirstValue(pos);
    343        }
    344
    345        @Override
    346        public int getInt() {
    347                return (int) getFirstValue();
    348        }
    349
    350        @Override
    351        public int getInt(final int i) {
    352                return (int) getFirstValue(i);
    353        }
    354
    355        @Override
    356        public int getInt(final int i, final int j) {
    357                return (int) getFirstValue(i, j);
    358        }
    359
    360        @Override
    361        public int getInt(final int... pos) {
    362                return (int) getFirstValue(pos);
    363        }
    364
    365        @Override
    366        public long getLong() {
    367                return (long) getFirstValue();
    368        }
    369
    370        @Override
    371        public long getLong(final int i) {
    372                return (long) getFirstValue(i);
    373        }
    374
    375        @Override
    376        public long getLong(final int i, final int j) {
    377                return (long) getFirstValue(i, j);
    378        }
    379
    380        @Override
    381        public long getLong(final int... pos) {
    382                return (long) getFirstValue(pos);
    383        }
    384
    385        @Override
    386        public float getFloat() {
    387                return (float) getFirstValue();
    388        }
    389
    390        @Override
    391        public float getFloat(final int i) {
    392                return (float) getFirstValue(i);
    393        }
    394
    395        @Override
    396        public float getFloat(final int i, final int j) {
    397                return (float) getFirstValue(i, j);
    398        }
    399
    400        @Override
    401        public float getFloat(final int... pos) {
    402                return (float) getFirstValue(pos);
    403        }
    404
    405        @Override
    406        public double getDouble() {
    407                return getFirstValue();
    408        }
    409
    410        @Override
    411        public double getDouble(final int i) {
    412                return getFirstValue(i);
    413        }
    414
    415        @Override
    416        public double getDouble(final int i, final int j) {
    417                return getFirstValue(i, j);
    418        }
    419
    420        @Override
    421        public double getDouble(final int... pos) {
    422                return getFirstValue(pos);
    423        }
    424
    425        @Override
    426        public void getDoubleArray(final double[] darray) {
    427                getDoubleArrayAbs(getFirst1DIndex(), darray);
    428        }
    429
    430        @Override
    431        public void getDoubleArray(final double[] darray, final int i) {
    432                getDoubleArrayAbs(get1DIndex(i), darray);
    433        }
    434
    435        @Override
    436        public void getDoubleArray(final double[] darray, final int i, final int j) {
    437                getDoubleArrayAbs(get1DIndex(i, j), darray);
    438        }
    439
    440        @Override
    441        public void getDoubleArray(final double[] darray, final int... pos) {
    442                getDoubleArrayAbs(get1DIndex(pos), darray);
    443        }
    444
    445        /**
    446         * @since 2.0
    447         */
    448        @SuppressWarnings("unchecked")
    449        protected StatisticsMetadata<double[]> getCompoundStats() {
    450                StatisticsMetadata<double[]> md = getFirstMetadata(StatisticsMetadata.class);
    451                if (md == null || md.isDirty()) {
    452                        md = new StatisticsMetadataImpl<double[]>();
    453                        md.initialize(this);
    454                        setMetadata(md);
    455                }
    456                return md;
    457        }
    458
    459        @Override
    460        public IntegerDataset argMax(int axis, boolean... ignoreInvalids) {
    461                logger.error("Cannot compare compound numbers");
    462                throw new UnsupportedOperationException("Cannot compare compound numbers");
    463        }
    464
    465        @Override
    466        public IntegerDataset argMin(int axis, boolean... ignoreInvalids) {
    467                logger.error("Cannot compare compound numbers");
    468                throw new UnsupportedOperationException("Cannot compare compound numbers");
    469        }
    470
    471        @Override
    472        public Number max(boolean... ignoreInvalids) {
    473                logger.error("Cannot compare compound numbers");
    474                throw new UnsupportedOperationException("Cannot compare compound numbers");
    475        }
    476
    477        @Override
    478        public CompoundDataset max(int axis, boolean... ignoreInvalids) {
    479                logger.error("Cannot compare compound numbers");
    480                throw new UnsupportedOperationException("Cannot compare compound numbers");
    481        }
    482
    483        @Override
    484        public Number min(boolean... ignoreInvalids) {
    485                logger.error("Cannot compare compound numbers");
    486                throw new UnsupportedOperationException("Cannot compare compound numbers");
    487        }
    488
    489        @Override
    490        public CompoundDataset min(int axis, boolean... ignoreInvalids) {
    491                logger.error("Cannot compare compound numbers");
    492                throw new UnsupportedOperationException("Cannot compare compound numbers");
    493        }
    494
    495
    496        @Override
    497        public int[] maxPos(boolean... ignoreNaNs) {
    498                logger.error("Cannot compare compound numbers");
    499                throw new UnsupportedOperationException("Cannot compare compound numbers");
    500        }
    501
    502        @Override
    503        public int[] minPos(boolean... ignoreNaNs) {
    504                logger.error("Cannot compare compound numbers");
    505                throw new UnsupportedOperationException("Cannot compare compound numbers");
    506        }
    507
    508        @Override
    509        public CompoundDataset peakToPeak(int axis, boolean... ignoreInvalids) {
    510                logger.error("Cannot compare compound numbers");
    511                throw new UnsupportedOperationException("Cannot compare compound numbers");
    512        }
    513
    514        @Override
    515        public double[] maxItem() {
    516                return getCompoundStats().getMaximum();
    517        }
    518
    519        @Override
    520        public double[] minItem() {
    521                return getCompoundStats().getMinimum();
    522        }
    523
    524        @Override
    525        public Object mean(boolean... ignoreInvalids) {
    526                return getCompoundStats().getMean();
    527        }
    528
    529        @Override
    530        public CompoundDataset mean(int axis, boolean... ignoreInvalids) {
    531                return (CompoundDataset) super.mean(axis, ignoreInvalids);
    532        }
    533
    534        @Override
    535        public CompoundDataset product(int axis, boolean... ignoreInvalids) {
    536                return (CompoundDataset) super.product(axis, ignoreInvalids);
    537        }
    538
    539        @Override
    540        public CompoundDataset rootMeanSquare(int axis, boolean... ignoreInvalids) {
    541                return (CompoundDataset) super.rootMeanSquare(axis, ignoreInvalids);
    542        }
    543
    544        @Override
    545        public CompoundDataset stdDeviation(int axis) {
    546                return (CompoundDataset) super.stdDeviation(axis, false);
    547        }
    548
    549        @Override
    550        public CompoundDataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
    551                return (CompoundDataset) super.stdDeviation(axis, isWholePopulation, ignoreInvalids);
    552        }
    553
    554        @Override
    555        public Object sum(boolean... ignoreInvalids) {
    556                return getCompoundStats().getSum();
    557        }
    558
    559        @Override
    560        public CompoundDataset sum(int axis, boolean... ignoreInvalids) {
    561                return (CompoundDataset) super.sum(axis, ignoreInvalids);
    562        }
    563
    564        @Override
    565        public double variance(boolean isWholePopulation, boolean... ignoreInvalids) {
    566                return getCompoundStats().getVariance(isWholePopulation, ignoreInvalids);
    567        }
    568
    569        @Override
    570        public CompoundDataset variance(int axis) {
    571                return (CompoundDataset) super.variance(axis, false);
    572        }
    573
    574        @Override
    575        public CompoundDataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
    576                return (CompoundDataset) super.variance(axis, isWholePopulation, ignoreInvalids);
    577        }
    578
    579        @Override
    580        public double rootMeanSquare(boolean... ignoreInvalids) {
    581                StatisticsMetadata<double[]> stats = getCompoundStats();
    582
    583                double[] mean = stats.getMean(ignoreInvalids);
    584                double result = 0;
    585                for (int i = 0; i < isize; i++) {
    586                        double m = mean[i];
    587                        result += m * m;
    588                }
    589                return Math.sqrt(result + stats.getVariance(true));
    590        }
    591
    592        /**
    593         * @return error
    594         */
    595        private CompoundDataset getInternalError() {
    596                ILazyDataset led = super.getErrors();
    597                if (led == null)
    598                        return null;
    599
    600                Dataset ed = null;
    601                try {
    602                        ed = DatasetUtils.sliceAndConvertLazyDataset(led);
    603                } catch (DatasetException e) {
    604                        logger.error("Could not get data from lazy dataset", e);
    605                }
    606
    607                CompoundDataset ced; // ensure it has the same number of elements
    608                if (!(ed instanceof CompoundDataset) || ed.getElementsPerItem() != isize) {
    609                        ced = new CompoundDoubleDataset(isize, true, ed);
    610                } else {
    611                        ced = (CompoundDataset) ed;
    612                }
    613
    614                if (led != ced) {
    615                        setErrors(ced); // set back
    616                }
    617                return ced;
    618        }
    619
    620        @Override
    621        public CompoundDataset getErrors() {
    622                CompoundDataset ed = getInternalError();
    623                if (ed == null)
    624                        return null;
    625
    626                return ed.getBroadcastView(shape);
    627        }
    628
    629        @Override
    630        public double getError(final int i) {
    631                return calcError(getInternalErrorArray(true, i));
    632        }
    633
    634        @Override
    635        public double getError(final int i, final int j) {
    636                return calcError(getInternalErrorArray(true, i, j));
    637        }
    638
    639        @Override
    640        public double getError(final int... pos) {
    641                return calcError(getInternalErrorArray(true, pos));
    642        }
    643
    644        private double calcError(double[] es) {
    645                if (es == null)
    646                        return 0;
    647
    648                // assume elements are independent
    649                double e = 0;
    650                for (int k = 0; k < isize; k++) {
    651                        e += es[k];
    652                }
    653
    654                return Math.sqrt(e);
    655        }
    656
    657        @Override
    658        public double[] getErrorArray(final int i) {
    659                return getInternalErrorArray(false, i);
    660        }
    661
    662        @Override
    663        public double[] getErrorArray(final int i, final int j) {
    664                return getInternalErrorArray(false, i, j);
    665        }
    666
    667        @Override
    668        public double[] getErrorArray(final int... pos) {
    669                return getInternalErrorArray(false, pos);
    670        }
    671
    672        private Dataset getInternalError(final boolean squared) {
    673                Dataset sed = squared ? getInternalSquaredError() : getInternalError();
    674                if (sed == null)
    675                        return null;
    676
    677                return sed.getBroadcastView(shape);
    678        }
    679
    680        private double[] getInternalErrorArray(final boolean squared, final int i) {
    681                Dataset sed = getInternalError(squared);
    682                if (sed == null)
    683                        return null;
    684
    685                double[] es;
    686                if (sed instanceof CompoundDoubleDataset) {
    687                        es = ((CompoundDoubleDataset) sed).getDoubleArray(i);
    688                        if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted
    689                                Arrays.fill(es, es[0]);
    690                        }
    691                } else {
    692                        es = new double[isize];
    693                        Arrays.fill(es, ((DoubleDataset) sed).getDouble(i));
    694                }
    695                return es;
    696        }
    697
    698        private double[] getInternalErrorArray(final boolean squared, final int i, final int j) {
    699                Dataset sed = getInternalError(squared);
    700                if (sed == null)
    701                        return null;
    702
    703                double[] es;
    704                if (sed instanceof CompoundDoubleDataset) {
    705                        es = ((CompoundDoubleDataset) sed).getDoubleArray(i, j);
    706                        if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted
    707                                Arrays.fill(es, es[0]);
    708                        }
    709                } else {
    710                        es = new double[isize];
    711                        Arrays.fill(es, ((DoubleDataset) sed).getDouble(i, j));
    712                }
    713                return es;
    714        }
    715
    716        private double[] getInternalErrorArray(final boolean squared, final int... pos) {
    717                Dataset sed = getInternalError(squared);
    718                if (sed == null)
    719                        return null;
    720
    721                double[] es = new double[isize];
    722                if (sed instanceof CompoundDoubleDataset) {
    723                        es = ((CompoundDoubleDataset) sed).getDoubleArray(pos);
    724                        if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted
    725                                Arrays.fill(es, es[0]);
    726                        }
    727                } else {
    728                        es = new double[isize];
    729                        Arrays.fill(es, ((DoubleDataset) sed).getDouble(pos));
    730                }
    731                return es;
    732        }
    733}
    734