    001/*-
    002 *******************************************************************************
    003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
    004 * All rights reserved. This program and the accompanying materials
    005 * are made available under the terms of the Eclipse Public License v1.0
    006 * which accompanies this distribution, and is available at
    007 * http://www.eclipse.org/legal/epl-v10.html
    008 *
    009 * Contributors:
    010 *    Peter Chang - initial API and implementation and/or initial documentation
    011 *******************************************************************************/
    012
    013package org.eclipse.january.dataset;
    014
    015import java.io.Serializable;
    016import java.lang.annotation.Annotation;
    017import java.lang.reflect.Array;
    018import java.lang.reflect.Field;
    019import java.util.ArrayList;
    020import java.util.Arrays;
    021import java.util.Collection;
    022import java.util.concurrent.ConcurrentMap;
    023import java.util.concurrent.ConcurrentHashMap;
    024import java.util.List;
    025import java.util.Map;
    026
    027import org.eclipse.january.DatasetException;
    028import org.eclipse.january.MetadataException;
    029import org.eclipse.january.metadata.Dirtiable;
    030import org.eclipse.january.metadata.ErrorMetadata;
    031import org.eclipse.january.metadata.IMetadata;
    032import org.eclipse.january.metadata.MetadataFactory;
    033import org.eclipse.january.metadata.MetadataType;
    034import org.eclipse.january.metadata.Reshapeable;
    035import org.eclipse.january.metadata.Sliceable;
    036import org.eclipse.january.metadata.Transposable;
    037import org.slf4j.Logger;
    038import org.slf4j.LoggerFactory;
    039
    040/**
    041 * Common base for both lazy and normal dataset implementations
    042 */
    043public abstract class LazyDatasetBase implements ILazyDataset, Serializable {
    044
    045        private static final long serialVersionUID = 767926846438976050L;
    046
    047        protected static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class);
    048
    049        protected static boolean catchExceptions;
    050
    051        static {
    052                /**
    053                 * Boolean to set to true if running jython scripts that utilise ScisoftPy in IDE
    054                 */
    055                catchExceptions = Boolean.getBoolean("run.in.eclipse");
    056        }
    057
    058        transient private boolean dirty = true; // indicate dirty state of metadata
    059        protected String name = "";
    060
    061        /**
    062         * The shape or dimensions of the dataset
    063         */
    064        protected int[] shape;
    065
    066        protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null;
    067
    068        /**
    069         * @return type of dataset item
    070         */
    071        abstract public int getDType();
    072
    073        @Override
    074        public Class<?> getElementClass() {
    075                return DTypeUtils.getElementClass(getDType());
    076        }
    077
    078        @Override
    079        public LazyDatasetBase clone() {
    080                return null;
    081        }
    082
    083        @Override
    084        public boolean equals(Object obj) {
    085                if (this == obj) {
    086                        return true;
    087                }
    088                if (obj == null) {
    089                        return false;
    090                }
    091                if (!getClass().equals(obj.getClass())) {
    092                        return false;
    093                }
    094
    095                LazyDatasetBase other = (LazyDatasetBase) obj;
    096                if (getDType() != other.getDType()) {
    097                        return false;
    098                }
    099                if (getElementsPerItem() != other.getElementsPerItem()) {
    100                        return false;
    101                }
    102                if (!Arrays.equals(shape, other.shape)) {
    103                        return false;
    104                }
    105                return true;
    106        }
    107
    108        @Override
    109        public int hashCode() {
    110                int hash = getDType() * 17 + getElementsPerItem();
    111                int rank = shape.length;
    112                for (int i = 0; i < rank; i++) {
    113                        hash = hash*17 + shape[i];
    114                }
    115                return hash;
    116        }
    117
    118        @Override
    119        public String getName() {
    120                return name;
    121        }
    122
    123        @Override
    124        public void setName(String name) {
    125                this.name = name;
    126        }
    127
    128        @Override
    129        public int[] getShape() {
    130                return shape.clone();
    131        }
    132
    133        @Override
    134        public int getRank() {
    135                return shape.length;
    136        }
    137
    138        /**
    139         * This method allows anything that dirties the dataset to clear various metadata values
    140         * so that the other methods can work correctly.
    141         */
    142        public void setDirty() {
    143                dirty = true;
    144        }
    145
    146        /**
    147         * Find first sub-interface of (or class that directly implements) MetadataType
    148         * @param clazz
    149         * @return sub-interface
    150         * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it
    151         */
    152        @SuppressWarnings("unchecked")
    153        public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) {
    154                if (clazz.equals(MetadataType.class)) {
    155                        throw new IllegalArgumentException("Cannot accept MetadataType");
    156                }
    157
    158                if (clazz.isInterface()) {
    159                        return clazz;
    160                }
    161
    162                if (clazz.isAnonymousClass()) { // special case
    163                        Class<?> s = clazz.getSuperclass();
    164                        if (!s.equals(Object.class)) {
    165                                // only use super class if it is not an anonymous class of an interface
    166                                clazz = (Class<? extends MetadataType>) s;
    167                        }
    168                }
    169
    170                for (Class<?> c : clazz.getInterfaces()) {
    171                        if (c.equals(MetadataType.class)) {
    172                                if (clazz.isAnonymousClass()) {
    173                                        throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType");
    174                                }
    175                                return clazz;
    176                        }
    177                        if (MetadataType.class.isAssignableFrom(c)) {
    178                                return (Class<? extends MetadataType>) c;
    179                        }
    180                }
    181
    182                Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class
    183                if (c != null) {
    184                        return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c);
    185                }
    186
    187                logger.error("Somehow the search for metadata type interface ended in a bad place");
    188                assert false; // should not be able to get here!!!
    189                return null;
    190        }
    191
    192        @Override
    193        public void setMetadata(MetadataType metadata) {
    194                addMetadata(metadata, true);
    195        }
    196
    197        @Override
    198        public void addMetadata(MetadataType metadata) {
    199                addMetadata(metadata, false);
    200        }
    201
    202        private synchronized void addMetadata(MetadataType metadata, boolean clear) {
    203                if (metadata == null)
    204                        return;
    205
    206                if (this.metadata == null) {
    207                        this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
    208                }
    209
    210                Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass());
    211                if (!this.metadata.containsKey(clazz)) {
    212                        this.metadata.put(clazz, new ArrayList<MetadataType>());
    213                } else if (clear) {
    214                        this.metadata.get(clazz).clear();
    215                }
    216                this.metadata.get(clazz).add(metadata);
    217
    218                // add for special case of sub-interfaces of IMetadata
    219                if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) {
    220                        clazz = IMetadata.class;
    221                        if (!this.metadata.containsKey(clazz)) {
    222                                this.metadata.put(clazz, new ArrayList<MetadataType>());
    223                        } else if (clear) {
    224                                this.metadata.get(clazz).clear();
    225                        }
    226                        this.metadata.get(clazz).add(metadata);
    227                }
    228        }
    229
    230        @Override
    231        @Deprecated
    232        public synchronized IMetadata getMetadata() {
    233                return getFirstMetadata(IMetadata.class);
    234        }
    235
    236        @SuppressWarnings("unchecked")
    237        @Override
    238        public synchronized <S extends MetadataType, T extends S> List<S> getMetadata(Class<T> clazz) throws MetadataException {
    239                if (metadata == null)
    240                        return null;
    241
    242                if (dirty) {
    243                        dirtyMetadata();
    244                        dirty = false;
    245                }
    246
    247                if (clazz == null) {
    248                        List<S> all = new ArrayList<S>();
    249                        for (Class<? extends MetadataType> c : metadata.keySet()) {
    250                                all.addAll((Collection<S>) metadata.get(c));
    251                        }
    252                        return all;
    253                }
    254
    255                return (List<S>) metadata.get(findMetadataTypeSubInterfaces(clazz));
    256        }
    257
    258        @Override
    259        public synchronized <S extends MetadataType, T extends S> S getFirstMetadata(Class<T> clazz) {
    260                try {
    261                        List<S> ml = getMetadata(clazz);
    262                        if (ml == null) return null;
    263                        for (S t : ml) {
    264                                if (clazz.isInstance(t)) return t;
    265                        }
    266                } catch (Exception e) {
    267                        logger.error("Get metadata failed!",e);
    268                }
    269
    270                return null;
    271        }
    272
    273        @Override
    274        public synchronized void clearMetadata(Class<? extends MetadataType> clazz) {
    275                if (metadata == null)
    276                        return;
    277
    278                if (clazz == null) {
    279                        metadata.clear();
    280                        return;
    281                }
    282
    283                List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz));
    284                if( list != null) {
    285                        list.clear();
    286                }
    287        }
    288
    289        /**
    290         * @since 2.0
    291         */
    292        protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() {
    293                return copyMetadata(metadata);
    294        }
    295
    296        /**
    297         * @since 2.0
    298         */
    299        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) {
    300                if (metadata == null)
    301                        return null;
    302
    303                ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
    304
    305                for (Class<? extends MetadataType> c : metadata.keySet()) {
    306                        List<MetadataType> l = metadata.get(c);
    307                        List<MetadataType> nl = new ArrayList<MetadataType>(l.size());
    308                        map.put(c, nl);
    309                        for (MetadataType m : l) {
    310                                nl.add(m.clone());
    311                        }
    312                }
    313                return map;
    314        }
    315
    316        interface MetadatasetAnnotationOperation {
    317                /**
    318                 * Process value of given field
    319                 * <p>
    320                 * When the field is not a container then the returned value
    321                 * may replace the old value
    322                 * @param f given field
    323                 * @param o value of field
    324                 * @return transformed field
    325                 */
    326                Object processField(Field f, Object o);
    327
    328                /**
    329                 * @return annotated class
    330                 */
    331                Class<? extends Annotation> getAnnClass();
    332
    333                /**
    334                 * @param axis
    335                 * @return number of dimensions to insert or remove
    336                 */
    337                int change(int axis);
    338
    339                /**
    340                 *
    341                 * @return rank or -1 to match
    342                 */
    343                int getNewRank();
    344
    345                /**
    346                 * Run on given lazy dataset
    347                 * @param lz
    348                 * @return
    349                 */
    350                ILazyDataset run(ILazyDataset lz);
    351        }
    352
    353        class MdsSlice implements MetadatasetAnnotationOperation {
    354                private boolean asView;
    355                private int[] start;
    356                private int[] stop;
    357                private int[] step;
    358                private int[] oShape;
    359                private long oSize;
    360
    361                public MdsSlice(boolean asView, final int[] start, final int[] stop, final int[] step, final int[] oShape) {
    362                        this.asView = asView;
    363                        this.start = start;
    364                        this.stop = stop;
    365                        this.step = step;
    366                        this.oShape = oShape;
    367                        oSize = ShapeUtils.calcLongSize(oShape);
    368                }
    369
    370                @Override
    371                public Object processField(Field field, Object o) {
    372                        return o;
    373                }
    374
    375                @Override
    376                public Class<? extends Annotation> getAnnClass() {
    377                        return Sliceable.class;
    378                }
    379
    380                @Override
    381                public int change(int axis) {
    382                        return 0;
    383                }
    384
    385                @Override
    386                public int getNewRank() {
    387                        return -1;
    388                }
    389
    390                @Override
    391                public ILazyDataset run(ILazyDataset lz) {
    392                        int rank = lz.getRank();
    393                        if (start.length != rank) {
    394                                throw new IllegalArgumentException("Slice dimensions do not match dataset!");
    395                        }
    396
    397                        int[] shape = lz.getShape();
    398                        int[] stt;
    399                        int[] stp;
    400                        int[] ste;
    401                        if (lz.getSize() == oSize) {
    402                                stt = start;
    403                                stp = stop;
    404                                ste = step;
    405                        } else {
    406                                stt = start.clone();
    407                                stp = stop.clone();
    408                                ste = step.clone();
    409                                for (int i = 0; i < rank; i++) {
    410                                        if (shape[i] >= oShape[i]) continue;
    411                                        if (shape[i] == 1) {
    412                                                stt[i] = 0;
    413                                                stp[i] = 1;
    414                                                ste[1] = 1;
    415                                        } else {
    416                                                throw new IllegalArgumentException("Sliceable dataset has invalid size!");
    417                                        }
    418                                }
    419                        }
    420
    421                        if (asView || (lz instanceof IDataset))
    422                                return lz.getSliceView(stt, stp, ste);
    423                        try {
    424                                return lz.getSlice(stt, stp, ste);
    425                        } catch (DatasetException e) {
    426                                logger.error("Could not slice dataset in metadata", e);
    427                                return null;
    428                        }
    429                }
    430        }
    431
    432        class MdsReshape implements MetadatasetAnnotationOperation {
    433                private boolean matchRank;
    434                private int[] oldShape;
    435                private int[] newShape;
    436                boolean onesOnly;
    437                int[] differences;
    438
    439                /*
    440                 * if only ones then record differences (insertions and deletions)
    441                 *
    442                 * if shape changing, find broadcasted dimensions and disallow
    443                 * merging that include those dimensions
    444                 */
    445                public MdsReshape(final int[] oldShape, final int[] newShape) {
    446                        this.oldShape = oldShape;
    447                        this.newShape = newShape;
    448                        differences = null;
    449                }
    450
    451                @Override
    452                public Object processField(Field field, Object o) {
    453                        Annotation a = field.getAnnotation(Reshapeable.class);
    454                        if (a != null) { // cannot be null
    455                                matchRank = ((Reshapeable) a).matchRank();
    456                        }
    457                        return o;
    458                }
    459
    460                @Override
    461                public Class<? extends Annotation> getAnnClass() {
    462                        return Reshapeable.class;
    463                }
    464
    465                @Override
    466                public int change(int axis) {
    467                        if (matchRank) {
    468                                if (differences == null)
    469                                        init();
    470
    471                                if (onesOnly) {
    472                                        return differences[axis];
    473                                }
    474                                throw new UnsupportedOperationException("TODO support other shape operations");
    475                        }
    476                        return 0;
    477                }
    478
    479                @Override
    480                public int getNewRank() {
    481                        return matchRank ? newShape.length : -1;
    482                }
    483
    484                private void init() {
    485                        int or = oldShape.length - 1;
    486                        int nr = newShape.length - 1;
    487                        if (or < 0 || nr < 0) { // zero-rank shapes
    488                                onesOnly = true;
    489                                differences = new int[1];
    490                                differences[0] = or < 0 ? nr + 1 : or + 1;
    491                                return;
    492                        }
    493                        int ob = 0;
    494                        int nb = 0;
    495                        onesOnly = true;
    496                        do {
    497                                while (oldShape[ob] == 1 && ob < or) {
    498                                        ob++; // next non-unit dimension
    499                                }
    500                                while (newShape[nb] == 1 && nb < nr) {
    501                                        nb++;
    502                                }
    503                                if (oldShape[ob++] != newShape[nb++]) {
    504                                        onesOnly = false;
    505                                        break;
    506                                }
    507                        } while (ob <= or && nb <= nr);
    508
    509                        ob = 0;
    510                        nb = 0;
    511                        differences = new int[or + 2];
    512                        if (onesOnly) {
    513                                // work out unit dimensions removed from or add to old
    514                                int j = 0;
    515                                do {
    516                                        if (oldShape[ob] != 1 && newShape[nb] != 1) {
    517                                                ob++;
    518                                                nb++;
    519                                        } else {
    520                                                while (oldShape[ob] == 1 && ob < or) {
    521                                                        ob++;
    522                                                        differences[j]--;
    523                                                }
    524                                                while (newShape[nb] == 1 && nb < nr) {
    525                                                        nb++;
    526                                                        differences[j]++;
    527                                                }
    528                                        }
    529                                        j++;
    530                                } while (ob <= or && nb <= nr && j <= or);
    531                                while (ob <= or && oldShape[ob] == 1) {
    532                                        ob++;
    533                                        differences[j]--;
    534                                }
    535                                while (nb <= nr && newShape[nb] == 1) {
    536                                        nb++;
    537                                        differences[j]++;
    538                                }
    539                        } else {
    540                                if (matchRank) {
    541                                        logger.error("Combining dimensions is currently not supported");
    542                                        throw new IllegalArgumentException("Combining dimensions is currently not supported");
    543                                }
    544                                // work out mapping: contiguous dimensions can be grouped or split
    545                                while (ob <= or && nb <= nr) {
    546                                        int ol = oldShape[ob];
    547                                        while (ol == 1 && ol <= or) {
    548                                                ob++;
    549                                                ol = oldShape[ob];
    550                                        }
    551                                        int oe = ob + 1;
    552                                        int nl = newShape[nb];
    553                                        while (nl == 1 && nl <= nr) {
    554                                                nb++;
    555                                                nl = newShape[nb];
    556                                        }
    557                                        int ne = nb + 1;
    558                                        if (ol < nl) {
    559                                                differences[ob] = 1;
    560                                                do { // case where new shape combines several dimensions into one dimension
    561                                                        if (oe == (or + 1)) {
    562                                                                break;
    563                                                        }
    564                                                        differences[oe] = 1;
    565                                                        ol *= oldShape[oe++];
    566                                                } while (ol < nl);
    567                                                differences[oe - 1] = oe - ob; // signal end with difference
    568                                                if (nl != ol) {
    569                                                        logger.error("Single dimension is incompatible with subshape");
    570                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
    571                                                }
    572                                        } else if (ol > nl) {
    573                                                do { // case where new shape spreads single dimension over several dimensions
    574                                                        if (ne == (nr + 1)) {
    575                                                                break;
    576                                                        }
    577                                                        nl *= newShape[ne++];
    578                                                } while (nl < ol);
    579                                                if (nl != ol) {
    580                                                        logger.error("Subshape is incompatible with single dimension");
    581                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
    582                                                }
    583
    584                                        }
    585
    586                                        ob = oe;
    587                                        nb = ne;
    588                                }
    589
    590                        }
    591                }
    592
    593                @Override
    594                public ILazyDataset run(ILazyDataset lz) {
    595                        if (differences == null)
    596                                init();
    597
    598                        int[] lshape = lz.getShape();
    599                        if (Arrays.equals(newShape, lshape)) {
    600                                return lz;
    601                        }
    602                        int or = lz.getRank();
    603                        int nr = newShape.length;
    604                        int[] nshape = new int[nr];
    605                        Arrays.fill(nshape, 1);
    606                        if (onesOnly) {
    607                                // ignore omit removed dimensions
    608                                for (int i = 0, si = 0, di = 0; i < (or+1) && si <= or && di < nr; i++) {
    609                                        int c = differences[i];
    610                                        if (c == 0) {
    611                                                nshape[di++] = lshape[si++];
    612                                        } else if (c > 0) {
    613                                                while (c-- > 0 && di < nr) {
    614                                                        di++;
    615                                                }
    616                                        } else if (c < 0) {
    617                                                si -= c; // remove dimensions by skipping forward in source array
    618                                        }
    619                                }
    620                        } else {
    621                                boolean[] broadcast = new boolean[or];
    622                                for (int ob = 0; ob < or; ob++) {
    623                                        broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1;
    624                                }
    625                                int osize = lz.getSize();
    626
    627                                // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...)
    628                                int ob = 0;
    629                                int nsize = 1;
    630                                for (int i = 0; i < nr; i++) {
    631                                        if (ob < or && broadcast[ob]) {
    632                                                if (differences[ob] != 0) {
    633                                                        logger.error("Metadata contains a broadcast axis which cannot be reshaped");
    634                                                        throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped");
    635                                                }
    636                                        } else {
    637                                                nshape[i] = nsize < osize ? newShape[i] : 1;
    638                                        }
    639                                        nsize *= nshape[i];
    640                                        ob++;
    641                                }
    642                        }
    643
    644                        ILazyDataset nlz = lz.getSliceView();
    645                        if (lz instanceof Dataset) {
    646                                nlz = ((Dataset) lz).reshape(nshape);
    647                        } else {
    648                                nlz = lz.getSliceView();
    649                                nlz.setShape(nshape);
    650                        }
    651                        return nlz;
    652                }
    653        }
    654
    655        class MdsTranspose implements MetadatasetAnnotationOperation {
    656                int[] map;
    657
    658                public MdsTranspose(final int[] axesMap) {
    659                        map = axesMap;
    660                }
    661
    662                @SuppressWarnings({ "rawtypes", "unchecked" })
    663                @Override
    664                public Object processField(Field f, Object o) {
    665                        // reorder arrays and lists according the axes map
    666                        if (o.getClass().isArray()) {
    667                                int l = Array.getLength(o);
    668                                if (l == map.length) {
    669                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
    670                                        for (int i = 0; i < l; i++) {
    671                                                Array.set(narray, i, Array.get(o, map[i]));
    672                                        }
    673                                        for (int i = 0; i < l; i++) {
    674                                                Array.set(o, i, Array.get(narray, i));
    675                                        }
    676                                }
    677                        } else if (o instanceof List<?>) {
    678                                List list = (List) o;
    679                                int l = list.size();
    680                                if (l == map.length) {
    681                                        Object narray = Array.newInstance(o.getClass().getComponentType(), l);
    682                                        for (int i = 0; i < l; i++) {
    683                                                Array.set(narray, i, list.get(map[i]));
    684                                        }
    685                                        list.clear();
    686                                        for (int i = 0; i < l; i++) {
    687                                                list.add(Array.get(narray, i));
    688                                        }
    689                                }
    690                        }
    691                        return o;
    692                }
    693
    694                @Override
    695                public Class<? extends Annotation> getAnnClass() {
    696                        return Transposable.class;
    697                }
    698
    699                @Override
    700                public int change(int axis) {
    701                        return 0;
    702                }
    703
    704                @Override
    705                public int getNewRank() {
    706                        return -1;
    707                }
    708
    709                @Override
    710                public ILazyDataset run(ILazyDataset lz) {
    711                        return lz.getTransposedView(map);
    712                }
    713        }
    714
    715        class MdsDirty implements MetadatasetAnnotationOperation {
    716
    717                @Override
    718                public Object processField(Field f, Object o) {
    719                        // throw exception if not boolean???
    720                        Class<?> t = f.getType();
    721                        if (t.equals(boolean.class) || t.equals(Boolean.class)) {
    722                                if (o.equals(false)) {
    723                                        o = true;
    724                                }
    725                        }
    726                        return o;
    727                }
    728
    729                @Override
    730                public Class<? extends Annotation> getAnnClass() {
    731                        return Dirtiable.class;
    732                }
    733
    734                @Override
    735                public int change(int axis) {
    736                        return 0;
    737                }
    738
    739                @Override
    740                public int getNewRank() {
    741                        return -1;
    742                }
    743
    744                @Override
    745                public ILazyDataset run(ILazyDataset lz) {
    746                        return lz;
    747                }
    748        }
    749
    750        /**
    751         * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced
    752         * dataset after cloning the metadata
    753         * @param asView if true then just a view
    754         * @param slice
    755         */
    756        protected void sliceMetadata(boolean asView, final SliceND slice) {
    757                processAnnotatedMetadata(new MdsSlice(asView, slice.getStart(), slice.getStop(), slice.getStep(), slice.getSourceShape()), true);
    758        }
    759
    760        /**
    761         * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing
    762         * or setting the shape
    763         *
    764         * @param newShape
    765         */
    766        protected void reshapeMetadata(final int[] oldShape, final int[] newShape) {
    767                processAnnotatedMetadata(new MdsReshape(oldShape, newShape), true);
    768        }
    769
    770        /**
    771         * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed
    772         * dataset after cloning the metadata
    773         * @param axesMap
    774         */
    775        protected void transposeMetadata(final int[] axesMap) {
    776                processAnnotatedMetadata(new MdsTranspose(axesMap), true);
    777        }
    778
    779        /**
    780         * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified
    781         * @since 2.0
    782         */
    783        protected void dirtyMetadata() {
    784                processAnnotatedMetadata(new MdsDirty(), true);
    785        }
    786
    787        @SuppressWarnings("unchecked")
    788        private void processAnnotatedMetadata(MetadatasetAnnotationOperation op, boolean throwException) {
    789                if (metadata == null)
    790                        return;
    791
    792                for (Class<? extends MetadataType> c : metadata.keySet()) {
    793                        for (MetadataType m : metadata.get(c)) {
    794                                if (m == null)
    795                                        continue;
    796
    797                                Class<? extends MetadataType> mc = m.getClass();
    798                                do { // iterate over super-classes
    799                                        processClass(op, m, mc, throwException);
    800                                        Class<?> sclazz = mc.getSuperclass();
    801                                        if (!MetadataType.class.isAssignableFrom(sclazz))
    802                                                break;
    803                                        mc = (Class<? extends MetadataType>) sclazz;
    804                                } while (true);
    805                        }
    806                }
    807        }
    808
    809        @SuppressWarnings({ "unchecked", "rawtypes" })
    810        private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc, boolean throwException) {
    811                for (Field f : mc.getDeclaredFields()) {
    812                        if (!f.isAnnotationPresent(op.getAnnClass()))
    813                                continue;
    814
    815                        try {
    816                                f.setAccessible(true);
    817                                Object o = f.get(m);
    818                                if (o == null)
    819                                        continue;
    820
    821                                Object no = op.processField(f, o);
    822                                if (no != o) {
    823                                        f.set(m, no);
    824                                        continue;
    825                                }
    826                                Object r = null;
    827                                if (o instanceof ILazyDataset) {
    828                                        try {
    829                                                f.set(m, op.run((ILazyDataset) o));
    830                                        } catch (Exception e) {
    831                                                logger.error("Problem processing " + o, e);
    832                                                if (!catchExceptions)
    833                                                        throw e;
    834                                        }
    835                                } else if (o.getClass().isArray()) {
    836                                        int l = Array.getLength(o);
    837                                        if (l <= 0)
    838                                                continue;
    839
    840                                        for (int i = 0; r == null && i < l; i++) {
    841                                                r = Array.get(o, i);
    842                                        }
    843                                        int n = op.getNewRank();
    844                                        if (r == null) {
    845                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
    846                                                        f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n));
    847                                                }
    848                                                continue;
    849                                        }
    850                                        if (n < 0)
    851                                                n = l;
    852                                        Object narray = Array.newInstance(r.getClass(), n);
    853                                        for (int i = 0, si = 0, di = 0; di < n && si < l; i++) {
    854                                                int c = op.change(i);
    855                                                if (c == 0) {
    856                                                        Array.set(narray, di++, processObject(op, Array.get(o, si++)));
    857                                                } else if (c > 0) {
    858                                                        di += c; // add nulls by skipping forward in destination array
    859                                                } else if (c < 0) {
    860                                                        si -= c; // remove dimensions by skipping forward in source array
    861                                                }
    862                                        }
    863                                        if (n == l) {
    864                                                for (int i = 0; i < l; i++) {
    865                                                        Array.set(o, i, Array.get(narray, i));
    866                                                }
    867                                        } else {
    868                                                f.set(m, narray);
    869                                        }
    870                                } else if (o instanceof List<?>) {
    871                                        List list = (List) o;
    872                                        int l = list.size();
    873                                        if (l <= 0)
    874                                                continue;
    875
    876                                        for (int i = 0; r == null && i < l; i++) {
    877                                                r = list.get(i);
    878                                        }
    879                                        int n = op.getNewRank();
    880                                        if (r == null) {
    881                                                if (n < 0 || n != l) { // all nulls be need to match rank as necessary
    882                                                        list.clear();
    883                                                        for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) {
    884                                                                list.add(null);
    885                                                        }
    886                                                }
    887                                                continue;
    888                                        }
    889
    890                                        if (n < 0)
    891                                                n = l;
    892                                        Object narray = Array.newInstance(r.getClass(), n);
    893                                        for (int i = 0, si = 0, di = 0; i < l && si < l; i++) {
    894                                                int c = op.change(i);
    895                                                if (c == 0) {
    896                                                        Array.set(narray, di++, processObject(op, list.get(si++)));
    897                                                } else if (c > 0) {
    898                                                        di += c; // add nulls by skipping forward in destination array
    899                                                } else if (c < 0) {
    900                                                        si -= c; // remove dimensions by skipping forward in source array
    901                                                }
    902                                        }
    903                                        list.clear();
    904                                        for (int i = 0; i < n; i++) {
    905                                                list.add(Array.get(narray, i));
    906                                        }
    907                                } else if (o instanceof Map<?,?>) {
    908                                        Map map = (Map) o;
    909                                        for (Object k : map.keySet()) {
    910                                                map.put(k, processObject(op, map.get(k)));
    911                                        }
    912                                }
    913                        } catch (Exception e) {
    914                                logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e);
    915                                if (throwException)
    916                                        throw new RuntimeException(e);
    917                        }
    918                }
    919        }
    920
    921        @SuppressWarnings({ "unchecked", "rawtypes" })
    922        private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception {
    923                if (o == null)
    924                        return o;
    925
    926                if (o instanceof ILazyDataset) {
    927                        try {
    928                                return op.run((ILazyDataset) o);
    929                        } catch (Exception e) {
    930                                logger.error("Problem processing " + o, e);
    931                                if (!catchExceptions)
    932                                        throw e;
    933                        }
    934                } else if (o.getClass().isArray()) {
    935                        int l = Array.getLength(o);
    936                        for (int i = 0; i < l; i++) {
    937                                Array.set(o, i, processObject(op, Array.get(o, i)));
    938                        }
    939                } else if (o instanceof List<?>) {
    940                        List list = (List) o;
    941                        for (int i = 0, imax = list.size(); i < imax; i++) {
    942                                list.set(i, processObject(op, list.get(i)));
    943                        }
    944                } else if (o instanceof Map<?,?>) {
    945                        Map map = (Map) o;
    946                        for (Object k : map.keySet()) {
    947                                map.put(k, processObject(op, map.get(k)));
    948                        }
    949                }
    950                return o;
    951        }
    952
    953        protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) {
    954                for (Class<? extends MetadataType> mc : oldMetadata.keySet()) {
    955                        metadata.put(mc, oldMetadata.get(mc));
    956                }
    957        }
    958
    959        @SuppressWarnings("deprecation")
    960        protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) {
    961                ILazyDataset d = null;
    962                if (blob instanceof ILazyDataset) {
    963                        d = (ILazyDataset) blob;
    964                        if (d instanceof IDataset) {
    965                                Dataset ed = DatasetUtils.convertToDataset((IDataset) d);
    966                                int is = ed.getElementsPerItem();
    967                                if (is != 1 && is != getElementsPerItem()) {
    968                                        throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset");
    969                                }
    970                                d = ed.cast(is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64);
    971                        } else if (!keepLazy) {
    972                                final int is = getElementsPerItem();
    973                                try {
    974                                        d = DatasetUtils.cast(d.getSlice(), is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64);
    975                                } catch (DatasetException e) {
    976                                        logger.error("Could not get data from lazy dataset", e);
    977                                        return null;
    978                                }
    979                        }
    980                } else {
    981                        final int is = getElementsPerItem();
    982                        if (is == 1) {
    983                                d = DatasetFactory.createFromObject(Dataset.FLOAT64, blob);
    984                        } else {
    985                                try {
    986                                        d = DatasetFactory.createFromObject(is, Dataset.ARRAYFLOAT64, blob);
    987                                } catch (IllegalArgumentException e) { // if only single value supplied try again
    988                                        d = DatasetFactory.createFromObject(Dataset.FLOAT64, blob);
    989                                }
    990                        }
    991                        if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) {
    992                                d.setShape(shape.clone());
    993                        }
    994                }
    995                List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape());
    996                d.setShape(s.get(0));
    997
    998                return d;
    999        }
    1000
    1001        @Override
    1002        public void setErrors(Serializable errors) {
    1003                if (shape == null) {
    1004                        throw new IllegalArgumentException("Cannot set errors for null dataset");
    1005                }
    1006                if (errors == null) {
    1007                        clearMetadata(ErrorMetadata.class);
    1008                        return;
    1009                }
    1010                if (errors == this) {
    1011                        logger.warn("Ignoring setting error to itself as this will lead to infinite recursion");
    1012                        return;
    1013                }
    1014
    1015                ILazyDataset errorData = createFromSerializable(errors, true);
    1016
    1017                ErrorMetadata emd = getErrorMetadata();
    1018                if (emd == null) {
    1019                        try {
    1020                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
    1021                                setMetadata(emd);
    1022                        } catch (MetadataException me) {
    1023                                logger.error("Could not create metadata", me);
    1024                        }
    1025                }
    1026                emd.setError(errorData);
    1027        }
    1028
    1029        protected ErrorMetadata getErrorMetadata() {
    1030                try {
    1031                        List<ErrorMetadata> el = getMetadata(ErrorMetadata.class);
    1032                        if (el != null && !el.isEmpty()) {
    1033                                 return el.get(0);
    1034                        }
    1035                } catch (Exception e) {
    1036                }
    1037                return null;
    1038        }
    1039
    1040        @Override
    1041        public ILazyDataset getErrors() {
    1042                ErrorMetadata emd = getErrorMetadata();
    1043                return emd == null ? null : emd.getError();
    1044        }
    1045
    1046        @Override
    1047        public boolean hasErrors() {
    1048                return LazyDatasetBase.this.getErrors() != null;
    1049        }
    1050
    1051        /**
    1052         * Check permutation axes
    1053         * @param shape
    1054         * @param axes
    1055         * @return cleaned up axes or null if trivial
    1056         */
    1057        public static int[] checkPermutatedAxes(int[] shape, int... axes) {
    1058                int rank = shape.length;
    1059
    1060                if (axes == null || axes.length == 0) {
    1061                        axes = new int[rank];
    1062                        for (int i = 0; i < rank; i++) {
    1063                                axes[i] = rank - 1 - i;
    1064                        }
    1065                }
    1066
    1067                if (axes.length != rank) {
    1068                        logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank);
    1069                        throw new IllegalArgumentException("axis permutation does not match shape of dataset");
    1070                }
    1071
    1072                // check all permutation values are within bounds
    1073                for (int d : axes) {
    1074                        if (d < 0 || d >= rank) {
    1075                                logger.error("axis permutation contains element {} outside rank of dataset", d);
    1076                                throw new IllegalArgumentException("axis permutation contains element outside rank of dataset");
    1077                        }
    1078                }
    1079
    1080                // check for a valid permutation (is this an unnecessary restriction?)
    1081                int[] perm = axes.clone();
    1082                Arrays.sort(perm);
    1083
    1084                for (int i = 0; i < rank; i++) {
    1085                        if (perm[i] != i) {
    1086                                logger.error("axis permutation is not valid: it does not contain complete set of axes");
    1087                                throw new IllegalArgumentException("axis permutation does not contain complete set of axes");
    1088                        }
    1089                }
    1090
    1091                if (Arrays.equals(axes, perm))
    1092                        return null; // signal identity or trivial permutation
    1093
    1094                return axes;
    1095        }
    1096}