001/*-
002 *******************************************************************************
003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *    Peter Chang - initial API and implementation and/or initial documentation
011 *******************************************************************************/
012
013package org.eclipse.january.dataset;
014
015import java.io.Serializable;
016import java.lang.reflect.Array;
017import java.text.Format;
018import java.util.ArrayList;
019import java.util.Arrays;
020import java.util.List;
021import java.util.concurrent.ConcurrentHashMap;
022import java.util.concurrent.ConcurrentMap;
023
024import org.eclipse.january.DatasetException;
025import org.eclipse.january.IMonitor;
026import org.eclipse.january.MetadataException;
027import org.eclipse.january.metadata.ErrorMetadata;
028import org.eclipse.january.metadata.MetadataFactory;
029import org.eclipse.january.metadata.MetadataType;
030import org.eclipse.january.metadata.StatisticsMetadata;
031import org.eclipse.january.metadata.internal.ErrorMetadataImpl;
032import org.eclipse.january.metadata.internal.StatisticsMetadataImpl;
033
034/**
035 * Generic container class for data 
036 * <p/>
037 * Each subclass has an array of primitive types, elements of this array are grouped or
038 * compounded to make items 
039 * <p/>
040 * Data items can be boolean, integer, float, complex float, vector float, etc
041 */
042public abstract class AbstractDataset extends LazyDatasetBase implements Dataset {
043        // pin UID to base class
044        private static final long serialVersionUID = Dataset.serialVersionUID;
045
046        protected int size; // number of items
047
048        transient protected AbstractDataset base; // is null when not a view
049        protected int[] stride; // can be null for row-major, contiguous datasets
050        protected int offset;
051
052        /**
053         * The data itself, held in a 1D array, but the object will wrap it to appear as possessing as many dimensions as
054         * wanted
055         */
056        protected Serializable odata = null;
057
058        /**
059         * Set aliased data as base data
060         */
061        abstract protected void setData();
062
063        /**
064         * Constructor required for serialisation.
065         */
066        public AbstractDataset() {
067        }
068
069        @Override
070        public synchronized Dataset synchronizedCopy() {
071                return clone();
072        }
073
074        @Override
075        public boolean equals(Object obj) {
076                if (this == obj) {
077                        return true;
078                }
079                if (obj == null) {
080                        return false;
081                }
082                if (!getClass().equals(obj.getClass())) {
083                        if (getRank() == 0) // for zero-rank datasets
084                                return obj.equals(getObjectAbs(0));
085                        return false;
086                }
087
088                Dataset other = (Dataset) obj;
089                if (getElementsPerItem() != other.getElementsPerItem())
090                        return false;
091                if (size != other.getSize())
092                        return false;
093                if (!Arrays.equals(shape, other.getShapeRef())) {
094                        return false;
095                }
096
097                return true;
098        }
099
100        @Override
101        public int hashCode() {
102                return getStats().getHash(shape);
103        }
104
105        @Override
106        abstract public AbstractDataset clone();
107
108        protected Format stringFormat = null;
109
110        @Override
111        public void setStringFormat(Format format) {
112                stringFormat = format;
113        }
114
115        @Override
116        public Dataset copy(final int dtype) {
117                if (getDType() == dtype) {
118                        return clone();
119                }
120                return DatasetUtils.copy(this, dtype);
121        }
122
123        @Override
124        public <T extends Dataset> T copy(Class<T> clazz) {
125                return DatasetUtils.copy(clazz, this);
126        }
127
128        @Override
129        public Dataset cast(final int dtype) {
130                if (getDType() == dtype) {
131                        return this;
132                }
133                return DatasetUtils.cast(this, dtype);
134        }
135
136        @Override
137        public <T extends Dataset> T cast(Class<T> clazz) {
138                return DatasetUtils.cast(clazz, this);
139        }
140
141        @Override
142        public Dataset cast(final boolean repeat, final int dtype, final int isize) {
143                if (getDType() == dtype && getElementsPerItem() == isize) {
144                        return this;
145                }
146                return DatasetUtils.cast(this, repeat, dtype, isize);
147        }
148
149        @Override
150        abstract public AbstractDataset getView(boolean deepCopyMetadata);
151
152        /**
153         * Copy fields from original to view
154         * @param orig
155         * @param view
156         * @param clone if true, then clone everything but bulk data
157         * @param cloneMetadata if true, clone metadata
158         */
159        protected static void copyToView(Dataset orig, AbstractDataset view, boolean clone, boolean cloneMetadata) {
160                view.name = orig.getName();
161                view.size = orig.getSize();
162                view.odata = orig.getBuffer();
163                view.offset = orig.getOffset();
164                view.base = orig instanceof AbstractDataset ? ((AbstractDataset) orig).base : null;
165
166                if (clone) {
167                        view.shape = orig.getShape();
168                        view.stride = orig instanceof AbstractDataset && ((AbstractDataset) orig).stride != null ?
169                                        ((AbstractDataset) orig).stride.clone() : null;
170                } else {
171                        view.shape = orig.getShapeRef();
172                        view.stride = orig instanceof AbstractDataset ? ((AbstractDataset) orig).stride : null;
173                }
174
175                view.metadata = getMetadataMap(orig, cloneMetadata);
176                int odtype = orig.getDType();
177                int vdtype = view.getDType();
178                if (odtype != vdtype) {
179                        view.setDirty();
180                }
181        }
182
183        /**
184         * @since 2.0
185         */
186        protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> getMetadataMap(Dataset a, boolean clone) {
187                if (a == null)
188                        return null;
189
190                List<MetadataType> all = null;
191                try {
192                        all = a.getMetadata(null);
193                } catch (Exception e) {
194                }
195                if (all == null)
196                        return null;
197
198                ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>();
199
200                for (MetadataType m : all) {
201                        if (m == null) {
202                                continue;
203                        }
204                        Class<? extends MetadataType> c = findMetadataTypeSubInterfaces(m.getClass());
205                        List<MetadataType> l = map.get(c);
206                        if (l == null) {
207                                l = new ArrayList<MetadataType>();
208                                map.put(c, l);
209                        }
210                        if (clone)
211                                m = m.clone();
212                        l.add(m);
213                }
214                return map;
215        }
216
217        @Override
218        public IntegerDataset getIndices() {
219                final IntegerDataset ret = DatasetUtils.indices(shape);
220                if (getName() != null) {
221                        ret.setName("Indices of " + getName());
222                }
223                return ret;
224        }
225
226        @Override
227        public Dataset getTransposedView(int... axes) {
228                axes = checkPermutatedAxes(shape, axes);
229
230                AbstractDataset t = getView(true);
231                if (axes == null || getRank() == 1)
232                        return t;
233
234                int rank = shape.length;
235                int[] tstride = new int[rank];
236                int[] toffset = new int[1];
237                int[] nshape = createStrides(new SliceND(shape), this, tstride, toffset);
238                int[] nstride = new int[rank];
239                for (int i = 0; i < rank; i++) {
240                        final int ax = axes[i];
241                        nstride[i] = tstride[ax];
242                        nshape[i] = shape[ax];
243                }
244                t.shape = nshape;
245                t.stride = nstride;
246                t.offset = toffset[0];
247                t.base = this;
248                t.setDirty();
249                t.transposeMetadata(axes);
250                return t;
251        }
252
253        @Override
254        public Dataset transpose(int... axes) {
255                Dataset t = getTransposedView(axes);
256                return t == null ? clone() : t.clone();
257        }
258
259        @Override
260        public Dataset swapAxes(int axis1, int axis2) {
261                int rank = shape.length;
262                if (axis1 < 0)
263                        axis1 += rank;
264                if (axis2 < 0)
265                        axis2 += rank;
266
267                if (axis1 < 0 || axis2 < 0 || axis1 >= rank || axis2 >= rank) {
268                        logger.error("Axis value invalid - out of range");
269                        throw new IllegalArgumentException("Axis value invalid - out of range");
270                }
271
272                if (rank == 1 || axis1 == axis2) {
273                        return this;
274                }
275
276                int[] axes = new int[rank];
277                for (int i = 0; i < rank; i++) {
278                        axes[i] = i;
279                }               
280
281                axes[axis1] = axis2;
282                axes[axis2] = axis1;
283                return getTransposedView(axes);
284        }
285
286        private boolean isContiguous() {
287                if (stride == null)
288                        return true;
289
290                if (offset != 0)
291                        return false;
292
293                int s = getElementsPerItem();
294                for (int j = getRank() - 1; j >= 0; j--) {
295                        if (stride[j] != s) {
296                                return false;
297                        }
298                        s *= shape[j];
299                }
300
301                return true;
302        }
303
304        @Override
305        public Dataset flatten() {
306                if (!isContiguous()) { // need to make a copy if not contiguous
307                        return clone().flatten();
308                }
309                return reshape(size);
310        }
311
312        /**
313         * Fill dataset from object at depth dimension
314         * @param obj
315         * @param depth
316         * @param pos position
317         */
318        protected void fillData(Object obj, final int depth, final int[] pos) {
319                if (obj == null) {
320                        int dtype = getDType();
321                        if (dtype == FLOAT32)
322                                set(Float.NaN, pos);
323                        else if (dtype == FLOAT64)
324                                set(Double.NaN, pos);
325                        return;
326                }
327
328                Class<?> clazz = obj.getClass();
329                if (obj instanceof List<?>) {
330                        List<?> jl = (List<?>) obj;
331                        int l = jl.size();
332                        for (int i = 0; i < l; i++) {
333                                Object lo = jl.get(i);
334                                fillData(lo, depth + 1, pos);
335                                pos[depth]++;
336                        }
337                        pos[depth] = 0;
338                } else if (clazz.isArray()) {
339                        int l = Array.getLength(obj);
340                        if (clazz.equals(odata.getClass())) {
341                                System.arraycopy(obj, 0, odata, get1DIndex(pos), l);
342                        } else if (clazz.getComponentType().isPrimitive()) {
343                                for (int i = 0; i < l; i++) {
344                                        set(Array.get(obj, i), pos);
345                                        pos[depth]++;
346                                }
347                                pos[depth] = 0;
348                        } else {
349                                for (int i = 0; i < l; i++) {
350                                        fillData(Array.get(obj, i), depth + 1, pos);
351                                        pos[depth]++;
352                                }
353                                pos[depth] = 0;
354                        }
355                } else if (obj instanceof IDataset) {
356                        boolean[] a = new boolean[shape.length];
357                        for (int i = depth; i < a.length; i++)
358                                a[i] = true;
359                        setSlice(obj, getSliceIteratorFromAxes(pos, a));
360                } else {
361                        set(obj, pos);
362                }
363        }
364
365        @Override
366        public IndexIterator getIterator(final boolean withPosition) {
367                if (stride != null) {
368                        return base.getSize() == 1  ? (withPosition ? new PositionIterator(offset, shape) :
369                                new SingleItemIterator(offset, size)) : new StrideIterator(shape, stride, offset);
370                }
371                return withPosition ? new ContiguousIteratorWithPosition(shape, size) : new ContiguousIterator(size);
372        }
373
374        @Override
375        public IndexIterator getIterator() {
376                return getIterator(false);
377        }
378
379        @Override
380        public PositionIterator getPositionIterator(final int... axes) {
381                return new PositionIterator(shape, axes);
382        }
383
384        @Override
385        public IndexIterator getSliceIterator(final int[] start, final int[] stop, final int[] step) {
386                return getSliceIterator(new SliceND(shape, start, stop, step));
387        }
388
389        /**
390         * @param slice
391         * @return an slice iterator that operates like an IndexIterator
392         */
393        public IndexIterator getSliceIterator(SliceND slice) {
394                if (ShapeUtils.calcLongSize(slice.getShape()) == 0) {
395                        return new NullIterator(shape, slice.getShape());
396                }
397                if (stride != null)
398                        return new StrideIterator(getElementsPerItem(), shape, stride, offset, slice);
399
400                return new SliceIterator(shape, size, slice);
401        }
402
403        @Override
404        public SliceIterator getSliceIteratorFromAxes(final int[] pos, boolean[] axes) {
405                int rank = shape.length;
406                int[] start;
407                int[] stop = new int[rank];
408                int[] step = new int[rank];
409
410                if (pos == null) {
411                        start = new int[rank];
412                } else if (pos.length == rank) {
413                        start = pos.clone();
414                } else {
415                        throw new IllegalArgumentException("pos array length is not equal to rank of dataset");
416                }
417                if (axes == null) {
418                        axes = new boolean[rank];
419                        Arrays.fill(axes, true);
420                } else if (axes.length != rank) {
421                        throw new IllegalArgumentException("axes array length is not equal to rank of dataset");
422                }
423
424                for (int i = 0; i < rank; i++) {
425                        if (axes[i]) {
426                                stop[i] = shape[i];
427                        } else {
428                                stop[i] = start[i] + 1;
429                        }
430                        step[i] = 1;
431                }
432                return (SliceIterator) getSliceIterator(start, stop, step);
433        }
434
435        @Override
436        public BooleanIterator getBooleanIterator(Dataset choice) {
437                return getBooleanIterator(choice, true);
438        }
439
440        @Override
441        public BooleanIterator getBooleanIterator(Dataset choice, boolean value) {
442                return new BooleanIterator(getIterator(), choice, value);
443        }
444
445        @Override
446        public Dataset getByBoolean(Dataset selection) {
447                checkCompatibility(selection);
448
449                final int length = ((Number) selection.sum()).intValue();
450                final int is = getElementsPerItem();
451                @SuppressWarnings("deprecation")
452                Dataset r = DatasetFactory.zeros(is, new int[] { length }, getDType());
453                BooleanIterator biter = getBooleanIterator(selection);
454
455                int i = 0;
456                while (biter.hasNext()) {
457                        r.setObjectAbs(i, getObjectAbs(biter.index));
458                        i += is;
459                }
460                return r;
461        }
462
463        @Override
464        public Dataset getBy1DIndex(IntegerDataset index) {
465                final int is = getElementsPerItem();
466                @SuppressWarnings("deprecation")
467                final Dataset r = DatasetFactory.zeros(is, index.getShape(), getDType());
468                final IntegerIterator iter = new IntegerIterator(index, size, is);
469
470                int i = 0;
471                while (iter.hasNext()) {
472                        r.setObjectAbs(i, getObjectAbs(iter.index));
473                        i += is;
474                }
475                return r;
476        }
477
478        @Override
479        public Dataset getByIndexes(final Object... indexes) {
480                final IntegersIterator iter = new IntegersIterator(shape, indexes);
481                final int is = getElementsPerItem();
482                @SuppressWarnings("deprecation")
483                final Dataset r = DatasetFactory.zeros(is, iter.getShape(), getDType());
484
485                final int[] pos = iter.getPos();
486                int i = 0;
487                while (iter.hasNext()) {
488                        r.setObjectAbs(i, getObject(pos));
489                        i += is;
490                }
491                return r;
492        }
493
494        @Override
495        public Class<?> getElementClass() {
496                return DTypeUtils.getElementClass(getDType());
497        }
498
499        @Override
500        public boolean hasFloatingPointElements() {
501                Class<?> cls = getElementClass();
502                return cls == Float.class || cls == Double.class;
503        }
504
505        @Override
506        public int getElementsPerItem() {
507                return DTypeUtils.getElementsPerItem(getDType());
508        }
509
510        @Override
511        public int getItemBytes() {
512                return DTypeUtils.getItemBytes(getDType(), getElementsPerItem());
513        }
514
515        @Override
516        public String getName() {
517                return name;
518        }
519
520        @Override
521        public void setName(final String name) {
522                this.name = name;
523        }
524
525        @Override
526        public int getSize() {
527                return size;
528        }
529
530        @Override
531        public int[] getShape() {
532                // make a copy of the dimensions data, and put that out
533                if (shape == null) {
534                        logger.warn("Shape is null!!!");
535                        return new int[] {};
536                }
537                return shape.clone();
538        }
539
540        @Override
541        public int getRank() {
542                return shape == null ? 0 : shape.length;
543        }
544
545        @Override
546        public int getNbytes() {
547                return getSize() * getItemBytes();
548        }
549
550        /**
551         * Check for -1 placeholder in shape and replace if necessary
552         * @param shape
553         * @param size
554         */
555        private void checkShape(int[] shape, int size) {
556                int rank = shape.length;
557                int found = -1;
558                int nsize = 1;
559                for (int i = 0; i < rank; i++) {
560                        int d = shape[i];
561                        if (d == -1) {
562                                if (found == -1) {
563                                        found = i;
564                                } else {
565                                        logger.error("Can only have one -1 placeholder in shape");
566                                        throw new IllegalArgumentException("Can only have one -1 placeholder in shape");
567                                }
568                        } else {
569                                nsize *= d;
570                        }
571                }
572                if (found >= 0) {
573                        shape[found] = size/nsize;
574                } else if (nsize != size && !(rank == 0 && size == 0)) {
575                        logger.error("New shape is not same size as old shape");
576                        throw new IllegalArgumentException("New size is not same as the old size. Old size is "+size+" new size is "+nsize+" and shape is "+Arrays.toString(shape));
577                }
578        }
579
580        @Override
581        public void setShape(final int... shape) {
582                int[] nshape = shape.clone();
583                checkShape(nshape, size);
584                if (Arrays.equals(this.shape, nshape)) {
585                        return;
586                }
587
588                if (stride != null) {
589                        // the only compatible shapes are ones where new dimensions are factors of old dimensions
590                        // or are combined adjacent old dimensions 
591                        int[] oshape = this.shape;
592                        int orank = oshape.length;
593                        int nrank = nshape.length;
594                        int diff = nrank - orank;
595                        int[] nstride = new int[nrank];
596                        boolean ones = true;
597                        // work forwards for broadcasting cases
598                        for (int i = 0, j = 0; i < orank || j < nrank;) {
599                                if (j >= diff && i < orank && j < nrank && oshape[i] == nshape[j]) {
600                                        nstride[j++] = stride[i++];
601                                } else if (j < nrank && nshape[j] == 1) {
602                                        nstride[j++] = 0;
603                                } else if (i < orank && oshape[i] == 1) {
604                                        i++;
605                                } else {
606                                        if (j < nrank)
607                                                j++;
608                                        if (i < orank)
609                                                i++;
610                                        ones = false;
611                                }
612                        }
613                        if (!ones) { // not just ones differ in shapes
614                                int[] ostride = stride;
615                                int ob = 0;
616                                int oe = 1;
617                                int nb = 0;
618                                int ne = 1;
619                                while (ob < orank && nb < nrank) {
620                                        int ol = oshape[ob];
621                                        int nl = nshape[nb];
622                                        
623                                        if (nl < ol) { // find group of shape dimensions that form common size
624                                                do { // case where new shape spreads single dimension over several dimensions
625                                                        if (ne == nrank) {
626                                                                break;
627                                                        }
628                                                        nl *= nshape[ne++];
629                                                } while (nl < ol);
630                                                if (nl != ol) {
631                                                        logger.error("Subshape is incompatible with single dimension");
632                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
633                                                }
634                                                int on = ne - 1;
635                                                while (nshape[on] == 1) {
636                                                        on--;
637                                                }
638
639                                                nstride[on] = ostride[ob];
640                                                for (int n = on - 1; n >= nb; n--) {
641                                                        if (nshape[n] == 1)
642                                                                continue;
643
644                                                        nstride[n] = nshape[on] * nstride[on];
645                                                        on = n;
646                                                }
647                                        } else if (ol < nl) {
648                                                do { // case where new shape combines several dimensions into one dimension
649                                                        if (oe == orank) {
650                                                                break;
651                                                        }
652                                                        ol *= oshape[oe++];
653                                                } while (ol < nl);
654                                                if (nl != ol) {
655                                                        logger.error("Single dimension is incompatible with subshape");
656                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
657                                                }
658
659                                                int oo = oe - 1;
660                                                while (oshape[oo] == 1) {
661                                                        oo--;
662                                                }
663                                                int os = ostride[oo];
664                                                for (int o = oo - 1; o >= ob; o--) {
665                                                        if (oshape[o] == 1)
666                                                                continue;
667                                                        if (ostride[o] != oshape[oo] * ostride[oo]) {
668                                                                logger.error("Subshape cannot be a non-contiguous view");
669                                                                throw new IllegalArgumentException("Subshape cannot be a non-contiguous view");
670                                                        }
671                                                        oo = o;
672                                                }
673                                                nstride[nb] = os;
674                                        } else {
675                                                nstride[nb] = ostride[ob];
676                                        }
677
678                                        ob = oe++;
679                                        nb = ne++;
680                                }
681                        }
682        
683                        stride = nstride;
684                }
685
686                setDirty();
687                if (this.shape != null) {
688                        reshapeMetadata(this.shape, nshape);
689                        this.shape = nshape;
690                }
691        }
692
693        @Override
694        public int[] getShapeRef() {
695                return shape;
696        }
697
698        @Override
699        public int getOffset() {
700                return offset;
701        }
702
703        @Override
704        public int[] getStrides() {
705                return stride;
706        }
707
708        @Override
709        public Serializable getBuffer() {
710                return odata;
711        }
712
713        @Override
714        public void overrideInternal(Serializable buffer, int... shape) {
715                if (buffer != null) {
716                        odata = buffer;
717                        setData();
718                        setDirty();
719                }
720        
721                if (shape != null) {
722                        this.shape = shape.clone();
723                        size = ShapeUtils.calcSize(this.shape);
724                }
725        }
726
727        /**
728         * Create a stride array from dataset
729         * @param a dataset
730         * @param offset output offset
731         * @return new strides
732         */
733        public static int[] createStrides(Dataset a, final int[] offset) {
734                return createStrides(a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), offset);
735        }
736
737        /**
738         * Create a stride array from dataset
739         * @param isize
740         * @param shape
741         * @param oStride original stride
742         * @param oOffset original offset (only used if there is an original stride)
743         * @param offset output offset
744         * @return new strides
745         */
746        public static int[] createStrides(final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] offset) {
747                int rank = shape.length;
748                final int[] stride;
749                if (oStride == null) {
750                        offset[0] = 0;
751                        stride = new int[rank];
752                        int s = isize;
753                        for (int j = rank - 1; j >= 0; j--) {
754                                stride[j] = s;
755                                s *= shape[j];
756                        }
757                } else {
758                        offset[0] = oOffset;
759                        stride = oStride.clone();
760                }
761                return stride;
762        }
763
764        /**
765         * Create a stride array from slice information and a dataset
766         * @param slice
767         * @param a dataset
768         * @param stride output stride
769         * @param offset output offset
770         * @return new shape
771         */
772        public static int[] createStrides(final SliceND slice, final Dataset a, final int[] stride, final int[] offset) {
773                return createStrides(slice, a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), stride, offset);
774        }
775
776        /**
777         * Create a stride array from slice and dataset information
778         * @param slice
779         * @param isize
780         * @param shape
781         * @param oStride original stride
782         * @param oOffset original offset (only used if there is an original stride)
783         * @param stride output stride
784         * @param offset output offset
785         * @return new shape
786         */
787        public static int[] createStrides(final SliceND slice, final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] stride, final int[] offset) {
788                int[] lstart = slice.getStart();
789                int[] lstep = slice.getStep();
790                int[] newShape = slice.getShape();
791                int rank = shape.length;
792
793                if (oStride == null) {
794                        int s = isize;
795                        offset[0] = 0;
796                        for (int j = rank - 1; j >= 0; j--) {
797                                stride[j] = s * lstep[j];
798                                offset[0] += s * lstart[j];
799                                s *= shape[j];
800                        }
801                } else {
802                        offset[0] = oOffset;
803                        for (int j = 0; j < rank; j++) {
804                                int s = oStride[j];
805                                stride[j] = lstep[j] * s;
806                                offset[0] += lstart[j] * s;
807                        }
808                }
809
810                return newShape;
811        }
812
813        @Override
814        public Dataset getBroadcastView(int... broadcastShape) {
815                AbstractDataset view = getView(true);
816                
817                if (!Arrays.equals(shape, broadcastShape)) {
818                        List<int[]> nShapes = BroadcastUtils.broadcastShapesToMax(broadcastShape, shape);
819                        view.setShape(nShapes.get(0));
820                        view.stride = BroadcastUtils.createBroadcastStrides(view, broadcastShape);
821                        view.base = this;
822                        view.shape = broadcastShape.clone();
823                        view.size = ShapeUtils.calcSize(broadcastShape);
824                        if (view.name == null || view.name.isEmpty()) {
825                                view.name = "Broadcast from " + Arrays.toString(shape);
826                        } else {
827                                view.name = "Broadcast of " + view.name + " from " + Arrays.toString(shape);
828                        }
829                }
830                return view;
831        }
832
833        @Override
834        public Dataset getSliceView(final int[] start, final int[] stop, final int[] step) {
835                return getSliceView(new SliceND(shape, start, stop, step));
836        }
837
838        @Override
839        public Dataset getSliceView(Slice... slice) {
840                if (slice == null || slice.length == 0) {
841                        return getView(true);
842                }
843
844                return getSliceView(new SliceND(shape, slice));
845        }
846
847        /**
848         * Get a slice of the dataset. The returned dataset is a view on a selection of items
849         * @param slice
850         * @return slice view
851         */
852        @Override
853        public Dataset getSliceView(SliceND slice) {
854                if (slice.isAll()) {
855                        return getView(true);
856                }
857
858                final int rank = shape.length;
859                int[] sStride = new int[rank];
860                int[] sOffset = new int[1];
861
862                int[] sShape = createStrides(slice, this, sStride, sOffset);
863        
864                AbstractDataset s = getView(false);
865                s.shape = sShape;
866                s.size = ShapeUtils.calcSize(sShape);
867                s.stride = sStride;
868                s.offset = sOffset[0];
869                s.base = this;
870
871                s.metadata = copyMetadata();
872                s.sliceMetadata(true, slice);
873
874                s.setDirty();
875                s.setName(name + BLOCK_OPEN + slice + BLOCK_CLOSE);
876
877                return s;
878        }
879
880        /**
881         * Get flattened view index of given position 
882         * @param pos
883         *            the integer array specifying the n-D position
884         * @return the index on the flattened dataset
885         */
886        private int getFlat1DIndex(final int[] pos) {
887                final int imax = pos.length;
888                if (imax == 0) {
889                        return 0;
890                }
891
892                return get1DIndexFromShape(pos);
893        }
894
895        /**
896         * @since 2.0
897         */
898        protected int getFirst1DIndex() {
899                if (shape == null) {
900                        throw new IllegalArgumentException("Cannot find an index from a null shape");
901                }
902                return stride == null ? 0 : offset;
903        }
904
905        @Override
906        public int get1DIndex(final int... n) {
907                if (n.length == 0 && shape.length == 0)
908                        return offset;
909
910                return stride == null ? get1DIndexFromShape(n) : get1DIndexFromStrides(n);
911        }
912
913        private static void throwAIOOBException(int i, int s, int d) {
914                throw new ArrayIndexOutOfBoundsException("Index (" + i + ") out of range [-" + s + "," + s
915                                + "] in dimension " + d);
916        }
917
918        /**
919         * @param i
920         * @return the index on the data array corresponding to that location
921         */
922        protected int get1DIndex(int i) {
923                if (shape == null) {
924                        throw new IllegalArgumentException("Cannot find an index from a null shape");
925                }
926                if (shape.length > 1) {
927                        logger.error("This dataset is not 1D but was addressed as such");
928                        throw new UnsupportedOperationException("This dataset is not 1D but was addressed as such");
929                }
930                if (i < 0) {
931                        i += shape[0];
932                }
933                if (i < 0 || i >= shape[0]) {
934                        throwAIOOBException(i, shape[0], 0);
935                }
936                return stride == null ? i : i*stride[0] + offset;
937        }
938
939        /**
940         * @param i
941         * @param j
942         * @return the index on the data array corresponding to that location
943         */
944        protected int get1DIndex(int i, int j) {
945                if (shape == null) {
946                        throw new IllegalArgumentException("Cannot find an index from a null shape");
947                }
948                if (shape.length != 2) {
949                        logger.error("This dataset is not 2D but was addressed as such");
950                        throw new UnsupportedOperationException("This dataset is not 2D but was addressed as such");
951                }
952                if (i < 0) {
953                        i += shape[0];
954                }
955                if (i < 0 || i >= shape[0]) {
956                        throwAIOOBException(i, shape[0], 0);
957                }
958                if (j < 0) {
959                        j += shape[1];
960                }
961                if (j < 0 || j >= shape[1]) {
962                        throwAIOOBException(i, shape[1], 1);
963                }
964                return stride == null ? i*shape[1] + j : i*stride[0] + j*stride[1] + offset;
965        }
966
967        protected int get1DIndexFromShape(final int[] n) {
968                return get1DIndexFromShape(shape, n);
969        }
970
971        protected static int get1DIndexFromShape(final int[] shape, final int[] n) {
972                if (shape == null) {
973                        throw new IllegalArgumentException("Cannot find an index from a null shape");
974                }
975                final int rank = shape.length;
976                if (rank != n.length) {
977                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
978                        logger.error(errMsg);
979                        throw new IllegalArgumentException(errMsg);
980                }
981                int index = 0;
982                for (int i = 0; i < rank; i++) {
983                        final int si = shape[i];
984                        int ni = n[i];
985                        if (ni < 0) {
986                                ni += si;
987                        }
988                        if (ni < 0 || ni >= si) {
989                                throwAIOOBException(ni, si, i);
990                        }
991                        index = index * si + ni;
992                }
993
994                return index;
995        }
996
997        private int get1DIndexFromStrides(final int[] n) {
998                return get1DIndexFromStrides(shape, stride, offset, n);
999        }
1000
1001        private static int get1DIndexFromStrides(final int[] shape, final int[] stride, final int offset, final int[] n) {
1002                if (shape == null) {
1003                        throw new IllegalArgumentException("Cannot find an index from a null shape");
1004                }
1005                final int rank = shape.length;
1006                if (rank != n.length) {
1007                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
1008                        logger.error(errMsg);
1009                        throw new IllegalArgumentException(errMsg);
1010                }
1011                int index = offset;
1012                for (int i = 0; i < rank; i++) {
1013                        final int st = stride[i];
1014                        if (st != 0) { // not broadcasted
1015                                final int si = shape[i];
1016                                int ni = n[i];
1017                                if (ni < 0) {
1018                                        ni += si;
1019                                }
1020                                if (ni < 0 || ni >= si) {
1021                                        throwAIOOBException(ni, si, i);
1022                                }
1023                                index += st * ni;
1024                        }
1025                }
1026                return index;
1027        }
1028
1029        @Override
1030        public int[] getNDPosition(final int n) {
1031                if (isIndexInRange(n)) {
1032                        throw new IllegalArgumentException("Index provided " + n
1033                                        + "is larger then the size of the containing array");
1034                }
1035
1036                return stride == null ? ShapeUtils.getNDPositionFromShape(n, shape) : getNDPositionFromStrides(n);
1037        }
1038
1039        private boolean isIndexInRange(final int n) {
1040                if (stride == null) {
1041                        return n >= size;
1042                }
1043                return n >= getBufferLength();
1044        }
1045
1046        /**
1047         * @return entire buffer length
1048         */
1049        abstract protected int getBufferLength();
1050
1051        private int[] getNDPositionFromStrides(int n) {
1052                n -= offset;
1053                int rank = shape.length;
1054                if (rank == 1) {
1055                        return new int[] { n / stride[0] };
1056                }
1057
1058                int[] output = new int[rank];
1059                int i = 0;
1060                while (i != n) { // TODO find more efficient way than this exhaustive search
1061                        int j = rank - 1;
1062                        for (; j >= 0; j--) {
1063                                output[j]++;
1064                                i += stride[j];
1065                                if (output[j] >= shape[j]) {
1066                                        output[j] = 0;
1067                                        i -= shape[j] * stride[j];
1068                                } else {
1069                                        break;
1070                                }
1071                        }
1072                        if (j == -1) {
1073                                logger.error("Index was not found in this strided dataset");
1074                                throw new IllegalArgumentException("Index was not found in this strided dataset");
1075                        }
1076                }
1077
1078                return output;
1079        }
1080
1081        @Override
1082        public int checkAxis(int axis) {
1083                return checkAxis(shape.length, axis);
1084        }
1085
1086        /**
1087         * Check that axis is in range [-rank,rank)
1088         * 
1089         * @param rank
1090         * @param axis
1091         * @return sanitized axis in range [0, rank)
1092         */
1093        protected static int checkAxis(int rank, int axis) {
1094                if (axis < 0) {
1095                        axis += rank;
1096                }
1097
1098                if (axis < 0 || axis >= rank) {
1099                        throw new IndexOutOfBoundsException("Axis " + axis + " given is out of range [0, " + rank + ")");
1100                }
1101                return axis;
1102        }
1103
1104        protected static final char BLOCK_OPEN = '[';
1105        protected static final char BLOCK_CLOSE = ']';
1106
1107        @Override
1108        public String toString() {
1109                return toString(false);
1110        }
1111
1112        @Override
1113        public String toString(boolean showData) {
1114                final int rank = shape == null ? 0 : shape.length;
1115                final StringBuilder out = new StringBuilder();
1116                if (DTypeUtils.isDTypeElemental(getDType())) {
1117                        out.append("Dataset ");
1118                } else {
1119                        out.append("Compound dataset (");
1120                        out.append(getElementsPerItem());
1121                        out.append(") ");
1122                }
1123
1124                if (!showData) {
1125                        if (name != null && name.length() > 0) {
1126                                out.append("'");
1127                                out.append(name);
1128                                out.append("' has shape ");
1129                        } else {
1130                                out.append("shape is ");
1131                        }
1132
1133                        out.append(BLOCK_OPEN);
1134                        if (rank > 0) {
1135                                out.append(shape[0]);
1136                        }
1137                        for (int i = 1; i < rank; i++) {
1138                                out.append(", " + shape[i]);
1139                        }
1140                        out.append(BLOCK_CLOSE);
1141                        return out.toString();
1142                }
1143
1144                if (size == 0) {
1145                        return out.toString();
1146                }
1147
1148                if (rank > 0) {
1149                        int[] pos = new int[rank];
1150                        final StringBuilder lead = new StringBuilder();
1151                        printBlocks(out, lead, 0, pos);
1152                } else {
1153                        out.append(getString());
1154                }
1155                return out.toString();
1156        }
1157
1158        /**
1159         * Limit to strings output via the toString() method
1160         */
1161        private static int maxStringLength = 120;
1162
1163        /**
1164         * Set maximum line length for toString() method
1165         * @param maxLineLength
1166         */
1167        public static void setMaxLineLength(int maxLineLength) {
1168                maxStringLength = maxLineLength;
1169        }
1170
1171        /**
1172         * @return maximum line length for toString() method
1173         */
1174        public static int getMaxLineLength() {
1175                return maxStringLength;
1176        }
1177
1178        /**
1179         * Limit to number of sub-blocks output via the toString() method
1180         */
1181        private static final int MAX_SUBBLOCKS = 6;
1182
1183        private final static String SEPARATOR = ",";
1184        private final static String SPACE = " ";
1185        private final static String ELLIPSIS = "...";
1186        private final static String NEWLINE = "\n";
1187
1188        /**
1189         * Make a line of output for last dimension of dataset
1190         * 
1191         * @param start
1192         * @return line
1193         */
1194        private StringBuilder makeLine(final int end, final int... start) {
1195                StringBuilder line = new StringBuilder();
1196                final int[] pos;
1197                if (end >= start.length) {
1198                        pos = Arrays.copyOf(start, end + 1);
1199                } else {
1200                        pos = start;
1201                }
1202                pos[end] = 0;
1203                line.append(BLOCK_OPEN);
1204                line.append(getString(pos));
1205
1206                final int length = shape[end];
1207
1208                // trim elements printed if length exceed estimate of maximum elements
1209                int excess = length - maxStringLength / 3; // space + number + separator
1210                if (excess > 0) {
1211                        int index = (length - excess) / 2;
1212                        for (int y = 1; y < index; y++) {
1213                                line.append(SEPARATOR + SPACE);
1214                                pos[end] = y;
1215                                line.append(getString(pos));
1216                        }
1217                        index = (length + excess) / 2;
1218                        for (int y = index; y < length; y++) {
1219                                line.append(SEPARATOR + SPACE);
1220                                pos[end] = y;
1221                                line.append(getString(pos));
1222                        }
1223                } else {
1224                        for (int y = 1; y < length; y++) {
1225                                line.append(SEPARATOR + SPACE);
1226                                pos[end] = y;
1227                                line.append(getString(pos));
1228                        }
1229                }
1230                line.append(BLOCK_CLOSE);
1231
1232                // trim string down to limit
1233                excess = line.length() - maxStringLength - ELLIPSIS.length() - 1;
1234                if (excess > 0) {
1235                        int index = line.substring(0, (line.length() - excess) / 2).lastIndexOf(SEPARATOR) + 2;
1236                        StringBuilder out = new StringBuilder(line.subSequence(0, index));
1237                        out.append(ELLIPSIS + SEPARATOR);
1238                        index = line.substring((line.length() + excess) / 2).indexOf(SEPARATOR) + (line.length() + excess) / 2 + 1;
1239                        out.append(line.subSequence(index, line.length()));
1240                        return out;
1241                }
1242
1243                return line;
1244        }
1245
1246        /**
1247         * recursive method to print blocks
1248         */
1249        private void printBlocks(final StringBuilder out, final StringBuilder lead, final int level, final int[] pos) {
1250                if (out.length() > 0) {
1251                        char last = out.charAt(out.length() - 1);
1252                        if (last != BLOCK_OPEN) {
1253                                out.append(lead);
1254                        }
1255                }
1256                final int end = getRank() - 1;
1257                if (level != end) {
1258                        out.append(BLOCK_OPEN);
1259                        int length = shape[level];
1260
1261                        // first sub-block
1262                        pos[level] = 0;
1263                        StringBuilder newlead = new StringBuilder(lead);
1264                        newlead.append(SPACE);
1265                        printBlocks(out, newlead, level + 1, pos);
1266                        if (length < 2) { // escape
1267                                out.append(BLOCK_CLOSE);
1268                                return;
1269                        }
1270
1271                        out.append(SEPARATOR + NEWLINE);
1272                        for (int i = level + 1; i < end; i++) {
1273                                out.append(NEWLINE);
1274                        }
1275
1276                        // middle sub-blocks
1277                        if (length < MAX_SUBBLOCKS) {
1278                                for (int x = 1; x < length - 1; x++) {
1279                                        pos[level] = x;
1280                                        printBlocks(out, newlead, level + 1, pos);
1281                                        if (end <= level + 1) {
1282                                                out.append(SEPARATOR + NEWLINE);
1283                                        } else {
1284                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
1285                                        }
1286                                }
1287                        } else {
1288                                final int excess = length - MAX_SUBBLOCKS;
1289                                int xmax = (length - excess) / 2;
1290                                for (int x = 1; x < xmax; x++) {
1291                                        pos[level] = x;
1292                                        printBlocks(out, newlead, level + 1, pos);
1293                                        if (end <= level + 1) {
1294                                                out.append(SEPARATOR + NEWLINE);
1295                                        } else {
1296                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
1297                                        }
1298                                }
1299                                out.append(newlead);
1300                                out.append(ELLIPSIS + SEPARATOR + NEWLINE);
1301                                xmax = (length + excess) / 2;
1302                                for (int x = xmax; x < length - 1; x++) {
1303                                        pos[level] = x;
1304                                        printBlocks(out, newlead, level + 1, pos);
1305                                        if (end <= level + 1) {
1306                                                out.append(SEPARATOR + NEWLINE);
1307                                        } else {
1308                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
1309                                        }
1310                                }
1311                        }
1312
1313                        // last sub-block
1314                        pos[level] = length - 1;
1315                        printBlocks(out, newlead, level + 1, pos);
1316                        out.append(BLOCK_CLOSE);
1317                } else {
1318                        out.append(makeLine(end, pos));
1319                }
1320        }
1321
1322        @Override
1323        public void setDirty() {
1324                dirtyMetadata();
1325        }
1326
1327        @Override
1328        public Dataset squeezeEnds() {
1329                return squeeze(true);
1330        }
1331
1332        @Override
1333        public Dataset squeeze() {
1334                return squeeze(false);
1335        }
1336
1337        @Override
1338        public Dataset squeeze(boolean onlyFromEnds) {
1339                final int[] tshape = ShapeUtils.squeezeShape(shape, onlyFromEnds);
1340                final int[] oshape = shape;
1341                if (stride == null) {
1342                        shape = tshape;
1343                } else {
1344                        int rank = shape.length;
1345                        int trank = tshape.length;
1346                        if (trank < rank) {
1347                                int[] tstride = new int[tshape.length];
1348                                if (onlyFromEnds) {
1349                                        for (int i = 0; i < rank; i++) {
1350                                                if (shape[i] != 1) {
1351                                                        for (int k = 0; k < trank; k++) {
1352                                                                tstride[k] = stride[i++];
1353                                                        }
1354                                                        break;
1355                                                }
1356                                        }
1357                                } else {
1358                                        int t = 0;
1359                                        for (int i = 0; i < rank; i++) {
1360                                                if (shape[i] != 1) {
1361                                                        tstride[t++] = stride[i];
1362                                                }
1363                                        }
1364                                }
1365                                shape = tshape;
1366                                stride = tstride;
1367                        }
1368                }
1369
1370                setDirty();
1371                reshapeMetadata(oshape, shape);
1372                return this;
1373        }
1374
1375        @Override
1376        public boolean isCompatibleWith(final ILazyDataset g) {
1377                return ShapeUtils.areShapesCompatible(shape, g.getShape());
1378        }
1379
1380        @Override
1381        public void checkCompatibility(final ILazyDataset g) throws IllegalArgumentException {
1382                ShapeUtils.checkCompatibility(this, g);
1383        }
1384
1385        @Override
1386        public Dataset reshape(final int... shape) {
1387                Dataset a;
1388                try {
1389                        a = getView(true);
1390                        a.setShape(shape);
1391                } catch (IllegalArgumentException e) {
1392                        a = clone();
1393                        a.setShape(shape);
1394                }
1395                return a;
1396        }
1397
1398        /**
1399         * @param start
1400         * @param stop
1401         * @param step
1402         * @return number of steps to take
1403         */
1404        protected static int calcSteps(final double start, final double stop, final double step) {
1405                return Math.max(0, (int) Math.ceil((stop - start) / step));
1406        }
1407
1408        @Override
1409        public boolean isComplex() {
1410                int type = getDType();
1411                return type == COMPLEX64 || type == COMPLEX128;
1412        }
1413
1414        @Override
1415        public Dataset getRealPart() {
1416                return this;
1417        }
1418
1419        @Override
1420        public Dataset getRealView() {
1421                return getView(true);
1422        }
1423
1424        @Override
1425        public Dataset getSlice(final int[] start, final int[] stop, final int[] step) {
1426                return getSlice(new SliceND(shape, start, stop, step));
1427        }
1428
1429        @Override
1430        public Dataset getSlice(Slice... slice) {
1431                return getSlice(new SliceND(shape, slice));
1432        }
1433
1434        @Override
1435        public Dataset getSlice(IMonitor monitor, Slice... slice) {
1436                return getSlice(slice);
1437        }
1438
1439        @Override
1440        public Dataset getSlice(IMonitor monitor, SliceND slice) {
1441                return getSlice(slice);
1442        }
1443
1444        @Override
1445        public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) {
1446                return getSlice(start, stop, step);
1447        }
1448
1449        /**
1450         * Get a slice of the dataset. The returned dataset is a copied selection of items
1451         * @param slice
1452         * @return The dataset of the sliced data
1453         */
1454        @Override
1455        public Dataset getSlice(final SliceND slice) {
1456                SliceIterator it = (SliceIterator) getSliceIterator(slice);
1457                AbstractDataset s = getSlice(it);
1458                s.metadata = copyMetadata();
1459                s.setDirty();
1460                s.sliceMetadata(true, slice);
1461                return s;
1462        }
1463
1464        /**
1465         * Get a slice of the dataset. The returned dataset is a copied selection of items
1466         * 
1467         * @param iterator Slice iterator
1468         * @return The dataset of the sliced data
1469         */
1470        abstract public AbstractDataset getSlice(final SliceIterator iterator);
1471
1472        @SuppressWarnings("deprecation")
1473        @Override
1474        public Dataset setSlice(final Object obj, final SliceND slice) {
1475                Dataset ds;
1476                if (obj instanceof Dataset) {
1477                        ds = (Dataset) obj;
1478                } else if (obj instanceof IDataset) {
1479                        ds = DatasetUtils.convertToDataset((IDataset) obj);
1480                } else {
1481                        int dtype = getDType();
1482                        if (dtype != Dataset.BOOL) {
1483                                dtype = DTypeUtils.getLargestDType(dtype);
1484                        }
1485                        ds = DatasetFactory.createFromObject(getElementsPerItem(), dtype, obj);
1486                }
1487
1488                return setSlicedView(getSliceView(slice), ds);
1489        }
1490
1491        @Override
1492        public Dataset setSlice(final Object obj, final int[] start, final int[] stop, final int[] step) {
1493                return setSlice(obj, new SliceND(shape, start, stop, step));
1494        }
1495
1496        /**
1497         * Set a view of current dataset to given dataset with broadcasting
1498         * @param view
1499         * @param d
1500         * @return this dataset
1501         */
1502        abstract Dataset setSlicedView(Dataset view, Dataset d);
1503
1504        @Override
1505        public Dataset setSlice(Object obj, Slice... slice) {
1506                if (slice == null || slice.length == 0) {
1507                        return setSlice(obj, new SliceND(shape));
1508                }
1509                return setSlice(obj, new SliceND(shape, slice));
1510        }
1511
1512        @Override
1513        public boolean all() {
1514                return Comparisons.allTrue(this);
1515        }
1516
1517        @Override
1518        public BooleanDataset all(final int axis) {
1519                return Comparisons.allTrue(this, axis);
1520        }
1521
1522        @Override
1523        public boolean any() {
1524                return Comparisons.anyTrue(this);
1525        }
1526
1527        @Override
1528        public BooleanDataset any(final int axis) {
1529                return Comparisons.anyTrue(this, axis);
1530        }
1531
1532        @Override
1533        public Dataset ifloorDivide(final Object o) {
1534                return idivide(o).ifloor();
1535        }
1536
1537        @Override
1538        public double residual(final Object o) {
1539                return residual(o, null, false);
1540        }
1541
1542        @Override
1543        public double residual(final Object o, boolean ignoreNaNs) {
1544                return residual(o, null, ignoreNaNs);
1545        }
1546
1547        /**
1548         * @since 2.0
1549         */
1550        @SuppressWarnings("unchecked")
1551        protected StatisticsMetadata<Number> getStats() {
1552                StatisticsMetadata<Number> md = getFirstMetadata(StatisticsMetadata.class);
1553                if (md == null || md.isDirty()) {
1554                        md = new StatisticsMetadataImpl<Number>();
1555                        md.initialize(this);
1556                        setMetadata(md);
1557                }
1558                return md;
1559        }
1560
1561        /**
1562         * @since 2.0
1563         */
1564        @SuppressWarnings("unchecked")
1565        protected StatisticsMetadata<String> getStringStats() {
1566                StatisticsMetadata<String> md = getFirstMetadata(StatisticsMetadata.class);
1567                if (md == null || md.isDirty()) {
1568                        md = new StatisticsMetadataImpl<String>();
1569                        md.initialize(this);
1570                        setMetadata(md);
1571                }
1572                return md;
1573        }
1574
1575        @Override
1576        public Number max(boolean... ignoreInvalids) {
1577                return getStats().getMaximum(ignoreInvalids);
1578        }
1579
1580        @Override
1581        public Dataset max(int axis, boolean... ignoreInvalids) {
1582                return getStats().getMaximum(axis, ignoreInvalids);
1583        }
1584
1585        @Override
1586        public Number min(boolean... ignoreInvalids) {
1587                return getStats().getMinimum(ignoreInvalids);
1588        }
1589
1590        @Override
1591        public Dataset min(int axis, boolean... ignoreInvalids) {
1592                return getStats().getMinimum(axis, ignoreInvalids);
1593        }
1594
1595        @Override
1596        public int argMax(boolean... ignoreInvalids) {
1597                return getFlat1DIndex(maxPos(ignoreInvalids));
1598        }
1599
1600        /**
1601         * @since 2.0
1602         */
1603        @Override
1604        public IntegerDataset argMax(int axis, boolean... ignoreInvalids) {
1605                return (IntegerDataset) getStats().getArgMaximum(axis, ignoreInvalids);
1606        }
1607
1608        @Override
1609        public int argMin(boolean... ignoreInvalids) {
1610                return getFlat1DIndex(minPos(ignoreInvalids));
1611        }
1612
1613        /**
1614         * @since 2.0
1615         */
1616        @Override
1617        public IntegerDataset argMin(int axis, boolean... ignoreInvalids) {
1618                return (IntegerDataset) getStats().getArgMinimum(axis, ignoreInvalids);
1619        }
1620
1621        @Override
1622        public Number peakToPeak(boolean... ignoreInvalids) {
1623                return DTypeUtils.fromDoubleToBiggestNumber(max(ignoreInvalids).doubleValue() - min(ignoreInvalids).doubleValue(), getDType());
1624        }
1625
1626        @Override
1627        public Dataset peakToPeak(int axis,  boolean... ignoreInvalids) {
1628                return Maths.subtract(max(axis, ignoreInvalids), min(axis, ignoreInvalids));
1629        }
1630
1631
1632        @Override
1633        public long count(boolean... ignoreInvalids) {
1634                return getStats().getCount(ignoreInvalids);
1635        }
1636
1637        @Override
1638        public Dataset count(int axis, boolean... ignoreInvalids) {
1639                return getStats().getCount(axis, ignoreInvalids);
1640        }
1641
1642        @Override
1643        public Object sum(boolean... ignoreInvalids) {
1644                return getStats().getSum(ignoreInvalids);
1645        }
1646
1647        @Override
1648        public Dataset sum(int axis, boolean... ignoreInvalids) {
1649                return getStats().getSum(axis, ignoreInvalids);
1650        }
1651
1652        @Override
1653        public Object product(boolean... ignoreInvalids) {
1654                return Stats.product(this, ignoreInvalids);
1655        }
1656
1657        @Override
1658        public Dataset product(int axis, boolean... ignoreInvalids) {
1659                return Stats.product(this, axis, ignoreInvalids);
1660        }
1661
1662        @Override
1663        public Object mean(boolean... ignoreInvalids) {
1664                return getStats().getMean(ignoreInvalids);
1665        }
1666
1667        @Override
1668        public Dataset mean(int axis, boolean... ignoreInvalids) {
1669                return getStats().getMean(axis, ignoreInvalids);
1670        }
1671
1672        @Override
1673        public double variance() {
1674                return variance(false);
1675        }
1676
1677        @Override
1678        public double variance(boolean isWholePopulation, boolean... ignoreInvalids) {
1679                return getStats().getVariance(isWholePopulation, ignoreInvalids);
1680        }
1681
1682        @Override
1683        public Dataset variance(int axis) {
1684                return getStats().getVariance(axis, false);
1685        }
1686
1687        @Override
1688        public Dataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
1689                return getStats().getVariance(axis, isWholePopulation, ignoreInvalids);
1690        }
1691
1692        @Override
1693        public double stdDeviation() {
1694                return Math.sqrt(variance());
1695        }
1696
1697        @Override
1698        public double stdDeviation(boolean isWholePopulation, boolean... ignoreInvalids) {
1699                return Math.sqrt(variance(isWholePopulation, ignoreInvalids));
1700        }
1701
1702        @Override
1703        public Dataset stdDeviation(int axis) {
1704                return Maths.sqrt(variance(axis, false));
1705        }
1706
1707        @Override
1708        public Dataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
1709                return Maths.sqrt(variance(axis, isWholePopulation, ignoreInvalids));
1710        }
1711
1712        @Override
1713        public double rootMeanSquare(boolean... ignoreInvalids) {
1714                StatisticsMetadata<Number> stats = getStats();
1715                final double mean = stats.getMean(ignoreInvalids).doubleValue();
1716                final double var = stats.getVariance(true, ignoreInvalids);
1717                return Math.sqrt(var + mean * mean);
1718        }
1719
1720        @Override
1721        public Dataset rootMeanSquare(int axis, boolean... ignoreInvalids) {
1722                StatisticsMetadata<Number> stats = getStats();
1723                Dataset v = stats.getVariance(axis, true, ignoreInvalids);
1724                Dataset m = stats.getMean(axis, ignoreInvalids);
1725                Dataset result = Maths.multiply(m, m);
1726                return Maths.sqrt(result.iadd(v));
1727        }
1728
1729        /**
1730         * Set item from compatible dataset in a direct and speedy way. Remember to setDirty afterwards.
1731         * 
1732         * @param dindex
1733         * @param sindex
1734         * @param src
1735         *            is the source data buffer
1736         */
1737        protected abstract void setItemDirect(final int dindex, final int sindex, final Object src);
1738
1739        /**
1740         * @return error broadcasted to current shape
1741         */
1742        private Dataset getBroadcastedInternalError() {
1743                if (shape == null) {
1744                        throw new IllegalArgumentException("Cannot get error for null dataset");
1745                }
1746                ILazyDataset led = super.getErrors();
1747                if (led == null)
1748                        return null;
1749
1750                Dataset ed = null;
1751                try {
1752                        ed = DatasetUtils.sliceAndConvertLazyDataset(led);
1753                } catch (DatasetException e) {
1754                        logger.error("Could not get data from lazy dataset", e);
1755                }
1756                if (led != ed) {
1757                        setErrors(ed); // set back
1758                }
1759
1760                return ed.getBroadcastView(shape);
1761        }
1762
1763        @Override
1764        public Dataset getErrors() {
1765                Dataset ed = getBroadcastedInternalError();
1766                if (ed == null)
1767                        return null;
1768
1769                return ed;
1770        }
1771
1772        @Override
1773        public double getError() {
1774                Dataset ed = getBroadcastedInternalError();
1775                if (ed == null)
1776                        return 0;
1777
1778                return ed.getDouble();
1779        }
1780
1781        @Override
1782        public double getError(final int i) {
1783                Dataset ed = getBroadcastedInternalError();
1784                if (ed == null)
1785                        return 0;
1786
1787                return ed.getDouble(i);
1788        }
1789
1790        @Override
1791        public double getError(final int i, final int j) {
1792                Dataset ed = getBroadcastedInternalError();
1793                if (ed == null)
1794                        return 0;
1795
1796                return ed.getDouble(i, j);
1797        }
1798
1799        @Override
1800        public double getError(int... pos) {
1801                Dataset ed = getBroadcastedInternalError();
1802                if (ed == null)
1803                        return 0;
1804
1805                return ed.getDouble(pos);
1806        }
1807
1808        @Override
1809        public double[] getErrorArray(final int i) {
1810                Dataset ed = getBroadcastedInternalError();
1811                if (ed == null)
1812                        return null;
1813
1814                return new double[] {getError(i)};
1815        }
1816
1817        @Override
1818        public double[] getErrorArray(final int i, final int j) {
1819                Dataset ed = getBroadcastedInternalError();
1820                if (ed == null)
1821                        return null;
1822
1823                return new double[] {getError(i, j)};
1824        }
1825
1826        @Override
1827        public double[] getErrorArray(int... pos) {
1828                Dataset ed = getBroadcastedInternalError();
1829                if (ed == null)
1830                        return null;
1831
1832                return new double[] {getError(pos)};
1833        }
1834
1835        protected Dataset getInternalSquaredError() {
1836                Dataset sed = getErrorBuffer().getBroadcastView(shape);
1837                return sed;
1838        }
1839
1840        @Override
1841        public Dataset getErrorBuffer() {
1842                ErrorMetadata emd = getErrorMetadata();
1843                if (emd == null)
1844                        return null;
1845
1846                if (!(emd instanceof ErrorMetadataImpl)) {
1847                        ILazyDataset led = emd.getError();
1848                        Dataset ed;
1849                        try {
1850                                ed = DatasetUtils.sliceAndConvertLazyDataset(led);
1851                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
1852                                setMetadata(emd);
1853                                emd.setError(ed);
1854                        } catch (MetadataException me) {
1855                                logger.error("Could not create metadata", me);
1856                        } catch (DatasetException e) {
1857                                logger.error("Could not get data from lazy dataset", e);
1858                        }
1859                }
1860
1861                return ((ErrorMetadataImpl) emd).getSquaredError();
1862        }
1863
1864        /**
1865         * Set a copy of the buffer that backs the (squared) error data
1866         * @param buffer can be null, anything that can be used to create a DoubleDataset or CompoundDoubleDataset
1867         */
1868        @Override
1869        public void setErrorBuffer(Serializable buffer) {
1870                if (shape == null) {
1871                        throw new IllegalArgumentException("Cannot set error buffer for null dataset");
1872                }
1873                if (buffer == null) {
1874                        clearMetadata(ErrorMetadata.class);
1875                        return;
1876                }
1877
1878                IDataset d = (IDataset) createFromSerializable(buffer, false);
1879                ErrorMetadata emd = getErrorMetadata();
1880                if (!(emd instanceof ErrorMetadataImpl)) {
1881                        try {
1882                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
1883                                setMetadata(emd);
1884                        } catch (MetadataException me) {
1885                                logger.error("Could not create metadata", me);
1886                        }
1887                }
1888                ((ErrorMetadataImpl) emd).setSquaredError(d);
1889        }
1890}