001/*-
002 *******************************************************************************
003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *    Peter Chang - initial API and implementation and/or initial documentation
011 *******************************************************************************/
012
013package org.eclipse.january.dataset;
014
015import java.io.IOException;
016import java.io.Serializable;
017import java.lang.annotation.Annotation;
018import java.lang.reflect.Field;
019import java.util.ArrayList;
020import java.util.Arrays;
021import java.util.HashMap;
022import java.util.List;
023import java.util.Map;
024
025import org.eclipse.january.DatasetException;
026import org.eclipse.january.IMonitor;
027import org.eclipse.january.io.ILazyLoader;
028import org.eclipse.january.metadata.MetadataFactory;
029import org.eclipse.january.metadata.MetadataType;
030import org.eclipse.january.metadata.OriginMetadata;
031import org.eclipse.january.metadata.Reshapeable;
032import org.eclipse.january.metadata.Sliceable;
033import org.eclipse.january.metadata.Transposable;
034
035public class LazyDataset extends LazyDatasetBase implements Serializable, Cloneable {
036        private static final long serialVersionUID = 2467865859867440242L;
037
038        protected int[]     oShape; // original shape
039        protected long      size;   // number of items
040        protected int       dtype;  // dataset type
041        protected int       isize;  // number of elements per item
042
043        protected ILazyLoader loader;
044        protected LazyDataset base = null; // used for transpose
045
046        // relative to loader or base
047        protected int         prepShape = 0; // prepending and post-pending 
048        protected int         postShape = 0; // changes to shape
049        protected int[]       begSlice = null; // slice begin
050        protected int[]       delSlice = null; // slice delta
051        protected int[]       map; // transposition map (same length as current shape)
052        protected Map<Class<? extends MetadataType>, List<MetadataType>> oMetadata = null;
053
054        /**
055         * Create a lazy dataset
056         * @param name
057         * @param dtype dataset type
058         * @param elements
059         * @param shape
060         * @param loader
061         */
062        public LazyDataset(String name, int dtype, int elements, int[] shape, ILazyLoader loader) {
063                this.name = name;
064                this.shape = shape.clone();
065                this.oShape = this.shape;
066                this.loader = loader;
067                this.dtype = dtype;
068                this.isize = elements;
069                try {
070                        size = ShapeUtils.calcLongSize(shape);
071                } catch (IllegalArgumentException e) {
072                        size = Long.MAX_VALUE; // this indicates that the entire dataset cannot be read in! 
073                }
074        }
075
076        /**
077         * Create a lazy dataset
078         * @param name
079         * @param dtype dataset type
080         * @param shape
081         * @param loader
082         */
083        public LazyDataset(String name, int dtype, int[] shape, ILazyLoader loader) {
084                this(name, dtype, 1, shape, loader);
085        }
086
087        /**
088         * Create a lazy dataset based on in-memory data (handy for testing)
089         * @param dataset
090         */
091        public static LazyDataset createLazyDataset(final Dataset dataset) {
092                return new LazyDataset(dataset.getName(), dataset.getDType(), dataset.getElementsPerItem(), dataset.getShape(),
093                new ILazyLoader() {
094                        private static final long serialVersionUID = -6725268922780517523L;
095
096                        final Dataset d = dataset;
097                        @Override
098                        public boolean isFileReadable() {
099                                return true;
100                        }
101
102                        @Override
103                        public Dataset getDataset(IMonitor mon, SliceND slice) throws IOException {
104                                return d.getSlice(mon, slice);
105                        }
106                });
107        }
108
109        /**
110         * Can return -1 for unknown
111         */
112        @Override
113        public int getDType() {
114                return dtype;
115        }
116
117        /**
118         * Can return -1 for unknown
119         */
120        @Override
121        public int getElementsPerItem() {
122                return isize;
123        }
124
125        @Override
126        public int getSize() {
127                return (int) size;
128        }
129
130        @Override
131        public String toString() {
132                StringBuilder out = new StringBuilder();
133
134                if (name != null && name.length() > 0) {
135                        out.append("Lazy dataset '");
136                        out.append(name);
137                        out.append("' has shape [");
138                } else {
139                        out.append("Lazy dataset shape is [");
140                }
141                int rank = shape == null ? 0 : shape.length;
142
143                if (rank > 0 && shape[0] >= 0) {
144                        out.append(shape[0]);
145                }
146                for (int i = 1; i < rank; i++) {
147                        out.append(", " + shape[i]);
148                }
149                out.append(']');
150
151                return out.toString();
152        }
153
154        @Override
155        public boolean equals(Object obj) {
156                if (!super.equals(obj))
157                        return false;
158
159                LazyDataset other = (LazyDataset) obj;
160                if (dtype != other.dtype) {
161                        return false;
162                }
163                if (isize != other.isize) {
164                        return false;
165                }
166
167                if (!Arrays.equals(shape, other.shape)) {
168                        return false;
169                }
170
171                if (loader != other.loader) {
172                        return false;
173                }
174
175                if (prepShape != other.prepShape) {
176                        return false;
177                }
178
179                if (postShape != other.postShape) {
180                        return false;
181                }
182
183                if (!Arrays.equals(begSlice, other.begSlice)) {
184                        return false;
185                }
186                if (!Arrays.equals(delSlice, other.delSlice)) {
187                        return false;
188                }
189                if (!Arrays.equals(map, other.map)) {
190                        return false;
191                }
192                return true;
193        }
194
195        @Override
196        public LazyDataset clone() {
197                LazyDataset ret = new LazyDataset(new String(name), dtype, isize, oShape, loader);
198                ret.shape = shape;
199                ret.size = size;
200                ret.prepShape = prepShape;
201                ret.postShape = postShape;
202                ret.begSlice = begSlice;
203                ret.delSlice = delSlice;
204                ret.map = map;
205                ret.base = base;
206                ret.metadata = copyMetadata();
207                ret.oMetadata = oMetadata;
208                return ret;
209        }
210
211        @Override
212        public void setShape(int... shape) {
213                setShapeInternal(shape);
214        }
215
216        @Override
217        public LazyDataset squeezeEnds() {
218                setShapeInternal(ShapeUtils.squeezeShape(shape, true));
219                return this;
220        }
221
222        @Override
223        public Dataset getSlice(int[] start, int[] stop, int[] step) throws DatasetException {
224                return getSlice(null, start, stop, step);
225        }
226
227        @Override
228        public Dataset getSlice(Slice... slice) throws DatasetException {
229                if (slice == null || slice.length == 0) {
230                        return getSlice(null, new SliceND(shape));
231                }
232                return getSlice(null, new SliceND(shape, slice));
233        }
234
235        @Override
236        public Dataset getSlice(SliceND slice) throws DatasetException {
237                return getSlice(null, slice);
238        }
239
240        @Override
241        public Dataset getSlice(IMonitor monitor, Slice... slice) throws DatasetException {
242                if (slice == null || slice.length == 0) {
243                        return getSlice(monitor, new SliceND(shape));
244                }
245                return getSlice(monitor, new SliceND(shape, slice));
246        }
247
248        @Override
249        public LazyDataset getSliceView(Slice... slice) {
250                if (slice == null || slice.length == 0) {
251                        return getSliceView(new SliceND(shape));
252                }
253                return getSliceView(new SliceND(shape, slice));
254        }
255
256        /**
257         * @param nShape
258         */
259        private void setShapeInternal(int... nShape) {
260                
261                long nsize = ShapeUtils.calcLongSize(nShape);
262                if (nsize != size) {
263                        throw new IllegalArgumentException("Size of new shape is not equal to current size");
264                }
265
266                if (nsize == 1) {
267                        shape = nShape.clone();
268                } else {
269                        int ob = -1; // first non-unit dimension
270                        int or = shape.length;
271                        for (int i = 0; i < or; i++) {
272                                if (shape[i] != 1) {
273                                        ob = i;
274                                        break;
275                                }
276                        }
277                        assert ob >= 0;
278                        int oe = -1; // last non-unit dimension
279                        for (int i = or - 1; i >= ob; i--) {
280                                if (shape[i] != 1) {
281                                        oe = i;
282                                        break;
283                                }
284                        }
285                        assert oe >= 0;
286                        oe++;
287        
288                        int nb = -1; // first non-unit dimension
289                        int nr = nShape.length;
290                        for (int i = 0; i < nr; i++) {
291                                if (nShape[i] != 1) {
292                                        nb = i;
293                                        break;
294                                }
295                        }
296        
297                        int i = ob;
298                        int j = nb;
299                        if (begSlice == null) {
300                                for (; i < oe && j < nr; i++, j++) {
301                                        if (shape[i] != nShape[j]) {
302                                                throw new IllegalArgumentException("New shape not allowed - can only change shape by adding or removing ones to ends of old shape");
303                                        }
304                                }
305                        } else {
306                                int[] nBegSlice = new int[nr];
307                                int[] nDelSlice = new int[nr];
308                                Arrays.fill(nDelSlice, 1);
309                                for (; i < oe && j < nr; i++, j++) {
310                                        if (shape[i] != nShape[j]) {
311                                                throw new IllegalArgumentException("New shape not allowed - can only change shape by adding or removing ones to ends of old shape");
312                                        }
313                                        nBegSlice[j] = begSlice[i];
314                                        nDelSlice[j] = delSlice[i];
315                                }
316                
317                                begSlice = nBegSlice;
318                                delSlice = nDelSlice;
319                        }
320                        prepShape += nb - ob;
321                        postShape += nr - oe;
322                }
323
324                if (metadata != null) {
325                        storeMetadata(metadata, Reshapeable.class);
326                        metadata = copyMetadata();
327                        reshapeMetadata(shape, nShape);
328                }
329                shape = nShape;
330        }
331
332        @Override
333        public LazyDataset getSliceView(int[] start, int[] stop, int[] step) {
334                return getSliceView(new SliceND(shape, start, stop, step));
335        }
336
337        @Override
338        public LazyDataset getSliceView(SliceND slice) {
339                LazyDataset view = clone();
340                if (slice.isAll()) {
341                        return view;
342                }
343
344                int[] lstart = slice.getStart();
345                int[] lstep  = slice.getStep();
346                final int rank = shape.length;
347
348                int[] nShape = slice.getShape();
349                view.shape = nShape;
350                view.size = ShapeUtils.calcLongSize(nShape);
351                if (begSlice == null) {
352                        view.begSlice = lstart.clone();
353                        view.delSlice = lstep.clone();
354                } else {
355                        view.begSlice = new int[rank];
356                        view.delSlice = new int[rank];
357                        for (int i = 0; i < rank; i++) {
358                                view.begSlice[i] = begSlice[i] + lstart[i] * delSlice[i];
359                                view.delSlice[i] = delSlice[i] * lstep[i];
360                        }
361                }
362                view.storeMetadata(metadata, Sliceable.class);
363                
364                view.sliceMetadata(true, slice);
365                return view;
366        }
367
368        @Override
369        public LazyDataset getTransposedView(int... axes) {
370                LazyDataset view = clone();
371
372                // everything now is seen through a map
373                axes = checkPermutatedAxes(shape, axes);
374                if (axes == null) {
375                        return view;
376                }
377
378                int r = shape.length;
379                view.shape = new int[r];
380                for (int i = 0; i < r; i++) {
381                        view.shape[i] = shape[axes[i]];
382                }
383
384                view.prepShape = 0;
385                view.postShape = 0;
386                view.begSlice = null;
387                view.delSlice = null;
388                view.map = axes;
389                view.base = this;
390                view.storeMetadata(metadata, Transposable.class);
391                view.transposeMetadata(axes);
392                return view;
393        }
394
395        @Override
396        public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) throws DatasetException {
397                return getSlice(monitor, new SliceND(shape, start, stop, step));
398        }
399
400        @Override
401        public Dataset getSlice(IMonitor monitor, SliceND slice) throws DatasetException {
402
403                if (loader != null && !loader.isFileReadable()) {
404                        return null; // TODO add interaction to use plot (or remote) server to load dataset
405                }
406
407                SliceND nslice = calcTrueSlice(slice);
408
409                Dataset a;
410                if (base != null) {
411                        a = base.getSlice(monitor, nslice);
412                } else {
413                        try {
414                                a = DatasetUtils.convertToDataset(loader.getDataset(monitor, nslice));
415                        } catch (IOException e) {
416                                logger.error("Problem getting {}: {}", String.format("slice %s %s %s from %s", Arrays.toString(slice.getStart()), Arrays.toString(slice.getStop()),
417                                                                Arrays.toString(slice.getStep()), loader), e);
418                                throw new DatasetException(e);
419                        }
420                        a.setName(name + AbstractDataset.BLOCK_OPEN + nslice.toString() + AbstractDataset.BLOCK_CLOSE);
421                        if (metadata != null && a instanceof LazyDatasetBase) {
422                                LazyDatasetBase ba = (LazyDatasetBase) a;
423                                ba.metadata = copyMetadata();
424                                if (oMetadata != null) {
425                                        ba.restoreMetadata(oMetadata);
426                                }
427                                //metadata axis may be larger than data
428                                if (!nslice.isAll() || nslice.getMaxShape() != nslice.getShape()) {
429                                        ba.sliceMetadata(true, nslice);
430                                }
431                        }
432                }
433                if (map != null) {
434                        a = a.getTransposedView(map);
435                }
436                if (slice != null) {
437                        a.setShape(slice.getShape());
438                }
439                a.addMetadata(MetadataFactory.createMetadata(OriginMetadata.class, this, nslice.convertToSlice(), oShape, null, name));
440                
441                return a;
442        }
443
444        // reverse transform
445        private int[] getOriginal(int[] values) {
446                if (values == null) {
447                        return null;
448                }
449                int r = values.length;
450                if (map == null || r < 2) {
451                        return values;
452                }
453                int[] ovalues = new int[r];
454                for (int i = 0; i < r; i++) {
455                        ovalues[map[i]] = values[i];
456                }
457                return ovalues;
458        }
459
460        protected final SliceND calcTrueSlice(SliceND slice) {
461                if (slice == null) {
462                        slice = new SliceND(shape);
463                }
464                int[] lstart = slice.getStart();
465                int[] lstop  = slice.getStop();
466                int[] lstep  = slice.getStep();
467
468                int[] nstart;
469                int[] nstop;
470                int[] nstep;
471
472                int r = base == null ? oShape.length : base.shape.length;
473                nstart = new int[r];
474                nstop = new int[r];
475                nstep = new int[r];
476                Arrays.fill(nstop, 1);
477                Arrays.fill(nstep, 1);
478                {
479                        int i = 0;
480                        int j = 0;
481                        if (prepShape < 0) { // ignore entries from new slice 
482                                i = -prepShape;
483                        } else if (prepShape > 0) {
484                                j = prepShape;
485                        }
486                        if (begSlice == null) {
487                                for (; i < r && j < shape.length; i++, j++) {
488                                        nstart[i] = lstart[j];
489                                        nstop[i]  = lstop[j];
490                                        int d = lstep[j];
491                                        if (d < 0 && nstop[i] < 0) { // need to wrap around further
492                                                int l = base == null ? oShape[j]: base.shape[j];
493                                                nstop[i] -= l;
494                                        }
495                                        nstep[i]  = d;
496                                }
497                        } else {
498                                for (; i < r && j < shape.length; i++, j++) {
499                                        int b = begSlice[j];
500                                        int d = delSlice[j];
501                                        nstart[i] = b + lstart[j] * d;
502                                        nstop[i]  = b + (lstop[j] - 1) * d + (d >= 0 ? 1 : -1);
503                                        if (d < 0 && nstop[i] < 0) { // need to wrap around further
504                                                int l = base == null ? oShape[j]: base.shape[j];
505                                                nstop[i] -= l;
506                                        }
507                                        nstep[i]  = lstep[j] * d;
508                                }
509                        }
510                        if (map != null) {
511                                nstart = getOriginal(nstart);
512                                nstop  = getOriginal(nstop);
513                                nstep  = getOriginal(nstep);
514                        }
515                }
516
517                return createSlice(nstart, nstop, nstep);
518        }
519
520        protected final IDataset transformInput(IDataset data) {
521                if (map == null) {
522                        return data;
523                }
524                return data.getTransposedView(map);
525        }
526
527        protected SliceND createSlice(int[] nstart, int[] nstop, int[] nstep) {
528                return new SliceND(base == null ? oShape : base.shape, nstart, nstop, nstep);
529        }
530
531        /**
532         * Store metadata items that has given annotation
533         * @param origMetadata
534         * @param aclazz
535         */
536        private void storeMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> origMetadata, Class<? extends Annotation> aclazz) {
537                List<Class<? extends MetadataType>> mclazzes = findAnnotatedMetadata(aclazz);
538                if (mclazzes.size() == 0) {
539                        return;
540                }
541
542                if (oMetadata == null) {
543                        oMetadata = new HashMap<Class<? extends MetadataType>, List<MetadataType>>();
544                }
545                for (Class<? extends MetadataType> mc : mclazzes) {
546                        if (oMetadata.containsKey(mc)) {
547                                continue; // do not overwrite original
548                        }
549
550                        List<MetadataType> l = origMetadata.get(mc);
551                        List<MetadataType> nl = new ArrayList<MetadataType>(l.size());
552                        for (MetadataType m : l) {
553                                nl.add(m.clone());
554                        }
555                        oMetadata.put(mc, nl);
556                }
557        }
558
559        @SuppressWarnings("unchecked")
560        private List<Class<? extends MetadataType>> findAnnotatedMetadata(Class<? extends Annotation> aclazz) {
561                List<Class<? extends MetadataType>> mclazzes = new ArrayList<Class<? extends MetadataType>>();
562                if (metadata == null) {
563                        return mclazzes;
564                }
565
566                for (Class<? extends MetadataType> c : metadata.keySet()) {
567                        boolean hasAnn = false;
568                        for (MetadataType m : metadata.get(c)) {
569                                if (m == null) {
570                                        continue;
571                                }
572
573                                Class<? extends MetadataType> mc = m.getClass();
574                                do { // iterate over super-classes
575                                        for (Field f : mc.getDeclaredFields()) {
576                                                if (f.isAnnotationPresent(aclazz)) {
577                                                        hasAnn = true;
578                                                        break;
579                                                }
580                                        }
581                                        Class<?> sclazz = mc.getSuperclass();
582                                        if (!MetadataType.class.isAssignableFrom(sclazz)) {
583                                                break;
584                                        }
585                                        mc = (Class<? extends MetadataType>) sclazz;
586                                } while (!hasAnn);
587                                if (hasAnn) {
588                                        break;
589                                }
590                        }
591                        if (hasAnn) {
592                                mclazzes.add(c);
593                        }
594                }
595                return mclazzes;
596        }
597
598        /**
599         * Gets the maximum size of a slice of a dataset in a given dimension
600         * which should normally fit in memory. Note that it might be possible
601         * to get more in memory, this is a conservative estimate and seems to
602         * almost always work at the size returned; providing Xmx is less than
603         * the physical memory.
604         * 
605         * To get more in memory increase -Xmx setting or use an expression
606         * which calls a rolling function (like rmean) instead of slicing directly
607         * to memory.
608         * 
609         * @param lazySet
610         * @param dimension
611         * @return maximum size of dimension that can be sliced.
612         */
613        public static int getMaxSliceLength(ILazyDataset lazySet, int dimension) {
614                // size in bytes of each item
615                final double size = DTypeUtils.getItemBytes(DTypeUtils.getDTypeFromClass(lazySet.getElementClass()), lazySet.getElementsPerItem());
616                
617                // Max in bytes takes into account our minimum requirement
618                final double max  = Math.max(Runtime.getRuntime().totalMemory(), Runtime.getRuntime().maxMemory());
619                
620                // Firstly if the whole dataset it likely to fit in memory, then we allow it.
621                // Space specified in bytes per item available
622                final double space = max/lazySet.getSize();
623
624                // If we have room for this whole dataset, then fine
625                int[] shape = lazySet.getShape();
626                if (space >= size) {
627                        return shape[dimension];
628                }
629
630                // Otherwise estimate what we can fit in, conservatively.
631                // First get size of one slice, see it that fits, if not, still return 1
632                double sizeOneSlice = size; // in bytes
633                for (int dim = 0; dim < shape.length; dim++) {
634                        if (dim == dimension) {
635                                continue;
636                        }
637                        sizeOneSlice *= shape[dim];
638                }
639                double avail = max / sizeOneSlice;
640                if (avail < 1) {
641                        return 1;
642                }
643
644                // We fudge this to leave some room
645                return (int) Math.floor(avail/4d);
646        }
647}