001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.IOException; 016import java.io.Serializable; 017import java.lang.annotation.Annotation; 018import java.lang.reflect.Field; 019import java.util.ArrayList; 020import java.util.Arrays; 021import java.util.HashMap; 022import java.util.List; 023import java.util.Map; 024 025import org.eclipse.january.DatasetException; 026import org.eclipse.january.IMonitor; 027import org.eclipse.january.io.ILazyLoader; 028import org.eclipse.january.metadata.MetadataFactory; 029import org.eclipse.january.metadata.MetadataType; 030import org.eclipse.january.metadata.OriginMetadata; 031import org.eclipse.january.metadata.Reshapeable; 032import org.eclipse.january.metadata.Sliceable; 033import org.eclipse.january.metadata.Transposable; 034 035public class LazyDataset extends LazyDatasetBase implements Serializable, Cloneable { 036 private static final long serialVersionUID = 2467865859867440242L; 037 038 protected int[] oShape; // original shape 039 protected long size; // number of items 040 protected int dtype; // dataset type 041 protected int isize; // number of elements per item 042 043 protected ILazyLoader loader; 044 protected LazyDataset base = null; // used for transpose 045 046 // relative to loader or base 047 protected int prepShape = 0; // prepending and post-pending 048 protected int postShape = 0; // changes to shape 049 protected int[] begSlice = null; // slice begin 050 protected int[] delSlice = null; // slice delta 051 protected int[] map; // transposition map (same length as current shape) 052 protected Map<Class<? extends MetadataType>, List<MetadataType>> oMetadata = null; 053 054 /** 055 * Create a lazy dataset 056 * @param name 057 * @param dtype dataset type 058 * @param elements 059 * @param shape 060 * @param loader 061 */ 062 public LazyDataset(String name, int dtype, int elements, int[] shape, ILazyLoader loader) { 063 this.name = name; 064 this.shape = shape.clone(); 065 this.oShape = this.shape; 066 this.loader = loader; 067 this.dtype = dtype; 068 this.isize = elements; 069 try { 070 size = ShapeUtils.calcLongSize(shape); 071 } catch (IllegalArgumentException e) { 072 size = Long.MAX_VALUE; // this indicates that the entire dataset cannot be read in! 073 } 074 } 075 076 /** 077 * Create a lazy dataset 078 * @param name 079 * @param dtype dataset type 080 * @param shape 081 * @param loader 082 */ 083 public LazyDataset(String name, int dtype, int[] shape, ILazyLoader loader) { 084 this(name, dtype, 1, shape, loader); 085 } 086 087 /** 088 * Create a lazy dataset based on in-memory data (handy for testing) 089 * @param dataset 090 */ 091 public static LazyDataset createLazyDataset(final Dataset dataset) { 092 return new LazyDataset(dataset.getName(), dataset.getDType(), dataset.getElementsPerItem(), dataset.getShape(), 093 new ILazyLoader() { 094 private static final long serialVersionUID = -6725268922780517523L; 095 096 final Dataset d = dataset; 097 @Override 098 public boolean isFileReadable() { 099 return true; 100 } 101 102 @Override 103 public Dataset getDataset(IMonitor mon, SliceND slice) throws IOException { 104 return d.getSlice(mon, slice); 105 } 106 }); 107 } 108 109 /** 110 * Can return -1 for unknown 111 */ 112 @Override 113 public int getDType() { 114 return dtype; 115 } 116 117 /** 118 * Can return -1 for unknown 119 */ 120 @Override 121 public int getElementsPerItem() { 122 return isize; 123 } 124 125 @Override 126 public int getSize() { 127 return (int) size; 128 } 129 130 @Override 131 public String toString() { 132 StringBuilder out = new StringBuilder(); 133 134 if (name != null && name.length() > 0) { 135 out.append("Lazy dataset '"); 136 out.append(name); 137 out.append("' has shape ["); 138 } else { 139 out.append("Lazy dataset shape is ["); 140 } 141 int rank = shape == null ? 0 : shape.length; 142 143 if (rank > 0 && shape[0] >= 0) { 144 out.append(shape[0]); 145 } 146 for (int i = 1; i < rank; i++) { 147 out.append(", " + shape[i]); 148 } 149 out.append(']'); 150 151 return out.toString(); 152 } 153 154 @Override 155 public boolean equals(Object obj) { 156 if (!super.equals(obj)) 157 return false; 158 159 LazyDataset other = (LazyDataset) obj; 160 if (dtype != other.dtype) { 161 return false; 162 } 163 if (isize != other.isize) { 164 return false; 165 } 166 167 if (!Arrays.equals(shape, other.shape)) { 168 return false; 169 } 170 171 if (loader != other.loader) { 172 return false; 173 } 174 175 if (prepShape != other.prepShape) { 176 return false; 177 } 178 179 if (postShape != other.postShape) { 180 return false; 181 } 182 183 if (!Arrays.equals(begSlice, other.begSlice)) { 184 return false; 185 } 186 if (!Arrays.equals(delSlice, other.delSlice)) { 187 return false; 188 } 189 if (!Arrays.equals(map, other.map)) { 190 return false; 191 } 192 return true; 193 } 194 195 @Override 196 public LazyDataset clone() { 197 LazyDataset ret = new LazyDataset(new String(name), dtype, isize, oShape, loader); 198 ret.shape = shape; 199 ret.size = size; 200 ret.prepShape = prepShape; 201 ret.postShape = postShape; 202 ret.begSlice = begSlice; 203 ret.delSlice = delSlice; 204 ret.map = map; 205 ret.base = base; 206 ret.metadata = copyMetadata(); 207 ret.oMetadata = oMetadata; 208 return ret; 209 } 210 211 @Override 212 public void setShape(int... shape) { 213 setShapeInternal(shape); 214 } 215 216 @Override 217 public LazyDataset squeezeEnds() { 218 setShapeInternal(ShapeUtils.squeezeShape(shape, true)); 219 return this; 220 } 221 222 @Override 223 public Dataset getSlice(int[] start, int[] stop, int[] step) throws DatasetException { 224 return getSlice(null, start, stop, step); 225 } 226 227 @Override 228 public Dataset getSlice(Slice... slice) throws DatasetException { 229 if (slice == null || slice.length == 0) { 230 return getSlice(null, new SliceND(shape)); 231 } 232 return getSlice(null, new SliceND(shape, slice)); 233 } 234 235 @Override 236 public Dataset getSlice(SliceND slice) throws DatasetException { 237 return getSlice(null, slice); 238 } 239 240 @Override 241 public Dataset getSlice(IMonitor monitor, Slice... slice) throws DatasetException { 242 if (slice == null || slice.length == 0) { 243 return getSlice(monitor, new SliceND(shape)); 244 } 245 return getSlice(monitor, new SliceND(shape, slice)); 246 } 247 248 @Override 249 public LazyDataset getSliceView(Slice... slice) { 250 if (slice == null || slice.length == 0) { 251 return getSliceView(new SliceND(shape)); 252 } 253 return getSliceView(new SliceND(shape, slice)); 254 } 255 256 /** 257 * @param nShape 258 */ 259 private void setShapeInternal(int... nShape) { 260 261 long nsize = ShapeUtils.calcLongSize(nShape); 262 if (nsize != size) { 263 throw new IllegalArgumentException("Size of new shape is not equal to current size"); 264 } 265 266 if (nsize == 1) { 267 shape = nShape.clone(); 268 } else { 269 int ob = -1; // first non-unit dimension 270 int or = shape.length; 271 for (int i = 0; i < or; i++) { 272 if (shape[i] != 1) { 273 ob = i; 274 break; 275 } 276 } 277 assert ob >= 0; 278 int oe = -1; // last non-unit dimension 279 for (int i = or - 1; i >= ob; i--) { 280 if (shape[i] != 1) { 281 oe = i; 282 break; 283 } 284 } 285 assert oe >= 0; 286 oe++; 287 288 int nb = -1; // first non-unit dimension 289 int nr = nShape.length; 290 for (int i = 0; i < nr; i++) { 291 if (nShape[i] != 1) { 292 nb = i; 293 break; 294 } 295 } 296 297 int i = ob; 298 int j = nb; 299 if (begSlice == null) { 300 for (; i < oe && j < nr; i++, j++) { 301 if (shape[i] != nShape[j]) { 302 throw new IllegalArgumentException("New shape not allowed - can only change shape by adding or removing ones to ends of old shape"); 303 } 304 } 305 } else { 306 int[] nBegSlice = new int[nr]; 307 int[] nDelSlice = new int[nr]; 308 Arrays.fill(nDelSlice, 1); 309 for (; i < oe && j < nr; i++, j++) { 310 if (shape[i] != nShape[j]) { 311 throw new IllegalArgumentException("New shape not allowed - can only change shape by adding or removing ones to ends of old shape"); 312 } 313 nBegSlice[j] = begSlice[i]; 314 nDelSlice[j] = delSlice[i]; 315 } 316 317 begSlice = nBegSlice; 318 delSlice = nDelSlice; 319 } 320 prepShape += nb - ob; 321 postShape += nr - oe; 322 } 323 324 if (metadata != null) { 325 storeMetadata(metadata, Reshapeable.class); 326 metadata = copyMetadata(); 327 reshapeMetadata(shape, nShape); 328 } 329 shape = nShape; 330 } 331 332 @Override 333 public LazyDataset getSliceView(int[] start, int[] stop, int[] step) { 334 return getSliceView(new SliceND(shape, start, stop, step)); 335 } 336 337 @Override 338 public LazyDataset getSliceView(SliceND slice) { 339 LazyDataset view = clone(); 340 if (slice.isAll()) { 341 return view; 342 } 343 344 int[] lstart = slice.getStart(); 345 int[] lstep = slice.getStep(); 346 final int rank = shape.length; 347 348 int[] nShape = slice.getShape(); 349 view.shape = nShape; 350 view.size = ShapeUtils.calcLongSize(nShape); 351 if (begSlice == null) { 352 view.begSlice = lstart.clone(); 353 view.delSlice = lstep.clone(); 354 } else { 355 view.begSlice = new int[rank]; 356 view.delSlice = new int[rank]; 357 for (int i = 0; i < rank; i++) { 358 view.begSlice[i] = begSlice[i] + lstart[i] * delSlice[i]; 359 view.delSlice[i] = delSlice[i] * lstep[i]; 360 } 361 } 362 view.storeMetadata(metadata, Sliceable.class); 363 364 view.sliceMetadata(true, slice); 365 return view; 366 } 367 368 @Override 369 public LazyDataset getTransposedView(int... axes) { 370 LazyDataset view = clone(); 371 372 // everything now is seen through a map 373 axes = checkPermutatedAxes(shape, axes); 374 if (axes == null) { 375 return view; 376 } 377 378 int r = shape.length; 379 view.shape = new int[r]; 380 for (int i = 0; i < r; i++) { 381 view.shape[i] = shape[axes[i]]; 382 } 383 384 view.prepShape = 0; 385 view.postShape = 0; 386 view.begSlice = null; 387 view.delSlice = null; 388 view.map = axes; 389 view.base = this; 390 view.storeMetadata(metadata, Transposable.class); 391 view.transposeMetadata(axes); 392 return view; 393 } 394 395 @Override 396 public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) throws DatasetException { 397 return getSlice(monitor, new SliceND(shape, start, stop, step)); 398 } 399 400 @Override 401 public Dataset getSlice(IMonitor monitor, SliceND slice) throws DatasetException { 402 403 if (loader != null && !loader.isFileReadable()) { 404 return null; // TODO add interaction to use plot (or remote) server to load dataset 405 } 406 407 SliceND nslice = calcTrueSlice(slice); 408 409 Dataset a; 410 if (base != null) { 411 a = base.getSlice(monitor, nslice); 412 } else { 413 try { 414 a = DatasetUtils.convertToDataset(loader.getDataset(monitor, nslice)); 415 } catch (IOException e) { 416 logger.error("Problem getting {}: {}", String.format("slice %s %s %s from %s", Arrays.toString(slice.getStart()), Arrays.toString(slice.getStop()), 417 Arrays.toString(slice.getStep()), loader), e); 418 throw new DatasetException(e); 419 } 420 a.setName(name + AbstractDataset.BLOCK_OPEN + nslice.toString() + AbstractDataset.BLOCK_CLOSE); 421 if (metadata != null && a instanceof LazyDatasetBase) { 422 LazyDatasetBase ba = (LazyDatasetBase) a; 423 ba.metadata = copyMetadata(); 424 if (oMetadata != null) { 425 ba.restoreMetadata(oMetadata); 426 } 427 //metadata axis may be larger than data 428 if (!nslice.isAll() || nslice.getMaxShape() != nslice.getShape()) { 429 ba.sliceMetadata(true, nslice); 430 } 431 } 432 } 433 if (map != null) { 434 a = a.getTransposedView(map); 435 } 436 if (slice != null) { 437 a.setShape(slice.getShape()); 438 } 439 a.addMetadata(MetadataFactory.createMetadata(OriginMetadata.class, this, nslice.convertToSlice(), oShape, null, name)); 440 441 return a; 442 } 443 444 // reverse transform 445 private int[] getOriginal(int[] values) { 446 if (values == null) { 447 return null; 448 } 449 int r = values.length; 450 if (map == null || r < 2) { 451 return values; 452 } 453 int[] ovalues = new int[r]; 454 for (int i = 0; i < r; i++) { 455 ovalues[map[i]] = values[i]; 456 } 457 return ovalues; 458 } 459 460 protected final SliceND calcTrueSlice(SliceND slice) { 461 if (slice == null) { 462 slice = new SliceND(shape); 463 } 464 int[] lstart = slice.getStart(); 465 int[] lstop = slice.getStop(); 466 int[] lstep = slice.getStep(); 467 468 int[] nstart; 469 int[] nstop; 470 int[] nstep; 471 472 int r = base == null ? oShape.length : base.shape.length; 473 nstart = new int[r]; 474 nstop = new int[r]; 475 nstep = new int[r]; 476 Arrays.fill(nstop, 1); 477 Arrays.fill(nstep, 1); 478 { 479 int i = 0; 480 int j = 0; 481 if (prepShape < 0) { // ignore entries from new slice 482 i = -prepShape; 483 } else if (prepShape > 0) { 484 j = prepShape; 485 } 486 if (begSlice == null) { 487 for (; i < r && j < shape.length; i++, j++) { 488 nstart[i] = lstart[j]; 489 nstop[i] = lstop[j]; 490 int d = lstep[j]; 491 if (d < 0 && nstop[i] < 0) { // need to wrap around further 492 int l = base == null ? oShape[j]: base.shape[j]; 493 nstop[i] -= l; 494 } 495 nstep[i] = d; 496 } 497 } else { 498 for (; i < r && j < shape.length; i++, j++) { 499 int b = begSlice[j]; 500 int d = delSlice[j]; 501 nstart[i] = b + lstart[j] * d; 502 nstop[i] = b + (lstop[j] - 1) * d + (d >= 0 ? 1 : -1); 503 if (d < 0 && nstop[i] < 0) { // need to wrap around further 504 int l = base == null ? oShape[j]: base.shape[j]; 505 nstop[i] -= l; 506 } 507 nstep[i] = lstep[j] * d; 508 } 509 } 510 if (map != null) { 511 nstart = getOriginal(nstart); 512 nstop = getOriginal(nstop); 513 nstep = getOriginal(nstep); 514 } 515 } 516 517 return createSlice(nstart, nstop, nstep); 518 } 519 520 protected final IDataset transformInput(IDataset data) { 521 if (map == null) { 522 return data; 523 } 524 return data.getTransposedView(map); 525 } 526 527 protected SliceND createSlice(int[] nstart, int[] nstop, int[] nstep) { 528 return new SliceND(base == null ? oShape : base.shape, nstart, nstop, nstep); 529 } 530 531 /** 532 * Store metadata items that has given annotation 533 * @param origMetadata 534 * @param aclazz 535 */ 536 private void storeMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> origMetadata, Class<? extends Annotation> aclazz) { 537 List<Class<? extends MetadataType>> mclazzes = findAnnotatedMetadata(aclazz); 538 if (mclazzes.size() == 0) { 539 return; 540 } 541 542 if (oMetadata == null) { 543 oMetadata = new HashMap<Class<? extends MetadataType>, List<MetadataType>>(); 544 } 545 for (Class<? extends MetadataType> mc : mclazzes) { 546 if (oMetadata.containsKey(mc)) { 547 continue; // do not overwrite original 548 } 549 550 List<MetadataType> l = origMetadata.get(mc); 551 List<MetadataType> nl = new ArrayList<MetadataType>(l.size()); 552 for (MetadataType m : l) { 553 nl.add(m.clone()); 554 } 555 oMetadata.put(mc, nl); 556 } 557 } 558 559 @SuppressWarnings("unchecked") 560 private List<Class<? extends MetadataType>> findAnnotatedMetadata(Class<? extends Annotation> aclazz) { 561 List<Class<? extends MetadataType>> mclazzes = new ArrayList<Class<? extends MetadataType>>(); 562 if (metadata == null) { 563 return mclazzes; 564 } 565 566 for (Class<? extends MetadataType> c : metadata.keySet()) { 567 boolean hasAnn = false; 568 for (MetadataType m : metadata.get(c)) { 569 if (m == null) { 570 continue; 571 } 572 573 Class<? extends MetadataType> mc = m.getClass(); 574 do { // iterate over super-classes 575 for (Field f : mc.getDeclaredFields()) { 576 if (f.isAnnotationPresent(aclazz)) { 577 hasAnn = true; 578 break; 579 } 580 } 581 Class<?> sclazz = mc.getSuperclass(); 582 if (!MetadataType.class.isAssignableFrom(sclazz)) { 583 break; 584 } 585 mc = (Class<? extends MetadataType>) sclazz; 586 } while (!hasAnn); 587 if (hasAnn) { 588 break; 589 } 590 } 591 if (hasAnn) { 592 mclazzes.add(c); 593 } 594 } 595 return mclazzes; 596 } 597 598 /** 599 * Gets the maximum size of a slice of a dataset in a given dimension 600 * which should normally fit in memory. Note that it might be possible 601 * to get more in memory, this is a conservative estimate and seems to 602 * almost always work at the size returned; providing Xmx is less than 603 * the physical memory. 604 * 605 * To get more in memory increase -Xmx setting or use an expression 606 * which calls a rolling function (like rmean) instead of slicing directly 607 * to memory. 608 * 609 * @param lazySet 610 * @param dimension 611 * @return maximum size of dimension that can be sliced. 612 */ 613 public static int getMaxSliceLength(ILazyDataset lazySet, int dimension) { 614 // size in bytes of each item 615 final double size = DTypeUtils.getItemBytes(DTypeUtils.getDTypeFromClass(lazySet.getElementClass()), lazySet.getElementsPerItem()); 616 617 // Max in bytes takes into account our minimum requirement 618 final double max = Math.max(Runtime.getRuntime().totalMemory(), Runtime.getRuntime().maxMemory()); 619 620 // Firstly if the whole dataset it likely to fit in memory, then we allow it. 621 // Space specified in bytes per item available 622 final double space = max/lazySet.getSize(); 623 624 // If we have room for this whole dataset, then fine 625 int[] shape = lazySet.getShape(); 626 if (space >= size) { 627 return shape[dimension]; 628 } 629 630 // Otherwise estimate what we can fit in, conservatively. 631 // First get size of one slice, see it that fits, if not, still return 1 632 double sizeOneSlice = size; // in bytes 633 for (int dim = 0; dim < shape.length; dim++) { 634 if (dim == dimension) { 635 continue; 636 } 637 sizeOneSlice *= shape[dim]; 638 } 639 double avail = max / sizeOneSlice; 640 if (avail < 1) { 641 return 1; 642 } 643 644 // We fudge this to leave some room 645 return (int) Math.floor(avail/4d); 646 } 647}