001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.Serializable; 016import java.lang.reflect.Array; 017import java.text.Format; 018import java.util.Arrays; 019import java.util.List; 020 021import org.eclipse.january.DatasetException; 022import org.eclipse.january.IMonitor; 023import org.eclipse.january.MetadataException; 024import org.eclipse.january.metadata.ErrorMetadata; 025import org.eclipse.january.metadata.MetadataFactory; 026import org.eclipse.january.metadata.StatisticsMetadata; 027import org.eclipse.january.metadata.internal.ErrorMetadataImpl; 028import org.eclipse.january.metadata.internal.StatisticsMetadataImpl; 029import org.slf4j.Logger; 030import org.slf4j.LoggerFactory; 031 032/** 033 * Generic container class for data 034 * <p> 035 * Each subclass has an array of primitive types, elements of this array are grouped or 036 * compounded to make items 037 * <p> 038 * Data items can be boolean, integer, float, complex float, vector float, etc 039 */ 040public abstract class AbstractDataset extends LazyDatasetBase implements Dataset { 041 // pin UID to base class 042 private static final long serialVersionUID = Dataset.serialVersionUID; 043 044 private static final Logger logger = LoggerFactory.getLogger(AbstractDataset.class); 045 046 protected int size; // number of items 047 048 transient protected AbstractDataset base; // is null when not a view 049 protected int[] stride; // can be null for row-major, contiguous datasets 050 protected int offset; 051 052 /** 053 * The data itself, held in a 1D array, but the object will wrap it to appear as possessing as many dimensions as 054 * wanted 055 */ 056 protected Serializable odata = null; 057 058 /** 059 * Set aliased data as base data 060 */ 061 abstract protected void setData(); 062 063 /** 064 * Constructor required for serialisation. 065 */ 066 public AbstractDataset() { 067 } 068 069 @Override 070 public synchronized Dataset synchronizedCopy() { 071 return clone(); 072 } 073 074 @Override 075 public Class<?> getElementClass() { 076 return InterfaceUtils.getElementClass(getClass()); 077 } 078 079 @Override 080 public int getDType() { 081 return DTypeUtils.getDType(getClass()); 082 } 083 084 @Override 085 public int hashCode() { 086 return getStats().getHash(shape); 087 } 088 089 @Override 090 abstract public AbstractDataset clone(); 091 092 protected Format stringFormat = null; 093 094 @Override 095 public void setStringFormat(Format format) { 096 stringFormat = format; 097 } 098 099 @Override 100 public Dataset copy(final int dtype) { 101 return copy(DTypeUtils.getInterface(dtype)); 102 } 103 104 @Override 105 public <T extends Dataset> T copy(Class<T> clazz) { 106 return DatasetUtils.copy(clazz, this); 107 } 108 109 @Override 110 public Dataset cast(final int dtype) { 111 return cast(DTypeUtils.getInterface(dtype)); 112 } 113 114 @SuppressWarnings("unchecked") 115 @Override 116 public <T extends Dataset> T cast(Class<T> clazz) { 117 if (clazz.isInstance(this)) { 118 return (T) this; 119 } 120 return DatasetUtils.cast(clazz, this); 121 } 122 123 @SuppressWarnings("unchecked") 124 @Override 125 public <T extends Dataset> T cast(final int isize, Class<T> clazz, final boolean repeat) { 126 if (clazz.isInstance(this) && getElementsPerItem() == isize) { 127 return (T) this; 128 } 129 return DatasetUtils.cast(isize, clazz, this, repeat); 130 } 131 132 @Override 133 public Dataset cast(final boolean repeat, final int dtype, final int isize) { 134 return cast(isize, DTypeUtils.getInterface(dtype), repeat); 135 } 136 137 @Override 138 abstract public AbstractDataset getView(boolean deepCopyMetadata); 139 140 /** 141 * Copy fields from original to view 142 * @param orig original 143 * @param view destination 144 * @param clone if true, then clone everything but bulk data 145 * @param cloneMetadata if true, clone metadata 146 */ 147 protected static void copyToView(Dataset orig, AbstractDataset view, boolean clone, boolean cloneMetadata) { 148 view.name = orig.getName(); 149 view.size = orig.getSize(); 150 view.odata = orig.getBuffer(); 151 view.offset = orig.getOffset(); 152 AbstractDataset a = orig instanceof AbstractDataset ? (AbstractDataset) orig : null; 153 view.base = a == null ? null : a.base; 154 155 int[] s = a == null ? null : a.stride; 156 if (clone) { 157 view.shape = orig.getShapeRef() == null ? null : orig.getShape(); 158 view.stride = s == null ? null : s.clone(); 159 } else { 160 view.shape = orig.getShapeRef(); 161 view.stride = s; 162 } 163 164 view.metadata = getMetadataMap(orig, cloneMetadata); 165 Class<? extends Dataset> oc = InterfaceUtils.findSubInterface(orig.getClass()); 166 Class<? extends Dataset> vc = InterfaceUtils.findSubInterface(view.getClass()); 167 if (!oc.equals(vc)) { 168 view.setDirty(); 169 } 170 } 171 172 @Override 173 public IntegerDataset getIndices() { 174 final IntegerDataset ret = DatasetUtils.indices(shape); 175 if (getName() != null) { 176 ret.setName("Indices of " + getName()); 177 } 178 return ret; 179 } 180 181 @Override 182 public Dataset getTransposedView(int... axes) { 183 axes = checkPermutatedAxes(shape, axes); 184 185 AbstractDataset t = getView(true); 186 if (axes == null || getRank() == 1) 187 return t; 188 189 int rank = shape.length; 190 int[] tstride = new int[rank]; 191 int[] toffset = new int[1]; 192 int[] nshape = createStrides(new SliceND(shape), this, tstride, toffset); 193 int[] nstride = new int[rank]; 194 for (int i = 0; i < rank; i++) { 195 final int ax = axes[i]; 196 nstride[i] = tstride[ax]; 197 nshape[i] = shape[ax]; 198 } 199 t.shape = nshape; 200 t.stride = nstride; 201 t.offset = toffset[0]; 202 t.base = this; 203 t.setDirty(); 204 t.transposeMetadata(axes); 205 return t; 206 } 207 208 @Override 209 public Dataset transpose(int... axes) { 210 Dataset t = getTransposedView(axes); 211 return t == null ? clone() : t.clone(); 212 } 213 214 @Override 215 public Dataset swapAxes(int axis1, int axis2) { 216 int rank = shape.length; 217 axis1 = ShapeUtils.checkAxis(rank, axis1); 218 axis2 = ShapeUtils.checkAxis(rank, axis2); 219 220 if (rank == 1 || axis1 == axis2) { 221 return this; 222 } 223 224 int[] axes = new int[rank]; 225 for (int i = 0; i < rank; i++) { 226 axes[i] = i; 227 } 228 229 axes[axis1] = axis2; 230 axes[axis2] = axis1; 231 return getTransposedView(axes); 232 } 233 234 boolean isContiguous() { 235 if (stride == null) 236 return true; 237 238 if (offset != 0) 239 return false; 240 241 int s = getElementsPerItem(); 242 for (int j = getRank() - 1; j >= 0; j--) { 243 if (stride[j] != s) { 244 return false; 245 } 246 s *= shape[j]; 247 } 248 249 return true; 250 } 251 252 @Override 253 public Dataset flatten() { 254 if (!isContiguous()) { // need to make a copy if not contiguous 255 return clone().flatten(); 256 } 257 return reshape(size); 258 } 259 260 /** 261 * Fill dataset from object at depth dimension 262 * @param obj fill value 263 * @param depth dimension 264 * @param pos position 265 */ 266 protected void fillData(Object obj, final int depth, final int[] pos) { 267 if (obj == null) { 268 Class<?> c = InterfaceUtils.getElementClass(getClass()); 269 if (Float.class.equals(c)) { 270 set(Float.NaN, pos); 271 } else if (Float.class.equals(c)) { 272 set(Double.NaN, pos); 273 } 274 return; 275 } 276 277 Class<?> clazz = obj.getClass(); 278 if (obj instanceof List<?>) { 279 List<?> jl = (List<?>) obj; 280 int l = jl.size(); 281 for (int i = 0; i < l; i++) { 282 Object lo = jl.get(i); 283 fillData(lo, depth + 1, pos); 284 pos[depth]++; 285 } 286 pos[depth] = 0; 287 } else if (clazz.isArray()) { 288 int l = Array.getLength(obj); 289 if (clazz.equals(odata.getClass())) { 290 System.arraycopy(obj, 0, odata, get1DIndex(pos), l); 291 } else if (clazz.getComponentType().isPrimitive()) { 292 for (int i = 0; i < l; i++) { 293 set(Array.get(obj, i), pos); 294 pos[depth]++; 295 } 296 pos[depth] = 0; 297 } else { 298 for (int i = 0; i < l; i++) { 299 fillData(Array.get(obj, i), depth + 1, pos); 300 pos[depth]++; 301 } 302 pos[depth] = 0; 303 } 304 } else if (obj instanceof IDataset) { 305 boolean[] a = new boolean[shape.length]; 306 Arrays.fill(a, depth, a.length, true); 307 setSlice(obj, getSliceIteratorFromAxes(pos, a)); 308 } else { 309 set(obj, pos); 310 } 311 } 312 313 @Override 314 public IndexIterator getIterator(final boolean withPosition) { 315 if (stride != null) { 316 return base.getSize() == 1 ? (withPosition ? new PositionIterator(offset, shape) : 317 new SingleItemIterator(offset, size)) : new StrideIterator(shape, stride, offset); 318 } 319 if (shape == null) { 320 return new NullIterator(); 321 } 322 323 return withPosition ? new ContiguousIteratorWithPosition(shape, size) : new ContiguousIterator(size); 324 } 325 326 @Override 327 public IndexIterator getIterator() { 328 return getIterator(false); 329 } 330 331 @Override 332 public PositionIterator getPositionIterator(final int... axes) { 333 return new PositionIterator(shape, axes); 334 } 335 336 @Override 337 public IndexIterator getSliceIterator(final int[] start, final int[] stop, final int[] step) { 338 return internalGetSliceIterator(new SliceND(shape, start, stop, step)); 339 } 340 341 /** 342 * @param slice an n-D slice 343 * @return an slice iterator that operates like an IndexIterator 344 */ 345 public IndexIterator getSliceIterator(SliceND slice) { 346 checkSliceND(slice); 347 return internalGetSliceIterator(slice); 348 } 349 350 /** 351 * @param slice an n-D slice 352 * @return an slice iterator that operates like an IndexIterator 353 */ 354 protected IndexIterator internalGetSliceIterator(SliceND slice) { 355 if (ShapeUtils.calcLongSize(slice.getShape()) == 0) { 356 return new NullIterator(shape, slice.getShape()); 357 } 358 if (stride != null) { 359 return new StrideIterator(getElementsPerItem(), shape, stride, offset, slice); 360 } 361 return new SliceIterator(shape, size, slice); 362 } 363 364 @Override 365 public SliceIterator getSliceIteratorFromAxes(final int[] pos, boolean[] axes) { 366 int rank = shape.length; 367 int[] start; 368 int[] stop = new int[rank]; 369 int[] step = new int[rank]; 370 371 if (pos == null) { 372 start = new int[rank]; 373 } else if (pos.length == rank) { 374 start = pos.clone(); 375 } else { 376 throw new IllegalArgumentException("pos array length is not equal to rank of dataset"); 377 } 378 if (axes == null) { 379 axes = new boolean[rank]; 380 Arrays.fill(axes, true); 381 } else if (axes.length != rank) { 382 throw new IllegalArgumentException("axes array length is not equal to rank of dataset"); 383 } 384 385 for (int i = 0; i < rank; i++) { 386 if (axes[i]) { 387 stop[i] = shape[i]; 388 } else { 389 stop[i] = start[i] + 1; 390 } 391 step[i] = 1; 392 } 393 return (SliceIterator) getSliceIterator(start, stop, step); 394 } 395 396 @Override 397 public BooleanIterator getBooleanIterator(Dataset choice) { 398 return getBooleanIterator(choice, true); 399 } 400 401 @Override 402 public BooleanIterator getBooleanIterator(Dataset choice, boolean value) { 403 return BooleanIterator.createIterator(value, this, choice, this); 404 } 405 406 @Override 407 public Dataset getByBoolean(Dataset selection) { 408 checkCompatibility(selection); 409 410 final int length = ((Number) selection.sum()).intValue(); 411 final int is = getElementsPerItem(); 412 Dataset r = DatasetFactory.zeros(is, getClass(), length); 413 BooleanIterator biter = getBooleanIterator(selection); 414 415 int i = 0; 416 while (biter.hasNext()) { 417 r.setObjectAbs(i, getObjectAbs(biter.index)); 418 i += is; 419 } 420 return r; 421 } 422 423 @Override 424 public Dataset getBy1DIndex(IntegerDataset index) { 425 final int is = getElementsPerItem(); 426 final Dataset r = DatasetFactory.zeros(is, getClass(), index.getShapeRef()); 427 final IntegerIterator iter = new IntegerIterator(index, size, is); 428 429 int i = 0; 430 while (iter.hasNext()) { 431 r.setObjectAbs(i, getObjectAbs(iter.index)); 432 i += is; 433 } 434 return r; 435 } 436 437 @Override 438 public Dataset getByIndexes(final Object... indexes) { 439 final IntegersIterator iter = new IntegersIterator(shape, indexes); 440 final int is = getElementsPerItem(); 441 final Dataset r = DatasetFactory.zeros(is, getClass(), iter.getShape()); 442 443 final int[] pos = iter.getPos(); 444 int i = 0; 445 while (iter.hasNext()) { 446 r.setObjectAbs(i, getObject(pos)); 447 i += is; 448 } 449 return r; 450 } 451 452 @Override 453 public boolean hasFloatingPointElements() { 454 Class<?> cls = getElementClass(); 455 return cls == Float.class || cls == Double.class; 456 } 457 458 @Override 459 public int getElementsPerItem() { 460 return InterfaceUtils.getElementsPerItem(getClass()); 461 } 462 463 @Override 464 public int getItemBytes() { 465 return InterfaceUtils.getItemBytes(getElementsPerItem(), getClass()); 466 } 467 468 @Override 469 public int getSize() { 470 return size; 471 } 472 473 @Override 474 public int[] getShape() { 475 // make a copy of the dimensions data, and put that out 476 if (shape == null) { 477 logger.warn("Shape is null!!!"); 478 return new int[] {}; 479 } 480 return shape.clone(); 481 } 482 483 @Override 484 public int getRank() { 485 return shape == null ? 0 : shape.length; 486 } 487 488 @Override 489 public int getNbytes() { 490 return getSize() * getItemBytes(); 491 } 492 493 /** 494 * Check for -1 placeholder in shape and replace if necessary 495 * @param shape to use 496 * @param size expected size 497 */ 498 private void checkShape(int[] shape, int size) { 499 if (shape == null) { 500 if (size == 0) { 501 return; 502 } 503 logger.error("New shape must not be null for nonzero-sized dataset"); 504 throw new IllegalArgumentException("New shape must not be null for nonzero-sized dataset"); 505 } 506 int rank = shape.length; 507 int found = -1; 508 int nsize = 1; 509 for (int i = 0; i < rank; i++) { 510 int d = shape[i]; 511 if (d == -1) { 512 if (found == -1) { 513 found = i; 514 } else { 515 logger.error("Can only have one -1 placeholder in shape"); 516 throw new IllegalArgumentException("Can only have one -1 placeholder in shape"); 517 } 518 } else { 519 nsize *= d; 520 } 521 } 522 if (found >= 0) { 523 shape[found] = size/nsize; 524 } else if (nsize != size && !(rank == 0 && size == 0)) { 525 logger.error("New shape is not same size as old shape"); 526 throw new IllegalArgumentException("New size is not same as the old size. Old size is "+size+" new size is "+nsize+" and shape is "+Arrays.toString(shape)); 527 } 528 } 529 530 @Override 531 public void setShape(final int... shape) { 532 int[] nshape = shape == null ? null : shape.clone(); 533 checkShape(nshape, size); 534 if (Arrays.equals(this.shape, nshape)) { 535 return; 536 } 537 538 if (stride != null) { 539 // the only compatible shapes are ones where new dimensions are factors of old dimensions 540 // or are combined adjacent old dimensions 541 int[] oshape = this.shape; 542 int orank = oshape.length; 543 int nrank = nshape.length; 544 int diff = nrank - orank; 545 int[] nstride = new int[nrank]; 546 boolean ones = true; 547 // work forwards for broadcasting cases 548 for (int i = 0, j = 0; i < orank || j < nrank;) { 549 if (j >= diff && i < orank && j < nrank && oshape[i] == nshape[j]) { 550 nstride[j++] = stride[i++]; 551 } else if (j < nrank && nshape[j] == 1) { 552 nstride[j++] = 0; 553 } else if (i < orank && oshape[i] == 1) { 554 i++; 555 } else { 556 if (j < nrank) 557 j++; 558 if (i < orank) 559 i++; 560 ones = false; 561 } 562 } 563 if (!ones) { // not just ones differ in shapes 564 int[] ostride = stride; 565 int ob = 0; 566 int oe = 1; 567 int nb = 0; 568 int ne = 1; 569 while (ob < orank && nb < nrank) { 570 int ol = oshape[ob]; 571 int nl = nshape[nb]; 572 573 if (nl < ol) { // find group of shape dimensions that form common size 574 do { // case where new shape spreads single dimension over several dimensions 575 if (ne == nrank) { 576 break; 577 } 578 nl *= nshape[ne++]; 579 } while (nl < ol); 580 if (nl != ol) { 581 logger.error("Subshape is incompatible with single dimension"); 582 throw new IllegalArgumentException("Subshape is incompatible with single dimension"); 583 } 584 int on = ne - 1; 585 while (nshape[on] == 1) { 586 on--; 587 } 588 589 nstride[on] = ostride[ob]; 590 for (int n = on - 1; n >= nb; n--) { 591 if (nshape[n] == 1) 592 continue; 593 594 nstride[n] = nshape[on] * nstride[on]; 595 on = n; 596 } 597 } else if (ol < nl) { 598 do { // case where new shape combines several dimensions into one dimension 599 if (oe == orank) { 600 break; 601 } 602 ol *= oshape[oe++]; 603 } while (ol < nl); 604 if (nl != ol) { 605 logger.error("Single dimension is incompatible with subshape"); 606 throw new IllegalArgumentException("Single dimension is incompatible with subshape"); 607 } 608 609 int oo = oe - 1; 610 while (oshape[oo] == 1) { 611 oo--; 612 } 613 int os = ostride[oo]; 614 for (int o = oo - 1; o >= ob; o--) { 615 if (oshape[o] == 1) 616 continue; 617 if (ostride[o] != oshape[oo] * ostride[oo]) { 618 logger.error("Subshape cannot be a non-contiguous view"); 619 throw new IllegalArgumentException("Subshape cannot be a non-contiguous view"); 620 } 621 oo = o; 622 } 623 nstride[nb] = os; 624 } else { 625 nstride[nb] = ostride[ob]; 626 } 627 628 ob = oe++; 629 nb = ne++; 630 } 631 } 632 633 stride = nstride; 634 } 635 636 setDirty(); 637 if (this.shape != null && metadata != null) { 638 reshapeMetadata(this.shape, nshape); 639 } 640 this.shape = nshape; 641 } 642 643 @Override 644 public int[] getShapeRef() { 645 return shape; 646 } 647 648 @Override 649 public int getOffset() { 650 return offset; 651 } 652 653 @Override 654 public int[] getStrides() { 655 return stride; 656 } 657 658 @Override 659 public Serializable getBuffer() { 660 return odata; 661 } 662 663 @Override 664 public void overrideInternal(Serializable buffer, int... shape) { 665 if (buffer != null) { 666 odata = buffer; 667 setData(); 668 setDirty(); 669 } 670 671 if (shape != null) { 672 this.shape = shape.clone(); 673 size = ShapeUtils.calcSize(this.shape); 674 } 675 } 676 677 /** 678 * Create a stride array from dataset 679 * @param a dataset 680 * @param offset output offset 681 * @return new strides 682 */ 683 public static int[] createStrides(Dataset a, final int[] offset) { 684 return createStrides(a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), offset); 685 } 686 687 /** 688 * Create a stride array from dataset 689 * @param isize item size 690 * @param shape to use 691 * @param oStride original stride 692 * @param oOffset original offset (only used if there is an original stride) 693 * @param offset output offset 694 * @return new strides 695 */ 696 public static int[] createStrides(final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] offset) { 697 int rank = shape.length; 698 final int[] stride; 699 if (oStride == null) { 700 offset[0] = 0; 701 stride = new int[rank]; 702 int s = isize; 703 for (int j = rank - 1; j >= 0; j--) { 704 stride[j] = s; 705 s *= shape[j]; 706 } 707 } else { 708 offset[0] = oOffset; 709 stride = oStride.clone(); 710 } 711 return stride; 712 } 713 714 /** 715 * Create a stride array from slice information and a dataset 716 * @param slice an n-D slice 717 * @param a dataset 718 * @param stride output stride 719 * @param offset output offset 720 * @return new shape 721 */ 722 public static int[] createStrides(final SliceND slice, final Dataset a, final int[] stride, final int[] offset) { 723 return createStrides(slice, a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), stride, offset); 724 } 725 726 /** 727 * Create a stride array from slice and dataset information 728 * @param slice an n-D slice 729 * @param isize item size 730 * @param shape to use 731 * @param oStride original stride 732 * @param oOffset original offset (only used if there is an original stride) 733 * @param stride output stride 734 * @param offset output offset 735 * @return new shape 736 */ 737 public static int[] createStrides(final SliceND slice, final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] stride, final int[] offset) { 738 int[] lstart = slice.getStart(); 739 int[] lstep = slice.getStep(); 740 int[] newShape = slice.getShape(); 741 int rank = shape.length; 742 743 if (oStride == null) { 744 int s = isize; 745 offset[0] = 0; 746 for (int j = rank - 1; j >= 0; j--) { 747 stride[j] = s * lstep[j]; 748 offset[0] += s * lstart[j]; 749 s *= shape[j]; 750 } 751 } else { 752 offset[0] = oOffset; 753 for (int j = 0; j < rank; j++) { 754 int s = oStride[j]; 755 stride[j] = lstep[j] * s; 756 offset[0] += lstart[j] * s; 757 } 758 } 759 760 return newShape; 761 } 762 763 @Override 764 public Dataset getBroadcastView(int... broadcastShape) { 765 AbstractDataset view = getView(true); 766 767 if (!Arrays.equals(shape, broadcastShape)) { 768 List<int[]> nShapes = BroadcastUtils.broadcastShapesToMax(broadcastShape, shape); 769 view.setShape(nShapes.get(0)); 770 view.stride = BroadcastUtils.createBroadcastStrides(view, broadcastShape); 771 view.base = this; 772 view.shape = broadcastShape.clone(); 773 view.size = ShapeUtils.calcSize(broadcastShape); 774 if (view.name == null || view.name.isEmpty()) { 775 view.name = "Broadcast from " + Arrays.toString(shape); 776 } else { 777 view.name = "Broadcast of " + view.name + " from " + Arrays.toString(shape); 778 } 779 } 780 return view; 781 } 782 783 @Override 784 public Dataset getSliceView(final int[] start, final int[] stop, final int[] step) { 785 return internalGetSliceView(new SliceND(shape, start, stop, step)); 786 } 787 788 @Override 789 public Dataset getSliceView(Slice... slice) { 790 if (slice == null || slice.length == 0) { 791 return getView(true); 792 } 793 794 return internalGetSliceView(new SliceND(shape, slice)); 795 } 796 797 /** 798 * Get a slice of the dataset. The returned dataset is a view on a selection of items 799 * @param slice an n-D slice 800 * @return slice view 801 */ 802 @Override 803 public Dataset getSliceView(SliceND slice) { 804 checkSliceND(slice); 805 return internalGetSliceView(slice); 806 } 807 808 private Dataset internalGetSliceView(SliceND slice) { 809 if (slice.isAll()) { 810 return getView(true); 811 } 812 813 final int rank = shape.length; 814 int[] sStride = new int[rank]; 815 int[] sOffset = new int[1]; 816 817 int[] sShape = createStrides(slice, this, sStride, sOffset); 818 819 AbstractDataset s = getView(false); 820 s.shape = sShape; 821 s.size = ShapeUtils.calcSize(sShape); 822 s.stride = sStride; 823 s.offset = sOffset[0]; 824 s.base = this; 825 826 s.metadata = copyMetadata(); 827 s.sliceMetadata(true, slice); 828 829 s.setDirty(); 830 s.setName(name + BLOCK_OPEN + slice + BLOCK_CLOSE); 831 832 return s; 833 } 834 835 /** 836 * Get flattened view index of given position 837 * @param pos 838 * the integer array specifying the n-D position 839 * @return the index on the flattened dataset 840 */ 841 private int getFlat1DIndex(final int[] pos) { 842 final int imax = pos.length; 843 if (imax == 0) { 844 return 0; 845 } 846 847 return get1DIndexFromShape(pos); 848 } 849 850 /** 851 * @return index of first element 852 * @since 2.0 853 */ 854 protected int getFirst1DIndex() { 855 if (shape == null) { 856 throw new IllegalArgumentException("Cannot find an index from a null shape"); 857 } 858 return stride == null ? 0 : offset; 859 } 860 861 @Override 862 public int get1DIndex(final int... n) { 863 if (n.length == 0 && shape.length == 0) 864 return offset; 865 866 return stride == null ? get1DIndexFromShape(n) : get1DIndexFromStrides(n); 867 } 868 869 private static void throwAIOOBException(int i, int s, int d) { 870 throw new ArrayIndexOutOfBoundsException("Index (" + i + ") out of range [-" + s + "," + s 871 + "] in dimension " + d); 872 } 873 874 /** 875 * @param i position in first dimension 876 * @return the index on the data array corresponding to that location 877 */ 878 protected int get1DIndex(int i) { 879 if (shape == null) { 880 throw new IllegalArgumentException("Cannot find an index from a null shape"); 881 } 882 if (shape.length > 1) { 883 logger.error("This dataset is not 1D but was addressed as such"); 884 throw new UnsupportedOperationException("This dataset is not 1D but was addressed as such"); 885 } 886 if (i < 0) { 887 i += shape[0]; 888 } 889 if (i < 0 || i >= shape[0]) { 890 throwAIOOBException(i, shape[0], 0); 891 } 892 return stride == null ? i : i*stride[0] + offset; 893 } 894 895 /** 896 * @param i position in first dimension 897 * @param j position in second dimension 898 * @return the index on the data array corresponding to that location 899 */ 900 protected int get1DIndex(int i, int j) { 901 if (shape == null) { 902 throw new IllegalArgumentException("Cannot find an index from a null shape"); 903 } 904 if (shape.length != 2) { 905 logger.error("This dataset is not 2D but was addressed as such"); 906 throw new UnsupportedOperationException("This dataset is not 2D but was addressed as such"); 907 } 908 if (i < 0) { 909 i += shape[0]; 910 } 911 if (i < 0 || i >= shape[0]) { 912 throwAIOOBException(i, shape[0], 0); 913 } 914 if (j < 0) { 915 j += shape[1]; 916 } 917 if (j < 0 || j >= shape[1]) { 918 throwAIOOBException(i, shape[1], 1); 919 } 920 return stride == null ? i*shape[1] + j : i*stride[0] + j*stride[1] + offset; 921 } 922 923 protected int get1DIndexFromShape(final int[] n) { 924 return get1DIndexFromShape(shape, n); 925 } 926 927 protected static int get1DIndexFromShape(final int[] shape, final int[] n) { 928 if (shape == null) { 929 throw new IllegalArgumentException("Cannot find an index from a null shape"); 930 } 931 final int rank = shape.length; 932 if (rank != n.length) { 933 String errMsg = String.format("Number of position values must be equal to rank of %d", rank); 934 logger.error(errMsg); 935 throw new IllegalArgumentException(errMsg); 936 } 937 int index = 0; 938 for (int i = 0; i < rank; i++) { 939 final int si = shape[i]; 940 int ni = n[i]; 941 if (ni < 0) { 942 ni += si; 943 } 944 if (ni < 0 || ni >= si) { 945 throwAIOOBException(ni, si, i); 946 } 947 index = index * si + ni; 948 } 949 950 return index; 951 } 952 953 private int get1DIndexFromStrides(final int[] n) { 954 return get1DIndexFromStrides(shape, stride, offset, n); 955 } 956 957 private static int get1DIndexFromStrides(final int[] shape, final int[] stride, final int offset, final int[] n) { 958 if (shape == null) { 959 throw new IllegalArgumentException("Cannot find an index from a null shape"); 960 } 961 final int rank = shape.length; 962 if (rank != n.length) { 963 String errMsg = String.format("Number of position values must be equal to rank of %d", rank); 964 logger.error(errMsg); 965 throw new IllegalArgumentException(errMsg); 966 } 967 int index = offset; 968 for (int i = 0; i < rank; i++) { 969 final int st = stride[i]; 970 if (st != 0) { // not broadcasted 971 final int si = shape[i]; 972 int ni = n[i]; 973 if (ni < 0) { 974 ni += si; 975 } 976 if (ni < 0 || ni >= si) { 977 throwAIOOBException(ni, si, i); 978 } 979 index += st * ni; 980 } 981 } 982 return index; 983 } 984 985 @Override 986 public int[] getNDPosition(final int n) { 987 if (isIndexInRange(n)) { 988 throw new IllegalArgumentException("Index provided " + n 989 + "is larger then the size of the containing array"); 990 } 991 992 return stride == null ? ShapeUtils.getNDPositionFromShape(n, shape) : getNDPositionFromStrides(n); 993 } 994 995 private boolean isIndexInRange(final int n) { 996 if (stride == null) { 997 return n >= size; 998 } 999 return n >= getBufferLength(); 1000 } 1001 1002 /** 1003 * @return entire buffer length 1004 */ 1005 abstract protected int getBufferLength(); 1006 1007 private int[] getNDPositionFromStrides(int n) { 1008 n -= offset; 1009 int rank = shape.length; 1010 if (rank == 1) { 1011 return new int[] { n / stride[0] }; 1012 } 1013 1014 int[] output = new int[rank]; 1015 int i = 0; 1016 while (i != n) { // TODO find more efficient way than this exhaustive search 1017 int j = rank - 1; 1018 for (; j >= 0; j--) { 1019 output[j]++; 1020 i += stride[j]; 1021 if (output[j] >= shape[j]) { 1022 output[j] = 0; 1023 i -= shape[j] * stride[j]; 1024 } else { 1025 break; 1026 } 1027 } 1028 if (j == -1) { 1029 logger.error("Index was not found in this strided dataset"); 1030 throw new IllegalArgumentException("Index was not found in this strided dataset"); 1031 } 1032 } 1033 1034 return output; 1035 } 1036 1037 @Override 1038 public int checkAxis(int axis) { 1039 return ShapeUtils.checkAxis(shape.length, axis); 1040 } 1041 1042 @Deprecated 1043 protected static int checkAxis(int rank, int axis) { 1044 return ShapeUtils.checkAxis(rank, axis); 1045 } 1046 1047 protected static final char BLOCK_OPEN = '['; 1048 protected static final char BLOCK_CLOSE = ']'; 1049 1050 @Override 1051 public String toString() { 1052 final int rank = shape == null ? 0 : shape.length; 1053 final StringBuilder out = new StringBuilder(); 1054 1055 if (InterfaceUtils.isElemental(getClass())) { 1056 out.append("Dataset "); 1057 } else { 1058 out.append("Compound dataset ("); 1059 out.append(getElementsPerItem()); 1060 out.append(") "); 1061 } 1062 1063 if (name != null && name.length() > 0) { 1064 out.append("'"); 1065 out.append(name); 1066 out.append("' has shape "); 1067 } else { 1068 out.append("shape is "); 1069 } 1070 1071 out.append(BLOCK_OPEN); 1072 if (rank > 0) { 1073 out.append(shape[0]); 1074 } 1075 for (int i = 1; i < rank; i++) { 1076 out.append(", " + shape[i]); 1077 } 1078 out.append(BLOCK_CLOSE); 1079 return out.toString(); 1080 } 1081 1082 @Override 1083 public String toString(boolean showData) { 1084 if (!showData) { 1085 return toString(); 1086 } 1087 1088 if (size == 0) { 1089 return "[]"; 1090 } 1091 1092 final int rank = shape == null ? 0 : shape.length; 1093 final StringBuilder out = new StringBuilder(); 1094 1095 if (rank > 0) { 1096 int[] pos = new int[rank]; 1097 final StringBuilder lead = new StringBuilder(); 1098 printBlocks(out, lead, 0, pos); 1099 } else { 1100 out.append(getString()); 1101 } 1102 return out.toString(); 1103 } 1104 1105 /** 1106 * Limit to strings output via the toString() method 1107 */ 1108 private static int maxStringLength = 120; 1109 1110 /** 1111 * Set maximum line length for toString() method 1112 * @param maxLineLength limit on length of line 1113 */ 1114 public static void setMaxLineLength(int maxLineLength) { 1115 maxStringLength = maxLineLength; 1116 } 1117 1118 /** 1119 * @return maximum line length for toString() method 1120 */ 1121 public static int getMaxLineLength() { 1122 return maxStringLength; 1123 } 1124 1125 /** 1126 * Limit to number of sub-blocks output via the toString() method 1127 */ 1128 private static final int MAX_SUBBLOCKS = 6; 1129 1130 private final static String SEPARATOR = ","; 1131 private final static String SPACE = " "; 1132 private final static String ELLIPSIS = "..."; 1133 private final static String NEWLINE = "\n"; 1134 1135 /** 1136 * Make a line of output for last dimension of dataset 1137 * @param end 1138 * @param start 1139 * @return line 1140 */ 1141 private StringBuilder makeLine(final int end, final int[] start) { 1142 StringBuilder line = new StringBuilder(); 1143 final int[] pos; 1144 if (end >= start.length) { 1145 pos = Arrays.copyOf(start, end + 1); 1146 } else { 1147 pos = start; 1148 } 1149 pos[end] = 0; 1150 line.append(BLOCK_OPEN); 1151 line.append(getString(pos)); 1152 1153 final int length = shape[end]; 1154 1155 // trim elements printed if length exceed estimate of maximum elements 1156 int excess = length - maxStringLength / 3; // space + number + separator 1157 int midIndex = -1; 1158 if (excess > 0) { 1159 int index = (length - excess) / 2; 1160 for (int y = 1; y < index; y++) { 1161 line.append(SEPARATOR + SPACE); 1162 pos[end] = y; 1163 line.append(getString(pos)); 1164 } 1165 midIndex = line.length() + 2; 1166 index = (length + excess) / 2; 1167 for (int y = index; y < length; y++) { 1168 line.append(SEPARATOR + SPACE); 1169 pos[end] = y; 1170 line.append(getString(pos)); 1171 } 1172 } else { 1173 for (int y = 1; y < length; y++) { 1174 line.append(SEPARATOR + SPACE); 1175 pos[end] = y; 1176 line.append(getString(pos)); 1177 } 1178 } 1179 line.append(BLOCK_CLOSE); 1180 1181 // trim string down to limit 1182 int lineLength = line.length(); 1183 excess = lineLength - maxStringLength - ELLIPSIS.length() - 1; 1184 if (excess > 0) { 1185 int index = (lineLength - excess) / 2; 1186 if (midIndex > 0 && index > midIndex) { 1187 index = midIndex; 1188 } else { 1189 index = line.lastIndexOf(SEPARATOR, index) + 2; 1190 } 1191 StringBuilder out = new StringBuilder(line.subSequence(0, index)); 1192 out.append(ELLIPSIS + SEPARATOR); 1193 index = (lineLength + excess) / 2; 1194 if (midIndex > 0 && index <= midIndex) { 1195 index = midIndex - 1; 1196 } else { 1197 index = line.indexOf(SEPARATOR, index) + 1; 1198 } 1199 out.append(line.subSequence(index, lineLength)); 1200 return out; 1201 } else if (midIndex > 0) { // add ellipsis 1202 StringBuilder out = new StringBuilder(line.subSequence(0, midIndex)); 1203 out.append(ELLIPSIS + SEPARATOR + SPACE); 1204 out.append(line.subSequence(midIndex, lineLength)); 1205 return out; 1206 } 1207 1208 return line; 1209 } 1210 1211 /** 1212 * recursive method to print blocks 1213 */ 1214 private void printBlocks(final StringBuilder out, final StringBuilder lead, final int level, final int[] pos) { 1215 if (out.length() > 0) { 1216 char last = out.charAt(out.length() - 1); 1217 if (last != BLOCK_OPEN) { 1218 out.append(lead); 1219 } 1220 } 1221 final int end = getRank() - 1; 1222 if (level != end) { 1223 out.append(BLOCK_OPEN); 1224 int length = shape[level]; 1225 1226 // first sub-block 1227 pos[level] = 0; 1228 StringBuilder newlead = new StringBuilder(lead); 1229 newlead.append(SPACE); 1230 printBlocks(out, newlead, level + 1, pos); 1231 if (length < 2) { // escape 1232 out.append(BLOCK_CLOSE); 1233 return; 1234 } 1235 1236 out.append(SEPARATOR + NEWLINE); 1237 for (int i = level + 1; i < end; i++) { 1238 out.append(NEWLINE); 1239 } 1240 1241 // middle sub-blocks 1242 if (length < MAX_SUBBLOCKS) { 1243 for (int x = 1; x < length - 1; x++) { 1244 pos[level] = x; 1245 printBlocks(out, newlead, level + 1, pos); 1246 if (end <= level + 1) { 1247 out.append(SEPARATOR + NEWLINE); 1248 } else { 1249 out.append(SEPARATOR + NEWLINE + NEWLINE); 1250 } 1251 } 1252 } else { 1253 final int excess = length - MAX_SUBBLOCKS; 1254 int xmax = (length - excess) / 2; 1255 for (int x = 1; x < xmax; x++) { 1256 pos[level] = x; 1257 printBlocks(out, newlead, level + 1, pos); 1258 if (end <= level + 1) { 1259 out.append(SEPARATOR + NEWLINE); 1260 } else { 1261 out.append(SEPARATOR + NEWLINE + NEWLINE); 1262 } 1263 } 1264 out.append(newlead); 1265 out.append(ELLIPSIS + SEPARATOR + NEWLINE); 1266 xmax = (length + excess) / 2; 1267 for (int x = xmax; x < length - 1; x++) { 1268 pos[level] = x; 1269 printBlocks(out, newlead, level + 1, pos); 1270 if (end <= level + 1) { 1271 out.append(SEPARATOR + NEWLINE); 1272 } else { 1273 out.append(SEPARATOR + NEWLINE + NEWLINE); 1274 } 1275 } 1276 } 1277 1278 // last sub-block 1279 pos[level] = length - 1; 1280 printBlocks(out, newlead, level + 1, pos); 1281 out.append(BLOCK_CLOSE); 1282 } else { 1283 out.append(makeLine(end, pos)); 1284 } 1285 } 1286 1287 @Override 1288 public Dataset squeezeEnds() { 1289 return squeeze(true); 1290 } 1291 1292 @Override 1293 public Dataset squeeze() { 1294 return squeeze(false); 1295 } 1296 1297 @Override 1298 public Dataset squeeze(boolean onlyFromEnds) { 1299 final int[] tshape = ShapeUtils.squeezeShape(shape, onlyFromEnds); 1300 final int[] oshape = shape; 1301 if (stride == null) { 1302 shape = tshape; 1303 } else { 1304 int rank = shape.length; 1305 int trank = tshape.length; 1306 if (trank < rank) { 1307 int[] tstride = new int[tshape.length]; 1308 if (onlyFromEnds) { 1309 for (int i = 0; i < rank; i++) { 1310 if (shape[i] != 1) { 1311 for (int k = 0; k < trank; k++) { 1312 tstride[k] = stride[i++]; 1313 } 1314 break; 1315 } 1316 } 1317 } else { 1318 int t = 0; 1319 for (int i = 0; i < rank; i++) { 1320 if (shape[i] != 1) { 1321 tstride[t++] = stride[i]; 1322 } 1323 } 1324 } 1325 shape = tshape; 1326 stride = tstride; 1327 } 1328 } 1329 1330 setDirty(); 1331 reshapeMetadata(oshape, shape); 1332 return this; 1333 } 1334 1335 @Override 1336 public boolean isCompatibleWith(final ILazyDataset g) { 1337 return ShapeUtils.areShapesCompatible(shape, g.getShape()); 1338 } 1339 1340 @Override 1341 public void checkCompatibility(final ILazyDataset g) throws IllegalArgumentException { 1342 ShapeUtils.checkCompatibility(this, g); 1343 } 1344 1345 @Override 1346 public Dataset reshape(final int... shape) { 1347 Dataset a; 1348 try { 1349 a = getView(true); 1350 a.setShape(shape); 1351 } catch (IllegalArgumentException e) { 1352 a = clone(); 1353 a.setShape(shape); 1354 } 1355 return a; 1356 } 1357 1358 /** 1359 * @param start begin 1360 * @param stop exclusive end 1361 * @param step number to skip 1362 * @return number of steps to take 1363 */ 1364 protected static int calcSteps(final double start, final double stop, final double step) { 1365 return Math.max(0, (int) Math.ceil((stop - start) / step)); 1366 } 1367 1368 @Override 1369 public boolean isComplex() { 1370 return false; 1371 } 1372 1373 @Override 1374 public Dataset getRealPart() { 1375 return this; 1376 } 1377 1378 @Override 1379 public Dataset getRealView() { 1380 return getView(true); 1381 } 1382 1383 @Override 1384 public Dataset getSlice(final int[] start, final int[] stop, final int[] step) { 1385 return internalGetSlice(new SliceND(shape, start, stop, step)); 1386 } 1387 1388 @Override 1389 public Dataset getSlice(Slice... slice) { 1390 return internalGetSlice(new SliceND(shape, slice)); 1391 } 1392 1393 @Override 1394 public Dataset getSlice(IMonitor monitor, Slice... slice) { 1395 return getSlice(slice); 1396 } 1397 1398 @Override 1399 public Dataset getSlice(IMonitor monitor, SliceND slice) { 1400 return getSlice(slice); 1401 } 1402 1403 @Override 1404 public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) { 1405 return getSlice(start, stop, step); 1406 } 1407 1408 /** 1409 * Get a slice of the dataset. The returned dataset is a copied selection of items 1410 * @param slice an n-D slice 1411 * @return The dataset of the sliced data 1412 */ 1413 @Override 1414 public Dataset getSlice(final SliceND slice) { 1415 checkSliceND(slice); 1416 return internalGetSlice(slice); 1417 } 1418 1419 private Dataset internalGetSlice(final SliceND slice) { 1420 SliceIterator it = (SliceIterator) internalGetSliceIterator(slice); 1421 AbstractDataset s = getSlice(it); 1422 s.metadata = copyMetadata(); 1423 s.setDirty(); 1424 s.sliceMetadata(true, slice); 1425 return s; 1426 } 1427 1428 /** 1429 * Get a slice of the dataset. The returned dataset is a copied selection of items 1430 * 1431 * @param iterator Slice iterator 1432 * @return The dataset of the sliced data 1433 */ 1434 abstract public AbstractDataset getSlice(final SliceIterator iterator); 1435 1436 @Override 1437 public Dataset setSlice(final Object obj, final SliceND slice) { 1438 checkSliceND(slice); 1439 return internalSetSlice(obj, slice); 1440 } 1441 1442 private Dataset internalSetSlice(final Object obj, final SliceND slice) { 1443 Dataset ds; 1444 if (obj instanceof Dataset) { 1445 ds = (Dataset) obj; 1446 } else if (obj instanceof IDataset) { 1447 ds = DatasetUtils.convertToDataset((IDataset) obj); 1448 } else { 1449 Class<? extends Dataset> dClass = getClass(); 1450 if (!BooleanDataset.class.equals(dClass)) { 1451 dClass = InterfaceUtils.getLargestInterface(this); 1452 } 1453 ds = DatasetFactory.createFromObject(getElementsPerItem(), dClass, obj); 1454 } 1455 1456 return setSlicedView(getSliceView(slice), ds); 1457 } 1458 1459 @Override 1460 public Dataset setSlice(final Object obj, final int[] start, final int[] stop, final int[] step) { 1461 return internalSetSlice(obj, new SliceND(shape, start, stop, step)); 1462 } 1463 1464 /** 1465 * Set a view of current dataset to given dataset with broadcasting 1466 * @param view destination 1467 * @param d source of data 1468 * @return this dataset 1469 */ 1470 abstract Dataset setSlicedView(Dataset view, Dataset d); 1471 1472 @Override 1473 public Dataset setSlice(Object obj, Slice... slice) { 1474 if (slice == null || slice.length == 0) { 1475 return internalSetSlice(obj, new SliceND(shape)); 1476 } 1477 return internalSetSlice(obj, new SliceND(shape, slice)); 1478 } 1479 1480 @Override 1481 public boolean all() { 1482 return Comparisons.allTrue(this); 1483 } 1484 1485 @Override 1486 public BooleanDataset all(final int axis) { 1487 return Comparisons.allTrue(this, axis); 1488 } 1489 1490 @Override 1491 public boolean any() { 1492 return Comparisons.anyTrue(this); 1493 } 1494 1495 @Override 1496 public BooleanDataset any(final int axis) { 1497 return Comparisons.anyTrue(this, axis); 1498 } 1499 1500 @Override 1501 public Dataset ifloorDivide(final Object o) { 1502 return idivide(o).ifloor(); 1503 } 1504 1505 @Override 1506 public double residual(final Object o) { 1507 return residual(o, null, false); 1508 } 1509 1510 @Override 1511 public double residual(final Object o, boolean ignoreNaNs) { 1512 return residual(o, null, ignoreNaNs); 1513 } 1514 1515 /** 1516 * @return statistics 1517 * @since 2.0 1518 */ 1519 @SuppressWarnings("unchecked") 1520 protected StatisticsMetadata<Number> getStats() { 1521 StatisticsMetadata<Number> md = getFirstMetadata(StatisticsMetadata.class); 1522 if (md == null || md.isDirty(this)) { 1523 md = new StatisticsMetadataImpl<Number>(); 1524 md.initialize(this); 1525 setMetadata(md); 1526 } 1527 return md; 1528 } 1529 1530 /** 1531 * @return statistics 1532 * @since 2.0 1533 */ 1534 @SuppressWarnings("unchecked") 1535 protected StatisticsMetadata<String> getStringStats() { 1536 StatisticsMetadata<String> md = getFirstMetadata(StatisticsMetadata.class); 1537 if (md == null || md.isDirty(this)) { 1538 md = new StatisticsMetadataImpl<String>(); 1539 md.initialize(this); 1540 setMetadata(md); 1541 } 1542 return md; 1543 } 1544 1545 @Override 1546 public Number max(boolean... ignoreInvalids) { 1547 return getStats().getMaximum(ignoreInvalids); 1548 } 1549 1550 @Override 1551 public Dataset max(int axis, boolean... ignoreInvalids) { 1552 return getStats().getMaximum(axis, ignoreInvalids); 1553 } 1554 1555 @Override 1556 public Dataset max(int[] axes, boolean... ignoreInvalids) { 1557 return getStats().getMaximum(axes, ignoreInvalids); 1558 } 1559 1560 @Override 1561 public Number min(boolean... ignoreInvalids) { 1562 return getStats().getMinimum(ignoreInvalids); 1563 } 1564 1565 @Override 1566 public Dataset min(int axis, boolean... ignoreInvalids) { 1567 return getStats().getMinimum(axis, ignoreInvalids); 1568 } 1569 1570 @Override 1571 public Dataset min(int[] axes, boolean... ignoreInvalids) { 1572 return getStats().getMinimum(axes, ignoreInvalids); 1573 } 1574 1575 @Override 1576 public int argMax(boolean... ignoreInvalids) { 1577 return getFlat1DIndex(maxPos(ignoreInvalids)); 1578 } 1579 1580 /** 1581 * @since 2.0 1582 */ 1583 @Override 1584 public IntegerDataset argMax(int axis, boolean... ignoreInvalids) { 1585 return (IntegerDataset) getStats().getArgMaximum(axis, ignoreInvalids); 1586 } 1587 1588 @Override 1589 public int argMin(boolean... ignoreInvalids) { 1590 return getFlat1DIndex(minPos(ignoreInvalids)); 1591 } 1592 1593 /** 1594 * @since 2.0 1595 */ 1596 @Override 1597 public IntegerDataset argMin(int axis, boolean... ignoreInvalids) { 1598 return (IntegerDataset) getStats().getArgMinimum(axis, ignoreInvalids); 1599 } 1600 1601 @Override 1602 public Number peakToPeak(boolean... ignoreInvalids) { 1603 return InterfaceUtils.fromDoubleToBiggestNumber(getClass(), max(ignoreInvalids).doubleValue() - min(ignoreInvalids).doubleValue()); 1604 } 1605 1606 @Override 1607 public Dataset peakToPeak(int axis, boolean... ignoreInvalids) { 1608 return Maths.subtract(max(axis, ignoreInvalids), min(axis, ignoreInvalids)); 1609 } 1610 1611 @Override 1612 public Dataset peakToPeak(int[] axes, boolean... ignoreInvalids) { 1613 return Maths.subtract(max(axes, ignoreInvalids), min(axes, ignoreInvalids)); 1614 } 1615 1616 @Override 1617 public long count(boolean... ignoreInvalids) { 1618 return getStats().getCount(ignoreInvalids); 1619 } 1620 1621 @Override 1622 public Dataset count(int axis, boolean... ignoreInvalids) { 1623 return getStats().getCount(axis, ignoreInvalids); 1624 } 1625 1626 @Override 1627 public Dataset count(int[] axes, boolean... ignoreInvalids) { 1628 return getStats().getCount(axes, ignoreInvalids); 1629 } 1630 1631 @Override 1632 public Object sum(boolean... ignoreInvalids) { 1633 return InterfaceUtils.toBiggestNumber(getClass(), getStats().getSum(ignoreInvalids)); 1634 } 1635 1636 @Override 1637 public Dataset sum(int axis, boolean... ignoreInvalids) { 1638 return getStats().getSum(axis, ignoreInvalids); 1639 } 1640 1641 @Override 1642 public Dataset sum(int[] axes, boolean... ignoreInvalids) { 1643 return getStats().getSum(axes, ignoreInvalids); 1644 } 1645 1646 @Override 1647 public Object product(boolean... ignoreInvalids) { 1648 return Stats.product(this, ignoreInvalids); 1649 } 1650 1651 @Override 1652 public Dataset product(int axis, boolean... ignoreInvalids) { 1653 return Stats.product(this, axis, ignoreInvalids); 1654 } 1655 1656 @Override 1657 public Dataset product(int[] axes, boolean... ignoreInvalids) { 1658 return Stats.product(this, axes, ignoreInvalids); 1659 } 1660 1661 @Override 1662 public Object mean(boolean... ignoreInvalids) { 1663 return getStats().getMean(ignoreInvalids); 1664 } 1665 1666 @Override 1667 public Dataset mean(int axis, boolean... ignoreInvalids) { 1668 return getStats().getMean(axis, ignoreInvalids); 1669 } 1670 1671 @Override 1672 public Dataset mean(int[] axes, boolean... ignoreInvalids) { 1673 return getStats().getMean(axes, ignoreInvalids); 1674 } 1675 1676 @Override 1677 public double variance() { 1678 return variance(false); 1679 } 1680 1681 @Override 1682 public double variance(boolean isWholePopulation, boolean... ignoreInvalids) { 1683 return getStats().getVariance(isWholePopulation, ignoreInvalids); 1684 } 1685 1686 @Override 1687 public Dataset variance(int axis) { 1688 return getStats().getVariance(axis, false); 1689 } 1690 1691 @Override 1692 public Dataset variance(int[] axes) { 1693 return getStats().getVariance(axes, false); 1694 } 1695 1696 @Override 1697 public Dataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) { 1698 return getStats().getVariance(axis, isWholePopulation, ignoreInvalids); 1699 } 1700 1701 @Override 1702 public Dataset variance(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) { 1703 return getStats().getVariance(axes, isWholePopulation, ignoreInvalids); 1704 } 1705 1706 @Override 1707 public double stdDeviation() { 1708 return Math.sqrt(variance()); 1709 } 1710 1711 @Override 1712 public double stdDeviation(boolean isWholePopulation, boolean... ignoreInvalids) { 1713 return Math.sqrt(variance(isWholePopulation, ignoreInvalids)); 1714 } 1715 1716 @Override 1717 public Dataset stdDeviation(int axis) { 1718 return Maths.sqrt(variance(axis, false)); 1719 } 1720 1721 @Override 1722 public Dataset stdDeviation(int[] axes) { 1723 return Maths.sqrt(variance(axes, false)); 1724 } 1725 1726 @Override 1727 public Dataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) { 1728 return Maths.sqrt(variance(axis, isWholePopulation, ignoreInvalids)); 1729 } 1730 1731 @Override 1732 public Dataset stdDeviation(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) { 1733 return Maths.sqrt(variance(axes, isWholePopulation, ignoreInvalids)); 1734 } 1735 1736 @Override 1737 public double rootMeanSquare(boolean... ignoreInvalids) { 1738 StatisticsMetadata<Number> stats = getStats(); 1739 final double mean = stats.getMean(ignoreInvalids).doubleValue(); 1740 final double var = stats.getVariance(true, ignoreInvalids); 1741 return Math.sqrt(var + mean * mean); 1742 } 1743 1744 @Override 1745 public Dataset rootMeanSquare(int axis, boolean... ignoreInvalids) { 1746 StatisticsMetadata<Number> stats = getStats(); 1747 Dataset v = stats.getVariance(axis, true, ignoreInvalids); 1748 Dataset m = stats.getMean(axis, ignoreInvalids); 1749 Dataset result = Maths.multiply(m, m); 1750 return Maths.sqrt(result.iadd(v)); 1751 } 1752 1753 @Override 1754 public Dataset rootMeanSquare(int[] axes, boolean... ignoreInvalids) { 1755 StatisticsMetadata<Number> stats = getStats(); 1756 Dataset v = stats.getVariance(axes, true, ignoreInvalids); 1757 Dataset m = stats.getMean(axes, ignoreInvalids); 1758 Dataset result = Maths.multiply(m, m); 1759 return Maths.sqrt(result.iadd(v)); 1760 } 1761 1762 /** 1763 * Set item from compatible dataset in a direct and speedy way. Remember to setDirty afterwards. 1764 * 1765 * @param dindex destination index 1766 * @param sindex source index 1767 * @param src 1768 * is the source data buffer 1769 */ 1770 protected abstract void setItemDirect(final int dindex, final int sindex, final Object src); 1771 1772 /** 1773 * @return error broadcasted to current shape 1774 */ 1775 private Dataset getBroadcastedInternalError() { 1776 if (shape == null) { 1777 throw new IllegalArgumentException("Cannot get error for null dataset"); 1778 } 1779 ILazyDataset led = super.getErrors(); 1780 if (led == null) 1781 return null; 1782 1783 Dataset ed = null; 1784 try { 1785 ed = DatasetUtils.sliceAndConvertLazyDataset(led); 1786 } catch (DatasetException e) { 1787 logger.error("Could not get data from lazy dataset", e); 1788 } 1789 if (led != ed) { 1790 setErrors(ed); // set back 1791 } 1792 1793 return ed.getBroadcastView(shape); 1794 } 1795 1796 @Override 1797 public Dataset getErrors() { 1798 Dataset ed = getBroadcastedInternalError(); 1799 if (ed == null) 1800 return null; 1801 1802 return ed; 1803 } 1804 1805 @Override 1806 public double getError() { 1807 Dataset ed = getBroadcastedInternalError(); 1808 if (ed == null) 1809 return 0; 1810 1811 return ed.getDouble(); 1812 } 1813 1814 @Override 1815 public double getError(final int i) { 1816 Dataset ed = getBroadcastedInternalError(); 1817 if (ed == null) 1818 return 0; 1819 1820 return ed.getDouble(i); 1821 } 1822 1823 @Override 1824 public double getError(final int i, final int j) { 1825 Dataset ed = getBroadcastedInternalError(); 1826 if (ed == null) 1827 return 0; 1828 1829 return ed.getDouble(i, j); 1830 } 1831 1832 @Override 1833 public double getError(int... pos) { 1834 Dataset ed = getBroadcastedInternalError(); 1835 if (ed == null) 1836 return 0; 1837 1838 return ed.getDouble(pos); 1839 } 1840 1841 @Override 1842 public double[] getErrorArray(final int i) { 1843 Dataset ed = getBroadcastedInternalError(); 1844 if (ed == null) 1845 return null; 1846 1847 return new double[] {getError(i)}; 1848 } 1849 1850 @Override 1851 public double[] getErrorArray(final int i, final int j) { 1852 Dataset ed = getBroadcastedInternalError(); 1853 if (ed == null) 1854 return null; 1855 1856 return new double[] {getError(i, j)}; 1857 } 1858 1859 @Override 1860 public double[] getErrorArray(int... pos) { 1861 Dataset ed = getBroadcastedInternalError(); 1862 if (ed == null) 1863 return null; 1864 1865 return new double[] {getError(pos)}; 1866 } 1867 1868 protected Dataset getInternalSquaredError() { 1869 Dataset sed = getErrorBuffer().getBroadcastView(shape); 1870 return sed; 1871 } 1872 1873 @Override 1874 public Dataset getErrorBuffer() { 1875 ErrorMetadata emd = getErrorMetadata(); 1876 if (emd == null) 1877 return null; 1878 1879 if (!(emd instanceof ErrorMetadataImpl)) { 1880 ILazyDataset led = emd.getError(); 1881 Dataset ed; 1882 try { 1883 ed = DatasetUtils.sliceAndConvertLazyDataset(led); 1884 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1885 setMetadata(emd); 1886 emd.setError(ed); 1887 } catch (MetadataException me) { 1888 logger.error("Could not create metadata", me); 1889 } catch (DatasetException e) { 1890 logger.error("Could not get data from lazy dataset", e); 1891 } 1892 } 1893 1894 return ((ErrorMetadataImpl) emd).getSquaredError(); 1895 } 1896 1897 /** 1898 * Set a copy of the buffer that backs the (squared) error data 1899 * @param buffer can be null, anything that can be used to create a DoubleDataset or CompoundDoubleDataset 1900 */ 1901 @Override 1902 public void setErrorBuffer(Serializable buffer) { 1903 if (shape == null) { 1904 throw new IllegalArgumentException("Cannot set error buffer for null dataset"); 1905 } 1906 if (buffer == null) { 1907 clearMetadata(ErrorMetadata.class); 1908 return; 1909 } 1910 1911 IDataset d = (IDataset) createFromSerializable(buffer, false); 1912 ErrorMetadata emd = getErrorMetadata(); 1913 if (!(emd instanceof ErrorMetadataImpl)) { 1914 try { 1915 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1916 setMetadata(emd); 1917 } catch (MetadataException me) { 1918 logger.error("Could not create metadata", me); 1919 } 1920 } 1921 ((ErrorMetadataImpl) emd).setSquaredError(d); 1922 } 1923}