001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.Serializable; 016import java.lang.annotation.Annotation; 017import java.lang.reflect.Array; 018import java.lang.reflect.Field; 019import java.util.ArrayList; 020import java.util.Arrays; 021import java.util.Collection; 022import java.util.concurrent.ConcurrentMap; 023import java.util.concurrent.ConcurrentHashMap; 024import java.util.List; 025import java.util.Map; 026 027import org.eclipse.january.DatasetException; 028import org.eclipse.january.MetadataException; 029import org.eclipse.january.metadata.Dirtiable; 030import org.eclipse.january.metadata.ErrorMetadata; 031import org.eclipse.january.metadata.IMetadata; 032import org.eclipse.january.metadata.MetadataFactory; 033import org.eclipse.january.metadata.MetadataType; 034import org.eclipse.january.metadata.Reshapeable; 035import org.eclipse.january.metadata.Sliceable; 036import org.eclipse.january.metadata.Transposable; 037import org.slf4j.Logger; 038import org.slf4j.LoggerFactory; 039 040/** 041 * Common base for both lazy and normal dataset implementations 042 */ 043public abstract class LazyDatasetBase implements ILazyDataset, Serializable { 044 045 private static final long serialVersionUID = 767926846438976050L; 046 047 protected static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class); 048 049 protected static boolean catchExceptions; 050 051 static { 052 /** 053 * Boolean to set to true if running jython scripts that utilise ScisoftPy in IDE 054 */ 055 catchExceptions = Boolean.getBoolean("run.in.eclipse"); 056 } 057 058 protected String name = ""; 059 060 /** 061 * The shape or dimensions of the dataset 062 */ 063 protected int[] shape; 064 065 protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null; 066 067 /** 068 * @return type of dataset item 069 */ 070 abstract public int getDType(); 071 072 @Override 073 public Class<?> getElementClass() { 074 return DTypeUtils.getElementClass(getDType()); 075 } 076 077 @Override 078 public LazyDatasetBase clone() { 079 return null; 080 } 081 082 @Override 083 public boolean equals(Object obj) { 084 if (this == obj) { 085 return true; 086 } 087 if (obj == null) { 088 return false; 089 } 090 if (!getClass().equals(obj.getClass())) { 091 return false; 092 } 093 094 LazyDatasetBase other = (LazyDatasetBase) obj; 095 if (getDType() != other.getDType()) { 096 return false; 097 } 098 if (getElementsPerItem() != other.getElementsPerItem()) { 099 return false; 100 } 101 if (!Arrays.equals(shape, other.shape)) { 102 return false; 103 } 104 return true; 105 } 106 107 @Override 108 public int hashCode() { 109 int hash = getDType() * 17 + getElementsPerItem(); 110 int rank = shape.length; 111 for (int i = 0; i < rank; i++) { 112 hash = hash*17 + shape[i]; 113 } 114 return hash; 115 } 116 117 @Override 118 public String getName() { 119 return name; 120 } 121 122 @Override 123 public void setName(String name) { 124 this.name = name; 125 } 126 127 @Override 128 public int[] getShape() { 129 return shape.clone(); 130 } 131 132 @Override 133 public int getRank() { 134 return shape.length; 135 } 136 137 /** 138 * Find first sub-interface of (or class that directly implements) MetadataType 139 * @param clazz 140 * @return sub-interface 141 * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it 142 */ 143 @SuppressWarnings("unchecked") 144 public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) { 145 if (clazz.equals(MetadataType.class)) { 146 throw new IllegalArgumentException("Cannot accept MetadataType"); 147 } 148 149 if (clazz.isInterface()) { 150 return clazz; 151 } 152 153 if (clazz.isAnonymousClass()) { // special case 154 Class<?> s = clazz.getSuperclass(); 155 if (!s.equals(Object.class)) { 156 // only use super class if it is not an anonymous class of an interface 157 clazz = (Class<? extends MetadataType>) s; 158 } 159 } 160 161 for (Class<?> c : clazz.getInterfaces()) { 162 if (c.equals(MetadataType.class)) { 163 if (clazz.isAnonymousClass()) { 164 throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType"); 165 } 166 return clazz; 167 } 168 if (MetadataType.class.isAssignableFrom(c)) { 169 return (Class<? extends MetadataType>) c; 170 } 171 } 172 173 Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class 174 if (c != null) { 175 return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c); 176 } 177 178 logger.error("Somehow the search for metadata type interface ended in a bad place"); 179 assert false; // should not be able to get here!!! 180 return null; 181 } 182 183 @Override 184 public void setMetadata(MetadataType metadata) { 185 addMetadata(metadata, true); 186 } 187 188 @Override 189 public void addMetadata(MetadataType metadata) { 190 addMetadata(metadata, false); 191 } 192 193 private synchronized void addMetadata(MetadataType metadata, boolean clear) { 194 if (metadata == null) 195 return; 196 197 if (this.metadata == null) { 198 this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 199 } 200 201 Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass()); 202 if (!this.metadata.containsKey(clazz)) { 203 this.metadata.put(clazz, new ArrayList<MetadataType>()); 204 } else if (clear) { 205 this.metadata.get(clazz).clear(); 206 } 207 this.metadata.get(clazz).add(metadata); 208 209 // add for special case of sub-interfaces of IMetadata 210 if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) { 211 clazz = IMetadata.class; 212 if (!this.metadata.containsKey(clazz)) { 213 this.metadata.put(clazz, new ArrayList<MetadataType>()); 214 } else if (clear) { 215 this.metadata.get(clazz).clear(); 216 } 217 this.metadata.get(clazz).add(metadata); 218 } 219 } 220 221 @Override 222 @Deprecated 223 public synchronized IMetadata getMetadata() { 224 return getFirstMetadata(IMetadata.class); 225 } 226 227 @SuppressWarnings("unchecked") 228 @Override 229 public synchronized <S extends MetadataType, T extends S> List<S> getMetadata(Class<T> clazz) throws MetadataException { 230 if (metadata == null) 231 return null; 232 233 if (clazz == null) { 234 List<S> all = new ArrayList<S>(); 235 for (Class<? extends MetadataType> c : metadata.keySet()) { 236 all.addAll((Collection<S>) metadata.get(c)); 237 } 238 return all; 239 } 240 241 return (List<S>) metadata.get(findMetadataTypeSubInterfaces(clazz)); 242 } 243 244 @Override 245 public synchronized <S extends MetadataType, T extends S> S getFirstMetadata(Class<T> clazz) { 246 try { 247 List<S> ml = getMetadata(clazz); 248 if (ml == null) return null; 249 for (S t : ml) { 250 if (clazz.isInstance(t)) return t; 251 } 252 } catch (Exception e) { 253 logger.error("Get metadata failed!",e); 254 } 255 256 return null; 257 } 258 259 @Override 260 public synchronized void clearMetadata(Class<? extends MetadataType> clazz) { 261 if (metadata == null) 262 return; 263 264 if (clazz == null) { 265 metadata.clear(); 266 return; 267 } 268 269 List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz)); 270 if( list != null) { 271 list.clear(); 272 } 273 } 274 275 /** 276 * @since 2.0 277 */ 278 protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() { 279 return copyMetadata(metadata); 280 } 281 282 /** 283 * @since 2.0 284 */ 285 protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) { 286 if (metadata == null) 287 return null; 288 289 ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 290 291 for (Class<? extends MetadataType> c : metadata.keySet()) { 292 List<MetadataType> l = metadata.get(c); 293 List<MetadataType> nl = new ArrayList<MetadataType>(l.size()); 294 map.put(c, nl); 295 for (MetadataType m : l) { 296 nl.add(m.clone()); 297 } 298 } 299 return map; 300 } 301 302 interface MetadatasetAnnotationOperation { 303 /** 304 * Process value of given field 305 * <p> 306 * When the field is not a container then the returned value 307 * may replace the old value 308 * @param f given field 309 * @param o value of field 310 * @return transformed field 311 */ 312 Object processField(Field f, Object o); 313 314 /** 315 * @return annotated class 316 */ 317 Class<? extends Annotation> getAnnClass(); 318 319 /** 320 * @param axis 321 * @return number of dimensions to insert or remove 322 */ 323 int change(int axis); 324 325 /** 326 * 327 * @return rank or -1 to match 328 */ 329 int getNewRank(); 330 331 /** 332 * Run on given lazy dataset 333 * @param lz 334 * @return 335 */ 336 ILazyDataset run(ILazyDataset lz); 337 } 338 339 class MdsSlice implements MetadatasetAnnotationOperation { 340 private boolean asView; 341 private int[] start; 342 private int[] stop; 343 private int[] step; 344 private int[] oShape; 345 private long oSize; 346 347 public MdsSlice(boolean asView, final int[] start, final int[] stop, final int[] step, final int[] oShape) { 348 this.asView = asView; 349 this.start = start; 350 this.stop = stop; 351 this.step = step; 352 this.oShape = oShape; 353 oSize = ShapeUtils.calcLongSize(oShape); 354 } 355 356 @Override 357 public Object processField(Field field, Object o) { 358 return o; 359 } 360 361 @Override 362 public Class<? extends Annotation> getAnnClass() { 363 return Sliceable.class; 364 } 365 366 @Override 367 public int change(int axis) { 368 return 0; 369 } 370 371 @Override 372 public int getNewRank() { 373 return -1; 374 } 375 376 @Override 377 public ILazyDataset run(ILazyDataset lz) { 378 int rank = lz.getRank(); 379 if (start.length != rank) { 380 throw new IllegalArgumentException("Slice dimensions do not match dataset!"); 381 } 382 383 int[] shape = lz.getShape(); 384 int[] stt; 385 int[] stp; 386 int[] ste; 387 if (lz.getSize() == oSize) { 388 stt = start; 389 stp = stop; 390 ste = step; 391 } else { 392 stt = start.clone(); 393 stp = stop.clone(); 394 ste = step.clone(); 395 for (int i = 0; i < rank; i++) { 396 if (shape[i] >= oShape[i]) continue; 397 if (shape[i] == 1) { 398 stt[i] = 0; 399 stp[i] = 1; 400 ste[1] = 1; 401 } else { 402 throw new IllegalArgumentException("Sliceable dataset has invalid size!"); 403 } 404 } 405 } 406 407 if (asView || (lz instanceof IDataset)) 408 return lz.getSliceView(stt, stp, ste); 409 try { 410 return lz.getSlice(stt, stp, ste); 411 } catch (DatasetException e) { 412 logger.error("Could not slice dataset in metadata", e); 413 return null; 414 } 415 } 416 } 417 418 class MdsReshape implements MetadatasetAnnotationOperation { 419 private boolean matchRank; 420 private int[] oldShape; 421 private int[] newShape; 422 boolean onesOnly; 423 int[] differences; 424 425 /* 426 * if only ones then record differences (insertions and deletions) 427 * 428 * if shape changing, find broadcasted dimensions and disallow 429 * merging that include those dimensions 430 */ 431 public MdsReshape(final int[] oldShape, final int[] newShape) { 432 this.oldShape = oldShape; 433 this.newShape = newShape; 434 differences = null; 435 } 436 437 @Override 438 public Object processField(Field field, Object o) { 439 Annotation a = field.getAnnotation(Reshapeable.class); 440 if (a != null) { // cannot be null 441 matchRank = ((Reshapeable) a).matchRank(); 442 } 443 return o; 444 } 445 446 @Override 447 public Class<? extends Annotation> getAnnClass() { 448 return Reshapeable.class; 449 } 450 451 @Override 452 public int change(int axis) { 453 if (matchRank) { 454 if (differences == null) 455 init(); 456 457 if (onesOnly) { 458 return differences[axis]; 459 } 460 throw new UnsupportedOperationException("TODO support other shape operations"); 461 } 462 return 0; 463 } 464 465 @Override 466 public int getNewRank() { 467 return matchRank ? newShape.length : -1; 468 } 469 470 private void init() { 471 int or = oldShape.length - 1; 472 int nr = newShape.length - 1; 473 if (or < 0 || nr < 0) { // zero-rank shapes 474 onesOnly = true; 475 differences = new int[1]; 476 differences[0] = or < 0 ? nr + 1 : or + 1; 477 return; 478 } 479 int ob = 0; 480 int nb = 0; 481 onesOnly = true; 482 do { 483 while (oldShape[ob] == 1 && ob < or) { 484 ob++; // next non-unit dimension 485 } 486 while (newShape[nb] == 1 && nb < nr) { 487 nb++; 488 } 489 if (oldShape[ob++] != newShape[nb++]) { 490 onesOnly = false; 491 break; 492 } 493 } while (ob <= or && nb <= nr); 494 495 ob = 0; 496 nb = 0; 497 differences = new int[or + 2]; 498 if (onesOnly) { 499 // work out unit dimensions removed from or add to old 500 int j = 0; 501 do { 502 if (oldShape[ob] != 1 && newShape[nb] != 1) { 503 ob++; 504 nb++; 505 } else { 506 while (oldShape[ob] == 1 && ob < or) { 507 ob++; 508 differences[j]--; 509 } 510 while (newShape[nb] == 1 && nb < nr) { 511 nb++; 512 differences[j]++; 513 } 514 } 515 j++; 516 } while (ob <= or && nb <= nr && j <= or); 517 while (ob <= or && oldShape[ob] == 1) { 518 ob++; 519 differences[j]--; 520 } 521 while (nb <= nr && newShape[nb] == 1) { 522 nb++; 523 differences[j]++; 524 } 525 } else { 526 if (matchRank) { 527 logger.error("Combining dimensions is currently not supported"); 528 throw new IllegalArgumentException("Combining dimensions is currently not supported"); 529 } 530 // work out mapping: contiguous dimensions can be grouped or split 531 while (ob <= or && nb <= nr) { 532 int ol = oldShape[ob]; 533 while (ol == 1 && ol <= or) { 534 ob++; 535 ol = oldShape[ob]; 536 } 537 int oe = ob + 1; 538 int nl = newShape[nb]; 539 while (nl == 1 && nl <= nr) { 540 nb++; 541 nl = newShape[nb]; 542 } 543 int ne = nb + 1; 544 if (ol < nl) { 545 differences[ob] = 1; 546 do { // case where new shape combines several dimensions into one dimension 547 if (oe == (or + 1)) { 548 break; 549 } 550 differences[oe] = 1; 551 ol *= oldShape[oe++]; 552 } while (ol < nl); 553 differences[oe - 1] = oe - ob; // signal end with difference 554 if (nl != ol) { 555 logger.error("Single dimension is incompatible with subshape"); 556 throw new IllegalArgumentException("Single dimension is incompatible with subshape"); 557 } 558 } else if (ol > nl) { 559 do { // case where new shape spreads single dimension over several dimensions 560 if (ne == (nr + 1)) { 561 break; 562 } 563 nl *= newShape[ne++]; 564 } while (nl < ol); 565 if (nl != ol) { 566 logger.error("Subshape is incompatible with single dimension"); 567 throw new IllegalArgumentException("Subshape is incompatible with single dimension"); 568 } 569 570 } 571 572 ob = oe; 573 nb = ne; 574 } 575 576 } 577 } 578 579 @Override 580 public ILazyDataset run(ILazyDataset lz) { 581 if (differences == null) 582 init(); 583 584 int[] lshape = lz.getShape(); 585 if (Arrays.equals(newShape, lshape)) { 586 return lz; 587 } 588 int or = lz.getRank(); 589 int nr = newShape.length; 590 int[] nshape = new int[nr]; 591 Arrays.fill(nshape, 1); 592 if (onesOnly) { 593 // ignore omit removed dimensions 594 for (int i = 0, si = 0, di = 0; i < (or+1) && si <= or && di < nr; i++) { 595 int c = differences[i]; 596 if (c == 0) { 597 nshape[di++] = lshape[si++]; 598 } else if (c > 0) { 599 while (c-- > 0 && di < nr) { 600 di++; 601 } 602 } else if (c < 0) { 603 si -= c; // remove dimensions by skipping forward in source array 604 } 605 } 606 } else { 607 boolean[] broadcast = new boolean[or]; 608 for (int ob = 0; ob < or; ob++) { 609 broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1; 610 } 611 int osize = lz.getSize(); 612 613 // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...) 614 int ob = 0; 615 int nsize = 1; 616 for (int i = 0; i < nr; i++) { 617 if (ob < or && broadcast[ob]) { 618 if (differences[ob] != 0) { 619 logger.error("Metadata contains a broadcast axis which cannot be reshaped"); 620 throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped"); 621 } 622 } else { 623 nshape[i] = nsize < osize ? newShape[i] : 1; 624 } 625 nsize *= nshape[i]; 626 ob++; 627 } 628 } 629 630 ILazyDataset nlz = lz.getSliceView(); 631 if (lz instanceof Dataset) { 632 nlz = ((Dataset) lz).reshape(nshape); 633 } else { 634 nlz = lz.getSliceView(); 635 nlz.setShape(nshape); 636 } 637 return nlz; 638 } 639 } 640 641 class MdsTranspose implements MetadatasetAnnotationOperation { 642 int[] map; 643 644 public MdsTranspose(final int[] axesMap) { 645 map = axesMap; 646 } 647 648 @SuppressWarnings({ "rawtypes", "unchecked" }) 649 @Override 650 public Object processField(Field f, Object o) { 651 // reorder arrays and lists according the axes map 652 if (o.getClass().isArray()) { 653 int l = Array.getLength(o); 654 if (l == map.length) { 655 Object narray = Array.newInstance(o.getClass().getComponentType(), l); 656 for (int i = 0; i < l; i++) { 657 Array.set(narray, i, Array.get(o, map[i])); 658 } 659 for (int i = 0; i < l; i++) { 660 Array.set(o, i, Array.get(narray, i)); 661 } 662 } 663 } else if (o instanceof List<?>) { 664 List list = (List) o; 665 int l = list.size(); 666 if (l == map.length) { 667 Object narray = Array.newInstance(o.getClass().getComponentType(), l); 668 for (int i = 0; i < l; i++) { 669 Array.set(narray, i, list.get(map[i])); 670 } 671 list.clear(); 672 for (int i = 0; i < l; i++) { 673 list.add(Array.get(narray, i)); 674 } 675 } 676 } 677 return o; 678 } 679 680 @Override 681 public Class<? extends Annotation> getAnnClass() { 682 return Transposable.class; 683 } 684 685 @Override 686 public int change(int axis) { 687 return 0; 688 } 689 690 @Override 691 public int getNewRank() { 692 return -1; 693 } 694 695 @Override 696 public ILazyDataset run(ILazyDataset lz) { 697 return lz.getTransposedView(map); 698 } 699 } 700 701 class MdsDirty implements MetadatasetAnnotationOperation { 702 703 @Override 704 public Object processField(Field f, Object o) { 705 // throw exception if not boolean??? 706 Class<?> t = f.getType(); 707 if (t.equals(boolean.class) || t.equals(Boolean.class)) { 708 if (o.equals(false)) { 709 o = true; 710 } 711 } 712 return o; 713 } 714 715 @Override 716 public Class<? extends Annotation> getAnnClass() { 717 return Dirtiable.class; 718 } 719 720 @Override 721 public int change(int axis) { 722 return 0; 723 } 724 725 @Override 726 public int getNewRank() { 727 return -1; 728 } 729 730 @Override 731 public ILazyDataset run(ILazyDataset lz) { 732 return lz; 733 } 734 } 735 736 /** 737 * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced 738 * dataset after cloning the metadata 739 * @param asView if true then just a view 740 * @param slice 741 */ 742 protected void sliceMetadata(boolean asView, final SliceND slice) { 743 processAnnotatedMetadata(new MdsSlice(asView, slice.getStart(), slice.getStop(), slice.getStep(), slice.getSourceShape()), true); 744 } 745 746 /** 747 * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing 748 * or setting the shape 749 * 750 * @param newShape 751 */ 752 protected void reshapeMetadata(final int[] oldShape, final int[] newShape) { 753 processAnnotatedMetadata(new MdsReshape(oldShape, newShape), true); 754 } 755 756 /** 757 * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed 758 * dataset after cloning the metadata 759 * @param axesMap 760 */ 761 protected void transposeMetadata(final int[] axesMap) { 762 processAnnotatedMetadata(new MdsTranspose(axesMap), true); 763 } 764 765 /** 766 * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified 767 * @since 2.0 768 */ 769 protected void dirtyMetadata() { 770 processAnnotatedMetadata(new MdsDirty(), true); 771 } 772 773 @SuppressWarnings("unchecked") 774 private void processAnnotatedMetadata(MetadatasetAnnotationOperation op, boolean throwException) { 775 if (metadata == null) 776 return; 777 778 for (Class<? extends MetadataType> c : metadata.keySet()) { 779 for (MetadataType m : metadata.get(c)) { 780 if (m == null) 781 continue; 782 783 Class<? extends MetadataType> mc = m.getClass(); 784 do { // iterate over super-classes 785 processClass(op, m, mc, throwException); 786 Class<?> sclazz = mc.getSuperclass(); 787 if (!MetadataType.class.isAssignableFrom(sclazz)) 788 break; 789 mc = (Class<? extends MetadataType>) sclazz; 790 } while (true); 791 } 792 } 793 } 794 795 @SuppressWarnings({ "unchecked", "rawtypes" }) 796 private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc, boolean throwException) { 797 for (Field f : mc.getDeclaredFields()) { 798 if (!f.isAnnotationPresent(op.getAnnClass())) 799 continue; 800 801 try { 802 f.setAccessible(true); 803 Object o = f.get(m); 804 if (o == null) 805 continue; 806 807 Object no = op.processField(f, o); 808 if (no != o) { 809 f.set(m, no); 810 continue; 811 } 812 Object r = null; 813 if (o instanceof ILazyDataset) { 814 try { 815 f.set(m, op.run((ILazyDataset) o)); 816 } catch (Exception e) { 817 logger.error("Problem processing " + o, e); 818 if (!catchExceptions) 819 throw e; 820 } 821 } else if (o.getClass().isArray()) { 822 int l = Array.getLength(o); 823 if (l <= 0) 824 continue; 825 826 for (int i = 0; r == null && i < l; i++) { 827 r = Array.get(o, i); 828 } 829 int n = op.getNewRank(); 830 if (r == null) { 831 if (n < 0 || n != l) { // all nulls be need to match rank as necessary 832 f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n)); 833 } 834 continue; 835 } 836 if (n < 0) 837 n = l; 838 Object narray = Array.newInstance(r.getClass(), n); 839 for (int i = 0, si = 0, di = 0; di < n && si < l; i++) { 840 int c = op.change(i); 841 if (c == 0) { 842 Array.set(narray, di++, processObject(op, Array.get(o, si++))); 843 } else if (c > 0) { 844 di += c; // add nulls by skipping forward in destination array 845 } else if (c < 0) { 846 si -= c; // remove dimensions by skipping forward in source array 847 } 848 } 849 if (n == l) { 850 for (int i = 0; i < l; i++) { 851 Array.set(o, i, Array.get(narray, i)); 852 } 853 } else { 854 f.set(m, narray); 855 } 856 } else if (o instanceof List<?>) { 857 List list = (List) o; 858 int l = list.size(); 859 if (l <= 0) 860 continue; 861 862 for (int i = 0; r == null && i < l; i++) { 863 r = list.get(i); 864 } 865 int n = op.getNewRank(); 866 if (r == null) { 867 if (n < 0 || n != l) { // all nulls be need to match rank as necessary 868 list.clear(); 869 for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) { 870 list.add(null); 871 } 872 } 873 continue; 874 } 875 876 if (n < 0) 877 n = l; 878 Object narray = Array.newInstance(r.getClass(), n); 879 for (int i = 0, si = 0, di = 0; i < l && si < l; i++) { 880 int c = op.change(i); 881 if (c == 0) { 882 Array.set(narray, di++, processObject(op, list.get(si++))); 883 } else if (c > 0) { 884 di += c; // add nulls by skipping forward in destination array 885 } else if (c < 0) { 886 si -= c; // remove dimensions by skipping forward in source array 887 } 888 } 889 list.clear(); 890 for (int i = 0; i < n; i++) { 891 list.add(Array.get(narray, i)); 892 } 893 } else if (o instanceof Map<?,?>) { 894 Map map = (Map) o; 895 for (Object k : map.keySet()) { 896 map.put(k, processObject(op, map.get(k))); 897 } 898 } 899 } catch (Exception e) { 900 logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e); 901 if (throwException) 902 throw new RuntimeException(e); 903 } 904 } 905 } 906 907 @SuppressWarnings({ "unchecked", "rawtypes" }) 908 private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception { 909 if (o == null) 910 return o; 911 912 if (o instanceof ILazyDataset) { 913 try { 914 return op.run((ILazyDataset) o); 915 } catch (Exception e) { 916 logger.error("Problem processing " + o, e); 917 if (!catchExceptions) 918 throw e; 919 } 920 } else if (o.getClass().isArray()) { 921 int l = Array.getLength(o); 922 for (int i = 0; i < l; i++) { 923 Array.set(o, i, processObject(op, Array.get(o, i))); 924 } 925 } else if (o instanceof List<?>) { 926 List list = (List) o; 927 for (int i = 0, imax = list.size(); i < imax; i++) { 928 list.set(i, processObject(op, list.get(i))); 929 } 930 } else if (o instanceof Map<?,?>) { 931 Map map = (Map) o; 932 for (Object k : map.keySet()) { 933 map.put(k, processObject(op, map.get(k))); 934 } 935 } 936 return o; 937 } 938 939 protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) { 940 for (Class<? extends MetadataType> mc : oldMetadata.keySet()) { 941 metadata.put(mc, oldMetadata.get(mc)); 942 } 943 } 944 945 @SuppressWarnings("deprecation") 946 protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) { 947 ILazyDataset d = null; 948 if (blob instanceof ILazyDataset) { 949 d = (ILazyDataset) blob; 950 if (d instanceof IDataset) { 951 Dataset ed = DatasetUtils.convertToDataset((IDataset) d); 952 int is = ed.getElementsPerItem(); 953 if (is != 1 && is != getElementsPerItem()) { 954 throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset"); 955 } 956 d = ed.cast(is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64); 957 } else if (!keepLazy) { 958 final int is = getElementsPerItem(); 959 try { 960 d = DatasetUtils.cast(d.getSlice(), is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64); 961 } catch (DatasetException e) { 962 logger.error("Could not get data from lazy dataset", e); 963 return null; 964 } 965 } 966 } else { 967 final int is = getElementsPerItem(); 968 if (is == 1) { 969 d = DatasetFactory.createFromObject(Dataset.FLOAT64, blob); 970 } else { 971 try { 972 d = DatasetFactory.createFromObject(is, Dataset.ARRAYFLOAT64, blob); 973 } catch (IllegalArgumentException e) { // if only single value supplied try again 974 d = DatasetFactory.createFromObject(Dataset.FLOAT64, blob); 975 } 976 } 977 if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) { 978 d.setShape(shape.clone()); 979 } 980 } 981 List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape()); 982 d.setShape(s.get(0)); 983 984 return d; 985 } 986 987 @Override 988 public void setErrors(Serializable errors) { 989 if (shape == null) { 990 throw new IllegalArgumentException("Cannot set errors for null dataset"); 991 } 992 if (errors == null) { 993 clearMetadata(ErrorMetadata.class); 994 return; 995 } 996 if (errors == this) { 997 logger.warn("Ignoring setting error to itself as this will lead to infinite recursion"); 998 return; 999 } 1000 1001 ILazyDataset errorData = createFromSerializable(errors, true); 1002 1003 ErrorMetadata emd = getErrorMetadata(); 1004 if (emd == null) { 1005 try { 1006 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1007 setMetadata(emd); 1008 } catch (MetadataException me) { 1009 logger.error("Could not create metadata", me); 1010 } 1011 } 1012 emd.setError(errorData); 1013 } 1014 1015 protected ErrorMetadata getErrorMetadata() { 1016 try { 1017 List<ErrorMetadata> el = getMetadata(ErrorMetadata.class); 1018 if (el != null && !el.isEmpty()) { 1019 return el.get(0); 1020 } 1021 } catch (Exception e) { 1022 } 1023 return null; 1024 } 1025 1026 @Override 1027 public ILazyDataset getErrors() { 1028 ErrorMetadata emd = getErrorMetadata(); 1029 return emd == null ? null : emd.getError(); 1030 } 1031 1032 @Override 1033 public boolean hasErrors() { 1034 return LazyDatasetBase.this.getErrors() != null; 1035 } 1036 1037 /** 1038 * Check permutation axes 1039 * @param shape 1040 * @param axes 1041 * @return cleaned up axes or null if trivial 1042 */ 1043 public static int[] checkPermutatedAxes(int[] shape, int... axes) { 1044 int rank = shape.length; 1045 1046 if (axes == null || axes.length == 0) { 1047 axes = new int[rank]; 1048 for (int i = 0; i < rank; i++) { 1049 axes[i] = rank - 1 - i; 1050 } 1051 } 1052 1053 if (axes.length != rank) { 1054 logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank); 1055 throw new IllegalArgumentException("axis permutation does not match shape of dataset"); 1056 } 1057 1058 // check all permutation values are within bounds 1059 for (int d : axes) { 1060 if (d < 0 || d >= rank) { 1061 logger.error("axis permutation contains element {} outside rank of dataset", d); 1062 throw new IllegalArgumentException("axis permutation contains element outside rank of dataset"); 1063 } 1064 } 1065 1066 // check for a valid permutation (is this an unnecessary restriction?) 1067 int[] perm = axes.clone(); 1068 Arrays.sort(perm); 1069 1070 for (int i = 0; i < rank; i++) { 1071 if (perm[i] != i) { 1072 logger.error("axis permutation is not valid: it does not contain complete set of axes"); 1073 throw new IllegalArgumentException("axis permutation does not contain complete set of axes"); 1074 } 1075 } 1076 1077 if (Arrays.equals(axes, perm)) 1078 return null; // signal identity or trivial permutation 1079 1080 return axes; 1081 } 1082}