001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.Serializable; 016import java.lang.reflect.Array; 017import java.text.Format; 018import java.util.Arrays; 019import java.util.List; 020 021import org.eclipse.january.DatasetException; 022import org.eclipse.january.IMonitor; 023import org.eclipse.january.MetadataException; 024import org.eclipse.january.metadata.ErrorMetadata; 025import org.eclipse.january.metadata.MetadataFactory; 026import org.eclipse.january.metadata.StatisticsMetadata; 027import org.eclipse.january.metadata.internal.ErrorMetadataImpl; 028import org.eclipse.january.metadata.internal.StatisticsMetadataImpl; 029 030/** 031 * Generic container class for data 032 * <p> 033 * Each subclass has an array of primitive types, elements of this array are grouped or 034 * compounded to make items 035 * <p> 036 * Data items can be boolean, integer, float, complex float, vector float, etc 037 */ 038public abstract class AbstractDataset extends LazyDatasetBase implements Dataset { 039 // pin UID to base class 040 private static final long serialVersionUID = Dataset.serialVersionUID; 041 042 protected int size; // number of items 043 044 transient protected AbstractDataset base; // is null when not a view 045 protected int[] stride; // can be null for row-major, contiguous datasets 046 protected int offset; 047 048 /** 049 * The data itself, held in a 1D array, but the object will wrap it to appear as possessing as many dimensions as 050 * wanted 051 */ 052 protected Serializable odata = null; 053 054 /** 055 * Set aliased data as base data 056 */ 057 abstract protected void setData(); 058 059 /** 060 * Constructor required for serialisation. 061 */ 062 public AbstractDataset() { 063 } 064 065 @Override 066 public synchronized Dataset synchronizedCopy() { 067 return clone(); 068 } 069 070 @Override 071 public int hashCode() { 072 return getStats().getHash(shape); 073 } 074 075 @Override 076 abstract public AbstractDataset clone(); 077 078 protected Format stringFormat = null; 079 080 @Override 081 public void setStringFormat(Format format) { 082 stringFormat = format; 083 } 084 085 @Override 086 public Dataset copy(final int dtype) { 087 return copy(DTypeUtils.getInterface(dtype)); 088 } 089 090 @Override 091 public <T extends Dataset> T copy(Class<T> clazz) { 092 return DatasetUtils.copy(clazz, this); 093 } 094 095 @Override 096 public Dataset cast(final int dtype) { 097 return cast(DTypeUtils.getInterface(dtype)); 098 } 099 100 @SuppressWarnings("unchecked") 101 @Override 102 public <T extends Dataset> T cast(Class<T> clazz) { 103 if (getClass().equals(clazz)) { 104 return (T) this; 105 } 106 return DatasetUtils.cast(clazz, this); 107 } 108 109 /** 110 * {@inheritDoc Dataset#cast(int, Class, boolean)} 111 * @since 2.3 112 */ 113 @SuppressWarnings("unchecked") 114 @Override 115 public <T extends Dataset> T cast(final int isize, Class<T> clazz, final boolean repeat) { 116 if (getClass().equals(clazz) && getElementsPerItem() == isize) { 117 return (T) this; 118 } 119 return DatasetUtils.cast(isize, clazz, this, repeat); 120 } 121 122 @Override 123 public Dataset cast(final boolean repeat, final int dtype, final int isize) { 124 return cast(isize, DTypeUtils.getInterface(dtype), repeat); 125 } 126 127 @Override 128 abstract public AbstractDataset getView(boolean deepCopyMetadata); 129 130 /** 131 * Copy fields from original to view 132 * @param orig 133 * @param view 134 * @param clone if true, then clone everything but bulk data 135 * @param cloneMetadata if true, clone metadata 136 */ 137 protected static void copyToView(Dataset orig, AbstractDataset view, boolean clone, boolean cloneMetadata) { 138 view.name = orig.getName(); 139 view.size = orig.getSize(); 140 view.odata = orig.getBuffer(); 141 view.offset = orig.getOffset(); 142 view.base = orig instanceof AbstractDataset ? ((AbstractDataset) orig).base : null; 143 144 if (clone) { 145 view.shape = orig.getShapeRef() == null ? null : orig.getShape(); 146 view.stride = orig instanceof AbstractDataset && ((AbstractDataset) orig).stride != null ? 147 ((AbstractDataset) orig).stride.clone() : null; 148 } else { 149 view.shape = orig.getShapeRef(); 150 view.stride = orig instanceof AbstractDataset ? ((AbstractDataset) orig).stride : null; 151 } 152 153 view.metadata = getMetadataMap(orig, cloneMetadata); 154 int odtype = orig.getDType(); 155 int vdtype = view.getDType(); 156 if (odtype != vdtype) { 157 view.setDirty(); 158 } 159 } 160 161 @Override 162 public IntegerDataset getIndices() { 163 final IntegerDataset ret = DatasetUtils.indices(shape); 164 if (getName() != null) { 165 ret.setName("Indices of " + getName()); 166 } 167 return ret; 168 } 169 170 @Override 171 public Dataset getTransposedView(int... axes) { 172 axes = checkPermutatedAxes(shape, axes); 173 174 AbstractDataset t = getView(true); 175 if (axes == null || getRank() == 1) 176 return t; 177 178 int rank = shape.length; 179 int[] tstride = new int[rank]; 180 int[] toffset = new int[1]; 181 int[] nshape = createStrides(new SliceND(shape), this, tstride, toffset); 182 int[] nstride = new int[rank]; 183 for (int i = 0; i < rank; i++) { 184 final int ax = axes[i]; 185 nstride[i] = tstride[ax]; 186 nshape[i] = shape[ax]; 187 } 188 t.shape = nshape; 189 t.stride = nstride; 190 t.offset = toffset[0]; 191 t.base = this; 192 t.setDirty(); 193 t.transposeMetadata(axes); 194 return t; 195 } 196 197 @Override 198 public Dataset transpose(int... axes) { 199 Dataset t = getTransposedView(axes); 200 return t == null ? clone() : t.clone(); 201 } 202 203 @Override 204 public Dataset swapAxes(int axis1, int axis2) { 205 int rank = shape.length; 206 axis1 = ShapeUtils.checkAxis(rank, axis1); 207 axis2 = ShapeUtils.checkAxis(rank, axis2); 208 209 if (rank == 1 || axis1 == axis2) { 210 return this; 211 } 212 213 int[] axes = new int[rank]; 214 for (int i = 0; i < rank; i++) { 215 axes[i] = i; 216 } 217 218 axes[axis1] = axis2; 219 axes[axis2] = axis1; 220 return getTransposedView(axes); 221 } 222 223 boolean isContiguous() { 224 if (stride == null) 225 return true; 226 227 if (offset != 0) 228 return false; 229 230 int s = getElementsPerItem(); 231 for (int j = getRank() - 1; j >= 0; j--) { 232 if (stride[j] != s) { 233 return false; 234 } 235 s *= shape[j]; 236 } 237 238 return true; 239 } 240 241 @Override 242 public Dataset flatten() { 243 if (!isContiguous()) { // need to make a copy if not contiguous 244 return clone().flatten(); 245 } 246 return reshape(size); 247 } 248 249 /** 250 * Fill dataset from object at depth dimension 251 * @param obj 252 * @param depth 253 * @param pos position 254 */ 255 protected void fillData(Object obj, final int depth, final int[] pos) { 256 if (obj == null) { 257 Class<?> c = InterfaceUtils.getElementClass(getClass()); 258 if (Float.class.equals(c)) { 259 set(Float.NaN, pos); 260 } else if (Float.class.equals(c)) { 261 set(Double.NaN, pos); 262 } 263 return; 264 } 265 266 Class<?> clazz = obj.getClass(); 267 if (obj instanceof List<?>) { 268 List<?> jl = (List<?>) obj; 269 int l = jl.size(); 270 for (int i = 0; i < l; i++) { 271 Object lo = jl.get(i); 272 fillData(lo, depth + 1, pos); 273 pos[depth]++; 274 } 275 pos[depth] = 0; 276 } else if (clazz.isArray()) { 277 int l = Array.getLength(obj); 278 if (clazz.equals(odata.getClass())) { 279 System.arraycopy(obj, 0, odata, get1DIndex(pos), l); 280 } else if (clazz.getComponentType().isPrimitive()) { 281 for (int i = 0; i < l; i++) { 282 set(Array.get(obj, i), pos); 283 pos[depth]++; 284 } 285 pos[depth] = 0; 286 } else { 287 for (int i = 0; i < l; i++) { 288 fillData(Array.get(obj, i), depth + 1, pos); 289 pos[depth]++; 290 } 291 pos[depth] = 0; 292 } 293 } else if (obj instanceof IDataset) { 294 boolean[] a = new boolean[shape.length]; 295 Arrays.fill(a, depth, a.length, true); 296 setSlice(obj, getSliceIteratorFromAxes(pos, a)); 297 } else { 298 set(obj, pos); 299 } 300 } 301 302 @Override 303 public IndexIterator getIterator(final boolean withPosition) { 304 if (stride != null) { 305 return base.getSize() == 1 ? (withPosition ? new PositionIterator(offset, shape) : 306 new SingleItemIterator(offset, size)) : new StrideIterator(shape, stride, offset); 307 } 308 if (shape == null) { 309 return new NullIterator(shape, shape); 310 } 311 312 return withPosition ? new ContiguousIteratorWithPosition(shape, size) : new ContiguousIterator(size); 313 } 314 315 @Override 316 public IndexIterator getIterator() { 317 return getIterator(false); 318 } 319 320 @Override 321 public PositionIterator getPositionIterator(final int... axes) { 322 return new PositionIterator(shape, axes); 323 } 324 325 @Override 326 public IndexIterator getSliceIterator(final int[] start, final int[] stop, final int[] step) { 327 return getSliceIterator(new SliceND(shape, start, stop, step)); 328 } 329 330 /** 331 * @param slice 332 * @return an slice iterator that operates like an IndexIterator 333 */ 334 public IndexIterator getSliceIterator(SliceND slice) { 335 if (ShapeUtils.calcLongSize(slice.getShape()) == 0) { 336 return new NullIterator(shape, slice.getShape()); 337 } 338 if (stride != null) { 339 return new StrideIterator(getElementsPerItem(), shape, stride, offset, slice); 340 } 341 return new SliceIterator(shape, size, slice); 342 } 343 344 @Override 345 public SliceIterator getSliceIteratorFromAxes(final int[] pos, boolean[] axes) { 346 int rank = shape.length; 347 int[] start; 348 int[] stop = new int[rank]; 349 int[] step = new int[rank]; 350 351 if (pos == null) { 352 start = new int[rank]; 353 } else if (pos.length == rank) { 354 start = pos.clone(); 355 } else { 356 throw new IllegalArgumentException("pos array length is not equal to rank of dataset"); 357 } 358 if (axes == null) { 359 axes = new boolean[rank]; 360 Arrays.fill(axes, true); 361 } else if (axes.length != rank) { 362 throw new IllegalArgumentException("axes array length is not equal to rank of dataset"); 363 } 364 365 for (int i = 0; i < rank; i++) { 366 if (axes[i]) { 367 stop[i] = shape[i]; 368 } else { 369 stop[i] = start[i] + 1; 370 } 371 step[i] = 1; 372 } 373 return (SliceIterator) getSliceIterator(start, stop, step); 374 } 375 376 @Override 377 public BooleanIterator getBooleanIterator(Dataset choice) { 378 return getBooleanIterator(choice, true); 379 } 380 381 @Override 382 public BooleanIterator getBooleanIterator(Dataset choice, boolean value) { 383 return BooleanIterator.createIterator(value, this, choice, this); 384 } 385 386 @Override 387 public Dataset getByBoolean(Dataset selection) { 388 checkCompatibility(selection); 389 390 final int length = ((Number) selection.sum()).intValue(); 391 final int is = getElementsPerItem(); 392 Dataset r = DatasetFactory.zeros(is, getClass(), length); 393 BooleanIterator biter = getBooleanIterator(selection); 394 395 int i = 0; 396 while (biter.hasNext()) { 397 r.setObjectAbs(i, getObjectAbs(biter.index)); 398 i += is; 399 } 400 return r; 401 } 402 403 @Override 404 public Dataset getBy1DIndex(IntegerDataset index) { 405 final int is = getElementsPerItem(); 406 final Dataset r = DatasetFactory.zeros(is, getClass(), index.getShapeRef()); 407 final IntegerIterator iter = new IntegerIterator(index, size, is); 408 409 int i = 0; 410 while (iter.hasNext()) { 411 r.setObjectAbs(i, getObjectAbs(iter.index)); 412 i += is; 413 } 414 return r; 415 } 416 417 @Override 418 public Dataset getByIndexes(final Object... indexes) { 419 final IntegersIterator iter = new IntegersIterator(shape, indexes); 420 final int is = getElementsPerItem(); 421 final Dataset r = DatasetFactory.zeros(is, getClass(), iter.getShape()); 422 423 final int[] pos = iter.getPos(); 424 int i = 0; 425 while (iter.hasNext()) { 426 r.setObjectAbs(i, getObject(pos)); 427 i += is; 428 } 429 return r; 430 } 431 432 @Override 433 public boolean hasFloatingPointElements() { 434 Class<?> cls = getElementClass(); 435 return cls == Float.class || cls == Double.class; 436 } 437 438 @Override 439 public int getElementsPerItem() { 440 return DTypeUtils.getElementsPerItem(getDType()); 441 } 442 443 @Override 444 public int getItemBytes() { 445 return DTypeUtils.getItemBytes(getDType(), getElementsPerItem()); 446 } 447 448 @Override 449 public int getSize() { 450 return size; 451 } 452 453 @Override 454 public int[] getShape() { 455 // make a copy of the dimensions data, and put that out 456 if (shape == null) { 457 logger.warn("Shape is null!!!"); 458 return new int[] {}; 459 } 460 return shape.clone(); 461 } 462 463 @Override 464 public int getRank() { 465 return shape == null ? 0 : shape.length; 466 } 467 468 @Override 469 public int getNbytes() { 470 return getSize() * getItemBytes(); 471 } 472 473 /** 474 * Check for -1 placeholder in shape and replace if necessary 475 * @param shape 476 * @param size 477 */ 478 private void checkShape(int[] shape, int size) { 479 int rank = shape.length; 480 int found = -1; 481 int nsize = 1; 482 for (int i = 0; i < rank; i++) { 483 int d = shape[i]; 484 if (d == -1) { 485 if (found == -1) { 486 found = i; 487 } else { 488 logger.error("Can only have one -1 placeholder in shape"); 489 throw new IllegalArgumentException("Can only have one -1 placeholder in shape"); 490 } 491 } else { 492 nsize *= d; 493 } 494 } 495 if (found >= 0) { 496 shape[found] = size/nsize; 497 } else if (nsize != size && !(rank == 0 && size == 0)) { 498 logger.error("New shape is not same size as old shape"); 499 throw new IllegalArgumentException("New size is not same as the old size. Old size is "+size+" new size is "+nsize+" and shape is "+Arrays.toString(shape)); 500 } 501 } 502 503 @Override 504 public void setShape(final int... shape) { 505 int[] nshape = shape.clone(); 506 checkShape(nshape, size); 507 if (Arrays.equals(this.shape, nshape)) { 508 return; 509 } 510 511 if (stride != null) { 512 // the only compatible shapes are ones where new dimensions are factors of old dimensions 513 // or are combined adjacent old dimensions 514 int[] oshape = this.shape; 515 int orank = oshape.length; 516 int nrank = nshape.length; 517 int diff = nrank - orank; 518 int[] nstride = new int[nrank]; 519 boolean ones = true; 520 // work forwards for broadcasting cases 521 for (int i = 0, j = 0; i < orank || j < nrank;) { 522 if (j >= diff && i < orank && j < nrank && oshape[i] == nshape[j]) { 523 nstride[j++] = stride[i++]; 524 } else if (j < nrank && nshape[j] == 1) { 525 nstride[j++] = 0; 526 } else if (i < orank && oshape[i] == 1) { 527 i++; 528 } else { 529 if (j < nrank) 530 j++; 531 if (i < orank) 532 i++; 533 ones = false; 534 } 535 } 536 if (!ones) { // not just ones differ in shapes 537 int[] ostride = stride; 538 int ob = 0; 539 int oe = 1; 540 int nb = 0; 541 int ne = 1; 542 while (ob < orank && nb < nrank) { 543 int ol = oshape[ob]; 544 int nl = nshape[nb]; 545 546 if (nl < ol) { // find group of shape dimensions that form common size 547 do { // case where new shape spreads single dimension over several dimensions 548 if (ne == nrank) { 549 break; 550 } 551 nl *= nshape[ne++]; 552 } while (nl < ol); 553 if (nl != ol) { 554 logger.error("Subshape is incompatible with single dimension"); 555 throw new IllegalArgumentException("Subshape is incompatible with single dimension"); 556 } 557 int on = ne - 1; 558 while (nshape[on] == 1) { 559 on--; 560 } 561 562 nstride[on] = ostride[ob]; 563 for (int n = on - 1; n >= nb; n--) { 564 if (nshape[n] == 1) 565 continue; 566 567 nstride[n] = nshape[on] * nstride[on]; 568 on = n; 569 } 570 } else if (ol < nl) { 571 do { // case where new shape combines several dimensions into one dimension 572 if (oe == orank) { 573 break; 574 } 575 ol *= oshape[oe++]; 576 } while (ol < nl); 577 if (nl != ol) { 578 logger.error("Single dimension is incompatible with subshape"); 579 throw new IllegalArgumentException("Single dimension is incompatible with subshape"); 580 } 581 582 int oo = oe - 1; 583 while (oshape[oo] == 1) { 584 oo--; 585 } 586 int os = ostride[oo]; 587 for (int o = oo - 1; o >= ob; o--) { 588 if (oshape[o] == 1) 589 continue; 590 if (ostride[o] != oshape[oo] * ostride[oo]) { 591 logger.error("Subshape cannot be a non-contiguous view"); 592 throw new IllegalArgumentException("Subshape cannot be a non-contiguous view"); 593 } 594 oo = o; 595 } 596 nstride[nb] = os; 597 } else { 598 nstride[nb] = ostride[ob]; 599 } 600 601 ob = oe++; 602 nb = ne++; 603 } 604 } 605 606 stride = nstride; 607 } 608 609 setDirty(); 610 if (this.shape != null && metadata != null) { 611 reshapeMetadata(this.shape, nshape); 612 } 613 this.shape = nshape; 614 } 615 616 @Override 617 public int[] getShapeRef() { 618 return shape; 619 } 620 621 @Override 622 public int getOffset() { 623 return offset; 624 } 625 626 @Override 627 public int[] getStrides() { 628 return stride; 629 } 630 631 @Override 632 public Serializable getBuffer() { 633 return odata; 634 } 635 636 @Override 637 public void overrideInternal(Serializable buffer, int... shape) { 638 if (buffer != null) { 639 odata = buffer; 640 setData(); 641 setDirty(); 642 } 643 644 if (shape != null) { 645 this.shape = shape.clone(); 646 size = ShapeUtils.calcSize(this.shape); 647 } 648 } 649 650 /** 651 * Create a stride array from dataset 652 * @param a dataset 653 * @param offset output offset 654 * @return new strides 655 */ 656 public static int[] createStrides(Dataset a, final int[] offset) { 657 return createStrides(a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), offset); 658 } 659 660 /** 661 * Create a stride array from dataset 662 * @param isize 663 * @param shape 664 * @param oStride original stride 665 * @param oOffset original offset (only used if there is an original stride) 666 * @param offset output offset 667 * @return new strides 668 */ 669 public static int[] createStrides(final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] offset) { 670 int rank = shape.length; 671 final int[] stride; 672 if (oStride == null) { 673 offset[0] = 0; 674 stride = new int[rank]; 675 int s = isize; 676 for (int j = rank - 1; j >= 0; j--) { 677 stride[j] = s; 678 s *= shape[j]; 679 } 680 } else { 681 offset[0] = oOffset; 682 stride = oStride.clone(); 683 } 684 return stride; 685 } 686 687 /** 688 * Create a stride array from slice information and a dataset 689 * @param slice 690 * @param a dataset 691 * @param stride output stride 692 * @param offset output offset 693 * @return new shape 694 */ 695 public static int[] createStrides(final SliceND slice, final Dataset a, final int[] stride, final int[] offset) { 696 return createStrides(slice, a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), stride, offset); 697 } 698 699 /** 700 * Create a stride array from slice and dataset information 701 * @param slice 702 * @param isize 703 * @param shape 704 * @param oStride original stride 705 * @param oOffset original offset (only used if there is an original stride) 706 * @param stride output stride 707 * @param offset output offset 708 * @return new shape 709 */ 710 public static int[] createStrides(final SliceND slice, final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] stride, final int[] offset) { 711 int[] lstart = slice.getStart(); 712 int[] lstep = slice.getStep(); 713 int[] newShape = slice.getShape(); 714 int rank = shape.length; 715 716 if (oStride == null) { 717 int s = isize; 718 offset[0] = 0; 719 for (int j = rank - 1; j >= 0; j--) { 720 stride[j] = s * lstep[j]; 721 offset[0] += s * lstart[j]; 722 s *= shape[j]; 723 } 724 } else { 725 offset[0] = oOffset; 726 for (int j = 0; j < rank; j++) { 727 int s = oStride[j]; 728 stride[j] = lstep[j] * s; 729 offset[0] += lstart[j] * s; 730 } 731 } 732 733 return newShape; 734 } 735 736 @Override 737 public Dataset getBroadcastView(int... broadcastShape) { 738 AbstractDataset view = getView(true); 739 740 if (!Arrays.equals(shape, broadcastShape)) { 741 List<int[]> nShapes = BroadcastUtils.broadcastShapesToMax(broadcastShape, shape); 742 view.setShape(nShapes.get(0)); 743 view.stride = BroadcastUtils.createBroadcastStrides(view, broadcastShape); 744 view.base = this; 745 view.shape = broadcastShape.clone(); 746 view.size = ShapeUtils.calcSize(broadcastShape); 747 if (view.name == null || view.name.isEmpty()) { 748 view.name = "Broadcast from " + Arrays.toString(shape); 749 } else { 750 view.name = "Broadcast of " + view.name + " from " + Arrays.toString(shape); 751 } 752 } 753 return view; 754 } 755 756 @Override 757 public Dataset getSliceView(final int[] start, final int[] stop, final int[] step) { 758 return getSliceView(new SliceND(shape, start, stop, step)); 759 } 760 761 @Override 762 public Dataset getSliceView(Slice... slice) { 763 if (slice == null || slice.length == 0) { 764 return getView(true); 765 } 766 767 return getSliceView(new SliceND(shape, slice)); 768 } 769 770 /** 771 * Get a slice of the dataset. The returned dataset is a view on a selection of items 772 * @param slice 773 * @return slice view 774 */ 775 @Override 776 public Dataset getSliceView(SliceND slice) { 777 if (slice.isAll()) { 778 return getView(true); 779 } 780 781 final int rank = shape.length; 782 int[] sStride = new int[rank]; 783 int[] sOffset = new int[1]; 784 785 int[] sShape = createStrides(slice, this, sStride, sOffset); 786 787 AbstractDataset s = getView(false); 788 s.shape = sShape; 789 s.size = ShapeUtils.calcSize(sShape); 790 s.stride = sStride; 791 s.offset = sOffset[0]; 792 s.base = this; 793 794 s.metadata = copyMetadata(); 795 s.sliceMetadata(true, slice); 796 797 s.setDirty(); 798 s.setName(name + BLOCK_OPEN + slice + BLOCK_CLOSE); 799 800 return s; 801 } 802 803 /** 804 * Get flattened view index of given position 805 * @param pos 806 * the integer array specifying the n-D position 807 * @return the index on the flattened dataset 808 */ 809 private int getFlat1DIndex(final int[] pos) { 810 final int imax = pos.length; 811 if (imax == 0) { 812 return 0; 813 } 814 815 return get1DIndexFromShape(pos); 816 } 817 818 /** 819 * @since 2.0 820 */ 821 protected int getFirst1DIndex() { 822 if (shape == null) { 823 throw new IllegalArgumentException("Cannot find an index from a null shape"); 824 } 825 return stride == null ? 0 : offset; 826 } 827 828 @Override 829 public int get1DIndex(final int... n) { 830 if (n.length == 0 && shape.length == 0) 831 return offset; 832 833 return stride == null ? get1DIndexFromShape(n) : get1DIndexFromStrides(n); 834 } 835 836 private static void throwAIOOBException(int i, int s, int d) { 837 throw new ArrayIndexOutOfBoundsException("Index (" + i + ") out of range [-" + s + "," + s 838 + "] in dimension " + d); 839 } 840 841 /** 842 * @param i 843 * @return the index on the data array corresponding to that location 844 */ 845 protected int get1DIndex(int i) { 846 if (shape == null) { 847 throw new IllegalArgumentException("Cannot find an index from a null shape"); 848 } 849 if (shape.length > 1) { 850 logger.error("This dataset is not 1D but was addressed as such"); 851 throw new UnsupportedOperationException("This dataset is not 1D but was addressed as such"); 852 } 853 if (i < 0) { 854 i += shape[0]; 855 } 856 if (i < 0 || i >= shape[0]) { 857 throwAIOOBException(i, shape[0], 0); 858 } 859 return stride == null ? i : i*stride[0] + offset; 860 } 861 862 /** 863 * @param i 864 * @param j 865 * @return the index on the data array corresponding to that location 866 */ 867 protected int get1DIndex(int i, int j) { 868 if (shape == null) { 869 throw new IllegalArgumentException("Cannot find an index from a null shape"); 870 } 871 if (shape.length != 2) { 872 logger.error("This dataset is not 2D but was addressed as such"); 873 throw new UnsupportedOperationException("This dataset is not 2D but was addressed as such"); 874 } 875 if (i < 0) { 876 i += shape[0]; 877 } 878 if (i < 0 || i >= shape[0]) { 879 throwAIOOBException(i, shape[0], 0); 880 } 881 if (j < 0) { 882 j += shape[1]; 883 } 884 if (j < 0 || j >= shape[1]) { 885 throwAIOOBException(i, shape[1], 1); 886 } 887 return stride == null ? i*shape[1] + j : i*stride[0] + j*stride[1] + offset; 888 } 889 890 protected int get1DIndexFromShape(final int[] n) { 891 return get1DIndexFromShape(shape, n); 892 } 893 894 protected static int get1DIndexFromShape(final int[] shape, final int[] n) { 895 if (shape == null) { 896 throw new IllegalArgumentException("Cannot find an index from a null shape"); 897 } 898 final int rank = shape.length; 899 if (rank != n.length) { 900 String errMsg = String.format("Number of position values must be equal to rank of %d", rank); 901 logger.error(errMsg); 902 throw new IllegalArgumentException(errMsg); 903 } 904 int index = 0; 905 for (int i = 0; i < rank; i++) { 906 final int si = shape[i]; 907 int ni = n[i]; 908 if (ni < 0) { 909 ni += si; 910 } 911 if (ni < 0 || ni >= si) { 912 throwAIOOBException(ni, si, i); 913 } 914 index = index * si + ni; 915 } 916 917 return index; 918 } 919 920 private int get1DIndexFromStrides(final int[] n) { 921 return get1DIndexFromStrides(shape, stride, offset, n); 922 } 923 924 private static int get1DIndexFromStrides(final int[] shape, final int[] stride, final int offset, final int[] n) { 925 if (shape == null) { 926 throw new IllegalArgumentException("Cannot find an index from a null shape"); 927 } 928 final int rank = shape.length; 929 if (rank != n.length) { 930 String errMsg = String.format("Number of position values must be equal to rank of %d", rank); 931 logger.error(errMsg); 932 throw new IllegalArgumentException(errMsg); 933 } 934 int index = offset; 935 for (int i = 0; i < rank; i++) { 936 final int st = stride[i]; 937 if (st != 0) { // not broadcasted 938 final int si = shape[i]; 939 int ni = n[i]; 940 if (ni < 0) { 941 ni += si; 942 } 943 if (ni < 0 || ni >= si) { 944 throwAIOOBException(ni, si, i); 945 } 946 index += st * ni; 947 } 948 } 949 return index; 950 } 951 952 @Override 953 public int[] getNDPosition(final int n) { 954 if (isIndexInRange(n)) { 955 throw new IllegalArgumentException("Index provided " + n 956 + "is larger then the size of the containing array"); 957 } 958 959 return stride == null ? ShapeUtils.getNDPositionFromShape(n, shape) : getNDPositionFromStrides(n); 960 } 961 962 private boolean isIndexInRange(final int n) { 963 if (stride == null) { 964 return n >= size; 965 } 966 return n >= getBufferLength(); 967 } 968 969 /** 970 * @return entire buffer length 971 */ 972 abstract protected int getBufferLength(); 973 974 private int[] getNDPositionFromStrides(int n) { 975 n -= offset; 976 int rank = shape.length; 977 if (rank == 1) { 978 return new int[] { n / stride[0] }; 979 } 980 981 int[] output = new int[rank]; 982 int i = 0; 983 while (i != n) { // TODO find more efficient way than this exhaustive search 984 int j = rank - 1; 985 for (; j >= 0; j--) { 986 output[j]++; 987 i += stride[j]; 988 if (output[j] >= shape[j]) { 989 output[j] = 0; 990 i -= shape[j] * stride[j]; 991 } else { 992 break; 993 } 994 } 995 if (j == -1) { 996 logger.error("Index was not found in this strided dataset"); 997 throw new IllegalArgumentException("Index was not found in this strided dataset"); 998 } 999 } 1000 1001 return output; 1002 } 1003 1004 @Override 1005 public int checkAxis(int axis) { 1006 return ShapeUtils.checkAxis(shape.length, axis); 1007 } 1008 1009 @Deprecated 1010 protected static int checkAxis(int rank, int axis) { 1011 return ShapeUtils.checkAxis(rank, axis); 1012 } 1013 1014 protected static final char BLOCK_OPEN = '['; 1015 protected static final char BLOCK_CLOSE = ']'; 1016 1017 @Override 1018 public String toString() { 1019 final int rank = shape == null ? 0 : shape.length; 1020 final StringBuilder out = new StringBuilder(); 1021 1022 if (DTypeUtils.isDTypeElemental(getDType())) { 1023 out.append("Dataset "); 1024 } else { 1025 out.append("Compound dataset ("); 1026 out.append(getElementsPerItem()); 1027 out.append(") "); 1028 } 1029 1030 if (name != null && name.length() > 0) { 1031 out.append("'"); 1032 out.append(name); 1033 out.append("' has shape "); 1034 } else { 1035 out.append("shape is "); 1036 } 1037 1038 out.append(BLOCK_OPEN); 1039 if (rank > 0) { 1040 out.append(shape[0]); 1041 } 1042 for (int i = 1; i < rank; i++) { 1043 out.append(", " + shape[i]); 1044 } 1045 out.append(BLOCK_CLOSE); 1046 return out.toString(); 1047 } 1048 1049 @Override 1050 public String toString(boolean showData) { 1051 if (!showData) { 1052 return toString(); 1053 } 1054 1055 if (size == 0) { 1056 return "[]"; 1057 } 1058 1059 final int rank = shape == null ? 0 : shape.length; 1060 final StringBuilder out = new StringBuilder(); 1061 1062 if (rank > 0) { 1063 int[] pos = new int[rank]; 1064 final StringBuilder lead = new StringBuilder(); 1065 printBlocks(out, lead, 0, pos); 1066 } else { 1067 out.append(getString()); 1068 } 1069 return out.toString(); 1070 } 1071 1072 /** 1073 * Limit to strings output via the toString() method 1074 */ 1075 private static int maxStringLength = 120; 1076 1077 /** 1078 * Set maximum line length for toString() method 1079 * @param maxLineLength 1080 */ 1081 public static void setMaxLineLength(int maxLineLength) { 1082 maxStringLength = maxLineLength; 1083 } 1084 1085 /** 1086 * @return maximum line length for toString() method 1087 */ 1088 public static int getMaxLineLength() { 1089 return maxStringLength; 1090 } 1091 1092 /** 1093 * Limit to number of sub-blocks output via the toString() method 1094 */ 1095 private static final int MAX_SUBBLOCKS = 6; 1096 1097 private final static String SEPARATOR = ","; 1098 private final static String SPACE = " "; 1099 private final static String ELLIPSIS = "..."; 1100 private final static String NEWLINE = "\n"; 1101 1102 /** 1103 * Make a line of output for last dimension of dataset 1104 * 1105 * @param start 1106 * @return line 1107 */ 1108 private StringBuilder makeLine(final int end, final int[] start) { 1109 StringBuilder line = new StringBuilder(); 1110 final int[] pos; 1111 if (end >= start.length) { 1112 pos = Arrays.copyOf(start, end + 1); 1113 } else { 1114 pos = start; 1115 } 1116 pos[end] = 0; 1117 line.append(BLOCK_OPEN); 1118 line.append(getString(pos)); 1119 1120 final int length = shape[end]; 1121 1122 // trim elements printed if length exceed estimate of maximum elements 1123 int excess = length - maxStringLength / 3; // space + number + separator 1124 int midIndex = -1; 1125 if (excess > 0) { 1126 int index = (length - excess) / 2; 1127 for (int y = 1; y < index; y++) { 1128 line.append(SEPARATOR + SPACE); 1129 pos[end] = y; 1130 line.append(getString(pos)); 1131 } 1132 midIndex = line.length() + 2; 1133 index = (length + excess) / 2; 1134 for (int y = index; y < length; y++) { 1135 line.append(SEPARATOR + SPACE); 1136 pos[end] = y; 1137 line.append(getString(pos)); 1138 } 1139 } else { 1140 for (int y = 1; y < length; y++) { 1141 line.append(SEPARATOR + SPACE); 1142 pos[end] = y; 1143 line.append(getString(pos)); 1144 } 1145 } 1146 line.append(BLOCK_CLOSE); 1147 1148 // trim string down to limit 1149 int lineLength = line.length(); 1150 excess = lineLength - maxStringLength - ELLIPSIS.length() - 1; 1151 if (excess > 0) { 1152 int index = (lineLength - excess) / 2; 1153 if (midIndex > 0 && index > midIndex) { 1154 index = midIndex; 1155 } else { 1156 index = line.lastIndexOf(SEPARATOR, index) + 2; 1157 } 1158 StringBuilder out = new StringBuilder(line.subSequence(0, index)); 1159 out.append(ELLIPSIS + SEPARATOR); 1160 index = (lineLength + excess) / 2; 1161 if (midIndex > 0 && index <= midIndex) { 1162 index = midIndex - 1; 1163 } else { 1164 index = line.indexOf(SEPARATOR, index) + 1; 1165 } 1166 out.append(line.subSequence(index, lineLength)); 1167 return out; 1168 } else if (midIndex > 0) { // add ellipsis 1169 StringBuilder out = new StringBuilder(line.subSequence(0, midIndex)); 1170 out.append(ELLIPSIS + SEPARATOR + SPACE); 1171 out.append(line.subSequence(midIndex, lineLength)); 1172 return out; 1173 } 1174 1175 return line; 1176 } 1177 1178 /** 1179 * recursive method to print blocks 1180 */ 1181 private void printBlocks(final StringBuilder out, final StringBuilder lead, final int level, final int[] pos) { 1182 if (out.length() > 0) { 1183 char last = out.charAt(out.length() - 1); 1184 if (last != BLOCK_OPEN) { 1185 out.append(lead); 1186 } 1187 } 1188 final int end = getRank() - 1; 1189 if (level != end) { 1190 out.append(BLOCK_OPEN); 1191 int length = shape[level]; 1192 1193 // first sub-block 1194 pos[level] = 0; 1195 StringBuilder newlead = new StringBuilder(lead); 1196 newlead.append(SPACE); 1197 printBlocks(out, newlead, level + 1, pos); 1198 if (length < 2) { // escape 1199 out.append(BLOCK_CLOSE); 1200 return; 1201 } 1202 1203 out.append(SEPARATOR + NEWLINE); 1204 for (int i = level + 1; i < end; i++) { 1205 out.append(NEWLINE); 1206 } 1207 1208 // middle sub-blocks 1209 if (length < MAX_SUBBLOCKS) { 1210 for (int x = 1; x < length - 1; x++) { 1211 pos[level] = x; 1212 printBlocks(out, newlead, level + 1, pos); 1213 if (end <= level + 1) { 1214 out.append(SEPARATOR + NEWLINE); 1215 } else { 1216 out.append(SEPARATOR + NEWLINE + NEWLINE); 1217 } 1218 } 1219 } else { 1220 final int excess = length - MAX_SUBBLOCKS; 1221 int xmax = (length - excess) / 2; 1222 for (int x = 1; x < xmax; x++) { 1223 pos[level] = x; 1224 printBlocks(out, newlead, level + 1, pos); 1225 if (end <= level + 1) { 1226 out.append(SEPARATOR + NEWLINE); 1227 } else { 1228 out.append(SEPARATOR + NEWLINE + NEWLINE); 1229 } 1230 } 1231 out.append(newlead); 1232 out.append(ELLIPSIS + SEPARATOR + NEWLINE); 1233 xmax = (length + excess) / 2; 1234 for (int x = xmax; x < length - 1; x++) { 1235 pos[level] = x; 1236 printBlocks(out, newlead, level + 1, pos); 1237 if (end <= level + 1) { 1238 out.append(SEPARATOR + NEWLINE); 1239 } else { 1240 out.append(SEPARATOR + NEWLINE + NEWLINE); 1241 } 1242 } 1243 } 1244 1245 // last sub-block 1246 pos[level] = length - 1; 1247 printBlocks(out, newlead, level + 1, pos); 1248 out.append(BLOCK_CLOSE); 1249 } else { 1250 out.append(makeLine(end, pos)); 1251 } 1252 } 1253 1254 @Override 1255 public Dataset squeezeEnds() { 1256 return squeeze(true); 1257 } 1258 1259 @Override 1260 public Dataset squeeze() { 1261 return squeeze(false); 1262 } 1263 1264 @Override 1265 public Dataset squeeze(boolean onlyFromEnds) { 1266 final int[] tshape = ShapeUtils.squeezeShape(shape, onlyFromEnds); 1267 final int[] oshape = shape; 1268 if (stride == null) { 1269 shape = tshape; 1270 } else { 1271 int rank = shape.length; 1272 int trank = tshape.length; 1273 if (trank < rank) { 1274 int[] tstride = new int[tshape.length]; 1275 if (onlyFromEnds) { 1276 for (int i = 0; i < rank; i++) { 1277 if (shape[i] != 1) { 1278 for (int k = 0; k < trank; k++) { 1279 tstride[k] = stride[i++]; 1280 } 1281 break; 1282 } 1283 } 1284 } else { 1285 int t = 0; 1286 for (int i = 0; i < rank; i++) { 1287 if (shape[i] != 1) { 1288 tstride[t++] = stride[i]; 1289 } 1290 } 1291 } 1292 shape = tshape; 1293 stride = tstride; 1294 } 1295 } 1296 1297 setDirty(); 1298 reshapeMetadata(oshape, shape); 1299 return this; 1300 } 1301 1302 @Override 1303 public boolean isCompatibleWith(final ILazyDataset g) { 1304 return ShapeUtils.areShapesCompatible(shape, g.getShape()); 1305 } 1306 1307 @Override 1308 public void checkCompatibility(final ILazyDataset g) throws IllegalArgumentException { 1309 ShapeUtils.checkCompatibility(this, g); 1310 } 1311 1312 @Override 1313 public Dataset reshape(final int... shape) { 1314 Dataset a; 1315 try { 1316 a = getView(true); 1317 a.setShape(shape); 1318 } catch (IllegalArgumentException e) { 1319 a = clone(); 1320 a.setShape(shape); 1321 } 1322 return a; 1323 } 1324 1325 /** 1326 * @param start 1327 * @param stop 1328 * @param step 1329 * @return number of steps to take 1330 */ 1331 protected static int calcSteps(final double start, final double stop, final double step) { 1332 return Math.max(0, (int) Math.ceil((stop - start) / step)); 1333 } 1334 1335 @Override 1336 public boolean isComplex() { 1337 int type = getDType(); 1338 return type == COMPLEX64 || type == COMPLEX128; 1339 } 1340 1341 @Override 1342 public Dataset getRealPart() { 1343 return this; 1344 } 1345 1346 @Override 1347 public Dataset getRealView() { 1348 return getView(true); 1349 } 1350 1351 @Override 1352 public Dataset getSlice(final int[] start, final int[] stop, final int[] step) { 1353 return getSlice(new SliceND(shape, start, stop, step)); 1354 } 1355 1356 @Override 1357 public Dataset getSlice(Slice... slice) { 1358 return getSlice(new SliceND(shape, slice)); 1359 } 1360 1361 @Override 1362 public Dataset getSlice(IMonitor monitor, Slice... slice) { 1363 return getSlice(slice); 1364 } 1365 1366 @Override 1367 public Dataset getSlice(IMonitor monitor, SliceND slice) { 1368 return getSlice(slice); 1369 } 1370 1371 @Override 1372 public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) { 1373 return getSlice(start, stop, step); 1374 } 1375 1376 /** 1377 * Get a slice of the dataset. The returned dataset is a copied selection of items 1378 * @param slice 1379 * @return The dataset of the sliced data 1380 */ 1381 @Override 1382 public Dataset getSlice(final SliceND slice) { 1383 SliceIterator it = (SliceIterator) getSliceIterator(slice); 1384 AbstractDataset s = getSlice(it); 1385 s.metadata = copyMetadata(); 1386 s.setDirty(); 1387 s.sliceMetadata(true, slice); 1388 return s; 1389 } 1390 1391 /** 1392 * Get a slice of the dataset. The returned dataset is a copied selection of items 1393 * 1394 * @param iterator Slice iterator 1395 * @return The dataset of the sliced data 1396 */ 1397 abstract public AbstractDataset getSlice(final SliceIterator iterator); 1398 1399 @Override 1400 public Dataset setSlice(final Object obj, final SliceND slice) { 1401 Dataset ds; 1402 if (obj instanceof Dataset) { 1403 ds = (Dataset) obj; 1404 } else if (obj instanceof IDataset) { 1405 ds = DatasetUtils.convertToDataset((IDataset) obj); 1406 } else { 1407 Class<? extends Dataset> dClass = getClass(); 1408 if (!BooleanDataset.class.equals(dClass)) { 1409 dClass = DTypeUtils.getLargestDataset(dClass); 1410 } 1411 ds = DatasetFactory.createFromObject(getElementsPerItem(), dClass, obj); 1412 } 1413 1414 return setSlicedView(getSliceView(slice), ds); 1415 } 1416 1417 @Override 1418 public Dataset setSlice(final Object obj, final int[] start, final int[] stop, final int[] step) { 1419 return setSlice(obj, new SliceND(shape, start, stop, step)); 1420 } 1421 1422 /** 1423 * Set a view of current dataset to given dataset with broadcasting 1424 * @param view 1425 * @param d 1426 * @return this dataset 1427 */ 1428 abstract Dataset setSlicedView(Dataset view, Dataset d); 1429 1430 @Override 1431 public Dataset setSlice(Object obj, Slice... slice) { 1432 if (slice == null || slice.length == 0) { 1433 return setSlice(obj, new SliceND(shape)); 1434 } 1435 return setSlice(obj, new SliceND(shape, slice)); 1436 } 1437 1438 @Override 1439 public boolean all() { 1440 return Comparisons.allTrue(this); 1441 } 1442 1443 @Override 1444 public BooleanDataset all(final int axis) { 1445 return Comparisons.allTrue(this, axis); 1446 } 1447 1448 @Override 1449 public boolean any() { 1450 return Comparisons.anyTrue(this); 1451 } 1452 1453 @Override 1454 public BooleanDataset any(final int axis) { 1455 return Comparisons.anyTrue(this, axis); 1456 } 1457 1458 @Override 1459 public Dataset ifloorDivide(final Object o) { 1460 return idivide(o).ifloor(); 1461 } 1462 1463 @Override 1464 public double residual(final Object o) { 1465 return residual(o, null, false); 1466 } 1467 1468 @Override 1469 public double residual(final Object o, boolean ignoreNaNs) { 1470 return residual(o, null, ignoreNaNs); 1471 } 1472 1473 /** 1474 * @since 2.0 1475 */ 1476 @SuppressWarnings("unchecked") 1477 protected StatisticsMetadata<Number> getStats() { 1478 StatisticsMetadata<Number> md = getFirstMetadata(StatisticsMetadata.class); 1479 if (md == null || md.isDirty(this)) { 1480 md = new StatisticsMetadataImpl<Number>(); 1481 md.initialize(this); 1482 setMetadata(md); 1483 } 1484 return md; 1485 } 1486 1487 /** 1488 * @since 2.0 1489 */ 1490 @SuppressWarnings("unchecked") 1491 protected StatisticsMetadata<String> getStringStats() { 1492 StatisticsMetadata<String> md = getFirstMetadata(StatisticsMetadata.class); 1493 if (md == null || md.isDirty(this)) { 1494 md = new StatisticsMetadataImpl<String>(); 1495 md.initialize(this); 1496 setMetadata(md); 1497 } 1498 return md; 1499 } 1500 1501 @Override 1502 public Number max(boolean... ignoreInvalids) { 1503 return getStats().getMaximum(ignoreInvalids); 1504 } 1505 1506 @Override 1507 public Dataset max(int axis, boolean... ignoreInvalids) { 1508 return getStats().getMaximum(axis, ignoreInvalids); 1509 } 1510 1511 @Override 1512 public Dataset max(int[] axes, boolean... ignoreInvalids) { 1513 return getStats().getMaximum(axes, ignoreInvalids); 1514 } 1515 1516 @Override 1517 public Number min(boolean... ignoreInvalids) { 1518 return getStats().getMinimum(ignoreInvalids); 1519 } 1520 1521 @Override 1522 public Dataset min(int axis, boolean... ignoreInvalids) { 1523 return getStats().getMinimum(axis, ignoreInvalids); 1524 } 1525 1526 @Override 1527 public Dataset min(int[] axes, boolean... ignoreInvalids) { 1528 return getStats().getMinimum(axes, ignoreInvalids); 1529 } 1530 1531 @Override 1532 public int argMax(boolean... ignoreInvalids) { 1533 return getFlat1DIndex(maxPos(ignoreInvalids)); 1534 } 1535 1536 /** 1537 * @since 2.0 1538 */ 1539 @Override 1540 public IntegerDataset argMax(int axis, boolean... ignoreInvalids) { 1541 return (IntegerDataset) getStats().getArgMaximum(axis, ignoreInvalids); 1542 } 1543 1544 @Override 1545 public int argMin(boolean... ignoreInvalids) { 1546 return getFlat1DIndex(minPos(ignoreInvalids)); 1547 } 1548 1549 /** 1550 * @since 2.0 1551 */ 1552 @Override 1553 public IntegerDataset argMin(int axis, boolean... ignoreInvalids) { 1554 return (IntegerDataset) getStats().getArgMinimum(axis, ignoreInvalids); 1555 } 1556 1557 @Override 1558 public Number peakToPeak(boolean... ignoreInvalids) { 1559 return InterfaceUtils.fromDoubleToBiggestNumber(getClass(), max(ignoreInvalids).doubleValue() - min(ignoreInvalids).doubleValue()); 1560 } 1561 1562 @Override 1563 public Dataset peakToPeak(int axis, boolean... ignoreInvalids) { 1564 return Maths.subtract(max(axis, ignoreInvalids), min(axis, ignoreInvalids)); 1565 } 1566 1567 @Override 1568 public Dataset peakToPeak(int[] axes, boolean... ignoreInvalids) { 1569 return Maths.subtract(max(axes, ignoreInvalids), min(axes, ignoreInvalids)); 1570 } 1571 1572 @Override 1573 public long count(boolean... ignoreInvalids) { 1574 return getStats().getCount(ignoreInvalids); 1575 } 1576 1577 @Override 1578 public Dataset count(int axis, boolean... ignoreInvalids) { 1579 return getStats().getCount(axis, ignoreInvalids); 1580 } 1581 1582 @Override 1583 public Dataset count(int[] axes, boolean... ignoreInvalids) { 1584 return getStats().getCount(axes, ignoreInvalids); 1585 } 1586 1587 @Override 1588 public Object sum(boolean... ignoreInvalids) { 1589 return InterfaceUtils.toBiggestNumber(getClass(), getStats().getSum(ignoreInvalids)); 1590 } 1591 1592 @Override 1593 public Dataset sum(int axis, boolean... ignoreInvalids) { 1594 return getStats().getSum(axis, ignoreInvalids); 1595 } 1596 1597 @Override 1598 public Dataset sum(int[] axes, boolean... ignoreInvalids) { 1599 return getStats().getSum(axes, ignoreInvalids); 1600 } 1601 1602 @Override 1603 public Object product(boolean... ignoreInvalids) { 1604 return Stats.product(this, ignoreInvalids); 1605 } 1606 1607 @Override 1608 public Dataset product(int axis, boolean... ignoreInvalids) { 1609 return Stats.product(this, axis, ignoreInvalids); 1610 } 1611 1612 @Override 1613 public Dataset product(int[] axes, boolean... ignoreInvalids) { 1614 return Stats.product(this, axes, ignoreInvalids); 1615 } 1616 1617 @Override 1618 public Object mean(boolean... ignoreInvalids) { 1619 return getStats().getMean(ignoreInvalids); 1620 } 1621 1622 @Override 1623 public Dataset mean(int axis, boolean... ignoreInvalids) { 1624 return getStats().getMean(axis, ignoreInvalids); 1625 } 1626 1627 @Override 1628 public Dataset mean(int[] axes, boolean... ignoreInvalids) { 1629 return getStats().getMean(axes, ignoreInvalids); 1630 } 1631 1632 @Override 1633 public double variance() { 1634 return variance(false); 1635 } 1636 1637 @Override 1638 public double variance(boolean isWholePopulation, boolean... ignoreInvalids) { 1639 return getStats().getVariance(isWholePopulation, ignoreInvalids); 1640 } 1641 1642 @Override 1643 public Dataset variance(int axis) { 1644 return getStats().getVariance(axis, false); 1645 } 1646 1647 @Override 1648 public Dataset variance(int[] axes) { 1649 return getStats().getVariance(axes, false); 1650 } 1651 1652 @Override 1653 public Dataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) { 1654 return getStats().getVariance(axis, isWholePopulation, ignoreInvalids); 1655 } 1656 1657 @Override 1658 public Dataset variance(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) { 1659 return getStats().getVariance(axes, isWholePopulation, ignoreInvalids); 1660 } 1661 1662 @Override 1663 public double stdDeviation() { 1664 return Math.sqrt(variance()); 1665 } 1666 1667 @Override 1668 public double stdDeviation(boolean isWholePopulation, boolean... ignoreInvalids) { 1669 return Math.sqrt(variance(isWholePopulation, ignoreInvalids)); 1670 } 1671 1672 @Override 1673 public Dataset stdDeviation(int axis) { 1674 return Maths.sqrt(variance(axis, false)); 1675 } 1676 1677 @Override 1678 public Dataset stdDeviation(int[] axes) { 1679 return Maths.sqrt(variance(axes, false)); 1680 } 1681 1682 @Override 1683 public Dataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) { 1684 return Maths.sqrt(variance(axis, isWholePopulation, ignoreInvalids)); 1685 } 1686 1687 @Override 1688 public Dataset stdDeviation(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) { 1689 return Maths.sqrt(variance(axes, isWholePopulation, ignoreInvalids)); 1690 } 1691 1692 @Override 1693 public double rootMeanSquare(boolean... ignoreInvalids) { 1694 StatisticsMetadata<Number> stats = getStats(); 1695 final double mean = stats.getMean(ignoreInvalids).doubleValue(); 1696 final double var = stats.getVariance(true, ignoreInvalids); 1697 return Math.sqrt(var + mean * mean); 1698 } 1699 1700 @Override 1701 public Dataset rootMeanSquare(int axis, boolean... ignoreInvalids) { 1702 StatisticsMetadata<Number> stats = getStats(); 1703 Dataset v = stats.getVariance(axis, true, ignoreInvalids); 1704 Dataset m = stats.getMean(axis, ignoreInvalids); 1705 Dataset result = Maths.multiply(m, m); 1706 return Maths.sqrt(result.iadd(v)); 1707 } 1708 1709 @Override 1710 public Dataset rootMeanSquare(int[] axes, boolean... ignoreInvalids) { 1711 StatisticsMetadata<Number> stats = getStats(); 1712 Dataset v = stats.getVariance(axes, true, ignoreInvalids); 1713 Dataset m = stats.getMean(axes, ignoreInvalids); 1714 Dataset result = Maths.multiply(m, m); 1715 return Maths.sqrt(result.iadd(v)); 1716 } 1717 1718 /** 1719 * Set item from compatible dataset in a direct and speedy way. Remember to setDirty afterwards. 1720 * 1721 * @param dindex 1722 * @param sindex 1723 * @param src 1724 * is the source data buffer 1725 */ 1726 protected abstract void setItemDirect(final int dindex, final int sindex, final Object src); 1727 1728 /** 1729 * @return error broadcasted to current shape 1730 */ 1731 private Dataset getBroadcastedInternalError() { 1732 if (shape == null) { 1733 throw new IllegalArgumentException("Cannot get error for null dataset"); 1734 } 1735 ILazyDataset led = super.getErrors(); 1736 if (led == null) 1737 return null; 1738 1739 Dataset ed = null; 1740 try { 1741 ed = DatasetUtils.sliceAndConvertLazyDataset(led); 1742 } catch (DatasetException e) { 1743 logger.error("Could not get data from lazy dataset", e); 1744 } 1745 if (led != ed) { 1746 setErrors(ed); // set back 1747 } 1748 1749 return ed.getBroadcastView(shape); 1750 } 1751 1752 @Override 1753 public Dataset getErrors() { 1754 Dataset ed = getBroadcastedInternalError(); 1755 if (ed == null) 1756 return null; 1757 1758 return ed; 1759 } 1760 1761 @Override 1762 public double getError() { 1763 Dataset ed = getBroadcastedInternalError(); 1764 if (ed == null) 1765 return 0; 1766 1767 return ed.getDouble(); 1768 } 1769 1770 @Override 1771 public double getError(final int i) { 1772 Dataset ed = getBroadcastedInternalError(); 1773 if (ed == null) 1774 return 0; 1775 1776 return ed.getDouble(i); 1777 } 1778 1779 @Override 1780 public double getError(final int i, final int j) { 1781 Dataset ed = getBroadcastedInternalError(); 1782 if (ed == null) 1783 return 0; 1784 1785 return ed.getDouble(i, j); 1786 } 1787 1788 @Override 1789 public double getError(int... pos) { 1790 Dataset ed = getBroadcastedInternalError(); 1791 if (ed == null) 1792 return 0; 1793 1794 return ed.getDouble(pos); 1795 } 1796 1797 @Override 1798 public double[] getErrorArray(final int i) { 1799 Dataset ed = getBroadcastedInternalError(); 1800 if (ed == null) 1801 return null; 1802 1803 return new double[] {getError(i)}; 1804 } 1805 1806 @Override 1807 public double[] getErrorArray(final int i, final int j) { 1808 Dataset ed = getBroadcastedInternalError(); 1809 if (ed == null) 1810 return null; 1811 1812 return new double[] {getError(i, j)}; 1813 } 1814 1815 @Override 1816 public double[] getErrorArray(int... pos) { 1817 Dataset ed = getBroadcastedInternalError(); 1818 if (ed == null) 1819 return null; 1820 1821 return new double[] {getError(pos)}; 1822 } 1823 1824 protected Dataset getInternalSquaredError() { 1825 Dataset sed = getErrorBuffer().getBroadcastView(shape); 1826 return sed; 1827 } 1828 1829 @Override 1830 public Dataset getErrorBuffer() { 1831 ErrorMetadata emd = getErrorMetadata(); 1832 if (emd == null) 1833 return null; 1834 1835 if (!(emd instanceof ErrorMetadataImpl)) { 1836 ILazyDataset led = emd.getError(); 1837 Dataset ed; 1838 try { 1839 ed = DatasetUtils.sliceAndConvertLazyDataset(led); 1840 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1841 setMetadata(emd); 1842 emd.setError(ed); 1843 } catch (MetadataException me) { 1844 logger.error("Could not create metadata", me); 1845 } catch (DatasetException e) { 1846 logger.error("Could not get data from lazy dataset", e); 1847 } 1848 } 1849 1850 return ((ErrorMetadataImpl) emd).getSquaredError(); 1851 } 1852 1853 /** 1854 * Set a copy of the buffer that backs the (squared) error data 1855 * @param buffer can be null, anything that can be used to create a DoubleDataset or CompoundDoubleDataset 1856 */ 1857 @Override 1858 public void setErrorBuffer(Serializable buffer) { 1859 if (shape == null) { 1860 throw new IllegalArgumentException("Cannot set error buffer for null dataset"); 1861 } 1862 if (buffer == null) { 1863 clearMetadata(ErrorMetadata.class); 1864 return; 1865 } 1866 1867 IDataset d = (IDataset) createFromSerializable(buffer, false); 1868 ErrorMetadata emd = getErrorMetadata(); 1869 if (!(emd instanceof ErrorMetadataImpl)) { 1870 try { 1871 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1872 setMetadata(emd); 1873 } catch (MetadataException me) { 1874 logger.error("Could not create metadata", me); 1875 } 1876 } 1877 ((ErrorMetadataImpl) emd).setSquaredError(d); 1878 } 1879}