package org.das2.qds;
import java.text.ParseException;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Pattern;
import org.das2.datum.CacheTag;
import org.das2.datum.Datum;
import org.das2.datum.DatumRange;
import org.das2.datum.EnumerationUnits;
import org.das2.datum.InconvertibleUnitsException;
import org.das2.datum.Units;
import org.das2.datum.UnitsConverter;
import org.das2.datum.UnitsUtil;
import org.das2.qds.examples.Schemes;
import org.das2.util.LoggerManager;
//import static org.das2.qds.DataSetUtil.isConstant;
import org.das2.qds.ops.Ops;
/**
* Common expressions that apply semantics to QDataSet. Introduced
* to reduce a lot of repeated code, but also to make it clear where semantics
* are being applied.
* @author jbf
*/
public final class SemanticOps {
/**
* this is a utility class which cannot be instantiated.
*/
private SemanticOps() {
}
private static final Logger logger= LoggerManager.getLogger("qdataset.ops");
private static final String CLASSNAME= SemanticOps.class.getName();
/**
* returns the units found in the UNITS property of the dataset,
* or Units.dimensionless if it is not found.
* @param ds
* @return the units found in the dataset, or Units.dimensionless.
*/
public static Units getUnits(QDataSet ds) {
if ( ds==null ) {
throw new NullPointerException("ds is null"); // breakpoint here
}
Units u = (Units) ds.property(QDataSet.UNITS);
if ( u==null && ( ds.rank()>1 && ds.property(QDataSet.JOIN_0)!=null ) ) {
u= (Units) ds.slice(0).property(QDataSet.UNITS);
}
return u == null ? Units.dimensionless : u;
}
/**
* return the UnitsConverter that will convert data from src to the units of dst.
* @param src the dataset from which we get the original units.
* @param dst the dataset from which we get the destination units.
* @return the UnitsConverter
* @throws IllegalArgumentException
*/
public static UnitsConverter getUnitsConverter( QDataSet src, QDataSet dst ) {
Units usrc= getUnits(src);
Units udst= getUnits(dst);
return usrc.getConverter(udst);
}
/**
* returns the UnitsConverter, or IDENTITY if the converter cannot be found
* and one of the two units is dimensionless.
* @param src the dataset from which we get the original units.
* @param dst the dataset from which we get the destination units.
* @return the UnitsConverter
* @throws InconvertibleUnitsException when it just can't be done (EnumerationUnits and Ratiometric)
*/
public static UnitsConverter getLooseUnitsConverter( QDataSet src, QDataSet dst ) {
Units usrc= getUnits(src);
Units udst= getUnits(dst);
try {
return usrc.getConverter(udst);
} catch ( InconvertibleUnitsException ex ) {
if ( UnitsUtil.isRatioMeasurement(usrc) && UnitsUtil.isRatioMeasurement(udst) ) {
if ( Units.dimensionless==usrc || Units.dimensionless==udst ) {
return UnitsConverter.LOOSE_IDENTITY;
} else {
throw ex;
}
} else {
throw ex;
}
}
}
/**
* return the labels for a bundle dataset. For a rank 2 bundle, this
* will be found in BUNDLE_1, or legacy ones may have nominal data for DEPEND_1.
* For a rank 1 bundle this will be BUNDLE_0.
* @param ds rank 1 or rank 2 bundle.
* @return the column names.
*/
public static String[] getComponentNames(QDataSet ds) {
int n = ds.length(0);
QDataSet bdesc;
switch (ds.rank()) {
case 2:
bdesc= (QDataSet) ds.property(QDataSet.BUNDLE_1);
break;
case 1:
bdesc= (QDataSet) ds.property(QDataSet.BUNDLE_0);
break;
default:
bdesc= null;
break;
}
if ( bdesc!=null && bdesc.rank()==2 ) {
String[] result= new String[n];
for ( int i=0; i<n; i++ ) {
result[i]= (String) bdesc.property(QDataSet.NAME, i);
if ( result[i]==null ) result[i]="ch_"+i;
}
return result;
}
QDataSet labels;
if ( bdesc!=null && bdesc.rank()==1 ) {
labels= bdesc;
} else {
labels= (QDataSet) ds.property(QDataSet.DEPEND_1);
}
if (labels == null) {
String[] result = new String[n];
for (int i = 0; i < n; i++) {
result[i] = "ch_" + i;
}
return result;
} else {
Units u = getUnits(labels);
String[] slabels = new String[n];
//if ( labels.rank()>1 && isConstant(labels) ) {
// labels= labels.slice(0);
//}
for (int i = 0; i < n; i++) {
if ( labels.rank()>1 ) {
slabels[i] = "ch_" + i;
} else {
slabels[i] = String.valueOf(u.createDatum(labels.value(i)));
}
}
return slabels;
}
}
/**
* return the labels for a dataset using BUNDLE_1 and DEPEND_1. If BUNDLE_1 is defined and contains a LABEL,
* then it is used, otherwise a string value of the data is used, supporting legacy bundles, and if no DEPEND_1 is found
* then use the NAME properties from the bundle, or "ch_<em>i</em>" for each channel.
* @param ds the dataset, with a BUNDLE_1 or DEPEND_1 dimension which could be used.
* @return labels for each bundled dataset.
*/
public static String[] getComponentLabels(QDataSet ds) {
int n = ds.length(0);
QDataSet bdesc= (QDataSet) ds.property(QDataSet.BUNDLE_1);
String[] bundleLabels= new String[n];
String[] bundleNames= new String[n];
if ( bdesc!=null && bdesc.rank()==2 ) {
for ( int i=0; i<n; i++ ) {
bundleLabels[i]= (String) bdesc.property(QDataSet.LABEL, i);
bundleNames[i]= (String)bdesc.property(QDataSet.NAME, i);
}
}
QDataSet labels;
if ( bdesc!=null && bdesc.rank()==1 ) {
labels= bdesc;
} else {
labels= (QDataSet) ds.property(QDataSet.DEPEND_1);
}
String[] slabels = new String[n];
if (labels == null) {
for (int i = 0; i < n; i++) {
if ( bundleLabels[i]!=null ) {
slabels[i]= bundleLabels[i];
} else {
if ( bundleNames[i]!=null ) {
slabels[i]= bundleNames[i];
} else {
slabels[i]= "ch_"+i;
}
}
}
} else {
Units u = getUnits(labels);
for (int i = 0; i < n; i++) {
if ( bundleLabels[i]!=null ) {
slabels[i]= bundleLabels[i];
} else {
if ( labels.rank()>1 ) {
slabels[i] = "ch_"+i;
} else {
slabels[i] = String.valueOf(u.createDatum(labels.value(i)));
}
}
}
}
return slabels;
}
/**
* lookupUnits canonical units object, or allocate one.
* Examples include:<ul>
* <li>"nT" where it's already allocated,
* <li>"apples" where it allocates a new one, and
* <li>"seconds since 2011-12-21T00:00" where it uses lookupTimeUnits.
* </ul>
* @param sunits string identifier.
* @return canonical units object.
* @deprecated use Units.lookupUnits
*/
public static synchronized Units lookupUnits(String sunits) {
return Units.lookupUnits(sunits);
}
/**
* return canonical das2 unit for colloquial time.
* @param s
* @throws java.text.ParseException
* @deprecated use Units.lookupTimeLengthUnit
* @return
*/
public static Units lookupTimeLengthUnit(String s) throws ParseException {
return Units.lookupTimeLengthUnit(s);
}
/**
* lookupUnits canonical units object, or allocate one. If one is
* allocated, then parse for "<unit> since <datum>" and add conversion to
* "microseconds since 2000-001T00:00" (us2000). Note leap seconds are ignored
* in the returned units, so each day is 86400 seconds long.
* @param units string like "microseconds since 2000-001T00:00"
* @return a units object that implements.
* @throws java.text.ParseException
* @deprecated use Units.lookupTimeUnits
*/
public static synchronized Units lookupTimeUnits( String units ) throws ParseException {
return Units.lookupTimeUnits(units);
}
/**
* lookupUnits canonical units object, or allocate one. If one is
* allocated, then parse for "<unit> since <datum>" and add conversion to
* "microseconds since 2000-001T00:00." Note leap seconds are ignored!
* @param base the base time, for example 2000-001T00:00.
* @param offsetUnits the offset units for example microseconds. Positive values of the units will be since the base time.
* @return the unit.
* @deprecated use Units.lookupTimeUnits
*/
public static synchronized Units lookupTimeUnits( Datum base, Units offsetUnits ) {
return Units.lookupTimeUnits(base,offsetUnits);
}
public static boolean isRank1Bundle(QDataSet ds) {
if ( ds.rank()!=1 ) return false;
if ( ds.property(QDataSet.BUNDLE_0)!=null ) {
return true;
} else {
QDataSet dep= (QDataSet) ds.property(QDataSet.DEPEND_0);
if ( dep==null ) return false;
Units depu= getUnits(dep);
return depu instanceof EnumerationUnits;
}
}
/**
* Test for bundle scheme. Returns true if the BUNDLE_1 is set.
* @param ds
* @return true if the dataset is a bundle
*/
public static boolean isBundle(QDataSet ds) {
return ds.rank()==2 && ds.property(QDataSet.BUNDLE_1)!=null && !isRank2Waveform(ds);
}
/**
* Test for rank 2 waveform dataset, where DEPEND_1 is offset from DEPEND_0,
* and the data is the waveform. Other rules include:<ul>
* <li> DEPEND_1 must be at least 128 elements long.
* <li> DEPEND_1 must not be dimensionless.
* <li> if DEPEND_1 is in the same units as DEPEND_0, then DEPEND_0 can be ignored.
* @param fillDs
* @return
*/
public static boolean isRank2Waveform(QDataSet fillDs ) {
if ( fillDs.rank()==2 ) {
QDataSet dep0= (QDataSet) fillDs.property(QDataSet.DEPEND_0);
QDataSet dep1= (QDataSet) fillDs.property(QDataSet.DEPEND_1);
if ( dep0!=null && dep1!=null &&
( ( dep1.rank()==1 && dep1.length()>=QDataSet.MIN_WAVEFORM_LENGTH )
|| ( dep1.rank()==2 && dep1.length(0)>=QDataSet.MIN_WAVEFORM_LENGTH ) ) ) {
Units dep0units= SemanticOps.getUnits( dep0 );
Units dep1units= SemanticOps.getUnits( dep1 );
if ( dep0units!=Units.dimensionless && dep1units.isConvertibleTo( dep0units.getOffsetUnits() ) ) {
if ( dep0units!=dep1units ) {
return true;
} else {
return Units.seconds.isConvertibleTo(dep0units); // Only with time offsets is this allowed. (http://cdaweb.gsfc.nasa.gov/data/polar/uvi/uvi_k0/2001/po_k0_uvi_20010106_v01.cdf?IMAGE_DATA, slice0.)
}
} else if ( UnitsUtil.isTimeLocation(dep0units) && dep0units==dep1units ) {
return true;
}
}
}
return false;
}
/**
* Test for rank 3 dataset that is a join of rank 2 waveform datasets.
* @param ds
* @return
*/
public static boolean isRank3JoinOfRank2Waveform( QDataSet ds ) {
return ( ds.rank()==3 && isJoin(ds) && isRank2Waveform( ds.slice(0) ) );
}
/**
* See Ops.isLegacyBundle
* @param zds
* @return
*/
public static boolean isLegacyBundle( QDataSet zds ) {
if ( zds.rank()==2 ) {
QDataSet dep1= (QDataSet) zds.property(QDataSet.DEPEND_1);
if ( dep1!=null ) {
Units u= (Units) dep1.property(QDataSet.UNITS);
if ( u instanceof EnumerationUnits ) {
return true;
}
}
}
return false;
}
/**
* Test for bins scheme, where BINS_1 (or BINS_0) is set.
* This is where a two-element index is min, max.
* Note the BINS dimension must be the last index of the QDataSet.
* @param ds
* @return
*/
public static boolean isBins(QDataSet ds ) {
String binsProp= (String) ds.property( "BINS_"+(ds.rank()-1) );
boolean bins= binsProp!=null && ( QDataSet.VALUE_BINS_MIN_MAX.equals( binsProp ) || "min,maxInclusive".equals( binsProp ) );
return bins;
}
/**
* returns true if the dataset indicates that it is monotonically
* increasing. See DataSetUtil.isMonotonic.
* @param ds
* @return
*/
public static boolean isMonotonic( QDataSet ds ) {
return DataSetUtil.isMonotonic(ds);
}
/**
* returns true if the dataset is rank 2 or greater with the first dimension a join dimension.
* Note this does not return true for implicit joins, where JOIN_0 is not set.
* @param ds
* @return
*/
public static boolean isJoin( QDataSet ds ) {
return ds.rank()>1 && ds.property(QDataSet.JOIN_0)!=null;
}
/**
* returns the plane requested by name, or null if it does not exist.
* If the name is PLANE_i, then return PLANE_i, otherwise return
* the dataset with this name.
* Note QDataSet has the rule that if PLANE_i is null, then all PLANE_(i+1)
* must also be null.
* @param ds
* @param name
* @return
*/
public static QDataSet getPlanarView( QDataSet ds, String name ) {
if ( ds.property( QDataSet.PLANE_0 )==null ) return null; // typical case, get them out of here quickly
if ( name.equals("") ) throw new IllegalArgumentException("empty name");
if ( name.charAt(0)=='P' && Pattern.matches( "PLANE_(\\d|\\d\\d)", name ) ) {
return (QDataSet)ds.property(name);
}
int i=0;
while ( i<QDataSet.MAX_PLANE_COUNT ) {
QDataSet plane= (QDataSet) ds.property( "PLANE_"+i );
if ( plane==null ) {
return null;
} else {
String tname= (String) plane.property( QDataSet.NAME );
if ( tname==null ) {
System.err.println("unnamed plane in "+ds );
} else {
if ( name.equals(tname) ) {
return plane;
}
}
}
i++;
}
return null;
}
/**
* return the weights dataset, possibly creating one based on VALID_MIN
* VALID_MAX and FILL_VALUE properties. The weights dataset will have
* value zero where the data is invalid, and a non-zero weight where it is
* valid. DataSets may also contain a weights table with relative weights,
* but this is not uniformly supported.
* Note: this uses QDataSet.WEIGHTS_PLANE
* Note: calls org.das2.qds.DataSetUtil.weightsDataSet.
* @see org.das2.qds.ops.Ops#valid which is equivalent
* @see #cadenceCheck which detects for gaps in cadence.
* @param ds
* @return QDataSet with same geometry containing zeros and non-zeros.
*/
public static QDataSet weightsDataSet( QDataSet ds ) {
return DataSetUtil.weightsDataSet(ds);
}
/**
* return a reasonable tag width to use, or null if one cannot be
* guessed.
* @param ds the dataset containing data with a cadence.
* @param yds null or a dataset that may contain fill.
* @return
*/
public static Datum guessXTagWidth( QDataSet ds, QDataSet yds ) {
QDataSet cadence= DataSetUtil.guessCadenceNew( ds, yds );
return cadence==null ? null : DataSetUtil.asDatum( cadence );
}
/**
* return the dataset containing the x tags for the dataset. This
* is QDataSet.DEPEND_0, or if that's null then IndexGenDataSet(ds.length).
* For a bundle, this is just the 0th dataset.
* For a join, this is a join of the xtags datasets of each dataset.
* @param ds
* @return
*/
public static QDataSet xtagsDataSet( QDataSet ds ) {
QDataSet dep0= (QDataSet) ds.property(QDataSet.DEPEND_0);
if ( dep0!=null ) return dep0;
if ( isBundle(ds) && !isBins(ds) ){
return DataSetOps.unbundle(ds,0);
} else if ( isLegacyBundle(ds) ) {
return DataSetOps.unbundle(ds,0);
} else if ( isJoin(ds) && ds.rank()>2 ) { // support only RPWS rank 3 array of spectrograms.
QDataSet xds= xtagsDataSet( ds.slice(0) );
JoinDataSet result= new JoinDataSet(xds);
for ( int i=1; i<ds.length(); i++ ) {
result.join( xtagsDataSet(ds.slice(i) ) );
}
return result;
} else {
return new IndexGenDataSet(ds.length());
}
}
/**
* return the ytags for the simple table that is ds.<ul>
* <li>rank 2 spectrogram: Z[X,Y] → Y
* <li>bundle_1: ds[ :, [x,y,z] ] → y
* <li>[x,y,z] → y
* </ul>
* TODO: consider that these break duck typing goal, and really need a scheme
* to tell it how to get the dataset.
* @param ds the dataset
* @return the ytags
*/
public static QDataSet ytagsDataSet( QDataSet ds ) {
QDataSet dep1= (QDataSet) ds.property(QDataSet.DEPEND_1);
if ( dep1!=null ) {
if ( SemanticOps.getUnits(dep1) instanceof EnumerationUnits ) {
if ( dep1.length()==1 ) {
return DataSetOps.slice1(ds,0);// Juno returned a one-element
} else {
return DataSetOps.slice1(ds,1);
}
} else {
return dep1;
}
} else if ( isBundle(ds) ) {
if ( ds.length(0)==1 ) { // test003_008 would use rank2 dataSet[IsoDat...=1441,1] to indicate Y(T). http://www.rbsp-ect.lanl.gov/data_pub/autoplot/scripts/rbsp_ephem.jyds
return DataSetOps.unbundle(ds,0);
} else {
return DataSetOps.unbundle( ds, 1 );
}
} else if ( isLegacyBundle(ds) ) {
return DataSetOps.unbundle(ds,1);
} else if ( isJoin(ds)) {
QDataSet yds= ytagsDataSet( ds.slice(0) );
JoinDataSet result= new JoinDataSet(yds);
for ( int i=1; i<ds.length(); i++ ) {
result.join( ytagsDataSet(ds.slice(i) ) );
}
return result;
} else if ( ds.length()>0
&& ds.property(QDataSet.DEPEND_1)==null
&& ds.rank()>1 && ds.property(QDataSet.DEPEND_0,0)!=null ) { // For Juno pktid=91
if ( DataSetUtil.isQube(ds) ) {
return xtagsDataSet( ds.slice(0) );
} else {
QDataSet yds= xtagsDataSet( ds.slice(0) );
JoinDataSet result= new JoinDataSet(yds);
for ( int i=1; i<ds.length(); i++ ) {
result.join( xtagsDataSet(ds.slice(i) ) );
}
result.putProperty( QDataSet.UNITS, yds.slice(0).property( QDataSet.UNITS ) );
return result;
}
} else {
if ( ds.rank()==1 ) {
return ds;
} else if ( ds.rank()==2 && isBins(ds) ) {
return ds;
} else {
QDataSet result= (QDataSet) ds.property(QDataSet.DEPEND_1);
if ( result==null ) {
return new IndexGenDataSet(ds.length(0)); //TODO: this should support non-qubes with IndexGenDataSet.lastIndex
} else {
return result;
}
}
}
}
/**
* return the first table of the bundle containing x and y
* @param tds
* @param x
* @param y
* @return
*/
public static QDataSet getSimpleTableContaining( QDataSet tds, Datum x, Datum y ) {
if ( tds.rank()==2 ) {
return tds;
} else {
for ( int i=0; i<tds.length(); i++ ) {
QDataSet tds1= tds.slice(i);
QDataSet bounds= SemanticOps.bounds(tds1);
Units xunits= SemanticOps.getUnits( SemanticOps.xtagsDataSet(tds1) );
Units yunits= SemanticOps.getUnits( SemanticOps.ytagsDataSet(tds1) );
if ( yunits instanceof EnumerationUnits ) {
return DataSetOps.slice2( tds, 0 ); //TODO: code this nicely. This is some something comes up with Image datasets.
}
if ( bounds.value(0,0)<=x.doubleValue(xunits) && x.doubleValue(xunits)<bounds.value(0,1)
&& bounds.value(1,0)<=y.doubleValue(yunits) && y.doubleValue(yunits)<bounds.value(1,1) ) {
return tds1;
}
}
}
return null;
}
/**
* return the dataset that is dependent on others. For a bundle, we
* use DataSetOps.unbundleDefaultDataSet
* @param ds
* @return
*/
public static QDataSet getDependentDataSet( QDataSet ds ) {
QDataSet vds;
if ( !SemanticOps.isTableDataSet(ds) ) {
if ( ds.rank()==2 && SemanticOps.isBundle(ds) ) {
vds = DataSetOps.unbundleDefaultDataSet( ds );
} else {
vds = (QDataSet) ds;
}
return vds;
} else {
return ds;
}
}
/**
* return the bounds DS[ JOIN_0=x,y; BINS_1=min,maxInclusive ] of the datasets
* independent parameters. This is only implemented for:
* <table summary="">
* <tr><td>rank 2 Tables</td><td>extent(X),extent(Y) and Z is not represented</td></tr>
* <tr><td>rank 3 array of tables</td><td>extent(X),extent(Y) and Z is not represented</td></tr>
* <tr><td>rank 1 Y(X)</td><td>extent(X),extent(Y)</td></tr>
* <tr><td>not for rank 2 bundle dataset</td></tr>
* <tr><td>not for rank 1 bundle dataset</td></tr>
* </table>
* The zeroth dimension will be the physical dimension of the DEPEND_0 values. Or said another way:
* <table summary="">
* <tr><td>bounds[0,0]= X min</td><td>bounds[0,1] = X max</td><td>bounds.slice(0) is the extent of X<td></tr>
* <tr><td>bounds[1,0]= Y min</td><td>bounds[1,1] = Y max</td><td>bounds.slice(1) is the extent of Y<td></tr>
* </table>
*
* @param ds rank 2 dataset with BINS_1="min,maxInclusive"
* @throws IllegalArgumentException when the dataset scheme is not supported
* @return
*/
public static QDataSet bounds( QDataSet ds ) {
logger.entering( CLASSNAME, "bounds" );
QDataSet result= (QDataSet) DataSetAnnotations.getInstance().getAnnotation( ds, DataSetAnnotations.ANNOTATION_BOUNDS );
if ( result!=null ) {
logger.exiting( CLASSNAME, "bounds" );
return result;
}
QDataSet xrange;
QDataSet yrange;
if ( ds.rank()==2 ) {
if ( ds.property(QDataSet.BUNDLE_1)!=null && ds.property(QDataSet.DEPEND_1)==null && ds.property(QDataSet.BINS_1)==null ) {
xrange= Ops.extent( SemanticOps.xtagsDataSet(ds), null );
yrange= Ops.extent( SemanticOps.ytagsDataSet(ds), null );
} else {
xrange= Ops.extent( SemanticOps.xtagsDataSet(ds), null );
yrange= Ops.extent( SemanticOps.ytagsDataSet(ds), null );
}
} else if ( ds.rank()==3 ) {
QDataSet ds1= ds.slice(0);
xrange= Ops.extent( SemanticOps.xtagsDataSet(ds1), null );
yrange= Ops.extent( SemanticOps.ytagsDataSet(ds1), null );
for ( int i=1; i<ds.length(); i++ ) {
ds1= ds.slice(i);
xrange= Ops.extent( SemanticOps.xtagsDataSet(ds1), xrange );
yrange= Ops.extent( SemanticOps.ytagsDataSet(ds1), yrange );
}
} else if ( ds.rank()==1 ) {
if ( ds.property(QDataSet.BUNDLE_0)!=null ) {
throw new IllegalArgumentException("scheme not supported: "+ds );
} else {
xrange= Ops.extent( SemanticOps.xtagsDataSet(ds), null );
yrange= Ops.extent( SemanticOps.ytagsDataSet(ds), null );
}
} else {
throw new IllegalArgumentException("scheme not supported: "+ds );
}
JoinDataSet jds= (JoinDataSet) Ops.join( xrange, yrange );
jds.putProperty( QDataSet.BINS_1,"min,maxInclusive");
DataSetAnnotations.getInstance().putAnnotation( ds, DataSetAnnotations.ANNOTATION_BOUNDS, jds );
logger.exiting( CLASSNAME, "bounds" );
return jds;
// DDataSet result= DDataSet.createRank2(2,2);
// result.putValue(0,0,xrange.value(0));
// result.putValue(0,1,xrange.value(1));
// result.putValue(1,0,yrange.value(0));
// result.putValue(1,1,yrange.value(1));
// result.putProperty(QDataSet.BINS_1,"min,maxInclusive");
//
//return result;
}
/**
* returns true if the dataset is the scheme of a legacy TableDataSet
* @param ds
* @return
*/
public static boolean isTableDataSet(QDataSet ds) {
if ( ds.rank()==3 || isSimpleTableDataSet(ds) ) return true;
QDataSet dep1= (QDataSet) ds.property( QDataSet.DEPEND_1 );
if ( ds.rank()==2 ) {
if ( dep1!=null ) { // TODO: we should check for EnumerationUnits here...
return true; // Y tags can be rank2, making it a table dataset but not a simple one.
} else {
QDataSet bds= (QDataSet) ds.property(QDataSet.BUNDLE_1);
return bds == null;
}
}
return false;
}
/**
* returns true if the dataset is the scheme of a legacy TableDataSet
* with only one table. Note "Tables" have just one X unit and one Y unit,
* no bundles.
* Consider: rule about "duck typing": rules like !Ops.isBundle break this
* because it requires to be a simpleTable, you cannot be a bundle. LANL
* rich ASCII allows datasets to be both bundles and simple tables.
* @param ds
* @return
*/
public static boolean isSimpleTableDataSet(QDataSet ds) {
QDataSet dep1= (QDataSet) ds.property( QDataSet.DEPEND_1 );
if ( dep1!=null && dep1.rank()!=1 ) return false;
return ds.rank()==2 && ( ( dep1!=null && dep1.rank()==1 ) || !Ops.isBundle(ds) ) && !Ops.isLegacyBundle(ds);
}
/**
* returns true if the dataset is a bundle of rank 1 datasets. If no
* dependence is declared, it is assumed that the first one or two datasets
* are the independent datasets, and the last is the dependent.
*<blockquote><pre>
* X,Y --> Y(X)
* X,Y,Z --> Z(X,Y)
*</pre></blockquote>
*
* @param ds
* @return
*/
public static boolean isSimpleBundleDataSet( QDataSet ds ) {
return ds.rank()==2 && ( ds.property(QDataSet.BUNDLE_1)!=null );
}
/**
* returns true if the dataset is a time series. This is either something that has DEPEND_0 as a dataset with time
* location units, or a join of other datasets that are time series.
* @param ds
* @return
*/
public static boolean isTimeSeries( QDataSet ds ) {
if ( isJoin(ds) ) {
return isTimeSeries( ds.slice(0) );
} else {
QDataSet dep0= (QDataSet) ds.property(QDataSet.DEPEND_0);
return dep0!=null && UnitsUtil.isTimeLocation( SemanticOps.getUnits(dep0) );
}
}
/**
* returns the Double value of the number, preserving null and NaN.
* @param value
* @return
*/
public static Double doubleValue( Number value ) {
if ( value==null ) return null;
return value.doubleValue();
}
/**
* returns the value as a datum. Note this should be used with reservation,
* this is not very efficient when the operation is done many times.
* @param ds dataset providing the UNITS and VALID_MIN, VALID_MAX and FILL_VALUE.
* @param d the double.
* @return a datum representing the value.
*/
public static Datum getDatum( QDataSet ds, double d ) {
Units u = SemanticOps.getUnits(ds);
Double vmin= doubleValue( (Number) ds.property( QDataSet.VALID_MIN ) );
Double vmax= doubleValue( (Number) ds.property( QDataSet.VALID_MAX ) );
Double fill= doubleValue( (Number)ds.property( QDataSet.FILL_VALUE ) );
if ( vmin!=null ) if ( vmin>d ) return u.getFillDatum();
if ( vmax!=null ) if ( vmax<d ) return u.getFillDatum();
if ( fill!=null ) if ( fill==d ) return u.getFillDatum();
return u.createDatum(d);
}
/**
* return the parts of the dataset within the bounds. This assumes how
* the data is visualized, so for example see SemanticOps.xtagsDataSet
* for which dimensions correspond to x and y.
* @param ds the rank 1 or more dataset, including joins.
* @param xrange the range or null if no trimming should be done
* @param yrange the range or null if no trimming should be done
* @return the trimmed dataset.
*/
public static QDataSet trim( QDataSet ds, DatumRange xrange, DatumRange yrange ) {
int rank=ds.rank();
if ( ds.rank()==0 ) return ds;
if ( xrange==null && yrange==null ) return ds;
if ( rank==3 ) {
if ( isJoin(ds) ) {
JoinDataSet jds= new JoinDataSet(ds.rank());
for ( int i=0; i<ds.length(); i++ ) {
QDataSet t1= trim( ds.slice(i), xrange, yrange );
if ( t1.length()>0 ) {
jds.join( t1 );
}
}
DataSetUtil.putProperties( DataSetUtil.getProperties(ds), jds );
return jds;
} else if ( Schemes.isCompositeImage(ds) ) {
int[] size= Ops.size(ds);
TailBundleDataSet bds= new TailBundleDataSet(3);
for ( int i=0; i<size[2]; i++ ) {
QDataSet ch= Ops.slice2(ds,i);
ch= trim( ch, xrange, yrange );
bds.bundle(ch);
}
return bds;
} else {
throw new IllegalArgumentException("not supported: "+ds);
}
} else if ( rank==2 ) {
if ( isRank2Waveform(ds) ) {
QDataSet xds= SemanticOps.xtagsDataSet(ds);
QDataSet yds= SemanticOps.xtagsDataSet(ds.slice(0));
QDataSet ydsMax= DataSetUtil.asDataSet( yds.value( yds.length()-1 ), SemanticOps.getUnits(yds) );
QDataSet xinside= xrange==null ? null :
Ops.and( Ops.ge( Ops.add( xds, ydsMax ), DataSetUtil.asDataSet(xrange.min()) ), Ops.le( xds, DataSetUtil.asDataSet(xrange.max()) ) );
SubsetDataSet sds= new SubsetDataSet(ds);
if ( xinside!=null ) sds.applyIndex( 0, Ops.where(xinside) ); //TODO: consider the use of trim which would be more efficient.
return sds;
} else if ( isSimpleTableDataSet(ds) ) {
QDataSet xds= SemanticOps.xtagsDataSet(ds);
QDataSet yds= SemanticOps.xtagsDataSet(ds.slice(0));
QDataSet xinside= xrange==null ? null :
Ops.and( Ops.ge( xds, DataSetUtil.asDataSet(xrange.min()) ), Ops.le( xds, DataSetUtil.asDataSet(xrange.max()) ) );
QDataSet yinside= yrange==null ? null :
Ops.and( Ops.ge( yds, DataSetUtil.asDataSet(yrange.min()) ), Ops.le( yds, DataSetUtil.asDataSet(yrange.max()) ) );
SubsetDataSet sds= new SubsetDataSet(ds);
if ( xinside!=null ) sds.applyIndex( 0, Ops.where(xinside) ); //TODO: consider the use of trim which would be more efficient.
if ( yinside!=null ) sds.applyIndex( 1, Ops.where(yinside) );
return sds;
} else if ( isBundle(ds) ) {
QDataSet xds= SemanticOps.xtagsDataSet(ds);
QDataSet yds= SemanticOps.ytagsDataSet(ds);
QDataSet xinside= xrange==null ? null :
Ops.and( Ops.ge( xds, DataSetUtil.asDataSet(xrange.min()) ), Ops.le( xds, DataSetUtil.asDataSet(xrange.max()) ) );
QDataSet yinside= yrange==null ? null :
Ops.and( Ops.ge( yds, DataSetUtil.asDataSet(yrange.min()) ), Ops.le( yds, DataSetUtil.asDataSet(yrange.max()) ) );
QDataSet ok;
SubsetDataSet sds= new SubsetDataSet(ds);
if ( xrange==null && yrange==null ) {
return ds;
} else if ( xrange==null ) {
//ok= Ops.where( yinside );
//sds.applyIndex( 1, ok );
return ds; // this is because we can't easily search the ytags.
} else if ( yrange==null ) {
ok= Ops.where( xinside );
sds.applyIndex( 0, ok );
} else {
logger.fine( "yds is being ignored, not sure why...");
ok= Ops.where( Ops.and( xinside, yinside ) );
sds.applyIndex( 0, ok );
}
return sds;
} else { // copy over elements where
QDataSet xds= SemanticOps.xtagsDataSet(ds);
QDataSet yds= SemanticOps.getDependentDataSet(ds);
QDataSet xinside= xrange==null ? null :
Ops.and( Ops.ge( xds, DataSetUtil.asDataSet(xrange.min()) ), Ops.le( xds, DataSetUtil.asDataSet(xrange.max()) ) );
//QDataSet yinside= null; //yrange==null ? null :
//Ops.and( Ops.ge( yds, DataSetUtil.asDataSet(yrange.min()) ), Ops.le( yds, DataSetUtil.asDataSet(yrange.max()) ) );
QDataSet ok;
SubsetDataSet sds= new SubsetDataSet(ds);
if ( xrange==null && yrange==null ) {
return ds;
} else if ( xrange==null ) {
//ok= Ops.where( yinside );
//sds.applyIndex( 1, ok );
return ds; // this is because we can't easily search the ytags.
} else if ( yrange==null ) {
ok= Ops.where( xinside );
sds.applyIndex( 0, ok );
} else {
logger.fine( "yds is being ignored, not sure why...");
//ok= Ops.where( Ops.and( xinside, yinside ) );
ok= Ops.where( xinside );
sds.applyIndex( 0, ok );
}
return sds;
}
} else if ( rank==1 ) {
QDataSet xds= SemanticOps.xtagsDataSet(ds);
QDataSet yds= SemanticOps.getDependentDataSet(ds);
QDataSet xinside= null;
if ( DataSetUtil.isMonotonic(xds) && xds.property(QDataSet.FILL_VALUE)==null ) { // TODO: validmin validmax...
if ( xrange!=null ) {
int i= DataSetUtil.xTagBinarySearch( xds, xrange.min(), 0, xds.length()-1 );
if ( i<0 ) {
i= -1 * ( i + 1 );
}
int j= DataSetUtil.xTagBinarySearch( xds, xrange.max(), i, xds.length()-1 );
if ( j<0 ) {
j= -1 * ( j + 1 );
}
if ( yrange==null ) {
return ds.trim(i,j); // optimization for waveforms...
} else {
int[] back= new int[ xds.length() ];
if ( j==xds.length() ) j= xds.length()-1; // bugfix: if xrange.max is gt last point.
for ( int ii=i; ii<=j; ii++ ) {
back[ii]= 1;
}
xinside= IDataSet.wrap(back);
}
}
} else {
if ( xds.rank()==2 && SemanticOps.isBins(xds) ) {
QDataSet xmin= Ops.slice1( xds, 0 );
QDataSet xmax= Ops.slice1( xds, 1 );
xinside= xrange==null ? null :
Ops.and( Ops.ge( xmax, DataSetUtil.asDataSet(xrange.min()) ), Ops.le( xmin, DataSetUtil.asDataSet(xrange.max()) ) );
} else {
xinside= xrange==null ? null :
Ops.and( Ops.ge( xds, DataSetUtil.asDataSet(xrange.min()) ), Ops.le( xds, DataSetUtil.asDataSet(xrange.max()) ) );
}
}
QDataSet yinside= yrange==null ? null :
Ops.and( Ops.ge( yds, DataSetUtil.asDataSet(yrange.min()) ), Ops.le( yds, DataSetUtil.asDataSet(yrange.max()) ) );
QDataSet ok;
if ( xrange==null ) {
ok= Ops.where( yinside );
} else if ( yrange==null ) {
ok= Ops.where( xinside );
} else {
ok= Ops.where( Ops.and( xinside, yinside ) );
}
SubsetDataSet sds= new SubsetDataSet(ds);
sds.applyIndex( 0, ok );
return sds;
} else {
throw new IllegalArgumentException("not supported: "+ds);
}
}
/**
* return a dataset with 1's where the cadence following this measurement is acceptable, and 0's where
* there should be a break in the data. For example, here's some pseudocode:
*<blockquote><pre>
* findex= Ops.interpolate( xds, x )
* cadenceCheck= cadenceCheck(xds)
* r= where( cadenceCheck[floor(findex)] eq 0 )
* x[r]= fill
*</pre></blockquote>
* Presently this just uses guessXTagWidth to get the cadence, but this may allow a future version to support
* mode changes.
*
* The result is a dataset with the same length, and the last element is always 1.
*
* @see Ops#valid which checks for fill and valid_min, valid_max.
* @param tds rank 1 dataset of length N.
* @param ds dataset dependent on tds and used to detect valid measurements, or null.
* @return dataset with length N
*/
public static QDataSet cadenceCheck( QDataSet tds, QDataSet ds ) {
Datum cadence= guessXTagWidth( tds, ds );
cadence= cadence.multiply(1.1);
QDataSet diffs= Ops.diff(tds);
QDataSet result= (MutablePropertyDataSet) Ops.lt( diffs, DataSetUtil.asDataSet(cadence) ); // cheat cast
if ( !( result instanceof ArrayDataSet ) ) {
result= ArrayDataSet.copy(result);
}
ArrayDataSet aresult= ((ArrayDataSet)result);
ArrayDataSet one= ArrayDataSet.createRank1( aresult.getComponentType(), 1 );
one.putValue(0,1.0);
DataSetUtil.copyDimensionProperties( aresult,one );
result= ArrayDataSet.append( aresult, one );
result= Ops.link( tds, result );
return result;
}
private static final Map<String,Class> propertyTypes= new HashMap();
static {
propertyTypes.put( QDataSet.UNITS, Units.class );
propertyTypes.put( QDataSet.TYPICAL_MIN, Number.class );
propertyTypes.put( QDataSet.TYPICAL_MAX, Number.class );
propertyTypes.put( QDataSet.VALID_MIN, Number.class );
propertyTypes.put( QDataSet.VALID_MAX, Number.class );
propertyTypes.put( QDataSet.FILL_VALUE, Number.class );
propertyTypes.put( QDataSet.ELEMENT_DIMENSIONS, int[].class ); // this will probably cause grief, Integer[] vs int[]
propertyTypes.put( QDataSet.CACHE_TAG, org.das2.datum.CacheTag.class );
propertyTypes.put( QDataSet.CADENCE, QDataSet.class );
propertyTypes.put( QDataSet.DEPEND_0, QDataSet.class );
propertyTypes.put( QDataSet.DEPEND_1, QDataSet.class );
propertyTypes.put( QDataSet.DEPEND_2, QDataSet.class );
propertyTypes.put( QDataSet.DEPEND_3, QDataSet.class );
propertyTypes.put( QDataSet.BUNDLE_0, QDataSet.class );
propertyTypes.put( QDataSet.BUNDLE_1, QDataSet.class );
propertyTypes.put( QDataSet.DELTA_PLUS, QDataSet.class );
propertyTypes.put( QDataSet.DELTA_MINUS, QDataSet.class );
propertyTypes.put( QDataSet.BIN_PLUS, QDataSet.class );
propertyTypes.put( QDataSet.BIN_MINUS, QDataSet.class );
propertyTypes.put( QDataSet.QUBE, Boolean.class );
}
/**
* verify property types. For example, that UNITS is a org.das2.datum.Units, etc.
* Returns true for unrecognized property names (future expansion) and null. If
* throwException is true, then an IllegalArgumentException is thrown.
* @param prop the property name, e.g. QDataSet.CADENCE
* @param value the candidate value for the property.
* @param throwException if true, throw descriptive exception instead of returning false.
* @return true if the property type is okay.
*/
public static boolean checkPropertyType( String prop, Object value, boolean throwException ) {
Class typ= propertyTypes.get(prop);
if ( typ==null || value==null || typ.isAssignableFrom( value.getClass() ) ) {
return true;
} else {
if ( throwException ) {
String styp= typ.toString();
if ( typ==Number.class ) {
styp="Number";
} else if ( typ==QDataSet.class ) {
styp="QDataSet";
}
if ( value instanceof String ) {
throw new IllegalArgumentException("bad value for property "+prop+": \""+value+"\", expected "+styp );
} else {
throw new IllegalArgumentException("bad value for property "+prop+": "+value+", expected "+styp );
}
} else {
return false;
}
}
}
/**
* return the property type expected for the property.
* @param prop
* @return QDataSet, String, Number, Units, or CacheTag
*/
public static String getPropertyType( String prop ) {
Class c= propertyTypes.get(prop);
if ( c==null ) {
throw new IllegalArgumentException("Property not recognized: "+prop);
} else {
if ( c.isAssignableFrom( CacheTag.class ) ) {
return "CacheTag";
} else if ( c.isAssignableFrom( Number.class ) ) {
return "Number";
} else if ( c.isAssignableFrom( Boolean.class ) ) {
return "Boolean";
} else if ( c.isAssignableFrom( QDataSet.class ) ) {
return "QDataSet";
} else if ( c.isAssignableFrom( Units.class ) ) {
return "Units";
} else {
throw new IllegalArgumentException("Property not supported: "+prop);
}
}
}
}