/* * CdfFileDataSetDescriptor.java * * Created on August 12, 2005, 3:07 PM * * */ package org.autoplot.metatree; import org.das2.datum.DatumRange; import org.das2.datum.Units; import java.lang.reflect.Array; import java.text.ParseException; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; import org.das2.datum.UnitsUtil; import org.das2.util.LatexToGranny; import org.das2.qds.ArrayDataSet; import org.das2.qds.DataSetOps; import org.das2.qds.MutablePropertyDataSet; import org.das2.qds.QDataSet; import org.das2.qds.SemanticOps; import org.autoplot.datasource.MetadataModel; import org.autoplot.datasource.DataSourceUtil; import org.autoplot.datasource.LogNames; import org.das2.qds.DDataSet; import org.das2.qds.FDataSet; import org.das2.qds.ops.Ops; /** * Metadata model for ISTP conventions. For example, FIELDNAM is mapped to QDataSet.NAME, SCALEMIN is * mapped to TYPICAL_MIN, etc. When LaTeX fragments are found in axis titles * {@code (s.contains("^{") || s.contains("_{"))}, then this is converted * into Granny control strings. * * @author Jeremy * */ public class IstpMetadataModel extends MetadataModel { private static Logger logger= Logger.getLogger( LogNames.APDSS ); /** * Non-null, non-empty String if it is virtual. The string will be like "compute_magnitude(B_xyz_gse)" */ public static final String USER_PROP_VIRTUAL_FUNCTION= "FUNCTION"; public static final String USER_PROP_VIRTUAL_COMPONENT_= "COMPONENT_"; public static final Object VALUE_MIN= "MIN"; public static final Object VALUE_MAX= "MAX"; /** * returns the Entry that is convertible to double as a double. * @throws IllegalArgumentException for strings */ private static double doubleValue(Object o, Units units, Object minmax ) { return doubleValue(o, units, Double.NaN, minmax ); } /** * returns the Entry that is convertible to double as a double. * When there is an array, throw IllegalArgumentException. * Note this is used in CdfDataSource and other projects. * @param o the datum in double, int, String, array, etc. * @param units the units of the datum, result is returned in these units. * @param deflt the default value * @param minmax VALUE_MIN or VALUE_MAX or null. * @return the double or * @throws IllegalArgumentException for strings */ public static double doubleValue(Object o, Units units, double deflt, Object minmax ) { if (o == null) { return deflt; } if (o instanceof Number) { return ((Number) o).doubleValue(); } else if (o instanceof String) { String s = (String) o; if (s.startsWith("CDF_PARSE_EPOCH(")) { // hack for Onera CDFs try { // hack for Onera CDFs return units.parse(s.substring(16, s.length() - 1)).doubleValue(units); } catch (ParseException ex) { logger.log(Level.FINE, "unable to parse {0}", o); return deflt; } } else { try { return units.parse(DataSourceUtil.unquote((String) o)).doubleValue(units); } catch (ParseException ex) { try { return Double.parseDouble((String) o); } catch (NumberFormatException ex2) { logger.log(Level.FINE, "unable to parse {0}", o); return deflt; } } } } else { Class c = o.getClass(); if (c.isArray()) { if (units == Units.cdfEpoch && Array.getLength(o) == 2) { // kludge for Epoch16 double cdfEpoch = Array.getDouble(o, 0) * 1000 + Array.getDouble(o, 1) / 1e9; Units.cdfEpoch.createDatum(cdfEpoch); return cdfEpoch; } else { double v= Array.getDouble(o, 0); int n= Array.getLength(o); for ( int i=1; imin.doubleValue() ) min= vrange.min().doubleValue(units); if ( vrange.max().doubleValue(units)max.doubleValue() ) max= vrange.max().doubleValue(units); //vap+cdaweb:ds=IM_HK_FSW&id=BF_DramMbeCnt&timerange=2005-12-18 } if ( UnitsUtil.isNominalMeasurement(units) ) { logger.fine("valid range not used for ordinal units"); return null; } else { return new Number[] { min, max }; } } /** * Return the range from VALIDMIN to VALIDMAX. If the unit is an ordinal unit (see LABL_PTR_1), then return null. * Note QDataSet only allows times from 1000AD to 9000AD when Units are TimeLocationUnits. * Note this is used in CdfDataSource and other projects. * @param attrs the ISTP attributes * @param units the units for this variable, used to interpret doubles. * @return the range. */ public static DatumRange getValidRange(Map attrs, Units units) { double max = doubleValue(attrs.get("VALIDMAX"), units, Double.MAX_VALUE, VALUE_MAX ); double min = doubleValue(attrs.get("VALIDMIN"), units, -1e29, VALUE_MIN ); //TODO: remove limitation if ( units.isFill(min) ) min= min / 100; // kludge because DatumRanges cannot contain fill. if ( UnitsUtil.isTimeLocation(units) ) { DatumRange vrange= new DatumRange( 3.15569952E13, 2.840126112E14, Units.cdfEpoch ); // approx 1000AD to 9000AD if ( vrange.min().doubleValue(units)>min ) min= vrange.min().doubleValue(units); if ( vrange.max().doubleValue(units)max ) max= vrange.max().doubleValue(units); //vap+cdaweb:ds=IM_HK_FSW&id=BF_DramMbeCnt&timerange=2005-12-18 } if ( UnitsUtil.isNominalMeasurement(units) ) { logger.fine("valid range not used for ordinal units"); return null; } else { if ( min {1}", new Object[]{min, max}); return null; } } } /** * returns the range of the data by looking for the SCALEMIN/SCALEMAX params, * Checks for valid range when SCALETYP=log. * Note QDataSet only allows times from 1000AD to 9000AD when Units are TimeLocationUnits. */ private static DatumRange getRange(Map attrs, Units units) { DatumRange range; double min, max; if (attrs.containsKey("SCALEMIN") && attrs.containsKey("SCALEMAX")) { max = doubleValue(attrs.get("SCALEMAX"), units, VALUE_MAX ); min = doubleValue(attrs.get("SCALEMIN"), units, VALUE_MIN ); } else { if (attrs.containsKey("SCALEMAX")) { max = doubleValue(attrs.get("SCALEMAX"), units, VALUE_MAX ); min = 0; //TODO: really, this doesn't cause problems? } else if ( attrs.containsKey("SCALEMIN") && "log".equalsIgnoreCase( (String)attrs.get("SCALETYP") ) ){ min= doubleValue(attrs.get("SCALEMIN"), units, VALUE_MIN ); if ( min<=0 ) return null; double possibleMax= doubleValue(attrs.get("VALIDMAX"), units, VALUE_MAX ); if ( possibleMax/min>1e1 && possibleMax/min<1e3 ) { max= possibleMax; } else { max= min * 1e3; } } else { // bug 1063 Don't use CDF valid range for typical range logger.finer("SCALEMIN and SCALEMAX are missing"); return null; } } if ( UnitsUtil.isRatioMeasurement(units) && units.isFill(min) ) min= min / 100 ; // kludge because DatumRanges cannot contain -1e31 if ( maxmin ) min= vrange.min().doubleValue(units); if ( vrange.max().doubleValue(units)max ) max= vrange.max().doubleValue(units); //vap+cdaweb:ds=IM_HK_FSW&id=BF_DramMbeCnt&timerange=2005-12-18 } if ( UnitsUtil.isNominalMeasurement(units) ) { logger.fine("range not used for ordinal units"); return null; } else { if ( min {1}", new Object[]{min, max}); } else { logger.log(Level.FINE, "SCALEMIN and SCALEMAX are NaN and NaN" ); } return null; } } } /** * return null or the scale type if found. * @param attrs * @return */ private static String getScaleType(Map attrs) { String type = null; if (attrs.containsKey("SCALETYP") && attrs.get("SCALETYP") instanceof String ) { // CAA STAFF type = String.valueOf( attrs.get("SCALETYP") ).toLowerCase(); } return type; } /** * Interpret the ISTP metadata into QDataSet properties. * @param meta * @return */ @Override public Map properties(Map meta) { return properties( meta, true ); } /** * * @param meta ISTP metadata from CDF files. * @param doRecurse if true, then allow recursion for other properties. * @return */ private Map properties(Map meta, boolean doRecurse ) { Map attrs; if ( meta==null ) { logger.fine("null attributes, not expected to be seen"); attrs= Collections.emptyMap(); } else { attrs= new HashMap(meta); } String name= String.valueOf(attrs.get("FIELDNAM")); Map user= new LinkedHashMap<>(); Map properties = new LinkedHashMap<>(); String title= ""; String s; s= (String)attrs.get("Source_name"); if ( s!=null ) { int i= s.indexOf('>'); if ( i>-1 ) { title= title + s.substring(0,i).trim(); } } s= (String)attrs.get("Descriptor"); if ( s!=null ) { int i= s.indexOf('>'); if ( i>-1 ) { if ( title.length()>0 ) title= title+"/"; title= title + s.substring(0,i).trim(); } } if ( title.trim().length()>0 ) title= title+" "; // add two spaces to delimit S/C name and instrument from description s= (String)attrs.get("CATDESC"); if ( s!=null ) { if ( LatexToGranny.isLatex(s) ) { s= LatexToGranny.latexToGranny(s); } title= title + s.trim(); } if ( title.trim().length()>0 ) properties.put( QDataSet.TITLE, title.trim() ); s= (String)attrs.get("VAR_NOTES"); if ( s!=null ) { properties.put( QDataSet.DESCRIPTION, s ); } if (attrs.containsKey("DISPLAY_TYPE")) { String type = (String) attrs.get("DISPLAY_TYPE"); int i= type.indexOf('>'); String stype= i==-1 ? type : type.substring(0,i); if ( !stype.equals(stype.toLowerCase() ) ) { logger.log(Level.FINE, "DISPLAY_TYPE should be lower case ({0})", type); } if ( stype.equalsIgnoreCase("spectrogram") ) { type= "spectrogram"; } else if ( stype.equalsIgnoreCase("time_series" ) || type.equalsIgnoreCase("stack_plot") ) { type= "time_series"; // TODO: this will be "series" after reduction is put in. } properties.put(QDataSet.RENDER_TYPE, type); } if (attrs.containsKey("VIRTUAL") ) { String v= (String) attrs.get("VIRTUAL"); String function= (String)attrs.get("FUNCTION"); user.put( IstpMetadataModel.USER_PROP_VIRTUAL_FUNCTION, function ); for ( int i=0; i<4; i++ ) { if ( attrs.get("COMPONENT_"+i)!=null ) { user.put( IstpMetadataModel.USER_PROP_VIRTUAL_COMPONENT_ + i, attrs.get("COMPONENT_"+i) ); } else { break; } } } Units units=null; String sunits= ""; if (attrs.containsKey("UNITS")) { sunits = String.valueOf( attrs.get("UNITS") ); if ( LatexToGranny.isLatex(sunits) ) { sunits= LatexToGranny.latexToGranny(sunits); } } else { logger.log(Level.FINE, "UNITS are missing for {0}", name ); } if ( sunits.equals("") && attrs.containsKey("UNIT_PTR_VALUE") ) { QDataSet ss= (QDataSet)attrs.get("UNIT_PTR_VALUE"); if ( ss.rank()==1 ) { double s0= ss.value(0); boolean canUse= true; for ( int i=1; i"); double d= Double.parseDouble( siConversion.substring(0,i) ); if ( d==1e-9 ) { units= Units.cdfTT2000; sunits= units.toString(); } } } if ( units==null ) { try { units = Units.lookupUnits(DataSourceUtil.unquote(sunits)); } catch (IllegalArgumentException e) { units = Units.dimensionless; } } // we need to distinguish between ms and epoch times. boolean isMillis=false; Object ovalidMax= attrs.get("VALIDMAX"); Object ovalidMin= attrs.get("VALIDMIN"); if ( ovalidMax!=null && ovalidMin!=null && ovalidMax instanceof Number && ovalidMin instanceof Number && units==Units.milliseconds ) { double validMax= ((Number)ovalidMax).doubleValue(); double validMin= ((Number)ovalidMin).doubleValue(); isMillis= validMin=range.min().doubleValue(units) && fillVald<=range.max().doubleValue(units) ) { properties.put( QDataSet.FILL_VALUE, fillVal ); } } else if ( ofv!=null && ofv.getClass().isArray() ) { // try to reduce it to one number. Number fillVal= (Number) Array.get(ofv,0); int n= Array.getLength(ofv); for ( int i=1; i=range.min().doubleValue(units) && fillVald<=range.max().doubleValue(units) ) { properties.put( QDataSet.FILL_VALUE, fillVal ); } } } properties.put(QDataSet.SCALE_TYPE, getScaleType(attrs)); } catch (IllegalArgumentException ex) { logger.log( Level.SEVERE, ex.getMessage(), ex ); } if ( doRecurse ) { for (int i = 0; i < QDataSet.MAX_RANK; i++) { String key = "DEPEND_" + i; Object o= attrs.get(key); if ( o==null ) continue; if ( !( o instanceof Map ) ) { //new RuntimeException("String where Map was expected").printStackTrace(); //TODO: track this down: vap+cdf:https://cdaweb.gsfc.nasa.gov/istp_public/data/fast/ies/1998/fa_k0_ies_19980102_v02.cdf?ion_0 continue; } Map props = (Map) o; for ( int j=0; j