package org.autoplot.netCDF; import java.text.ParseException; import java.util.logging.Logger; import org.das2.datum.Units; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.logging.Level; import org.das2.datum.LoggerManager; import org.das2.datum.TimeParser; import org.das2.datum.UnitsConverter; import org.das2.util.monitor.NullProgressMonitor; import org.das2.util.monitor.ProgressMonitor; import org.das2.qds.AbstractDataSet; import org.das2.qds.DataSetUtil; import org.das2.qds.QDataSet; import org.autoplot.datasource.MetadataModel; import org.das2.qds.ops.Ops; import org.autoplot.metatree.IstpMetadataModel; import org.autoplot.metatree.MetadataUtil; import org.das2.datum.EnumerationUnits; import org.das2.datum.TimeUtil; import org.das2.datum.UnitsUtil; import ucar.ma2.DataType; import ucar.ma2.InvalidRangeException; import ucar.ma2.Range; import ucar.nc2.Variable; import ucar.nc2.Attribute; import ucar.nc2.AttributeContainer; import ucar.nc2.dataset.NetcdfDataset; /** * wraps a netCDF variable (or HDF5 variable) to present it as a QDataSet. * * @author jbf */ public class NetCdfVarDataSet extends AbstractDataSet { Variable v; double[] data; int[] shape; private static final Logger logger= LoggerManager.getLogger("apdss.netcdf"); public static NetCdfVarDataSet create( Variable variable, String constraint, NetcdfDataset ncfile, ProgressMonitor mon ) throws IOException { NetCdfVarDataSet result = new NetCdfVarDataSet( ); result.read(variable, ncfile, constraint, null, false, mon ); return result; } private NetCdfVarDataSet( ) { putProperty(QDataSet.QUBE, Boolean.TRUE); } private static String sliceConstraints( String constraints, int i ) { if ( constraints==null ) { return null; } else { if ( constraints.startsWith("[") && constraints.endsWith("]") ) { constraints= constraints.substring(1,constraints.length()-1); } String[] cc= constraints.split(","); if ( i>=cc.length ) { return null; } else if ( cc[i].equals(":") ) { return null; } else { return cc[i]; // TODO: this doesn't address depend variable that is rank 2, but this is not supported in NetCDF anyway. } } } /** * returns [ start, stop, stride ] or [ start, -1, -1 ] for slice. This is * provided to reduce code and for uniform behavior. * * See CdfJavaDataSource, which is where this was copied from. * @param constraint, such as "[0:100:2]" for even records between 0 and 100, non-inclusive. * @param recCount the number of records for when negative indeces are used. * @return [ start, stop, stride ] or [ start, -1, -1 ] for slice. * @throws java.text.ParseException */ public static long[] parseConstraint(String constraint, long recCount) throws ParseException { long[] result = new long[]{0, recCount, 1}; if (constraint == null) { return result; } else { if ( constraint.startsWith("[") && constraint.endsWith("]") ) { constraint= constraint.substring(1,constraint.length()-1); } try { String[] ss= constraint.split(":",-2); if ( ss.length>0 && ss[0].length()>0 ) { result[0]= Integer.parseInt(ss[0]); if ( result[0]<0 ) result[0]= recCount+result[0]; } if ( ss.length>1 && ss[1].length()>0 ) { result[1]= Integer.parseInt(ss[1]); if ( result[1]<0 ) result[1]= recCount+result[1]; } if ( ss.length>2 && ss[2].length()>0 ) { result[2]= Integer.parseInt(ss[2]); } if ( ss.length==1 ) { // slice result[1]= -1; result[2]= -1; } } catch ( NumberFormatException ex ) { throw new ParseException("expected integer: "+ex.toString(),0); } return result; } } private int sliceCount( boolean [] slice, int idim ) { int result= 0; for ( int i=0; i-1 ) { digitCount=-1; break; } // we found a non-digit preceeding a digit, so this isn't a block of digits like expected. } } switch (digitCount) { case 16: tp= TimeParser.create("$Y$j$H$M$S$(subsec,places=3)"); break; case 17: tp= TimeParser.create("$Y$m$d$H$M$S$(subsec,places=3)"); break; case -1: s= s.trim(); try { String t= TimeParser.iso8601String(s); tp= TimeParser.create(t); break; } catch ( IllegalArgumentException ex ) { return null; } default: } return tp; } /** * Read the NetCDF data, including DEPEND_0 if used, using the CDF conventions. * @param variable the NetCDF variable. * @param ncfile the NetCDF file. * @param constraints null, or string like "[0:10]" Note it's allowed for the constraint to not have [] because this is called recursively. * @param mm if non-null, a metadata model, like IstpMetadataModel, is asserted. If null, then any variable containing DEPEND_0 implies IstpMetadataModel. * @param mon * @throws IOException */ private void read( Variable variable, NetcdfDataset ncfile, String constraints, MetadataModel mm, boolean isDepend, ProgressMonitor mon) throws IOException { this.v= variable; if ( !mon.isStarted() ) mon.started(); //das2 bug: monitor blinks if we call started again here mon.setProgressMessage( "reading "+v.getNameAndDimensions() ); if ( mm==null ) { long t0= System.currentTimeMillis(); List vvs=ncfile.getVariables(); for ( Variable vv: vvs ) { if ( vv.findAttribute("DEPEND_0" )!=null ) { mm= new IstpMetadataModel(); break; } } logger.log(Level.FINER, "look for DEPEND_0 (ms):{0}", (System.currentTimeMillis()-t0)); } logger.finer("v.getShape()"); shape= v.getShape(); boolean[] slice= new boolean[shape.length]; ucar.ma2.Array a; if ( constraints!=null ) { if ( constraints.startsWith("[") && constraints.endsWith("]") ) { constraints= constraints.substring(1,constraints.length()-1); } try { String[] cc= constraints.split(","); List ranges= new ArrayList( v.getRanges() ); for ( int i=0; i1 ) properties.put( QDataSet.QUBE, Boolean.TRUE ); if ( v.getParentStructure()!=null ) { //TODO: this is probably wrong for structure of rank 2 data. shape= new int[] { data.length }; slice= new boolean[shape.length]; } boolean isCoordinateVariable= false; for ( int ir=0; ir attributes= new HashMap(); mon.setProgressMessage("reading attributes"); logger.finer("v.getAttributes()"); AttributeContainer attrs= v.attributes(); for ( Iterator i= attrs.iterator(); i.hasNext(); ) { Attribute attr= (Attribute) i.next(); if ( !attr.isArray() ) { if ( attr.isString() ) { attributes.put( attr.getName(), attr.getStringValue() ); } else { attributes.put( attr.getName(), String.valueOf( attr.getNumericValue() ) ); } } } Object lablPtr= attributes.get("LABL_PTR_1"); if ( lablPtr!=null && lablPtr instanceof String ) { Variable vv= ncfile.findVariable((String)lablPtr); if ( vv==null ) { logger.log(Level.WARNING, "unable to find variable: {0}", lablPtr); } else { if ( vv.getDataType()==DataType.CHAR && vv.getDimensions().size()==2 && shape[1]==vv.getDimension(0).getLength() ) { String[] ss= new String[vv.getDimension(0).getLength()]; char[][] arr= (char[][])vv.read().copyToNDJavaArray(); for ( int i=0; i istpProps= mm.properties(attributes); if ( properties.get( QDataSet.UNITS )==Units.us2000 ) { UnitsConverter uc= UnitsConverter.getConverter(Units.cdfEpoch, Units.us2000 ); if ( istpProps.containsKey(QDataSet.VALID_MIN) ) istpProps.put( QDataSet.VALID_MIN, uc.convert( (Number)istpProps.get(QDataSet.VALID_MIN ) ) ); if ( istpProps.containsKey(QDataSet.VALID_MAX) ) istpProps.put( QDataSet.VALID_MAX, uc.convert( (Number)istpProps.get(QDataSet.VALID_MAX ) ) ); if ( istpProps.containsKey(QDataSet.TYPICAL_MIN) ) istpProps.put( QDataSet.TYPICAL_MIN, uc.convert( (Number)istpProps.get(QDataSet.TYPICAL_MIN ) ) ); if ( istpProps.containsKey(QDataSet.TYPICAL_MAX) ) istpProps.put( QDataSet.TYPICAL_MAX, uc.convert( (Number)istpProps.get(QDataSet.TYPICAL_MAX ) ) ); istpProps.put(QDataSet.UNITS,Units.us2000); } if ( istpProps.containsKey(QDataSet.RENDER_TYPE) ) { String s= (String)istpProps.get(QDataSet.RENDER_TYPE); if ( s.equals("image") ) { logger.fine("removing DISPLAY_TYPE=image because it's incorrect"); istpProps.remove(QDataSet.RENDER_TYPE); } } if ( properties.containsKey(QDataSet.UNITS) ) { istpProps.remove("UNITS"); }; properties.putAll(istpProps); for ( int ir=0; ir newShape= new ArrayList(shape.length); for ( int i=0; i=data.length) { throw new IllegalArgumentException("index out of bounds"); } return data[ index ]; } @Override public double value( int i, int j, int k ) { //int index= i + shape[0] * j + shape[0] * shape[1] * k; int index= k + shape[2] * j + shape[2] * shape[1] * i; if ( index>=data.length) { throw new IllegalArgumentException("index out of bounds"); } return data[index]; } @Override public double value( int i, int j, int k, int l ) { int index= l + shape[3] * k + shape[3] * shape[2] * j + shape[3] * shape[2] * shape[1] * i; if ( index>=data.length) { throw new IllegalArgumentException("index out of bounds"); } return data[index]; } @Override public int length() { return shape[0]; } @Override public int length( int dim ) { return shape[1]; } @Override public int length( int dim0, int dim1 ) { return shape[2]; } @Override public int length( int dim0, int dim1, int dim2 ) { return shape[3]; } @Override public QDataSet trim(int start, int end) { return super.trim(start, end); // TODO: introduce offset so we don't need to copy. } @Override public QDataSet slice(int i) { return super.slice(i); } }