package org.autoplot.inline; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.text.ParseException; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.das2.datum.EnumerationUnits; import org.das2.datum.LoggerManager; import org.das2.datum.Units; import org.das2.util.monitor.ProgressMonitor; import org.python.core.PyList; import org.python.core.PyObject; import org.python.core.PyTuple; import org.python.util.PythonInterpreter; import org.das2.qds.BundleDataSet; import org.das2.qds.DDataSet; import org.das2.qds.DataSetOps; import org.das2.qds.DataSetUtil; import org.das2.qds.MutablePropertyDataSet; import org.das2.qds.QDataSet; import org.autoplot.datasource.AbstractDataSource; import org.autoplot.datasource.URISplit; import org.autoplot.datasource.capability.TimeSeriesBrowse; import org.das2.qds.ops.Ops; import org.das2.qds.util.DataSetBuilder; import org.autoplot.jythonsupport.JythonOps; import org.autoplot.jythonsupport.JythonUtil; import org.autoplot.jythonsupport.Util; import org.das2.datum.Datum; import org.das2.datum.DatumUtil; import org.das2.datum.InconvertibleUnitsException; import org.das2.qds.examples.Schemes; import org.python.core.Py; import org.python.core.PyException; /** * Data source used mostly for demonstrations and quick modifications * of data. It also implements the "Mash Up Tool" which is a special * case of vap+inline, where N datasets are loaded an a combination of these * datasets is returned. * @author jbf */ public class InlineDataSource extends AbstractDataSource { private static final Logger logger= LoggerManager.getLogger("jython.inline"); PythonInterpreter interp; TimeSeriesBrowse tsb=null; public InlineDataSource(URI uri) { super(uri); List script= new ArrayList<>(); String timerange= InlineDataSourceFactory.getScript( uri.toString(), script ); if ( timerange!=null ) { try { tsb= InlineTimeSeriesBrowse.create( uri.toString(),timerange ); addCapability( TimeSeriesBrowse.class, tsb ); } catch (ParseException ex) { logger.warning(ex.toString()); } } } /** * execute the expression. This can be a command, presumably, or an variable * name or an expression. * @param c expression or variable name. * @return the dataset resolved. * @throws Exception */ private MutablePropertyDataSet handleJythonExpression( String c ) throws Exception { logger.finest(c); PyObject result= evalCommand( interp,c ); QDataSet res; if (result instanceof PyList) { res = JythonOps.dataset((PyList) result); } else if ( result instanceof PyTuple && ((PyTuple)result).size()<3 ) { //JythonOps.coerce(result); too bad coerce doesn't do this already. PyTuple tres= (PyTuple)result; switch (tres.size()) { case 2: res= Ops.link( (QDataSet)tres.get(0), (QDataSet)tres.get(1) ); break; case 3: res= Ops.link( (QDataSet)tres.get(0), (QDataSet)tres.get(1), (QDataSet)tres.get(2) ); break; case 1: res= (QDataSet)tres.get(0); // how did this happen? Might as well support it break; default: throw new ParseException( "unable to parse command: "+c, 0 ); } } else { res = (QDataSet) result.__tojava__(QDataSet.class); if ( res==null ) { throw new IllegalArgumentException("expression is not a QDataSet: "+c); } } return DataSetOps.makePropertiesMutable(res); } /** * parse the string which contains a comma-delineated sequence of * data. If all elements are parseable as a * double, the result is a dimensionless array. If parseable as * times, the result is a time array. Otherwise the result is a * result has enumeration units for the ordinal values. * @param s formatted ds with only commas delineating datums * @return rank 1 dataset, or rank 0 if there are no commas. */ private MutablePropertyDataSet parseInlineDsSimple( String s ) { logger.log(Level.FINEST, "parseInlineDsSimple {0}", s); Units u= Units.dimensionless; Units tu= Units.us2000; EnumerationUnits eu= EnumerationUnits.create("default"); String[] ss2= s.split(","); DDataSet result= DDataSet.createRank1(ss2.length); boolean isTime= false; boolean isEnum= false; for (String ss21 : ss2) { try { if (!isTime && !isEnum) { try { u.parse(ss21); } catch ( InconvertibleUnitsException ex3 ) { Datum d= DatumUtil.lookupDatum(ss21); u= d.getUnits(); } } } catch (ParseException e) { isTime= true; if (!isEnum) { try { tu.parse(ss21); }catch (ParseException ex) { try { Datum d= DatumUtil.lookupDatum(ss21); u= d.getUnits(); isTime= false; } catch ( ParseException ex2 ) { isEnum= true; } } } } } try { for ( int j=0; j *
  • a list of variables like "aa,bb,cc" which means link the three together to make a dataset. *
  • an expression like "aa+bb" or "pow(aa,10)" *
  • a literal like "1,2,3,2,3,2,1" or "2000-02-02T02:02,2000-02-02T02:03,2000-02-02T02:04" * * @param s the line of code * @return the QDataSet * @throws Exception when the line cannot be interpreted. */ private MutablePropertyDataSet parseInlineDs( String s ) throws Exception { logger.log(Level.FINEST, "parseInlineDs {0}", s); if ( s.equals("None") || s.equals("null") || s.equals("") ) return null; String linkCommand=null; try { PyObject result; if ( Ops.isSafeName(s) ) { result= evalCommand( interp, s ); } else { linkCommand= "link( "+s + ")"; result= evalCommand( interp, linkCommand ); //wha? } QDataSet res = (QDataSet) result.__tojava__(QDataSet.class); return DataSetOps.makePropertiesMutable(res); } catch ( RuntimeException ex ) { // since we couldn't run the command, we know that wasn't it. // TODO: it would be nicer to try to parse the string a little instead. logger.log(Level.FINE, "failed to execute: {0}", (linkCommand!=null ? linkCommand : s ) ); logger.log( Level.FINE, ex.getMessage(), ex ); } try { return handleJythonExpression(s); } catch ( Exception ex ) { boolean isNotList= s.length()>0 && ( ( s.charAt(0)>='a' && s.charAt(0)<='z' ) || s.charAt(0)=='(' ); if ( isNotList ) { throw new IllegalArgumentException( "inline jython code raises exception: "+ex, ex ); } } String[] ss= s.split(";",-2); if ( ss.length>1 ) { // rank 2 BundleDataSet bds= BundleDataSet.createRank1Bundle(); String[] ss2= ss[0].split(","); int nds= ss2.length; // number per record // do each column of the rank two table. int nrec=ss.length; for ( int j=0; j * x=getDataSet("http://autoplot.org/data/autoplot.cdf?Magnitude&noDep=T")&y=getDataSet('http://autoplot.org/data/autoplot.cdf?BGSEc&slice1=2')&sqrt(x) * * @param s the string to split. * @param delim the delimiter to split on, for example the ampersand (&). * @param exclude1 for example the single quote (') * @param exclude2 for example the double quote (") Note URIs don't support these anyway. * @return the split. */ protected static String[] guardedSplit( String s, char delim, char exclude1, char exclude2 ) { return Util.guardedSplit(s, delim, exclude1, exclude2); } /** * return true if the name is a property name like DEPEND_0 or DELTA_PLUS. * @param n the name * @return true if the name is a property name */ private boolean isPropName( String n ) { return DataSetUtil.getPropertyType(n)!=null; } // http://autoplot.org/developer.inlineData //vap+inline:3,4;3,6;5,6 //vap+inline:2000-001T00:00,23.5;2000-002T00:00,23.5;2000-003T00:00,23.5 //vap+inline:1,2,3&DEPEND_0=1,2,3&DEPEND_0.UNITS=hours since 2000-001T00:00 //vap+inline:exp(findgen(20))&UNITS=eV&SCALE_TYPE=log&LABEL=Energy //vap+inline:ripples(1440)&DEPEND_0=timegen('2003-05-01','1 min',1440) //vap+inline:t=linspace(0,2*PI,200)&cos(2*t),sin(3*t),t @Override public QDataSet getDataSet( ProgressMonitor mon ) throws Exception { String s= tsb==null ? getURI() : tsb.getURI(); logger.log(Level.FINE, "getDataSet {0}", s ); logger.log( Level.FINER, "create interpreter"); interp= JythonUtil.createInterpreter(true); if ( ! org.autoplot.jythonsupport.Util.isLegacyImports() ) { // we need to always bring this in to support legacy URIs. logger.log( Level.FINER, "import the stuff we don't import automatically anymore"); try (InputStream in = org.autoplot.jythonsupport.Util.class.getResource("imports2017.py").openStream()) { interp.execfile( in, "imports2017.py"); } } List script= new ArrayList<>(); String timerange= InlineDataSourceFactory.getScript( s, script ); String[] ss= script.toArray(new String[script.size()]); // make sure timerange is set before any other calls. if ( timerange!=null ) { interp.set("timerange",timerange); } MutablePropertyDataSet ds= null; MutablePropertyDataSet bundle1= null; MutablePropertyDataSet[] depn= new MutablePropertyDataSet[4]; Map[] deppropn= new Map[4]; Map p= new LinkedHashMap<>(); if ( ss.length==1 && ss[0].equals("None") ) { logger.info("vap+inline:None is useful for testing"); return null; } mon.setTaskSize(ss.length); mon.started(); try { Pattern depPat= Pattern.compile("DEPEND_(\\d+)(\\.([A-Z]+))?"); Matcher m; for ( int i=0; i-1 && isPropName(arg.substring(0,ieq).trim()) ) { propName= arg.substring(0,ieq).trim(); } if ( propName!=null ) { // it's a directive String propValue= arg.substring(ieq+1).trim(); if ( (m=depPat.matcher(propName)).matches() ) { int idep= Integer.parseInt( m.group(1) ); if ( m.group(3)!=null ) { Map map= deppropn[idep]; if ( map==null ) { map= new HashMap(); deppropn[idep]= map; } map.put( m.group(3), propValue ); } else { depn[idep]= parseInlineDs(propValue); } } else if ( propName.startsWith("BUNDLE_1") ) { if ( propValue.equals("") || propValue.equals("None") || propValue.equals("null") ) { p.put(propName,propValue); } else { bundle1= parseInlineDs(propValue); } } else { if ( DataSetUtil.isDimensionProperty(propName) || propName.equals(QDataSet.RENDER_TYPE) || propName.equals(QDataSet.DELTA_PLUS) || propName.equals(QDataSet.DELTA_MINUS) ) { p.put(propName,propValue); } else { try { interp.set( "monitor", mon.getSubtaskMonitor(arg)); execCommand( interp,arg ); } catch ( Exception ex ) { throw ex; // https://sourceforge.net/p/autoplot/bugs/1376/ } } } } else if ( isAssignment(arg) ) { logger.log( Level.FINER, "assignment {0}", arg); interp.set( "monitor", mon.getSubtaskMonitor(arg)); arg= URISplit.uriDecode(arg); if ( arg.startsWith("timerange=") ) continue; try { execCommand( interp, arg ); } catch ( Exception ex ) { if ( ex instanceof PyException ) { Object o= ((PyException)ex).value.__tojava__(Exception.class); if ( o==Py.NoConversion ) { throw ex; } else { Exception ex2= (Exception)o; throw ex2; } } else { throw ex; } } } else { ds= parseInlineDs(arg); } } } finally { mon.finished(); } if ( ds==null ) { throw new IllegalArgumentException("URI don't contain anything to plot"); } for ( int idep=0; idep depp= deppropn[idep]; if ( depp==null ) continue; for ( Entry ent: depp.entrySet() ) { String prop= ent.getKey(); MutablePropertyDataSet dep0= depn[idep]; if ( dep0==null ) { throw new IllegalArgumentException( "DEPEND_"+idep+"."+prop+" specified, but no DEPEND_"+idep+" ds"); } String propValue= ent.getValue(); switch (prop) { case "UNITS": dep0.putProperty( prop,Units.lookupUnits(propValue)); break; case "FILL_VALUE": case "VALID_MIN": case "VALID_MAX": case "TYPICAL_MIN": case "TYPICAL_MAX": dep0.putProperty( prop,Double.parseDouble(propValue)); break; case "MONOTONIC": dep0.putProperty( prop, Boolean.parseBoolean(propValue) ); // True or TRUE break; default: // it would be nice if the same code handled DEPEND_0 and the dataset. dep0.putProperty(prop,propValue); break; } } } for ( Entry e: p.entrySet() ) { String prop= e.getKey(); String propValue= e.getValue(); ds= Ops.putProperty( ds, prop, propValue ); } if ( depn[0]==null ) { if ( bundle1==null && ds.rank()==2 && ds.length(1)==2 ) { //TODO: we should be able to use bundle dataset... kludge if ( Ops.isBundle(ds) ) { MutablePropertyDataSet xx= (MutablePropertyDataSet)DataSetOps.unbundle(ds,0) ; MutablePropertyDataSet zz= (MutablePropertyDataSet)DataSetOps.unbundle(ds,1); zz.putProperty( QDataSet.DEPEND_0, xx ); ds= zz; } else if ( ds instanceof BundleDataSet ) { // use unbundle to support TimeLocation and EnumerationUnits types BundleDataSet bds= (BundleDataSet)ds; MutablePropertyDataSet xx= (MutablePropertyDataSet) bds.unbundle(0); MutablePropertyDataSet zz= (MutablePropertyDataSet) bds.unbundle(ds.length(0)-1); if ( ds.property(QDataSet.RENDER_TYPE)!=null ) zz.putProperty(QDataSet.RENDER_TYPE,ds.property(QDataSet.RENDER_TYPE)); // vap+inline:0,0,100,100,0,0; 0,0,0,100,100,0&RENDER_TYPE=scatter zz.putProperty( QDataSet.DEPEND_0, xx ); ds= zz; } else if ( Schemes.isBoundingBox(ds) ) { // do nothing } else { MutablePropertyDataSet xx= DDataSet.copy(DataSetOps.slice1(ds,0)); MutablePropertyDataSet zz= DDataSet.copy(DataSetOps.slice1(ds,ds.length(0)-1)); DataSetUtil.copyDimensionProperties( ds, zz ); // we put these in the wrong place, fix this... if ( ds.property(QDataSet.RENDER_TYPE)!=null ) zz.putProperty(QDataSet.RENDER_TYPE,ds.property(QDataSet.RENDER_TYPE)); // vap+inline:0,0,100,100,0,0; 0,0,0,100,100,0&RENDER_TYPE=scatter zz.putProperty( QDataSet.DEPEND_0, xx ); ds= zz; } } } else { for ( int idep=0; idep<4; idep++ ) { ds.putProperty( "DEPEND_"+idep, depn[idep] ); } } if ( bundle1!=null ) { ds.putProperty( QDataSet.BUNDLE_1, bundle1 ); } return ds; } /** * execute the command line the interpreter, with a hook for security concerns. * @param interp the interpreter * @param arg the command to execute. */ private static void execCommand( PythonInterpreter interp, String arg ) { if ( arg.contains("execfile") ) { throw new IllegalArgumentException("inline commands cannot contain execfile"); } else if ( arg.contains("__import__") ) { throw new IllegalArgumentException("inline commands cannot contain __import__"); } interp.exec(arg); } private static PyObject evalCommand( PythonInterpreter interp, String arg ) { if ( arg.contains("execfile") ) { throw new IllegalArgumentException("inline commands cannot contain execfile"); } else if ( arg.contains("__import__") ) { throw new IllegalArgumentException("inline commands cannot contain __import__"); } return interp.eval(arg); } private boolean isAssignment(String arg) { int i= arg.indexOf("="); if ( i==-1 ) return false; String varNames= arg.substring(0,i); Pattern p= Pattern.compile("[a-zA-Z_][a-zA-Z_0-9]*\\s*"); if ( p.matcher(varNames).matches() ) { return true; } Pattern p2= Pattern.compile("\\(([a-zA-Z_][a-zA-Z_0-9]*)(\\,[a-zA-Z_][a-zA-Z_0-9]*)*\\)"); return p2.matcher(varNames).matches(); } }