package org.autoplot.jythonsupport; import java.awt.Color; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.LineNumberReader; import java.io.OutputStream; import java.io.Reader; import java.io.StringReader; import java.net.URI; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.das2.datum.Datum; import org.das2.datum.DatumRange; import org.das2.util.LoggerManager; import org.das2.util.monitor.NullProgressMonitor; import org.das2.util.monitor.ProgressMonitor; import org.python.core.FileUtil; import org.python.core.Py; import org.python.core.PyDictionary; import org.python.core.PyException; import org.python.core.PyFloat; import org.python.core.PyInteger; import org.python.core.PyJavaInstance; import org.python.core.PyList; import org.python.core.PyObject; import org.python.core.PyString; import org.python.core.PySyntaxError; import org.python.core.PySystemState; import org.python.parser.SimpleNode; import org.python.parser.ast.Assign; import org.python.parser.ast.Attribute; import org.python.parser.ast.Call; import org.python.parser.ast.If; import org.python.parser.ast.Module; import org.python.parser.ast.Name; import org.python.parser.ast.Subscript; import org.python.parser.ast.VisitorBase; import org.python.parser.ast.exprType; import org.python.parser.ast.stmtType; import org.python.util.InteractiveInterpreter; import org.python.util.PythonInterpreter; import org.autoplot.datasource.AutoplotSettings; import org.autoplot.datasource.DataSetURI; import org.python.core.PyTuple; /** * Utilities to support Jython scripting. * * @see org.autoplot.JythonUtil * @author jbf */ public class JythonUtil { private static final Logger logger = LoggerManager.getLogger("jython"); /** * create an interpreter object configured for Autoplot contexts: * * This also adds things to the python search path (see * getLocalJythonAutoplotLib) so imports will find them. * * @param sandbox limit symbols to safe symbols for server. * @return PythonInterpreter ready for commands. * @throws java.io.IOException */ public static InteractiveInterpreter createInterpreter(boolean sandbox) throws IOException { if (PySystemState.cachedir == null) { String autoplotData= AutoplotSettings.settings().resolveProperty(AutoplotSettings.PROP_AUTOPLOTDATA); System.setProperty("python.cachedir", autoplotData + "/pycache"); } /// http://www.gossamer-threads.com/lists/python/python/697524 org.python.core.PySystemState pySys = new org.python.core.PySystemState(); //pySys.setdefaultencoding("utf8"); //doesn't work with Jython2.2, try with 2.5 String[] loadClasses = new String[]{"glob.py", "autoplot2017.py", "autoplotapp2017.py"}; // these must be in the root of the interpretter search path. for (String pysrc : loadClasses) { if (pysrc.equals("glob.py")) { URL jarUrl = InteractiveInterpreter.class.getResource("/" + pysrc); if (jarUrl != null) { String f = getLocalJythonLib(); pySys.path.insert(0, new PyString(f)); } else { logger.log(Level.WARNING, "Couldn''t find jar containing {0}. See https://sourceforge.net/p/autoplot/bugs/576/", pysrc); } } else if (pysrc.equals("autoplotapp2017.py")) { String f = getLocalJythonAutoplotAppLib(); if (!pySys.path.contains(new PyString(f))) { // TODO possible bug here: PyString/String means local path is in there 4 times. pySys.path.insert(0, new PyString(f)); } } else { String f = getLocalJythonAutoplotLib(); if (!pySys.path.contains(new PyString(f))) { pySys.path.insert(0, new PyString(f)); } } } InteractiveInterpreter interp = new InteractiveInterpreter(null, pySys); // try { // System.err.println("java1-> "+interp.eval("java") ); // } catch ( Exception ex ) { // System.err.println("java1-> ok!" ); // } boolean loadAutoplotStuff = true; if (loadAutoplotStuff) { maybeLoadAdapters(); if (Util.isLegacyImports()) { URL imports = JythonOps.class.getResource("/imports2017.py"); if (imports == null) { throw new RuntimeException("unable to locate imports2017.py on classpath"); } else { logger.log(Level.FINE, "loading imports2017.py from {0}", imports); } InputStream in = imports.openStream(); // note this stream will load in another stream. byte[] bimports = FileUtil.readBytes(in); String simports = new String(bimports); logger.log(Level.FINE, simports); //InputStream in = imports.openStream(); try { interp.execfile(new ByteArrayInputStream(bimports), "/imports2017.py"); } finally { in.close(); } } } interp.set("dataset", new DatasetCommand()); interp.set("monitor", new NullProgressMonitor()); // try { // System.err.println("java2-> "+interp.eval("java") ); // } catch ( Exception ex ) { // System.err.println("java2-> ok!" ); // } return interp; } /** * set up the interp variables scripts will often use, such as PWD and * monitor. * * @param interp * @param pwd * @param resourceUri * @param paramsl * @param mon */ public static void setupInterp(PythonInterpreter interp, String pwd, String resourceUri, Map paramsl, ProgressMonitor mon) { interp.set("PWD", pwd); interp.exec("import autoplot2017 as autoplot"); interp.exec("autoplot.params=dict()"); for (Entry e : paramsl.entrySet()) { String s = e.getKey(); if (!s.equals("arg_0") && !s.equals("script")) { String sval = e.getValue(); sval = JythonUtil.maybeQuoteString(sval); logger.log(Level.FINE, "autoplot.params[''{0}'']={1}", new Object[]{s, sval}); interp.exec("autoplot.params['" + s + "']=" + sval); } } if (resourceUri != null) { interp.set("resourceURI", resourceUri); // legacy interp.exec("autoplot.params['" + "resourceURI" + "']=" + JythonUtil.maybeQuoteString(resourceUri)); } interp.set("monitor", mon); try (InputStream in = JythonOps.class.getResource("/autoplot2017.py").openStream()) { interp.execfile(in, "/autoplot2017.py"); // import everything into default namespace. } catch (IOException ex) { logger.log(Level.SEVERE, ex.getMessage(), ex); } } /** * transfer the contents of in to out. in and out are closed after the * operation. //TODO: other implementations of this exist... * * @param in * @param out * @throws IOException */ private static void transferStream(InputStream in, OutputStream out) throws IOException { byte[] buf = new byte[2048]; int n; try { n = in.read(buf); while (n > -1) { out.write(buf, 0, n); n = in.read(buf); } } finally { out.close(); in.close(); } } /** * ensure that the file has a parent writable directory. * * @param file * @return true if the folder could be made. */ private static boolean makeHomeFor(File file) { File f = file.getParentFile(); if (!f.exists()) { return f.mkdirs(); } else { return true; } } /** * copy everything out to autoplot_data/jython without going to web again. * The old code showed issues where autoplot.org could not be resolved. * * @return the item to add to the python search path. * @throws IOException */ private static String getLocalJythonLib() throws IOException { File ff2 = new File(AutoplotSettings.settings().resolveProperty(AutoplotSettings.PROP_AUTOPLOTDATA)); File ff3 = new File(ff2.toString() + "/jython"); File ff4 = new File(ff2.toString() + "/jython/zlib.py"); if (ff4.exists()) { return ff3.toString(); } synchronized (JythonUtil.class) { if (!ff3.exists()) { if (!ff3.mkdirs()) { throw new IOException("Unable to mkdirs " + ff3); } } } if (JythonUtil.class.getResource("/pylisting.txt") == null) { throw new IllegalArgumentException("unable to find pylisting.txt in application, which is needed to install Jython codes."); } else { logger.log(Level.FINE, "unpacking jython codes in {0}", JythonUtil.class.getResource("/pylisting.txt")); try (BufferedReader r = new BufferedReader(new InputStreamReader(JythonUtil.class.getResourceAsStream("/pylisting.txt")))) { String s = r.readLine(); while (s != null) { File ff5 = new File(ff3, s); logger.log(Level.FINER, "copy to local folder python code: {0}", s); InputStream in = JythonUtil.class.getResourceAsStream("/" + s); if (in == null) { throw new IllegalArgumentException("unable to find jython code which should be embedded in application: " + s); } if (s.contains("/")) { if (!makeHomeFor(ff5)) { throw new IOException("Unable to makeHomeFor " + ff5); } } try (FileOutputStream out = new FileOutputStream(ff5)) { transferStream(in, out); } finally { in.close(); if (new File(ff3, s).setReadOnly() == false) { logger.log(Level.FINER, "set read-only on file {0} failed", s); } if (new File(ff3, s).setWritable(true, true) == false) { logger.log(Level.FINER, "set write for user only on file {0} failed", s); } } s = r.readLine(); } } } logger.fine(" ...done"); return ff3.toString(); } /** * copy all the app stuff to autoplot_data/jython without going to web * again. * * @return the item to add to the python search path. * @throws IOException */ private static String getLocalJythonAutoplotAppLib() throws IOException { File ff2 = new File(AutoplotSettings.settings().resolveProperty(AutoplotSettings.PROP_AUTOPLOTDATA)); File ff3 = new File(ff2.toString() + "/jython"); File ff4 = new File(ff3.toString(), "pylistingapp2017.txt"); if (ff4.exists()) { return ff3.toString(); } synchronized (JythonUtil.class) { if (!ff3.exists()) { if (!ff3.mkdirs()) { throw new IOException("Unable to mkdirs " + ff3); } } } if (JythonUtil.class.getResource("/pylistingapp2017.txt") == null) { logger.log(Level.FINE, "unable to find pylistingapp2017.txt in application, assuming this is not the Autoplot client application."); } else { logger.log(Level.FINE, "unpacking jython codes in {0}", JythonUtil.class.getResource("/pylistingapp2017.txt")); try (BufferedReader r = new BufferedReader(new InputStreamReader(JythonUtil.class.getResourceAsStream("/pylistingapp2017.txt")))) { String s = r.readLine(); while (s != null) { int i = s.indexOf("#"); if (i > -1) { s = s.substring(0, i); } s = s.trim(); if (s.length() > 0) { File ff5 = new File(ff3, s); logger.log(Level.FINER, "copy to local folder python code: {0}", s); if (s.contains("/")) { if (!makeHomeFor(ff5)) { throw new IOException("Unable to makeHomeFor " + ff5); } } if (ff5.exists()) { logger.fine("already have file, skip..."); s = r.readLine(); continue; } InputStream in = JythonUtil.class.getResourceAsStream("/" + s); if (in == null) { throw new IllegalArgumentException("unable to find jython code which should be embedded in application: " + s); } //Re https://sourceforge.net/p/autoplot/bugs/1724/: //Really each file should be copied and then renamed. try (FileOutputStream out = new FileOutputStream(ff5)) { transferStream(in, out); } finally { in.close(); if (new File(ff3, s).setReadOnly() == false) { logger.log(Level.FINER, "set read-only on file {0} failed", s); } if (new File(ff3, s).setWritable(true, true) == false) { logger.log(Level.FINER, "set write for user only on file {0} failed", s); } } } s = r.readLine(); } } } return ff3.toString(); } /** * copy the two python files specific to Autoplot into the user's * autoplot_data/jython folder. This reads the version from the first line * of the autoplot2017.py. * * @return the item to add to the python search path. * @throws IOException */ private static String getLocalJythonAutoplotLib() throws IOException { File ff2 = new File(AutoplotSettings.settings().resolveProperty(AutoplotSettings.PROP_AUTOPLOTDATA)); File ff3 = new File(ff2.toString() + "/jython"); File ff4 = new File(ff3, "autoplot2017.py"); String vers = ""; // This is the version that Autoplot would like to find, and should be found within the Java class path. double currentVersion = 2.00; //rfe320 improved getParam support. if (ff4.exists()) { try (BufferedReader r = new BufferedReader(new FileReader(ff4))) { String line = r.readLine(); if (line != null) { Pattern versPattern = Pattern.compile("# autoplot2017.py v([\\d\\.]+) .*"); // must be parsable as a double. Matcher m = versPattern.matcher(line); if (m.matches()) { vers = m.group(1); } } } } if (logger.isLoggable(Level.FINE)) { logger.fine("== JythonUtil getLocalJythonAutoplotLib =="); logger.log(Level.FINE, "ff4.exists()={0}", ff4.exists()); logger.log(Level.FINE, "vers={0}", vers); logger.log(Level.FINE, "currentVersion={0}", currentVersion); } if (!ff4.exists() || vers.equals("") || Double.parseDouble(vers) < currentVersion) { logger.log(Level.FINE, "looking for version={0} of {1}, but didn''t find it.", new Object[]{currentVersion, ff4}); logger.log(Level.FINE, "doesn't seem like we have the right file, downloading..."); synchronized (JythonUtil.class) { if (!ff3.exists()) { if (!ff3.mkdir()) { throw new IOException("Unable to mkdir " + ff3); } } } String[] ss = new String[]{"autoplot2017.py", "autoplotapp2017.py"}; for (String s : ss) { try (InputStream in = JythonUtil.class.getResourceAsStream("/" + s); FileOutputStream out = new FileOutputStream(new File(ff3, s))) { transferStream(in, out); } if (new File(ff3, s).setReadOnly() == false) { logger.log(Level.FINER, "set read-only on file {0} failed", s); } if (new File(ff3, s).setWritable(true, true) == false) { logger.log(Level.FINER, "set write for user only on file {0} failed", s); } } } logger.fine(" ...done"); return ff3.toString(); } /** * check the script that it doesn't redefine symbol names like "str" * * @param uri, such as sftp://user@host/script.jy * @param errs an empty list where the errors can be logged. * @return true if an err is suspected. * @throws java.io.IOException */ public static boolean pythonLint(URI uri, List errs) throws IOException { LineNumberReader reader; File src = DataSetURI.getFile(uri, new NullProgressMonitor()); reader = new LineNumberReader(new BufferedReader(new FileReader(src))); try { return pythonLint(reader, errs); } finally { reader.close(); } } /** * check the script that it doesn't redefine symbol names like "str" * * @param reader, which will be not be closed here. * @param errs an empty list where the errors can be logged. * @return true if an err is suspected. * @throws java.io.IOException */ public static boolean pythonLint(LineNumberReader reader, List errs) throws IOException { String vnarg = "(\\s*)([a-zA-Z_][a-zA-Z0-9_]*)\\s*"; // any variable name VERIFIED Pattern assign = Pattern.compile(vnarg + "=.*"); InteractiveInterpreter interp = createInterpreter(true); String line = reader.readLine(); while (line != null) { Matcher m = assign.matcher(line); if (m.matches()) { String vname = m.group(2); String indent= m.group(1); try { if ( vname.equals("xrange") && indent.length()>0 ) { logger.fine("this is just xrange keyword in plot"); } else { PyObject po = interp.eval(vname); errs.add("" + reader.getLineNumber() + ": " + vname + "=" + po.__repr__()); } } catch (PyException ex) { // this is what we want } } line = reader.readLine(); } return errs.size() > 0; } private static boolean haveloadedAdapters = false; /** * load the adapters, once. */ private synchronized static void maybeLoadAdapters() { if (!haveloadedAdapters) { Py.getAdapter().addPostClass(new PyQDataSetAdapter()); Py.getAdapter().addPostClass(new PyDatumAdapter()); haveloadedAdapters = true; } } /** * TODO: this ought to remove the need for ParametersFormPanel. */ public static class Param { public String name; public String label; // the label for the variable used in the script public Object deft; public Object value; // the value if available, null means not present. public String doc; public List enums; // the allowed values public List examples; // example values /** * constraints for the value, such as:
    *
  • 'labels':['RBSP-A','RBSP-B'] labels for each enum. *
*/ public Map constraints; /** * The parameter type:
    *
  • T (TimeRange), *
  • A (String, but note a string with the values enumerated either T * or F is treated as a boolean.) *
  • F (Double or Integer, but note the values [0,1] imply it's a * boolean.), *
  • D (Datum), *
  • S (DatumRange), *
  • U (Dataset URI), *
  • L (URL), a file location, not a URI with parameters. *
  • or R (the resource URI) *
*/ public char type; @Override public String toString() { return name + "=" + deft; } } /** * scrape through the script looking for documentation declarations returns * an array, possibly containing:
    *
  • LABEL few words *
  • TITLE sentence *
  • DESCRIPTION short paragraph *
* This would originally look for lines like:
* # TITLE: Text Recombinator
* but this has been deprecated and scripts should use setScriptTitle * and setScriptDescription * * @param reader, open and ready to read, which will be closed. * @return the documentation found. * @throws java.io.IOException */ public static Map getDocumentation(BufferedReader reader) throws IOException { String line = reader.readLine(); StringBuilder scriptBuilder= new StringBuilder(); while ( line!=null ) { scriptBuilder.append(line).append('\n'); line= reader.readLine(); } String script= scriptBuilder.toString(); ScriptDescriptor sd= org.autoplot.jythonsupport.JythonUtil.describeScript( script, new HashMap() ); Map result = new HashMap<>(); if ( sd.getDescription().length()>0 ) result.put( "DESCRIPTION", sd.getDescription() ); if ( sd.getTitle().length()>0 ) result.put( "TITLE", sd.getTitle() ); if ( sd.getLabel().length()>0 ) result.put( "LABEL", sd.getLabel() ); if ( sd.getIconURL().length()>0 ) result.put( "ICONURL", sd.getIconURL() ); if ( result.isEmpty() ) { reader= new BufferedReader( new StringReader(script) ); String s = reader.readLine(); Pattern p = Pattern.compile("#\\s*([a-zA-Z]+)\\s*:(.*)"); while (s != null) { Matcher m = p.matcher(s); if (m.matches()) { String prop = m.group(1).toUpperCase(); String value = m.group(2).trim(); if (prop.equals("LABEL") || prop.equals("TITLE") || prop.equals("DESCRIPTION")) { result.put(prop, value); } } s = reader.readLine(); } reader.close(); } return result; } //there are a number of functions which take a trivial amount of time to execute and are needed for some scripts, such as the string.upper() function. //The commas are to guard against the id being a subset of another id ("lower," does not match "lowercase"). //TODO: update this after Python upgrade. private static final String[] okay = new String[]{"range,", "xrange,", "getParam,", "lower,", "upper,", "URI,", "URL,", "DatumRangeUtil,", "TimeParser", "str,", "int,", "long,", "float,", "datum,", "datumRange,"}; /** * return true if the function call is trivial to execute and can be * evaluated within a few milliseconds. * * @param sn * @return */ private static boolean trivialFunctionCall(SimpleNode sn) { if (sn instanceof Call) { Call c = (Call) sn; boolean klugdyOkay = false; String ss = c.func.toString(); for (String s : okay) { if (ss.contains(s)) { klugdyOkay = true; } } if (ss.startsWith("Attribute[")) { klugdyOkay = true; // parms.keys() } if (klugdyOkay == false) { if (ss.contains("TimeUtil") && ss.contains("now")) { klugdyOkay = true; } } logger.log(Level.FINER, "trivialFunctionCall={0} for {1}", new Object[]{klugdyOkay, c.func.toString()}); return klugdyOkay; } else { return false; } } private static class MyVisitorBase extends VisitorBase { boolean looksOkay = true; boolean visitNameFail = false; HashSet names = new HashSet(); MyVisitorBase(HashSet names) { this.names = names; } @Override public Object visitName(Name node) throws Exception { if (!names.contains(node.id)) { visitNameFail = true; } return super.visitName(node); //To change body of generated methods, choose Tools | Templates. } @Override public Object visitCall(Call node) throws Exception { return super.visitCall(node); //To change body of generated methods, choose Tools | Templates. } @Override protected Object unhandled_node(SimpleNode sn) throws Exception { return sn; } @Override public void traverse(SimpleNode sn) throws Exception { if (sn instanceof Call) { boolean newLooksOkay = trivialFunctionCall(sn); if (!newLooksOkay) { logger.log(Level.FINE, "looksOkay=False, {0}", sn); } looksOkay = newLooksOkay; } else if (sn instanceof Assign) { // TODO: I have to admit I don't understand what traverse means. I would have thought it was all nodes... Assign a = ((Assign) sn); exprType et = a.value; if (et instanceof Call) { boolean newLooksOkay = trivialFunctionCall(et); if (!newLooksOkay) { logger.log(Level.FINE, "looksOkay=False, {0}", sn); } looksOkay = newLooksOkay; } } else if (sn instanceof Name) { //visitName((Name)sn).id } } public boolean looksOkay() { return looksOkay; } /** * this contains a node whose name we can't resolve. * * @return */ public boolean visitNameFail() { return visitNameFail; } } /** * inspect the node to look for function calls that are not to the function * "getParam". This is awful code that will be rewritten when we upgrade * Python to 2.7. * * @param o * @param variableNames * @return */ private static boolean simplifyScriptToGetParamsOkayNoCalls(SimpleNode o, HashSet variableNames) { if (o instanceof Call) { Call c = (Call) o; if (!trivialFunctionCall(c)) { logger.finest(String.format("%04d simplify->false: %s", o.beginLine, o.toString())); return false; } } MyVisitorBase vb = new MyVisitorBase(variableNames); try { o.traverse(vb); logger.finest(String.format(" %04d simplify->%s: %s", o.beginLine, vb.looksOkay(), o)); return vb.looksOkay(); } catch (Exception ex) { logger.log(Level.SEVERE, ex.getMessage(), ex); } logger.finest(String.format("!! %04d simplify->false: %s", o.beginLine, o)); return false; } /** * can we resolve this node given the variable names we know? * * @param o * @param variableNames * @return true if the node can be resolved. */ private static boolean simplifyScriptToGetParamsCanResolve(SimpleNode o, HashSet variableNames) { //if ( o.beginLine>=617 && o.beginLine<619 ) { // System.err.println( "here at 617-ish"); //} if (o instanceof Name) { Name c = (Name) o; if (!variableNames.contains(c.id)) { logger.finest(String.format("%04d canResolve->false: %s", o.beginLine, o.toString())); return false; } } if (o instanceof Attribute) { Attribute at = (Attribute) o; while (at.value instanceof Attribute || at.value instanceof Subscript) { if (at.value instanceof Attribute) { at = (Attribute) at.value; } else { Subscript s = (Subscript) at.value; if (s.value instanceof Attribute) { at = (Attribute) s.value; } else { return false; // oh just give up... } } } if (at.value instanceof Name) { Name n = (Name) at.value; if (!variableNames.contains(n.id)) { return false; } } } MyVisitorBase vb = new MyVisitorBase(variableNames); try { o.traverse(vb); logger.finest(String.format(" %04d canResolve->%s: %s", o.beginLine, vb.visitNameFail, o)); return !vb.visitNameFail; } catch (Exception ex) { logger.log(Level.SEVERE, ex.getMessage(), ex); } logger.finest(String.format("!! %04d canResolve->false: %s", o.beginLine, o)); return false; } /** * put quotes around values that appear to be strings. We see if it's * parsable as a double or the keyword True or False. * * @param sval the string, for example "2015" "'2015'" or "2015-01-01" * @return 2015, "'2015'", "'2015-01-01'" */ public static String maybeQuoteString(String sval) { boolean isNumber = false; try { Double.parseDouble(sval); } catch (NumberFormatException ex) { isNumber = false; } if (!isNumber && !sval.equals("True") && !sval.equals("False")) { if (sval.length() == 0 || !(sval.startsWith("'") && sval.endsWith("'"))) { sval = String.format("'%s'", sval); } } return sval; } /** * pop off the quotes to get the text inside. * * @param sval "'2015-01-01'" * @return "2015-01-01" */ public static String maybeUnquoteString(String sval) { if ((sval.startsWith("'") && sval.endsWith("'")) || (sval.startsWith("\"") && sval.endsWith("\""))) { return sval.substring(1, sval.length() - 1); } else { return sval; } } /** * put each parameter into the dictionary autoplot.params. * * @param interp * @param paramsl */ public static void setParams(PythonInterpreter interp, Map paramsl) { interp.exec("import autoplot2017 as autoplot"); interp.exec("autoplot.params=dict()"); for (Entry e : paramsl.entrySet()) { String s = e.getKey(); if (s.length() == 0) { throw new IllegalArgumentException("param name is \"\": " + s); } if (!s.equals("arg_0") && !s.equals("script")) { try { String sval = e.getValue(); sval = maybeQuoteString(sval); logger.log(Level.FINE, "autoplot.params[''{0}'']={1}", new Object[]{s, sval}); interp.exec("autoplot.params['" + s + "']=" + sval); } catch (Exception ex) { logger.warning(ex.getMessage()); } } } } /** * return true if the call is a setScriptTitle or setScriptDescription call. * setScriptTitle( 'Batch Master Demo' ) * * @param o * @param variableNames * @return */ private static boolean isSetScriptCall(stmtType o, HashSet variableNames) { if (o instanceof org.python.parser.ast.Expr) { org.python.parser.ast.Expr expr = (org.python.parser.ast.Expr) o; if (expr.value instanceof Call) { Call c = ((Call) expr.value); if (c.func instanceof Name) { Name n = (Name) c.func; if (n.id.equals("setScriptTitle") || n.id.equals("setScriptDescription") || n.id.equals("setScriptLabel") || n.id.equals("setScriptIcon")) { return true; } } return false; } return false; } return false; } /** * return true if we can include this in the script without a huge * performance penalty. * * @param o * @return */ private static boolean simplifyScriptToGetParamsOkay(stmtType o, HashSet variableNames) { //if ( o.beginLine==607 ) { // leave this commented code as a reference for debugging // System.err.println("here at line "+o.beginLine); //} if ((o instanceof org.python.parser.ast.ImportFrom)) { return true; } if ((o instanceof org.python.parser.ast.Import)) { return true; } if ((o instanceof org.python.parser.ast.Assign)) { Assign a = (Assign) o; if (simplifyScriptToGetParamsOkayNoCalls(a.value, variableNames)) { if (!simplifyScriptToGetParamsCanResolve(a.value, variableNames)) { return false; } for (exprType target : a.targets) { exprType et = (exprType) target; if (et instanceof Name) { String id = ((Name) target).id; variableNames.add(id); logger.log(Level.FINEST, "assign to variable {0}", id); } else if (et instanceof Attribute) { Attribute at = (Attribute) et; while (at.value instanceof Attribute || at.value instanceof Subscript) { if (at.value instanceof Attribute) { at = (Attribute) at.value; } else { Subscript s = (Subscript) at.value; if (s.value instanceof Attribute) { at = (Attribute) s.value; } else { return false; // oh just give up... } } } if (at.value instanceof Name) { Name n = (Name) at.value; if (!variableNames.contains(n.id)) { return false; } } } } return true; } else { return false; } } if ((o instanceof org.python.parser.ast.If)) { return simplifyScriptToGetParamsOkayNoCalls(o, variableNames); } if ((o instanceof org.python.parser.ast.Print)) { return false; } logger.log(Level.FINEST, "not okay to simplify: {0}", o); return false; } private static void maybeAppendSort(String theLine, StringBuilder result) { int i = theLine.indexOf("getParam"); if (i != -1) { i = theLine.indexOf("="); String v = theLine.substring(0, i).trim(); int indent = theLine.indexOf(v); if (indent > 0) { result.append(theLine.substring(0, indent)); } result.append("sort_.append( \'").append(v).append("\')\n"); } } private static StringBuilder appendToResult(StringBuilder result, String line) { //if ( line.contains("sTimeBinSeconds") ) { // System.err.println("heresTimeBinSeconds"); //} result.append(line); return result; } /** * Extracts the parts of the program that get parameters or take a trivial * amount of time to execute. This may call itself recursively when if * blocks are encountered. See test038. * * @param ss the entire script. * @param stmts statements being processed. * @param variableNames variable names that have been resolved. * @param beginLine first line of the script being processed. * @param lastLine INCLUSIVE last line of the script being processed. * @param depth recursion depth, for debugging. * @return */ public static String simplifyScriptToGetParams(String[] ss, stmtType[] stmts, HashSet variableNames, int beginLine, int lastLine, int depth) { int acceptLine = -1; // first line to accept int currentLine = 0; // current line we are writing (0 is first line). StringBuilder result = new StringBuilder(); for (int istatement = 0; istatement < stmts.length; istatement++) { stmtType o = stmts[istatement]; logger.log(Level.FINER, "line {0}: {1}", new Object[]{o.beginLine, o.beginLine > 0 ? ss[o.beginLine - 1] : "(bad line number)"}); if (o.beginLine > 0) { beginLine = o.beginLine; } else { acceptLine = beginLine; // elif clause in autoplot-test038/lastSuccessfulBuild/artifact/test038_demoParms1.jy } if (beginLine > lastLine) { continue; } if (o instanceof org.python.parser.ast.If) { if (acceptLine > -1) { for (int i = acceptLine; i < beginLine; i++) { appendToResult(result, ss[i - 1]).append("\n"); } } If iff = (If) o; boolean includeBlock; if (simplifyScriptToGetParamsCanResolve(iff.test, variableNames)) { for (int i = beginLine; i < iff.body[0].beginLine; i++) { result.append(ss[i - 1]).append("\n"); } // write out the 'if' part includeBlock = true; } else { includeBlock = false; } int lastLine1; //lastLine1 is the last line of the "if" clause. if (iff.orelse != null && iff.orelse.length > 0) { if (iff.orelse[0].beginLine > 0) { lastLine1 = iff.orelse[0].beginLine - 2; // -2 is for the "else:" part. } else { if (iff.orelse[0] instanceof If) { lastLine1 = ((If) iff.orelse[0]).test.beginLine - 1; } else { logger.warning("failure to deal with another day..."); throw new RuntimeException("this case needs to be dealt with..."); } } } else if ((istatement + 1) < stmts.length) { lastLine1 = stmts[istatement + 1].beginLine - 1; } else { lastLine1 = lastLine; } if (includeBlock) { String ss1 = simplifyScriptToGetParams(ss, iff.body, variableNames, -1, lastLine1, depth + 1); if (ss1.length() == 0) { String line; if (iff.beginLine == 0 && beginLine > 0 && iff.body[0].beginLine > 0) { line = ss[iff.body[0].beginLine - 1]; } else { line = ss[iff.beginLine]; } String[] ss2 = line.split("\\S", -2); String indent = ss2[0]; result.append(indent).append("pass\n"); logger.fine("things have probably gone wrong..."); } else { appendToResult(result, ss1); } if (iff.orelse != null) { appendToResult(result, ss[lastLine1]); int lastLine2; if ((istatement + 1) < stmts.length) { lastLine2 = stmts[istatement + 1].beginLine - 1; } else { lastLine2 = lastLine; } String ss2 = simplifyScriptToGetParams(ss, iff.orelse, variableNames, lastLine1 + 2, lastLine2, depth + 1); if (ss2.length() > 0) { result.append("\n"); } appendToResult(result, ss2); if (ss2.length() == 0) { // we didn't add anything... String line; line = ss[iff.orelse[0].beginLine - 1]; String[] ss3 = line.split("\\S", -2); String indent = ss3[0]; result.append("\n").append(indent).append("pass\n"); } else { result.append("\n"); // write of the else or elif line } } } acceptLine = -1; } else { if (simplifyScriptToGetParamsOkay(o, variableNames)) { if (acceptLine < 0) { acceptLine = (o).beginLine; for (int i = currentLine + 1; i < acceptLine; i++) { result.append("\n"); currentLine = acceptLine; } } } else if (isSetScriptCall(o, variableNames)) { if (acceptLine < 0) { acceptLine = (o).beginLine; for (int i = currentLine + 1; i < acceptLine; i++) { result.append("\n"); currentLine = acceptLine; } } } else { if (acceptLine > -1) { int thisLine = (o).beginLine; for (int i = acceptLine; i < thisLine; i++) { appendToResult(result, ss[i - 1]).append("\n"); } appendToResult(result, "\n"); currentLine = thisLine; acceptLine = -1; } } } } if (acceptLine > -1) { int thisLine = lastLine; for (int i = acceptLine; i <= thisLine; i++) { appendToResult(result, ss[i - 1]).append("\n"); } } return result.toString(); } private static class VisitNamesVisitorBase extends VisitorBase { String name; List names; VisitNamesVisitorBase(String name) { this.name = name; names = new ArrayList(); } @Override public Object visitName(Name node) throws Exception { if (name.equals(node.id)) { names.add(node); } return super.visitName(node); //To change body of generated methods, choose Tools | Templates. } @Override public Object visitCall(Call node) throws Exception { return super.visitCall(node); //To change body of generated methods, choose Tools | Templates. } @Override protected Object unhandled_node(SimpleNode sn) throws Exception { return sn; } @Override public void traverse(SimpleNode sn) throws Exception { sn.traverse(this); } /** * return the nodes where the name is used. * * @return */ public List getNames() { return names; } } /** * get the nodes where the symbol is used. * * @param script the jython script which is parsed. * @param symbol the symbol to look for. * @return the AST nodes which contain location information. */ public static List showUsage(String script, String symbol) { Module n = (Module) org.python.core.parser.parse(script, "exec"); VisitNamesVisitorBase vb = new VisitNamesVisitorBase(symbol); for (stmtType st : n.body) { try { st.traverse(vb); } catch (Exception ex) { logger.log(Level.SEVERE, null, ex); } } return vb.getNames(); } /** * extracts the parts of the program that get parameters. * * @param script the entire python program * @param addSort if true, add parameters to keep track of the order that * getParam was called. This has no effect now. * @return the python program with lengthy calls removed, up to the last * getParam call. */ public static String simplifyScriptToGetParams(String script, boolean addSort) throws PySyntaxError { String[] ss = script.split("\n"); int lastLine = -1; for (int i = 0; i < ss.length; i++) { String line = ss[i]; int ich = line.indexOf('#'); if (ich > -1) { line = line.substring(0, ich); } if (line.contains("getParam")) { lastLine = i + 1; } else if (line.contains("setScriptTitle")) { lastLine = i + 1; } else if (line.contains("setScriptDescription")) { lastLine = i + 1; } else if (line.contains("setScriptLabel")) { lastLine = i + 1; } else if (line.contains("setScriptIcon")) { lastLine = i + 1; } } if (lastLine == -1) { return ""; } // check for continuation in last getParam call. while (ss.length > lastLine + 1 && ss[lastLine].trim().length() > 0 && Character.isWhitespace(ss[lastLine].charAt(0))) { lastLine++; } // Chris showed that a closing bracket or paren doesn't need to be indented. See test038/jydsCommentBug.jyds if (lastLine < ss.length) { String closeParenCheck = ss[lastLine].trim(); if (closeParenCheck.equals(")") || closeParenCheck.equals("]")) { lastLine++; } } HashSet variableNames = new HashSet(); variableNames.add("getParam"); // this is what allows the getParam calls to be included. variableNames.add("str"); // include casts. variableNames.add("int"); variableNames.add("long"); variableNames.add("float"); variableNames.add("datum"); variableNames.add("datumRange"); variableNames.add("URI"); variableNames.add("URL"); variableNames.add("True"); variableNames.add("False"); try { Module n = (Module) org.python.core.parser.parse(script, "exec"); return simplifyScriptToGetParams(ss, n.body, variableNames, 1, lastLine, 0); } catch (PySyntaxError ex) { throw ex; } } /** * support for the old getGetParams. Note this closes the reader. * * @param reader * @return * @throws IOException * @throws PySyntaxError */ public static List getGetParams(Reader reader) throws IOException, PySyntaxError { return getGetParams(null, readScript(reader), new HashMap()); } /** * Object containing a description of a script, containing its parameters * and title describing it. */ public static interface ScriptDescriptor { /** * a short label, suitable for a menu item. * @return a short label, suitable for a menu item. */ String getLabel(); /** * a one-line title, suitable for a GUI heading. * @return a one-line title, suitable for a GUI heading. */ String getTitle(); /** * sentence or paragraph documenting the script. * @return */ String getDescription(); /** * the web address of an icon for the script. * @return */ String getIconURL(); /** * the list of the script parameters, containing the type and default values. * @return */ List getParams(); //List getOutputParams(); // this should finally be done as well. } public static final ScriptDescriptor EMPTY = new ScriptDescriptor() { @Override public String getLabel() { return ""; } @Override public String getTitle() { return ""; } @Override public String getDescription() { return ""; } @Override public String getIconURL() { return ""; } @Override public List getParams() { return new ArrayList<>(); } }; public static ScriptDescriptor errorScriptDescriptor( final PySyntaxError ex ) { return new ScriptDescriptor() { @Override public String getLabel() { return "ERROR"; } @Override public String getTitle() { return "PySyntaxError"; } @Override public String getDescription() { return ex.traceback.dumpStack(); } @Override public String getIconURL() { return ""; } @Override public List getParams() { return Collections.emptyList(); } }; } /** * return the script description and arguments. * * @param script the script Jython code. * @param params any operator-defined values. * @return * @throws IOException */ public static ScriptDescriptor describeScript(String script, Map params) throws IOException { return describeScript( null, script, params ); } /** * return the script description and arguments. * @param env the environment, containing PWD and maybe DOM. * @param script the script Jython code. * @param params any operator-defined values. * @return a description of the script parameters and metadata. * @throws IOException */ public static ScriptDescriptor describeScript( Map env, String script, Map params) throws IOException { String prog; try { prog= simplifyScriptToGetParams(script, true); // removes calls to slow methods, and gets the essence of the controls of the script. } catch ( PySyntaxError ex ) { return errorScriptDescriptor(ex); } logger.log(Level.FINER, "Simplified script: {0}", prog); PythonInterpreter interp; try { interp = createInterpreter(true); } catch (IOException ex) { logger.log( Level.WARNING, ex.getMessage(), ex ); return EMPTY; } if (env != null) { for (Entry ent : env.entrySet()) { if (ent.getKey() == null) { logger.log(Level.WARNING, "parameter name was null"); } else if (ent.getValue() == null) { if (ent.getKey().equals("dom")) { logger.log(Level.FINE, "parameter \"dom\" value was set to null"); // Some scripts don't use dom. } else { logger.log(Level.WARNING, "parameter value was null"); } } else { logger.log(Level.FINER, "setting env {0} to {1}", new Object[]{ent.getKey(), ent.getValue()}); interp.set(ent.getKey(), ent.getValue()); } } } interp.set("autoplot2017._scriptLabel", ""); interp.set("autoplot2017._scriptTitle", ""); interp.set("autoplot2017._scriptDescription", ""); interp.set("autoplot2017._scriptIcon", ""); if ( params!=null ) { setParams(interp, params); } try { prog = JythonRefactory.fixImports(prog, ""); } catch (IOException ex) { Logger.getLogger(JythonUtil.class.getName()).log(Level.SEVERE, null, ex); } interp.exec(prog); interp.exec("import autoplot2017 as autoplot\n"); PyList sort = (PyList) interp.eval("autoplot._paramSort"); boolean altWhy = false; // I don't know why things are suddenly showing up in this other space. if (sort.isEmpty()) { try { sort = (PyList) interp.eval("_paramSort"); if (sort.size() > 0) { logger.warning("things are suddenly in the wrong space. This is because things are incorrectly imported."); altWhy = true; } } catch (PyException ex) { // good... } } final List result = new ArrayList(); for (int i = 0; i < sort.__len__(); i++) { String theParamName = (String) sort.get(i); PyList oo = (PyList) interp.eval("autoplot._paramMap['" + theParamName + "']"); if (altWhy) { oo = (PyList) interp.eval("_paramMap['" + (String) sort.get(i) + "']"); } Param p = new Param(); p.label = (String) sort.get(i); // should be name in the script if (p.label.startsWith("__")) { continue; // __doc__, __main__ symbols defined by Jython. } p.name = oo.__getitem__(0).toString(); // name in the URI p.deft = oo.__getitem__(1); p.doc = oo.__getitem__(2).toString(); PyObject oconstraints= oo.__getitem__(3); if (oconstraints instanceof PyList) { PyList pyList = ((PyList) oconstraints); List enums = new ArrayList(pyList.size()); for (int j = 0; j < pyList.size(); j++) { enums.add(j, pyList.get(j)); } p.enums = enums; } else if (oconstraints instanceof PyDictionary) { PyDictionary pyDict = ((PyDictionary) oconstraints); PyObject enumsObject; PyObject examplesObject; if (pyDict.has_key(new PyString("enum"))) { logger.warning("values should be used instead of enum"); enumsObject = pyDict.get(new PyString("enum")); } else if (pyDict.has_key(new PyString("values"))) { enumsObject = pyDict.get(new PyString("values")); } else { enumsObject = null; } if (pyDict.has_key(new PyString("examples"))) { examplesObject = pyDict.get(new PyString("examples")); } else { examplesObject = null; } Map constraints = new HashMap<>(); if ( enumsObject != null ) { if ( enumsObject instanceof PyList ) { PyList enumsList = (PyList) enumsObject; List enums = new ArrayList(enumsList.size()); for (int j = 0; j < enumsList.size(); j++) { enums.add(j, enumsList.get(j)); } p.enums = enums; PyObject labelsObject = pyDict.get(new PyString("labels")); if (labelsObject != null && labelsObject instanceof PyList) { PyList labelsList = (PyList) labelsObject; List labels = new ArrayList(labelsList.size()); for (int j = 0; j < labelsList.size(); j++) { labels.add(j, labelsList.get(j)); } constraints.put("labels", labels); } } else { logger.log(Level.WARNING, "should be a list: {0}", enumsObject); } } if ( examplesObject!=null ) { if ( examplesObject instanceof PyList ) { PyList examplesList = (PyList) examplesObject; List list = new ArrayList(examplesList.size()); for (int j = 0; j < examplesList.size(); j++) { list.add(j, examplesList.get(j)); } p.examples = list; PyObject labelsObject = pyDict.get(new PyString("labels")); if (labelsObject != null && labelsObject instanceof PyList) { PyList labelsList = (PyList) labelsObject; List labels = new ArrayList(labelsList.size()); for (int j = 0; j < labelsList.size(); j++) { labels.add(j, labelsList.get(j)); } constraints.put("labels", labels); } } else { logger.log(Level.WARNING, "should be a list: {0}", enumsObject); } } p.constraints = constraints; } p.value = params == null ? null : params.get(p.name); if (p.name.equals("resourceUri")) { p.name = "resourceURI"; // I will regret allowing for this sloppiness... } switch (p.name) { case "resourceURI": p.type = 'R'; p.deft = p.deft.toString(); if (p.value != null) { p.value = p.value.toString(); } break; case "timerange": p.type = 'T'; p.deft = p.deft.toString(); if (p.value != null) { p.value = p.value.toString(); } break; default: if (p.deft instanceof String) { p.type = 'A'; p.deft = p.deft.toString(); if (p.value != null) { p.value = p.value.toString(); } } else if (p.deft instanceof PyString) { p.type = 'A'; p.deft = p.deft.toString(); if (p.value != null) { p.value = p.value.toString(); } } else if (p.deft instanceof PyInteger) { //TODO: Consider if int types should be preserved. p.type = 'F'; p.deft = ((PyInteger) p.deft).__tojava__(int.class); if (p.value != null) { if (p.value.equals("False")) { p.value = 0; } else if (p.value.equals("True")) { p.value = 1; } else { p.value = Integer.parseInt(p.value.toString()); } } } else if (p.deft instanceof PyFloat) { p.type = 'F'; p.deft = ((PyFloat) p.deft).__tojava__(double.class); if (p.value != null) { p.value = Double.parseDouble(p.value.toString()); } } else if (p.deft instanceof PyJavaInstance) { Object pp = ((PyJavaInstance) p.deft).__tojava__(URI.class); if (pp == Py.NoConversion) { pp = ((PyJavaInstance) p.deft).__tojava__(Datum.class); if (pp == Py.NoConversion) { pp = ((PyJavaInstance) p.deft).__tojava__(DatumRange.class); if (pp == Py.NoConversion) { pp = ((PyJavaInstance) p.deft).__tojava__(URL.class); if (pp == Py.NoConversion) { pp = ((PyJavaInstance) p.deft).__tojava__(Color.class); p.type = 'C'; p.deft = pp; } else { p.type = 'L'; p.deft = pp; } } else { p.type = 'S'; p.deft = pp; } } else { p.type = 'D'; p.deft = pp; } } else { p.type = 'U'; p.deft = pp; } } break; } result.add(p); } final String label = String.valueOf(interp.eval("autoplot._scriptLabel")); final String title = String.valueOf(interp.eval("autoplot._scriptTitle")); final String description = String.valueOf(interp.eval("autoplot._scriptDescription")); final String icon = String.valueOf(interp.eval("autoplot._scriptIcon")); ScriptDescriptor sd = new ScriptDescriptor() { @Override public String getLabel() { return label; } @Override public String getTitle() { return title; } @Override public String getDescription() { return description; } @Override public String getIconURL() { return icon; } @Override public List getParams() { return result; } }; return sd; } /** * read all the lines of a script into a string. The reader will be closed. * * @param reader * @return * @throws IOException */ public static String readScript(Reader reader) throws IOException { String s; StringBuilder build = new StringBuilder(); try (BufferedReader breader = new BufferedReader(reader)) { s = breader.readLine(); while (s != null) { build.append(s).append("\n"); s = breader.readLine(); } } return build.toString(); } /** *

* scrape through the script looking for getParam calls. These are executed, * and we get labels and infer types from the defaults. For example,
* getParam( 'foo', 3.0 ) will always return a real and
* getParam( 'foo', 3 ) will always return an integer.
* * Other examples include:
* getParam( 'foo', 'A', '', [ 'A', 'B' ] ) constrains the values * to A or B
*

*

* Thinking about the future, people have asked that human-ready labels be * fixed to list selections. Constraints should be added to number * parameters to specify ranges. And last it would be nice to specify when a * parameter is ignored by the script (dA is not used is mode B is active). *
* getParam( 'foo', 3, '', { 'min':0, 'max':10 } ) might (Not * implemented) constrain ranges
* getParam( 'sc', 'A', '', [ 'A', 'B' ], { 'A':'big one', 'B':'little * one' } ) might (Not implemented) allow labels
* getParam( 'foo', 'dA', '', [], { '_ignoreIf':'sc==B' } ) might * (Not implemented) allow groups to be disabled when not active
*

*

* A few things the Autoplot script developer must know: *

    *
  • getParam calls can only contain literals, and each must be executable * as if it were the only line of code. This may be relaxed in the future. *
  • the entire getParam line must be on one line. This too may be * relaxed. *
*

* * @param script A string containing the entire Jython program. * @return list of parameter descriptions, in the order they were * encountered in the file. * @throws PyException */ public static List getGetParams(String script) throws PyException { return getGetParams(null,script, new HashMap()); } /** * look through the script, removing expensive calls to make a script that * can be executed to get a list of getParam calls. The user can provide a * list of current settings, so that the thread of execution is matched. * * @param script any jython script. * @param params user-specified values. * @return a list of parameters. * @throws PyException */ public static List getGetParams(String script, Map params) throws PyException { return getGetParams(null, script, params); } /** * look through the script, removing expensive calls to make a script that * can be executed to get a list of getParam calls. The user can provide a * list of current settings, so that the thread of execution is matched. * Each parameter is given a type code: **
{@code
     * R resourceURI, special string
     * T timerange, special string
     * A string
     * F float, double, or int
     * U URI
     * L URL
     * D Datum
     * S DatumRange
     * C Color
     *
     *}
* note "arg_0" "arg_1" are used to refer to positional (unnamed) * parameters. * * @param env any values which may be defined already, such as "dom" and "PWD" * @param script any jython script. * @param params user-specified values or null. * @return a list of parameters. * @throws PyException * @deprecated use describeScript * @see #describeScript(java.util.Map, java.lang.String, java.util.Map) */ public static List getGetParams(Map env, String script, Map params) throws PyException { String prog = simplifyScriptToGetParams(script, true); // removes calls to slow methods, and gets the essence of the controls of the script. logger.log(Level.FINER, "Simplified script: {0}", prog); PythonInterpreter interp; try { interp = createInterpreter(true); } catch (IOException ex) { return new ArrayList(); } if (env != null) { for (Entry ent : env.entrySet()) { if (ent.getKey() == null) { logger.log(Level.WARNING, "parameter name was null"); } else if (ent.getValue() == null) { if (ent.getKey().equals("dom")) { logger.log(Level.FINE, "parameter \"dom\" value was set to null"); // Some scripts don't use dom. } else { logger.log(Level.WARNING, "parameter value was null"); } } else { interp.set(ent.getKey(), ent.getValue()); } } } if (params != null) { // I don't think this has ever worked (jbf,20190923) setParams(interp, params); } try { prog = JythonRefactory.fixImports(prog, ""); } catch (IOException ex) { Logger.getLogger(JythonUtil.class.getName()).log(Level.SEVERE, null, ex); } interp.exec(prog); interp.exec("import autoplot2017 as autoplot\n"); PyList sort = (PyList) interp.eval("autoplot._paramSort"); boolean altWhy = false; // I don't know why things are suddenly showing up in this other space. if (sort.isEmpty()) { try { sort = (PyList) interp.eval("_paramSort"); if (sort.size() > 0) { logger.warning("things are suddenly in the wrong space. This is because things are incorrectly imported."); altWhy = true; } } catch (PyException ex) { // good... } } List result = new ArrayList(); for (int i = 0; i < sort.__len__(); i++) { String theParamName = (String) sort.get(i); PyList oo = (PyList) interp.eval("autoplot._paramMap['" + theParamName + "']"); if (altWhy) { oo = (PyList) interp.eval("_paramMap['" + (String) sort.get(i) + "']"); } Param p = new Param(); p.label = (String) sort.get(i); // should be name in the script if (p.label.startsWith("__")) { continue; // __doc__, __main__ symbols defined by Jython. } p.name = oo.__getitem__(0).toString(); // name in the URI p.deft = oo.__getitem__(1); p.doc = oo.__getitem__(2).toString(); if (oo.__getitem__(3) instanceof PyList) { PyList pyList = ((PyList) oo.__getitem__(3)); List enums = new ArrayList(pyList.size()); for (int j = 0; j < pyList.size(); j++) { enums.add(j, pyList.get(j)); } p.enums = enums; } else if (oo.__getitem__(3) instanceof PyDictionary) { PyDictionary pyDict = ((PyDictionary) oo.__getitem__(3)); PyObject enumsObject; if (pyDict.has_key(new PyString("enum"))) { enumsObject = pyDict.pop(new PyString("enum")); } else if (pyDict.has_key(new PyString("values"))) { enumsObject = pyDict.pop(new PyString("values")); } else { enumsObject = null; } Map constraints = new HashMap<>(); if (enumsObject != null && enumsObject instanceof PyList) { PyList enumsList = (PyList) enumsObject; List enums = new ArrayList(enumsList.size()); for (int j = 0; j < enumsList.size(); j++) { enums.add(j, enumsList.get(j)); } p.enums = enums; PyObject labelsObject = pyDict.pop(new PyString("labels")); if (labelsObject != null && labelsObject instanceof PyList) { PyList labelsList = (PyList) labelsObject; List labels = new ArrayList(labelsList.size()); for (int j = 0; j < labelsList.size(); j++) { labels.add(j, labelsList.get(j)); } constraints.put("labels", labels); } } p.constraints = constraints; } p.value = params == null ? null : params.get(p.name); if (p.name.equals("resourceUri")) { p.name = "resourceURI"; // I will regret allowing for this sloppiness... } switch (p.name) { case "resourceURI": p.type = 'R'; p.deft = p.deft.toString(); if (p.value != null) { p.value = p.value.toString(); } break; case "timerange": p.type = 'T'; p.deft = p.deft.toString(); if (p.value != null) { p.value = p.value.toString(); } break; default: if (p.deft instanceof String) { p.type = 'A'; p.deft = p.deft.toString(); if (p.value != null) { p.value = p.value.toString(); } } else if (p.deft instanceof PyString) { p.type = 'A'; p.deft = p.deft.toString(); if (p.value != null) { p.value = p.value.toString(); } } else if (p.deft instanceof PyInteger) { //TODO: Consider if int types should be preserved. p.type = 'F'; p.deft = ((PyInteger) p.deft).__tojava__(int.class); if (p.value != null) { if (p.value.equals("False")) { p.value = 0; } else if (p.value.equals("True")) { p.value = 1; } else { p.value = Integer.parseInt(p.value.toString()); } } } else if (p.deft instanceof PyFloat) { p.type = 'F'; p.deft = ((PyFloat) p.deft).__tojava__(double.class); if (p.value != null) { p.value = Double.parseDouble(p.value.toString()); } } else if (p.deft instanceof PyJavaInstance) { Object pp = ((PyJavaInstance) p.deft).__tojava__(URI.class); if (pp == Py.NoConversion) { pp = ((PyJavaInstance) p.deft).__tojava__(Datum.class); if (pp == Py.NoConversion) { pp = ((PyJavaInstance) p.deft).__tojava__(DatumRange.class); if (pp == Py.NoConversion) { pp = ((PyJavaInstance) p.deft).__tojava__(URL.class); if (pp == Py.NoConversion) { pp = ((PyJavaInstance) p.deft).__tojava__(Color.class); p.type = 'C'; p.deft = pp; } else { p.type = 'L'; p.deft = pp; } } else { p.type = 'S'; p.deft = pp; } } else { p.type = 'D'; p.deft = pp; } } else { p.type = 'U'; p.deft = pp; } } break; } result.add(p); } return result; } /** * return a list of the getDataSet calls, from index to simplified * getDataSet call. Experimental--interface may change * * @param env * @param script * @param params * @return * @throws PyException */ public static Map getGetDataSet(Map env, String script, Map params) throws PyException { String[] ss = script.split("\n"); for (int i = ss.length - 1; i >= 0; i--) { if (!ss[i].contains("getDataSet")) { ss[i] = ""; } else { break; } } StringBuilder prog1 = new StringBuilder(ss[0]); prog1.append("\n"); for (int i = 1; i < ss.length; i++) { prog1.append(ss[i]).append("\n"); } String prog = prog1.toString(); logger.log(Level.FINER, "Simplified script: {0}", prog); PythonInterpreter interp; try { interp = createInterpreter(true); } catch (IOException ex) { return Collections.emptyMap(); } if (env != null) { for (Entry ent : env.entrySet()) { if (ent.getKey() == null) { logger.log(Level.WARNING, "parameter name was null"); } else if (ent.getValue() == null) { logger.log(Level.WARNING, "parameter value was null"); } else { interp.set(ent.getKey(), ent.getValue()); } } } if (params != null) { setParams(interp, params); } interp.set("timerange", "timerange"); String redefineGDS = "gds={}\nngds=0\ndef getDataSet( uri, timerange='', map=0 ):\n global ngds\n global gdsi\n gds[ngds]=uri+' '+timerange\n ngds=ngds+1\n"; interp.exec(redefineGDS); try { interp.exec(prog); } catch (PyException ex) { logger.log(Level.WARNING, null, ex); throw ex; } Map result = new LinkedHashMap<>(); PyDictionary r = (PyDictionary) interp.get("gds"); for (Object k : r.keys()) { result.put(k.toString(), r.get(Py.java2py(k)).toString()); } //for ( Entry e : r.entrySet() ) { // result.put( e.getKey().toString(), e.getValue().toString() ); //} return result; } /** * return a Java Map for a Python dictionary. * * @param pd * @return */ public static Map pyDictionaryToMap(PyDictionary pd) { Map m = new LinkedHashMap<>(); for (Object tt : pd.items()) { String k = ((PyTuple) tt).get(0).toString(); Object o = ((PyTuple) tt).get(1); if (o instanceof PyString) { m.put(String.valueOf(k), o.toString()); } else if (o instanceof PyQDataSet) { m.put(String.valueOf(k), ((PyQDataSet) o).ds); } else if (o instanceof PyDatum) { m.put(String.valueOf(k), ((PyDatum) o).datum); } else if (o instanceof PyFloat) { m.put(String.valueOf(k), Py.tojava((PyObject) o, "java.lang.Double")); } else if (o instanceof PyDictionary) { m.put(String.valueOf(k), pyDictionaryToMap((PyDictionary) o)); } else { logger.log(Level.INFO, "assuming Java type where conversion is not implemented: {0}", o); m.put(String.valueOf(k), o); } } return m; } /** * scrape script for local variables, looking for assignments. The reader is * closed after reading. * * @param reader the source for the script. It is closed when the code * executes properly. * @return a map of the local variable name to the line containing it. * @throws java.io.IOException */ public static Map getLocals(BufferedReader reader) throws IOException { try { String s = reader.readLine(); Pattern assignPattern = Pattern.compile("\\s*([_a-zA-Z][_a-zA-Z0-9]*)\\s*=.*(#(.*))?"); Pattern defPattern = Pattern.compile("def .*"); boolean inDef = false; Map result = new LinkedHashMap<>(); // from ID to description while (s != null) { if (inDef == false) { Matcher defm = defPattern.matcher(s); if (defm.matches()) { inDef = true; } } else { if (s.length() > 0 && !Character.isWhitespace(s.charAt(0))) { Matcher defm = defPattern.matcher(s); inDef = defm.matches(); } } if (!inDef) { Matcher m = assignPattern.matcher(s); if (m.matches()) { if (m.group(3) != null) { result.put(m.group(1), m.group(3)); } else { result.put(m.group(1), s); } } } s = reader.readLine(); } return result; } finally { reader.close(); } } /** * return python code that is equivalent, except it has no side-effects like * plotting. This code is not exact, for example (a,b)= (1,2) is not * supported. This code is run to support completions. * * @param eval string containing the entire program. * @return the script as a string, with side-effects removed. * @deprecated this should not be used, because newer codes use the * fully-implemented Jython parser. */ public static String removeSideEffects(String eval) { BufferedReader reader = new BufferedReader(new StringReader(eval)); StringBuilder result = new StringBuilder(); try { String s = reader.readLine(); Pattern assignPattern = Pattern.compile("\\s*([_a-zA-Z][_a-zA-Z0-9]*)\\s*=.*(#(.*))?"); Pattern defPattern = Pattern.compile("def .*"); Pattern importPattern1 = Pattern.compile("from .*"); Pattern importPattern2 = Pattern.compile("import .*"); boolean inDef = false; while (s != null) { int comment = s.indexOf("#"); if (comment > -1) { s = s.substring(0, comment); } boolean sideEffect = true; if (s.length() > 1 && Character.isWhitespace(s.charAt(0))) { // just skip over routines. s = reader.readLine(); continue; } if (inDef == false) { Matcher defm = defPattern.matcher(s); if (defm.matches()) { inDef = true; sideEffect = false; } } else { Matcher defm = defPattern.matcher(s); if (defm.matches()) { if (sideEffect) { result.append(" pass\n"); } } if (s.length() > 0 && !Character.isWhitespace(s.charAt(0))) { inDef = defm.matches(); if (inDef) { sideEffect = false; //TODO: what about blank line, this isn't an "END" } } if (inDef && s.trim().equals("pass")) { // syntax error otherwise. sideEffect = false; } } if (!inDef) { Matcher m = assignPattern.matcher(s); if (m.matches()) { sideEffect = false; } else if (importPattern1.matcher(s).matches()) { sideEffect = false; } else if (importPattern2.matcher(s).matches()) { sideEffect = false; } } if (!sideEffect) { result.append(s).append("\n"); } s = reader.readLine(); } if (inDef) { result.append(" pass\n"); } } catch (IOException ex) { logger.log(Level.SEVERE, ex.getMessage(), ex); } finally { try { reader.close(); } catch (IOException ex) { logger.log(Level.SEVERE, ex.getMessage(), ex); } } return result.toString(); } /** * join the array using the delimiter join( ['a','b'], '_' ) -> a_b Note * Java 8 finally has a join, and this should be used when Java 8 is * available. * * @param list strings to join * @param delim * @return the joined string */ public static String join(String[] list, String delim) { return join(Arrays.asList(list), delim); } /** * join the array using the delimiter join( ['a','b'], '_' ) -> a_b Note * Java 8 finally has a join, and this should be used when Java 8 is * available. * * @param list strings to join * @param delim * @return the joined string */ public static String join(List list, String delim) { if (list.isEmpty()) { return ""; } else { StringBuilder result = new StringBuilder(list.get(0)); for (int i = 1; i < list.size(); i++) { result.append(delim).append(list.get(i)); } return result.toString(); } } // public static void main( String[] args ) throws IOException { // main_test1(args); // } // // /** // * test the getGetParams for a script, seeing if we can reduce // * and run the script within interactive time. // * // * @param file // * @throws Exception // */ // private static void doTestGetParams( String testId, String file ) { // long t0= System.currentTimeMillis(); // System.err.println("== test "+testId+": "+ file + " ==" ); // // try { // String script= JythonUtil.readScript( new FileReader(file) ); // String scrip= org.autoplot.jythonsupport.JythonUtil.simplifyScriptToGetParams(script,true); // File f= new File(file); // String fout= "./test038_"+f.getName(); // try ( FileWriter fw= new FileWriter(fout) ) { // fw.append(scrip); // } // List parms= org.autoplot.jythonsupport.JythonUtil.getGetParams( script ); // for ( Param p: parms ) { // System.err.println(p); // } // System.err.println( String.format( "read params in %d millis: %s\n", System.currentTimeMillis()-t0, file ) ); // } catch ( Exception ex ) { // logger.log(Level.WARNING,null,ex); // System.err.println( String.format( "failed within %d millis: %s\n", System.currentTimeMillis()-t0, file ) ); // } // // } // // public static void main_test1(String[] args ) throws FileNotFoundException { // doTestGetParams("006","/home/jbf/ct/hudson/script/test038/yab_20131003.jy"); //TODO: needs fixing // doTestGetParams("000","/home/jbf/ct/hudson/script/test038/trivial.jy"); // doTestGetParams("001","/home/jbf/ct/hudson/script/test038/demoParms0.jy"); // doTestGetParams("002","/home/jbf/ct/hudson/script/test038/demoParms1.jy"); // doTestGetParams("003","/home/jbf/ct/hudson/script/test038/demoParms.jy"); // doTestGetParams("004","/home/jbf/ct/hudson/script/test038/rbsp/emfisis/background_removal_wfr.jyds"); // } }