QDataSets are wrapped so that operators are overloaded.
*
a standard set of names are imported.
*
* This also adds things to the Jython search path (see
* getLocalJythonAutoplotLib) so imports will find them.
*
* @param sandbox limit symbols to safe symbols for server.
* @return PythonInterpreter ready for commands.
* @throws java.io.IOException
*/
public static InteractiveInterpreter createInterpreter(boolean sandbox) throws IOException {
if (PySystemState.cachedir == null) {
String autoplotData= AutoplotSettings.settings().resolveProperty(AutoplotSettings.PROP_AUTOPLOTDATA);
System.setProperty("python.cachedir", autoplotData + "/pycache");
}
/// http://www.gossamer-threads.com/lists/python/python/697524
org.python.core.PySystemState pySys = new org.python.core.PySystemState();
//pySys.setdefaultencoding("utf8"); //doesn't work with Jython2.2, try with 2.5
String[] loadClasses = new String[]{"glob.py", "autoplot2017.py", "autoplotapp2017.py"}; // these must be in the root of the interpretter search path.
for (String pysrc : loadClasses) {
switch (pysrc) {
case "glob.py":
URL jarUrl = InteractiveInterpreter.class.getResource("/" + pysrc);
if (jarUrl != null) {
String f = getLocalJythonLib();
pySys.path.insert(0, new PyString(f));
} else {
logger.log(Level.WARNING, "Couldn''t find jar containing {0}. See https://sourceforge.net/p/autoplot/bugs/576/", pysrc);
} break;
case "autoplotapp2017.py":
{
String f = getLocalJythonAutoplotAppLib();
if (!pySys.path.contains(new PyString(f))) { // TODO possible bug here: PyString/String means local path is in there 4 times.
pySys.path.insert(0, new PyString(f));
} break;
}
default:
{
String f = getLocalJythonAutoplotLib();
if (!pySys.path.contains(new PyString(f))) {
pySys.path.insert(0, new PyString(f));
} break;
}
}
}
InteractiveInterpreter interp = new InteractiveInterpreter(null, pySys);
// try {
// System.err.println("java1-> "+interp.eval("java") );
// } catch ( Exception ex ) {
// System.err.println("java1-> ok!" );
// }
boolean loadAutoplotStuff = true;
if (loadAutoplotStuff) {
maybeLoadAdapters();
if (Util.isLegacyImports()) {
URL imports = JythonOps.class.getResource("/imports2017.py");
if (imports == null) {
throw new RuntimeException("unable to locate imports2017.py on classpath");
} else {
logger.log(Level.FINE, "loading imports2017.py from {0}", imports);
}
InputStream in = imports.openStream(); // note this stream will load in another stream.
byte[] bimports = FileUtil.readBytes(in);
//InputStream in = imports.openStream();
try {
interp.execfile(new ByteArrayInputStream(bimports), "/imports2017.py");
} finally {
in.close();
}
}
}
interp.set("dataset", new DatasetCommand());
interp.set("getDataSet", new GetDataSetCommand() );
interp.set("getDataSets", new GetDataSetsCommand() );
interp.set("monitor", new NullProgressMonitor());
// try {
// System.err.println("java2-> "+interp.eval("java") );
// } catch ( Exception ex ) {
// System.err.println("java2-> ok!" );
// }
return interp;
}
/**
* set up the interp variables scripts will often use, such as PWD and
* monitor.
*
* @param interp
* @param pwd
* @param resourceUri
* @param paramsl
* @param mon
*/
public static void setupInterp(PythonInterpreter interp, String pwd, String resourceUri, Map paramsl, ProgressMonitor mon) {
interp.set("PWD", pwd);
interp.exec("import autoplot2017 as autoplot");
interp.exec("autoplot.params=dict()");
for (Entry e : paramsl.entrySet()) {
String s = e.getKey();
if (!s.equals("arg_0") && !s.equals("script")) {
String sval = e.getValue();
sval = JythonUtil.maybeQuoteString(sval);
logger.log(Level.FINE, "autoplot.params[''{0}'']={1}", new Object[]{s, sval});
interp.exec("autoplot.params['" + s + "']=" + sval);
}
}
if (resourceUri != null) {
interp.set("resourceURI", resourceUri); // legacy
interp.exec("autoplot.params['" + "resourceURI" + "']=" + JythonUtil.maybeQuoteString(resourceUri));
}
interp.set("monitor", mon);
try (InputStream in = JythonOps.class.getResource("/autoplot2017.py").openStream()) {
interp.execfile(in, "/autoplot2017.py"); // import everything into default namespace.
} catch (IOException ex) {
logger.log(Level.SEVERE, ex.getMessage(), ex);
}
}
/**
* transfer the contents of in to out. in and out are closed after the
* operation. //TODO: other implementations of this exist...
*
* @param in
* @param out
* @throws IOException
*/
private static void transferStream(InputStream in, OutputStream out) throws IOException {
byte[] buf = new byte[2048];
int n;
try {
n = in.read(buf);
while (n > -1) {
out.write(buf, 0, n);
n = in.read(buf);
}
} finally {
out.close();
in.close();
}
}
/**
* ensure that the file has a parent writable directory.
*
* @param file
* @return true if the folder could be made.
*/
private static boolean makeHomeFor(File file) {
File f = file.getParentFile();
if (!f.exists()) {
return f.mkdirs();
} else {
return true;
}
}
/**
* copy everything out to autoplot_data/jython without going to web again.
* The old code showed issues where autoplot.org could not be resolved.
*
* @return the item to add to the Jython search path.
* @throws IOException
*/
private static String getLocalJythonLib() throws IOException {
File ff2 = new File(AutoplotSettings.settings().resolveProperty(AutoplotSettings.PROP_AUTOPLOTDATA));
File ff3 = new File(ff2.toString() + "/jython");
File ff4 = new File(ff2.toString() + "/jython/zlib.py");
if (ff4.exists()) {
return ff3.toString();
}
synchronized (JythonUtil.class) {
if (!ff3.exists()) {
if (!ff3.mkdirs()) {
throw new IOException("Unable to mkdirs " + ff3);
}
}
}
if (JythonUtil.class.getResource("/pylisting.txt") == null) {
throw new IllegalArgumentException("unable to find pylisting.txt in application, which is needed to install Jython codes.");
} else {
logger.log(Level.FINE, "unpacking jython codes in {0}", JythonUtil.class.getResource("/pylisting.txt"));
try (BufferedReader r = new BufferedReader(new InputStreamReader(JythonUtil.class.getResourceAsStream("/pylisting.txt")))) {
String s = r.readLine();
while (s != null) {
File ff5 = new File(ff3, s);
logger.log(Level.FINER, "copy to local folder Jython code: {0}", s);
InputStream in = JythonUtil.class.getResourceAsStream("/" + s);
if (in == null) {
throw new IllegalArgumentException("unable to find Jython code which should be embedded in application: " + s);
}
if (s.contains("/")) {
if (!makeHomeFor(ff5)) {
throw new IOException("Unable to makeHomeFor " + ff5);
}
}
try (FileOutputStream out = new FileOutputStream(ff5)) {
transferStream(in, out);
} finally {
in.close();
if (new File(ff3, s).setReadOnly() == false) {
logger.log(Level.FINER, "set read-only on file {0} failed", s);
}
if (new File(ff3, s).setWritable(true, true) == false) {
logger.log(Level.FINER, "set write for user only on file {0} failed", s);
}
}
s = r.readLine();
}
}
}
logger.fine(" ...done");
return ff3.toString();
}
/**
* copy all the app stuff to autoplot_data/jython without going to web
* again.
*
* @return the item to add to the Jython search path.
* @throws IOException
*/
private static String getLocalJythonAutoplotAppLib() throws IOException {
File ff2 = new File(AutoplotSettings.settings().resolveProperty(AutoplotSettings.PROP_AUTOPLOTDATA));
File ff3 = new File(ff2.toString() + "/jython");
File ff4 = new File(ff3.toString(), "pylistingapp2017.txt");
if (ff4.exists()) {
return ff3.toString();
}
synchronized (JythonUtil.class) {
if (!ff3.exists()) {
if (!ff3.mkdirs()) {
throw new IOException("Unable to mkdirs " + ff3);
}
}
}
if (JythonUtil.class.getResource("/pylistingapp2017.txt") == null) {
logger.log(Level.FINE, "unable to find pylistingapp2017.txt in application, assuming this is not the Autoplot client application.");
} else {
logger.log(Level.FINE, "unpacking jython codes in {0}", JythonUtil.class.getResource("/pylistingapp2017.txt"));
try (BufferedReader r = new BufferedReader(new InputStreamReader(JythonUtil.class.getResourceAsStream("/pylistingapp2017.txt")))) {
String s = r.readLine();
while (s != null) {
int i = s.indexOf("#");
if (i > -1) {
s = s.substring(0, i);
}
s = s.trim();
if (s.length() > 0) {
File ff5 = new File(ff3, s);
logger.log(Level.FINER, "copy to local folder Jython code: {0}", s);
if (s.contains("/")) {
if (!makeHomeFor(ff5)) {
throw new IOException("Unable to makeHomeFor " + ff5);
}
}
if (ff5.exists()) {
logger.fine("already have file, skip...");
s = r.readLine();
continue;
}
InputStream in = JythonUtil.class.getResourceAsStream("/" + s);
if (in == null) {
throw new IllegalArgumentException("unable to find jython code which should be embedded in application: " + s);
}
//Re https://sourceforge.net/p/autoplot/bugs/1724/:
//Really each file should be copied and then renamed.
try (FileOutputStream out = new FileOutputStream(ff5)) {
transferStream(in, out);
} finally {
in.close();
if (new File(ff3, s).setReadOnly() == false) {
logger.log(Level.FINER, "set read-only on file {0} failed", s);
}
if (new File(ff3, s).setWritable(true, true) == false) {
logger.log(Level.FINER, "set write for user only on file {0} failed", s);
}
}
}
s = r.readLine();
}
}
}
return ff3.toString();
}
/**
* copy the two Jython files specific to Autoplot into the user's
* autoplot_data/jython folder. This reads the version from the first line
* of the autoplot2017.py.
*
* @return the item to add to the Jython search path.
* @throws IOException
*/
private static String getLocalJythonAutoplotLib() throws IOException {
File ff2 = new File(AutoplotSettings.settings().resolveProperty(AutoplotSettings.PROP_AUTOPLOTDATA));
File ff3 = new File(ff2.toString() + "/jython");
File ff4 = new File(ff3, "autoplot2017.py");
String vers = "";
// This is the version that Autoplot would like to find, and should be found within the Java class path.
double currentVersion = 2.00; //rfe320 improved getParam support.
if (ff4.exists()) {
try (BufferedReader r = new BufferedReader(new FileReader(ff4))) {
String line = r.readLine();
if (line != null) {
Pattern versPattern = Pattern.compile("# autoplot2017.py v([\\d\\.]+) .*"); // must be parsable as a double.
Matcher m = versPattern.matcher(line);
if (m.matches()) {
vers = m.group(1);
}
}
}
}
if (logger.isLoggable(Level.FINE)) {
logger.fine("== JythonUtil getLocalJythonAutoplotLib ==");
logger.log(Level.FINE, "ff4.exists()={0}", ff4.exists());
logger.log(Level.FINE, "vers={0}", vers);
logger.log(Level.FINE, "currentVersion={0}", currentVersion);
}
if (!ff4.exists() || vers.equals("") || Double.parseDouble(vers) < currentVersion) {
logger.log(Level.FINE, "looking for version={0} of {1}, but didn''t find it.", new Object[]{currentVersion, ff4});
logger.log(Level.FINE, "doesn't seem like we have the right file, downloading...");
synchronized (JythonUtil.class) {
if (!ff3.exists()) {
if (!ff3.mkdir()) {
throw new IOException("Unable to mkdir " + ff3);
}
}
}
String[] ss = new String[]{"autoplot2017.py", "autoplotapp2017.py"};
for (String s : ss) {
try (InputStream in = JythonUtil.class.getResourceAsStream("/" + s);
FileOutputStream out = new FileOutputStream(new File(ff3, s))) {
transferStream(in, out);
}
if (new File(ff3, s).setReadOnly() == false) {
logger.log(Level.FINER, "set read-only on file {0} failed", s);
}
if (new File(ff3, s).setWritable(true, true) == false) {
logger.log(Level.FINER, "set write for user only on file {0} failed", s);
}
}
}
logger.fine(" ...done");
return ff3.toString();
}
/**
* check the script that it doesn't redefine symbol names like "str"
*
* @param uri, such as sftp://user@host/script.jy
* @param errs an empty list where the errors can be logged.
* @return true if an err is suspected.
* @throws java.io.IOException
*/
public static boolean pythonLint(URI uri, List errs) throws IOException {
LineNumberReader reader;
File src = DataSetURI.getFile(uri, new NullProgressMonitor());
reader = new LineNumberReader(new BufferedReader(new FileReader(src)));
try {
return pythonLint(reader, errs);
} finally {
reader.close();
}
}
/**
* check the script that it doesn't redefine symbol names like "str"
*
* @param reader, which will be not be closed here.
* @param errs an empty list where the errors can be logged.
* @return true if an err is suspected.
* @throws java.io.IOException
*/
public static boolean pythonLint(LineNumberReader reader, List errs) throws IOException {
StringBuilder build= new StringBuilder();
String line;
while ( ( line= reader.readLine() )!=null ) {
build.append(line).append("\n");
}
String script= build.toString();
List ll= StaticCodeAnalysis.showReassignFunctionCall( script, true, null );
for ( int i=0; i 0;
}
private static boolean haveloadedAdapters = false;
/**
* load the adapters, once.
*/
private synchronized static void maybeLoadAdapters() {
if (!haveloadedAdapters) {
Py.getAdapter().addPostClass(new PyQDataSetAdapter());
Py.getAdapter().addPostClass(new PyDatumAdapter());
haveloadedAdapters = true;
}
}
/**
* scrape through the script looking for documentation declarations returns
* an map, possibly containing:
*
LABEL few words
*
TITLE sentence
*
DESCRIPTION short paragraph
*
* This would originally look for lines like:
* # TITLE: Text Recombinator
* but this has been deprecated and scripts should use setScriptTitle
* and setScriptDescription
*
* @param reader, open and ready to read, which will be closed.
* @return the documentation found.
* @see #getDocumentation(java.io.BufferedReader, java.net.URI)
* @throws java.io.IOException
*/
public static Map getDocumentation(BufferedReader reader ) throws IOException {
return getDocumentation( reader, null );
}
/**
* scrape through the script looking for documentation declarations returns
* an map, possibly containing:
*
LABEL few words
*
TITLE sentence
*
DESCRIPTION short paragraph
*
* This would originally look for lines like:
* # TITLE: Text Recombinator
* but this has been deprecated and scripts should use setScriptTitle
* and setScriptDescription
*
* @param reader, open and ready to read, which will be closed.
* @param resourceURI the location of the script to define PWD.
* @return the documentation found.
* @throws java.io.IOException
*/
public static Map getDocumentation(BufferedReader reader, URI resourceURI ) throws IOException {
String line = reader.readLine();
StringBuilder scriptBuilder= new StringBuilder();
while ( line!=null ) {
scriptBuilder.append(line).append('\n');
line= reader.readLine();
}
String script= scriptBuilder.toString();
Map env= new HashMap<>();
if ( resourceURI!=null ) {
URISplit split= URISplit.parse(resourceURI.toString());
env.put( "PWD", split.path );
}
ScriptDescriptor sd= org.autoplot.jythonsupport.JythonUtil.describeScript( env, script, null );
Map result = new HashMap<>();
if ( sd.getDescription().length()>0 ) result.put( "DESCRIPTION", sd.getDescription() );
if ( sd.getTitle().length()>0 ) result.put( "TITLE", sd.getTitle() );
if ( sd.getLabel().length()>0 ) result.put( "LABEL", sd.getLabel() );
if ( sd.getIconURL().length()>0 ) result.put( "ICONURL", sd.getIconURL() );
if ( result.isEmpty() ) {
reader= new BufferedReader( new StringReader(script) );
String s = reader.readLine();
Pattern p = Pattern.compile("#\\s*([a-zA-Z]+)\\s*:(.*)");
while (s != null) {
Matcher m = p.matcher(s);
if (m.matches()) {
String prop = m.group(1).toUpperCase();
String value = m.group(2).trim();
if (prop.equals("LABEL") || prop.equals("TITLE") || prop.equals("DESCRIPTION")) {
result.put(prop, value);
}
}
s = reader.readLine();
}
reader.close();
}
return result;
}
/**
* there are a number of functions which take a trivial amount of time to execute and are needed for some scripts, such as the
* string.upper() function. The commas are to guard against the id being a subset of another id ("lower," does not match
* "lowercase"). TODO: update this after Python upgrade.
* @see SimplifyScriptSupport#okay
*/
final static String[] okay = new String[] {
"range,", "xrange,", "irange,","map,","join,","len,","dict,","zip,",
"getParam,", "lower,", "upper,", "URI,", "URL,", "PWD,", "File,",
"setScriptDescription", "setScriptTitle", "setScriptLabel", "setScriptIcon",
"DatumRangeUtil,", "TimeParser,",
"str,", "int,", "long,", "float,", "datum,", "datumRange,","dataset,",
"indgen,", "findgen,","dindgen,",
"ones,", "zeros,",
"linspace,", "logspace,",
"dblarr,", "fltarr,", "strarr,", "intarr,", "bytarr,",
"ripples,", "split,",
"color,", "colorFromString,", "isinstance,",
"readConfiguration,"
};
/**
* return true if the function call is trivial to execute and can be
* evaluated within a few milliseconds.
*
* @param sn
* @return
*/
private static boolean trivialFunctionCall(SimpleNode sn) {
//there are a number of functions which take a trivial amount of time to execute and are needed for some scripts, such as
//the string.upper() function.
//The commas are to guard against the id being a subset of another id ("lower," does not match "lowercase").
//TODO: update this after Python upgrade.
if (sn instanceof Call) {
Call c = (Call) sn;
boolean klugdyOkay = false;
String ss = c.func.toString();
for (String s : okay) {
if (ss.contains(s)) {
klugdyOkay = true;
}
}
if (ss.startsWith("Attribute[")) {
klugdyOkay = true; // parms.keys()
}
if (klugdyOkay == false) {
if (ss.contains("TimeUtil") && ss.contains("now")) {
klugdyOkay = true;
}
}
logger.log(Level.FINER, "trivialFunctionCall={0} for {1}", new Object[]{klugdyOkay, c.func.toString()});
return klugdyOkay;
} else if ( sn instanceof org.python.parser.ast.Num ) {
return true;
} else if ( sn instanceof org.python.parser.ast.Name ) {
return true;
} else if ( sn instanceof org.python.parser.ast.Str ) {
return true;
} else {
return false;
}
}
private static class MyVisitorBase extends VisitorBase {
boolean looksOkay = true;
boolean visitNameFail = false;
HashSet names = new HashSet();
SimpleNode node;
MyVisitorBase(HashSet names, SimpleNode node ) {
this.names = names;
this.node = node; // for reference
if ( this.node.toString().contains("id=r_erg") ) {
System.err.println("HERE STOP 671");
}
}
@Override
public Object visitName(Name node) throws Exception {
if (!names.contains(node.id)) {
visitNameFail = true;
}
return null;
}
@Override
protected Object unhandled_node(SimpleNode sn) throws Exception {
return sn;
}
@Override
public void traverse(SimpleNode sn) throws Exception {
if (sn instanceof Call) {
boolean newLooksOkay = trivialFunctionCall(sn);
if (!newLooksOkay) {
logger.log(Level.FINE, "looksOkay=False, {0}", sn);
} else {
Call c= (Call)sn;
for ( exprType e: c.args ) {
if ( !simplifyScriptToGetParamsCanResolve( e, names ) ) {
newLooksOkay= false;
visitNameFail= true; // TODO: I don't understand why there are two variables...
}
}
}
looksOkay = newLooksOkay;
} else if (sn instanceof Assign) { // TODO: I have to admit I don't understand what traverse means. I would have thought it was all nodes...
Assign a = ((Assign) sn);
exprType et = a.value;
if (et instanceof Call) {
traverse( et );
}
} else if ( sn instanceof BinOp ) {
traverse(((BinOp) sn).left);
traverse(((BinOp) sn).right);
}
}
public boolean looksOkay() {
return looksOkay;
}
/**
* this contains a node whose name we can't resolve.
*
* @return
*/
public boolean visitNameFail() {
return visitNameFail;
}
}
/**
* inspect the node to look for function calls that are not to the function
* "getParam". This is awful code that will be rewritten when we upgrade
* Python to 2.7.
*
* @param o
* @param variableNames
* @return
*/
private static boolean simplifyScriptToGetParamsOkayNoCalls(SimpleNode o, HashSet variableNames) {
if (o instanceof Call) {
Call c = (Call) o;
if (!trivialFunctionCall(c)) {
logger.finest(String.format("%04d simplify->false: %s", o.beginLine, o.toString()));
return false;
}
}
MyVisitorBase vb = new MyVisitorBase(variableNames,o);
try {
o.traverse(vb);
logger.finest(String.format(" %04d simplify->%s: %s", o.beginLine, vb.looksOkay(), o));
return vb.looksOkay();
} catch (Exception ex) {
logger.log(Level.SEVERE, ex.getMessage(), ex);
}
logger.finest(String.format("!! %04d simplify->false: %s", o.beginLine, o));
return false;
}
/**
* can we resolve this node given the variable names we know?
*
* @param o
* @param variableNames
* @return true if the node can be resolved.
*/
private static boolean simplifyScriptToGetParamsCanResolve(SimpleNode o, HashSet variableNames) {
//if ( o.beginLine>=617 && o.beginLine<619 ) {
// System.err.println( "here at 617-ish");
//}
if (o instanceof Name) {
Name c = (Name) o;
if (!variableNames.contains(c.id)) {
logger.finest(String.format("%04d canResolve->false: %s", o.beginLine, o.toString()));
return false;
}
}
if (o instanceof Attribute) {
Attribute at = (Attribute) o;
while (at.value instanceof Attribute || at.value instanceof Subscript) {
if (at.value instanceof Attribute) {
at = (Attribute) at.value;
} else {
Subscript s = (Subscript) at.value;
if (s.value instanceof Attribute) {
at = (Attribute) s.value;
} else {
return false; // oh just give up...
}
}
}
if (at.value instanceof Name) {
Name n = (Name) at.value;
if (!variableNames.contains(n.id)) {
return false;
}
} else if ( at.value instanceof Call ) {
return simplifyScriptToGetParamsCanResolve( at.value, variableNames );
} else {
return false;
}
}
if ( o instanceof Call ) { // ds.property( QDataSet.DEPEND_0 )
Call c= (Call) o;
if ( c.func instanceof Attribute ) {
if ( !simplifyScriptToGetParamsCanResolve( c.func, variableNames ) ){
return false;
}
}
}
MyVisitorBase vb = new MyVisitorBase(variableNames,o);
try {
o.traverse(vb);
logger.finest(String.format(" %04d canResolve->%s: %s", o.beginLine, vb.visitNameFail, o));
return !vb.visitNameFail;
} catch (Exception ex) {
logger.log(Level.SEVERE, ex.getMessage(), ex);
}
logger.finest(String.format("!! %04d canResolve->false: %s", o.beginLine, o));
return false;
}
/**
* put quotes around values that appear to be strings. We see if it's
* parsable as a double or the keyword True or False.
*
* @param sval the string, for example "2015" "'2015'" or "2015-01-01"
* @return 2015, "'2015'", "'2015-01-01'"
*/
public static String maybeQuoteString(String sval) {
boolean isNumber = false;
try {
Double.parseDouble(sval);
} catch (NumberFormatException ex) {
isNumber = false;
}
if (!isNumber && !sval.equals("True") && !sval.equals("False")) {
if (sval.length() == 0 || !(sval.startsWith("'") && sval.endsWith("'"))) {
sval = String.format("'%s'", sval);
}
}
return sval;
}
/**
* pop off the quotes to get the text inside.
*
* @param sval "'2015-01-01'"
* @return "2015-01-01"
*/
public static String maybeUnquoteString(String sval) {
if ((sval.startsWith("'") && sval.endsWith("'"))
|| (sval.startsWith("\"") && sval.endsWith("\""))) {
return sval.substring(1, sval.length() - 1);
} else {
return sval;
}
}
/**
* put each parameter into the dictionary autoplot.params.
*
* @param interp
* @param paramsl
*/
public static void setParams(PythonInterpreter interp, Map paramsl) {
interp.exec("import autoplot2017 as autoplot");
interp.exec("autoplot.params=dict()");
for (Entry e : paramsl.entrySet()) {
String s = e.getKey();
if (s.length() == 0) {
throw new IllegalArgumentException("param name is \"\": " + s);
}
if (!s.equals("arg_0") && !s.equals("script")) {
try {
String sval = e.getValue();
sval = maybeQuoteString(sval);
logger.log(Level.FINE, "autoplot.params[''{0}'']={1}", new Object[]{s, sval});
interp.exec("autoplot.params['" + s + "']=" + sval);
} catch (Exception ex) {
logger.warning(ex.getMessage());
}
}
}
}
/**
* return true if the call is a setScriptTitle or setScriptDescription call.
* setScriptTitle( 'Batch Master Demo' )
*
* @param o
* @param variableNames
* @return
*/
private static boolean isSetScriptCall(stmtType o, HashSet variableNames) {
if (o instanceof org.python.parser.ast.Expr) {
org.python.parser.ast.Expr expr = (org.python.parser.ast.Expr) o;
if (expr.value instanceof Call) {
Call c = ((Call) expr.value);
if (c.func instanceof Name) {
Name n = (Name) c.func;
if (n.id.equals("setScriptTitle")
|| n.id.equals("setScriptDescription")
|| n.id.equals("setScriptLabel")
|| n.id.equals("setScriptIcon")) {
return true;
}
}
return false;
}
return false;
}
return false;
}
/**
* return true if we can include this in the script without a huge
* performance penalty.
*
* @param o
* @return
*/
private static boolean simplifyScriptToGetParamsOkay(stmtType o, HashSet variableNames) {
//if ( o.beginLine==607 ) { // leave this commented code as a reference for debugging
// System.err.println("here at line "+o.beginLine);
//}
if ((o instanceof org.python.parser.ast.ImportFrom)) {
return true;
}
if ((o instanceof org.python.parser.ast.Import)) {
return true;
}
if ((o instanceof org.python.parser.ast.Assign)) {
Assign a = (Assign) o;
for ( exprType a1 : a.targets ) {
if ( a1 instanceof Subscript ) {
Subscript ss= (Subscript)a1;
if ( !simplifyScriptToGetParamsCanResolve(ss.value,variableNames) ) {
return false;
}
if ( !simplifyScriptToGetParamsCanResolve(ss.slice,variableNames) ) {
return false;
}
}
}
if (simplifyScriptToGetParamsOkayNoCalls(a.value, variableNames)) {
if (!simplifyScriptToGetParamsCanResolve(a.value, variableNames)) {
return false;
}
for (exprType target : a.targets) {
exprType et = (exprType) target;
if (et instanceof Name) {
String id = ((Name) target).id;
variableNames.add(id);
logger.log(Level.FINEST, "assign to variable {0}", id);
} else if (et instanceof Attribute) {
Attribute at = (Attribute) et;
while (at.value instanceof Attribute || at.value instanceof Subscript) {
if (at.value instanceof Attribute) {
at = (Attribute) at.value;
} else {
Subscript s = (Subscript) at.value;
if (s.value instanceof Attribute) {
at = (Attribute) s.value;
} else {
return false; // oh just give up...
}
}
}
if (at.value instanceof Name) {
Name n = (Name) at.value;
if (!variableNames.contains(n.id)) {
return false;
}
}
}
}
return true;
} else {
return false;
}
}
if ((o instanceof org.python.parser.ast.If)) {
return simplifyScriptToGetParamsOkayNoCalls(o, variableNames);
}
if ((o instanceof org.python.parser.ast.Print)) {
return false;
}
if ((o instanceof org.python.parser.ast.Expr)) { // preserve constant strings which are effective documentation
org.python.parser.ast.Expr exp= (org.python.parser.ast.Expr)o;
if ( exp.value instanceof org.python.parser.ast.Str ) {
return true;
}
}
logger.log(Level.FINEST, "not okay to simplify: {0}", o);
return false;
}
private static void maybeAppendSort(String theLine, StringBuilder result) {
int i = theLine.indexOf("getParam");
if (i != -1) {
i = theLine.indexOf("=");
String v = theLine.substring(0, i).trim();
int indent = theLine.indexOf(v);
if (indent > 0) {
result.append(theLine.substring(0, indent));
}
result.append("sort_.append( \'").append(v).append("\')\n");
}
}
/**
* return the indentation string (spaces or tabs or empty) for the line.
* @param line a line of Jython code.
* @return the indentation
*/
protected static String indentForLine( String line ) {
String[] ss2 = line.split("\\S", -2);
String indent = ss2[0];
return indent;
}
/**
* handle where a line continues on to the next line because of indent,
* and then where a closing parenthesis might be found as well.
* @param lines
* @param iline
* @return the new line number
*/
protected static int handleContinue( String[] lines, int iline ) {
if ( iline==lines.length-1 ) {
return iline;
}
String theLine= lines[iline];
int i= theLine.indexOf("#");
if ( i>-1 ) theLine= theLine.substring(0,i);
if ( theLine.trim().endsWith(":") ) {
logger.finer("line opens to block");
}
String thisIndent= indentForLine(theLine);
String nextIndent= indentForLine(lines[iline+1]);
while ( ilinethisIndent.length() ) {
iline++;
nextIndent= indentForLine(lines[iline+1]);
}
if ( iline=ss.length ) {
throw new IllegalArgumentException("lastLine is >= number of lines");
}
if ( !ss[0].equals("# simplifyScriptToGetParams") ) {
throw new IllegalArgumentException("first line must be '# simplifyScriptToGetParams'");
}
int acceptLine = -1; // first line to accept
int currentLine = 1; // current line we are writing (1 is first line).
StringBuilder result = new StringBuilder();
for (int istatement = 0; istatement < stmts.length; istatement++) {
stmtType o = stmts[istatement];
String theLine= SimplifyScriptSupport.getSourceForStatement( ss, o );
//if ( stmts.length==109 && o.beginLine >50 ) {
// System.err.println("theLine: "+ o.beginLine + " " +theLine);
//}
int lineCount= theLine.split("\n",-2).length;
if ( depth==0 ) {
logger.finest(theLine); //breakpoint here.
//System.err.println(theLine);
}
logger.log( Level.FINER, "line {0}: {1}", new Object[] { o.beginLine, theLine } );
if ( o.beginLine>0 ) {
if ( beginLine<0 && istatement==0 ) acceptLine= o.beginLine;
if ( lineCount>1 ) {
beginLine= o.beginLine - (lineCount-1) ;
} else {
beginLine= o.beginLine;
}
} else {
acceptLine = beginLine; // elif clause in autoplot-test038/lastSuccessfulBuild/artifact/test038_demoParms1.jy
}
if (beginLine > lastLine) {
continue;
}
if ( o instanceof TryExcept ) {
//System.err.println("here try except");
acceptLine= -1;
continue;
}
if (o instanceof org.python.parser.ast.If) {
if (acceptLine > -1) {
for (int i = acceptLine; i < beginLine; i++) {
appendToResult(result, ss[i]).append("\n");
}
}
If iff = (If) o;
boolean includeBlock;
if (simplifyScriptToGetParamsCanResolve(iff.test, variableNames)) {
for (int i = beginLine; i < iff.body[0].beginLine; i++) {
result.append(ss[i]).append("\n");
} // write out the 'if' part
includeBlock = true;
} else {
includeBlock = false;
}
int lastLine1; //lastLine1 is the last line of the "if" clause.
int elseLine=-1;
if (iff.orelse != null && iff.orelse.length > 0) {
if (iff.orelse[0].beginLine > 0) {
lastLine1 = iff.orelse[0].beginLine - 1; // -1 is for the "else:" part.
if ( ss[lastLine1].trim().startsWith("else") ) {
elseLine= lastLine1;
lastLine1=lastLine1-1;
} else if ( ss[lastLine1].trim().startsWith("elif") ) {
elseLine= lastLine1;
lastLine1=lastLine1-1;
}
} else {
if (iff.orelse[0] instanceof If) {
elseLine = ((If) iff.orelse[0]).test.beginLine;
lastLine1= elseLine-1;
} else {
lastLine1= elseLine+1;
logger.fine("failure to deal with another day...");
//throw new RuntimeException("this case needs to be dealt with...");
}
}
} else if ((istatement + 1) < stmts.length) {
lastLine1 = stmts[istatement + 1].beginLine - 1;
} else {
lastLine1 = lastLine;
}
if (includeBlock) {
String ss1 = simplifyScriptToGetParams(ss, iff.body, variableNames, -1, lastLine1, depth + 1);
if (ss1.trim().length() == 0) {
String line;
if ( iff.body[0].beginLine > 0) {
line = ss[iff.body[0].beginLine];
} else {
line = ss[iff.beginLine];
}
String indent = indentForLine( line );
result.append(indent).append("pass # ").append(line).append("\n");
logger.fine("things have probably gone wrong...");
} else {
appendToResult(result, ss1);
}
if (iff.orelse != null) {
if ( elseLine>-1 ) {
appendToResult(result, ss[elseLine]);
} else {
//appendToResult(result, ss[iff.orelse[0].beginLine]);
logger.fine("failure #2 to deal with another day...");
}
int lastLine2;
if ((istatement + 1) < stmts.length) {
lastLine2 = stmts[istatement + 1].beginLine - 1;
} else {
lastLine2 = lastLine;
}
String ss2 = simplifyScriptToGetParams(ss, iff.orelse, variableNames, elseLine + 1, lastLine2, depth + 1);
if (ss2.length() > 0) {
result.append("\n");
}
appendToResult(result, ss2);
if (ss2.length() == 0) { // we didn't add anything...
String line;
line = ss[iff.orelse[0].beginLine];
String indent = indentForLine( line );
result.append("\n").append(indent).append("pass # ").append(line).append("\n");
} else {
result.append("\n"); // write of the else or elif line
}
}
}
acceptLine = -1;
} else {
if (simplifyScriptToGetParamsOkay(o, variableNames)) {
if (acceptLine < 0) {
acceptLine = beginLine;
for (int i = currentLine; i < acceptLine; i++) {
String indent= indentForLine( ss[i] );
int icomment= ss[i].indexOf("#");
if ( icomment>=0 ) {
String line= indent + spaces.substring(indent.length(),icomment)+ss[i].substring(icomment);
result.append(line).append("\n");
}
currentLine = acceptLine;
}
}
} else if (isSetScriptCall(o, variableNames)) {
if (acceptLine < 0) {
acceptLine = beginLine;
for (int i = currentLine + 1; i < acceptLine; i++) {
result.append("\n");
currentLine = acceptLine;
}
}
} else {
if (acceptLine > -1) {
int thisLine = getBeginLine( ss, o ) ;
String indent= indentForLine( ss[acceptLine] );
for (int i = acceptLine; i < thisLine; i++) {
if ( ss[i].contains("getDataSet") ) {
appendToResult(result, indent + "pass #1139 "+ ss[i]).append("\n"); // TODO: kludge--how did this work before???
} else {
appendToResult(result, ss[i]).append("\n");
}
}
appendToResult(result, "\n");
currentLine = o.beginLine;
acceptLine = -1;
}
}
}
}
if (acceptLine > -1) {
lastLine= handleContinue( ss, lastLine );
int thisLine = lastLine;
for (int i = acceptLine; i <= thisLine; i++) {
appendToResult(result, ss[i]).append("\n");
}
}
return result.toString();
}
/**
* there's a problem where multi-line strings and expressions have a begin line at the end not the beginning.
* @param ss the script which has been parsed into lines.
* @param o the AST statement
* @return the line of the beginning of the statement.
*/
public static int getBeginLine( String[] ss, stmtType o ) {
int beginLine= o.beginLine;
if ( o instanceof org.python.parser.ast.Expr ) {
org.python.parser.ast.Expr expr= (org.python.parser.ast.Expr)o;
int bl2= expr.value.beginLine;
if ( bl2 -1) {
line = line.substring(0, ich);
}
if (line.contains("getParam")) {
llogger.log(Level.FINER, "getParam at line {0}", ilineNum);
lastLine = ilineNum;
withinSimplifyLine= true;
} else if (line.contains("setScriptTitle")) {
llogger.log(Level.FINER, "setScriptTitle at line {0}", ilineNum);
lastLine = ilineNum;
withinSimplifyLine= true;
} else if (line.contains("setScriptDescription")) {
llogger.log(Level.FINER, "setScriptDescription at line {0}", ilineNum);
lastLine = ilineNum;
withinSimplifyLine= true;
} else if (line.contains("setScriptLabel")) {
llogger.log(Level.FINER, "setScriptLabel at line {0}", ilineNum);
lastLine = ilineNum;
withinSimplifyLine= true;
} else if (line.contains("setScriptIcon")) {
llogger.log(Level.FINER, "setScriptIcon at line {0}", ilineNum);
lastLine = ilineNum;
withinSimplifyLine= true;
} else {
if ( !withinTripleQuote ) {
withinSimplifyLine= false;
}
}
if ( line.contains("'''") ) {
if ( withinTripleQuote ) {
if ( !Character.isWhitespace(line.charAt(0)) && withinSimplifyLine ) {
lastLine = ilineNum;
}
}
if ( withinTripleQuote ) {
llogger.log(Level.FINER, "close triple quote at line {0}", ilineNum);
} else {
llogger.log(Level.FINER, "open triple quote at line {0}", ilineNum);
}
withinTripleQuote= !withinTripleQuote;
}
}
if (lastLine == -1) {
return "";
}
// check for continuation in last getParam call.
while (ss.length > lastLine + 1 && ss[lastLine].trim().length() > 0 && Character.isWhitespace(ss[lastLine].charAt(0))) {
lastLine++;
}
// Chris showed that a closing bracket or paren doesn't need to be indented. See test038/jydsCommentBug.jyds
if (lastLine < ss.length) {
String closeParenCheck = ss[lastLine].trim();
if (closeParenCheck.equals(")") || closeParenCheck.equals("]")) {
lastLine++;
}
}
HashSet variableNames = new HashSet();
variableNames.add("getParam"); // this is what allows the getParam calls to be included.
variableNames.add("map");
variableNames.add("str"); // include casts.
variableNames.add("int");
variableNames.add("long");
variableNames.add("float");
variableNames.add("datum");
variableNames.add("datumRange");
variableNames.add("URI");
variableNames.add("URL");
variableNames.add("True");
variableNames.add("False");
variableNames.add("range");
variableNames.add("xrange");
variableNames.add("map");
variableNames.add("dict");
variableNames.add("zip");
variableNames.add("PWD");
try {
Module n = (Module) org.python.core.parser.parse(script, "exec");
return simplifyScriptToGetParams(ss, n.body, variableNames, 1, lastLine, 0);
} catch (PySyntaxError ex) {
throw ex;
}
}
/**
* support for the old getGetParams. Note this closes the reader.
*
* @param reader
* @return
* @throws IOException
* @throws PySyntaxError
*/
public static List getGetParams(Reader reader) throws IOException, PySyntaxError {
return getGetParams(null, readScript(reader), new HashMap());
}
/**
* Object containing a description of a script, containing its parameters
* and title describing it.
*/
public static interface ScriptDescriptor {
/**
* a short label, suitable for a menu item.
* @return a short label, suitable for a menu item.
*/
String getLabel();
/**
* a one-line title, suitable for a GUI heading.
* @return a one-line title, suitable for a GUI heading.
*/
String getTitle();
/**
* sentence or paragraph documenting the script.
* @return
*/
String getDescription();
/**
* the web address of an icon for the script.
* @return
*/
String getIconURL();
/**
* the list of the script parameters, containing the type and default values.
* @return
*/
List getParams();
//List getOutputParams(); // this should finally be done as well.
}
public static final ScriptDescriptor EMPTY = new ScriptDescriptor() {
@Override
public String getLabel() {
return "";
}
@Override
public String getTitle() {
return "";
}
@Override
public String getDescription() {
return "";
}
@Override
public String getIconURL() {
return "";
}
@Override
public List getParams() {
return new ArrayList<>();
}
};
public static ScriptDescriptor errorScriptDescriptor( final PySyntaxError ex ) {
return new ScriptDescriptor() {
@Override
public String getLabel() {
return "ERROR";
}
@Override
public String getTitle() {
return "PySyntaxError";
}
@Override
public String getDescription() {
return ex.traceback.dumpStack();
}
@Override
public String getIconURL() {
return "";
}
@Override
public List getParams() {
return Collections.emptyList();
}
};
}
/**
* return the script description and arguments.
*
* @param script the script Jython code.
* @param params any operator-defined values.
* @return
* @throws IOException
*/
public static ScriptDescriptor describeScript(String script, Map params) throws IOException {
return describeScript( null, script, params );
}
private static PyObject getConstraintP( PyDictionary dict, PyObject key, PyObject deft ) {
PyObject o= dict.get(key,deft);
if ( Py.isInstance( o, deft.getType() ) ) {
return o;
} else {
throw new IllegalArgumentException("constaints contains the wrong type");
}
}
/**
* return the script description and arguments.
* @param env the environment, containing PWD and maybe DOM.
* @param script the script Jython code.
* @param params any operator-defined values.
* @return a description of the script parameters and metadata.
* @throws IOException
*/
public static ScriptDescriptor describeScript( Map env, String script, Map params) throws IOException {
String prog;
try {
prog= simplifyScriptToGetParams(script, true); // removes calls to slow methods, and gets the essence of the controls of the script.
} catch ( PySyntaxError ex ) {
return errorScriptDescriptor(ex);
}
//org.das2.util.FileUtil.writeStringToFile( new File("/home/jbf/tmp/simplified.jy"), prog );
logger.log(Level.FINER, "Simplified script: {0}", prog);
PythonInterpreter interp;
try {
interp = createInterpreter(true);
} catch (IOException ex) {
logger.log( Level.WARNING, ex.getMessage(), ex );
return EMPTY;
}
if (env != null) {
for (Entry ent : env.entrySet()) {
if (ent.getKey() == null) {
logger.log(Level.WARNING, "parameter name was null");
} else if (ent.getValue() == null) {
if (ent.getKey().equals("dom")) {
logger.log(Level.FINE, "parameter \"dom\" value was set to null"); // Some scripts don't use dom.
} else {
logger.log(Level.WARNING, "parameter value was null");
}
} else {
logger.log(Level.FINER, "setting env {0} to {1}", new Object[]{ent.getKey(), ent.getValue()});
interp.set(ent.getKey(), ent.getValue());
}
}
}
interp.set("autoplot2017._scriptLabel", "");
interp.set("autoplot2017._scriptTitle", "");
interp.set("autoplot2017._scriptDescription", "");
interp.set("autoplot2017._scriptIcon", "");
if ( params!=null ) {
setParams(interp, params);
}
try {
prog = JythonRefactory.fixImports(prog, "");
} catch (IOException ex) {
Logger.getLogger(JythonUtil.class.getName()).log(Level.SEVERE, null, ex);
}
interp.exec(prog);
interp.exec("import autoplot2017 as autoplot\n");
PyList sort = (PyList) interp.eval("autoplot._paramSort");
boolean altWhy = false; // I don't know why things are suddenly showing up in this other space.
if (sort.isEmpty()) {
try {
sort = (PyList) interp.eval("_paramSort");
if (sort.size() > 0) {
logger.warning("things are suddenly in the wrong space. This is because things are incorrectly imported.");
altWhy = true;
}
} catch (PyException ex) {
// good...
}
}
LinkedList jsort= new LinkedList<>();
for (int i = 0; i < sort.__len__(); i++) {
String item= (String)sort.get(i);
if ( jsort.contains(item) ) {
jsort.remove(item);
}
jsort.add(item); // move it to the end
}
final List result = new ArrayList();
for (int i = 0; i < jsort.size(); i++) {
String theParamName = (String) jsort.get(i);
PyList oo = (PyList) interp.eval("autoplot._paramMap['" + theParamName + "']");
if (altWhy) {
oo = (PyList) interp.eval("_paramMap['" + theParamName + "']");
}
Param p = new Param();
p.label = theParamName; // should be name in the script
if (p.label.startsWith("__")) {
continue; // __doc__, __main__ symbols defined by Jython.
}
p.name = oo.__getitem__(0).toString(); // name in the URI
p.deft = oo.__getitem__(1);
p.doc = oo.__getitem__(2).toString();
p.constraints= new HashMap<>(); // always have constraints so we don't need null check.
PyObject oconstraints= oo.__getitem__(3);
if (oconstraints instanceof PyList) {
PyList pyList = ((PyList) oconstraints);
List