package org.autoplot.jythonsupport; import java.lang.reflect.Array; import java.lang.reflect.Method; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.das2.jythoncompletion.JythonCompletionTask; import org.das2.qds.QDataSet; import org.das2.qds.ops.Ops; import org.python.core.PySyntaxError; import org.python.parser.ast.If; import org.python.parser.ast.Module; import org.python.parser.ast.stmtType; import org.das2.util.LoggerManager; import org.python.parser.SimpleNode; import org.python.parser.ast.Assert; import org.python.parser.ast.Assign; import org.python.parser.ast.Attribute; import org.python.parser.ast.BinOp; import org.python.parser.ast.Call; import org.python.parser.ast.Compare; import org.python.parser.ast.Dict; import org.python.parser.ast.Expr; import org.python.parser.ast.Index; import org.python.parser.ast.List; import org.python.parser.ast.Name; import org.python.parser.ast.Num; import org.python.parser.ast.Str; import org.python.parser.ast.Subscript; import org.python.parser.ast.Tuple; import org.python.parser.ast.UnaryOp; import org.python.parser.ast.VisitorBase; import org.python.parser.ast.aliasType; import org.python.parser.ast.exprType; import org.python.util.PythonInterpreter; /** * AST support for Jython completions. This is not meant to be thorough, but instead should be helpful when working with scripts. * * @author jbf * @see JythonUtil#simplifyScriptToGetParams(java.lang.String, boolean) */ public class SimplifyScriptSupport { private static final Logger logger = LoggerManager.getLogger("jython.simplify"); /** * eat away at the end of the script until it can be parsed * * @param script a Jython script. * @return the script with lines at the end removed such that the script can compile. * @see JythonCompletionTask#trimLinesToMakeValid(java.lang.String) */ public static String alligatorParse(String script) { logger.entering("SimplifyScriptSupport", "alligatorParse"); String[] ss = JythonUtil.splitCodeIntoLines(null, script); String scri = script; int lastLine = ss.length; boolean parseOkay= false; while (lastLine > 0) { scri = JythonUtil.join(Arrays.copyOfRange(ss, 0, lastLine), "\n"); try { org.python.core.parser.parse(scri, "exec"); parseOkay= true; break; } catch (Exception e) { logger.finest("fail to parse, no worries."); } lastLine--; } if ( parseOkay==false ) { scri= ""; } logger.exiting("SimplifyScriptSupport", "alligatorParse"); return scri; } /** * quick and dirty attempt to resolve tuple for format statement. * @param n * @param row * @param column * @param env * @return */ public static Object[] tryResolveTupleNode( SimpleNode n, int row, int column, Map env ) { Tuple t= (Tuple)n; Object[] result= new Object[t.elts.length]; for ( int i=0; i env ) { if ( n.beginLine==row ) { if ( n instanceof Assign ) { Assign a= (Assign)n; return tryResolveStringNode( a.value, row, column, env ); } else if ( n instanceof Str ) { return ((Str)n).s; } else if ( n instanceof Num ) { return String.valueOf(((Num)n).n); } else if ( n instanceof Expr ) { Expr e= (Expr)n; return tryResolveStringNode( e.value, row, column, env ); } else if ( n instanceof Call ) { Call e= (Call)n; if ( e.func instanceof Name && ((Name)e.func).id.equals("getParam") ) { if ( e.args.length>1 ) { String s= tryResolveStringNode( e.args[1], row, column, env ); if ( s!=null ) return s; } } else if ( e.func instanceof Name && ((Name)e.func).id.equals("str") ) { if ( e.args.length==1 ) { String s= tryResolveStringNode( e.args[0], row, column, env ); if ( s!=null ) return s; } } return null; } else if ( n instanceof Name ) { Name na= (Name)n; Object o= env.get( na.id ); if ( o instanceof String ) { return (String)o; } else { return null; } } else if ( n instanceof BinOp ) { BinOp e= (BinOp)n; String sleft= tryResolveStringNode( e.left, row, column, env ); String sright= tryResolveStringNode( e.right, row, column, env ); if ( sleft!=null && sright!=null && e.op==BinOp.Add ) { return sleft + sright; } else if ( sleft!=null && e.right instanceof Tuple && e.op==BinOp.Mod ) { //PythonInterpreter interp= new PythonInterpreter(null); Object[] ss= tryResolveTupleNode( e.right, row, column, env ); if ( ss!=null ) { sleft= sleft.replaceAll("\\%d", "%s"); // small cheat try { return String.format( sleft,ss ); } catch (Exception ex ) { logger.log( Level.INFO, ex.getMessage(), ex ); return null; } } else { return null; } } else { return null; } } else { return null; } } else { return null; } } /** * given the node n, try to resolve its string value, maybe by implementing some of the addition (concatenation). * This was introduced to support URI completions within Jython codes, allowing the filename to be a variable and * thus shortening lines. * @param n the AST * @param row the row of the caret * @param column the column of the caret * @param env any variables which have been identified as string values. * @return the string or null. */ public static String tryResolveStringNode( Module n, int row, int column, Map env ) { stmtType thet=null; for ( stmtType t : n.body ) { if ( t.beginLine>=row ) { thet= t; break; } if ( t instanceof Assign ) { Assign a= (Assign)t; if ( a.targets.length==1 && a.targets[0] instanceof Name ) { if ( a.value instanceof Str ) { env.put( ((Name)a.targets[0]).id, ((Str)a.value).s ); } else if ( a.value instanceof BinOp ) { String s= tryResolveStringNode( a.value, a.beginLine, a.beginColumn, env ); if ( s!=null ) { env.put( ((Name)a.targets[0]).id, s ); } } else if ( a.value instanceof Call ) { Call c= (Call)a.value; if ( c.func instanceof Name && ((Name)c.func).id.equals("getParam") ) { String s= tryResolveStringNode( a.value, a.beginLine, a.beginColumn, env ); if ( s!=null ) { env.put( ((Name)a.targets[0]).id, s ); } } } } } } // find the node within thet containing the carot. if ( thet instanceof Assign ) { exprType t= ((Assign)thet).value; if ( t instanceof Call ) { Call c= (Call)t; if ( c.func instanceof Name && ((Name)c.func).id.equals("getDataSet") ) { return tryResolveStringNode( c.args[0], row, column, env ); } } } return null; } /** * Remove parts of the script which are expensive so that the script can be run and completions offered. TODO: What is the * difference between this and simplifyScriptToCompletions? * * @param script Jython script * @return simplified version of the script. * @see #simplifyScriptToCompletions(java.lang.String) * @see https://github.com/autoplot/dev/tree/master/bugs/sf/1687 */ public static String removeSideEffects(String script) { script = alligatorParse(script); String[] ss = JythonUtil.splitCodeIntoLines("# simplifyScriptToGetCompletions", script); Module n; try { n = (Module) org.python.core.parser.parse(script, "exec"); } catch (Exception ex) { // do it again so we can debug. n = (Module) org.python.core.parser.parse(script, "exec"); } if (false && n.body.length > 0 && n.body[0].beginLine > n.beginLine) { logger.fine("shifting line numbers!"); int shift = n.body[0].beginLine - n.beginLine; // strange bug here. for (stmtType s : n.body) { s.beginLine -= shift; } } HashSet variableNames = new HashSet(); int ilastLine = ss.length - 1; return simplifyScriptToGetCompletions(ss, n.body, variableNames, 1, ilastLine, 0); } /** * useful for debugging * * @param result * @param line * @return */ private static StringBuilder appendToResult(StringBuilder result, String line) { result.append(line); return result; } /** * extracts the parts of the program that are quickly executed, generating a code which can be run and then queried for * completions. This uses a Jython syntax tree (AST), so the code must be free of syntax errors. This will remove lines from the * end of the code until the code compiles, in case the script has a partially defined def or class. * * @param script the entire python program * @return the python program with lengthy calls removed. * @see #removeSideEffects(java.lang.String) * @see JythonUtil#simplifyScriptToGetParams(java.lang.String, boolean) */ public static String simplifyScriptToCompletions(String script) throws PySyntaxError { if (script.trim().length() == 0) { return script; } script = alligatorParse(script); String[] ss = JythonUtil.splitCodeIntoLines("# simplifyScriptToGetCompletions", script); int lastLine = ss.length - 1; HashSet variableNames = new HashSet(); variableNames.add("getParam"); // this is what allows the getParam calls to be included. //variableNames.add("getDataSet"); // new class lookup is cleaner method for this. variableNames.add("str"); // include casts. variableNames.add("int"); variableNames.add("long"); variableNames.add("float"); variableNames.add("datum"); variableNames.add("datumRange"); variableNames.add("dataset"); variableNames.add("URI"); variableNames.add("URL"); variableNames.add("PWD"); variableNames.add("dom"); // TODO: only true for .jy scripts. variableNames.add("True"); variableNames.add("False"); try { Module n = null; int count = 4; PySyntaxError ex0 = null; while (lastLine > 0 && count > 0) { try { n = (Module) org.python.core.parser.parse(script, "exec"); break; } catch (PySyntaxError ex) { // pop off the last line and try again. if (ex0 == null) { ex0 = ex; } lastLine--; script = JythonUtil.join(Arrays.copyOf(ss, lastLine), "\n"); count--; } } if (n == null) { throw ex0; } String s = simplifyScriptToGetCompletions(ss, n.body, variableNames, 1, lastLine, 0); //s= GETDATASET_CODE + s; //s= "PWD='file:/tmp/'\n"+s; return s; } catch (PySyntaxError ex) { throw ex; } } private static String getIfBlock(String[] ss, stmtType[] body, HashSet variableNames, int firstLine, int lastLine1, int depth) { StringBuilder result = new StringBuilder(); String ss1 = simplifyScriptToGetCompletions(ss, body, variableNames, firstLine, lastLine1, depth + 1); if (ss1.length() == 0) { // String line; // if ( firstLine==0 && iff.beginLine>0 ) { // line= ss[body[0].beginLine-1]; // } else { // line= ss[iff.beginLine]; // } Pattern p = Pattern.compile("(\\s*)(\\S*).*"); Matcher m = p.matcher(ss[firstLine]); String indent; if (m.matches()) { indent = m.group(1); } else { indent = ""; } result.append(indent).append("pass ## SimplifyScriptSupport.getIfBlock \n"); logger.fine("things have probably gone wrong..."); } else { appendToResult(result, ss1); } return result.toString(); } /** * Using the stmtType get the line, or lines. If the last line contains a single triple-quote, we need to kludge a little and * look for the preceding triple-quote in previous lines. * * @param ss * @param o * @return */ public static String getSourceForStatement(String[] ss, stmtType o) { if (o.beginLine == 0) { return "(bad line number)"; } int endLine= o.beginLine; int beginLine= endLine; if ( o instanceof Expr ) { Expr e = (Expr)o; if ( e.value.beginLine= ss.length) { throw new IllegalArgumentException("lastLine is >= number of lines"); } if (!ss[0].equals("# simplifyScriptToGetCompletions")) { throw new IllegalArgumentException("first line must be '# simplifyScriptToGetCompletions'"); } Map importedNames = new LinkedHashMap<>(); importedNames.put("Color", "java.awt"); importedNames.put("DatumRange", "org.das2.datum" ); importedNames.put("Units", "org.das2.datum" ); importedNames.put("DatumRangeUtil","org.das2.datum"); importedNames.put("TimeUtil","org.das2.datum"); importedNames.put("URL", "java.net"); importedNames.put("URI", "java.net"); importedNames.put("TimeParser", "org.das2.datum" ); int acceptLine = -1; // first line to accept int currentLine = beginLine; // current line we are writing. StringBuilder result = new StringBuilder(); for (int istatement = 0; istatement < stmts.length; istatement++) { stmtType o = stmts[istatement]; String theLine = getSourceForStatement(ss, o); int lineCount = theLine.split("\n", -2).length; if (depth == 0) { logger.finest(theLine); //breakpoint here. //System.err.println(theLine); } logger.log(Level.FINER, "line {0}: {1}", new Object[]{o.beginLine, theLine}); if (o.beginLine > 0) { if (beginLine < 0 && istatement == 0) { acceptLine = o.beginLine; } if (lineCount > 1) { beginLine = o.beginLine - (lineCount - 1); } else { beginLine = o.beginLine; } } else { acceptLine = beginLine; // elif clause in autoplot-test038/lastSuccessfulBuild/artifact/test038_demoParms1.jy } if (beginLine > lastLine) { continue; } if (o instanceof Assign ) { if ( !simplifyScriptToGetCompletionsOkay(o, variableNames,importedNames) ) { // check for method calls where we know the type. Assign a = (Assign) o; String cl = maybeIdentifyType(a, importedNames); if (cl != null) { if (acceptLine > -1) { for (int i = acceptLine; i < beginLine; i++) { appendToResult(result, ss[i]).append("\n"); } } if ( cl.contains("\n") ) { String[] cls= cl.split("\n"); for ( String cl1: cls ) { appendToResult(result, getIndent(theLine) + cl1).append("\n") ; } } else { appendToResult(result, getIndent(theLine) + cl).append("\n") ; } acceptLine = -1; continue; } } else { Assign a = (Assign) o; String cl = maybeIdentifyType(a, importedNames); if (cl != null) { if (acceptLine > -1) { for (int i = acceptLine; i < beginLine; i++) { appendToResult(result, ss[i]).append("\n"); } } appendToResult(result, getIndent(theLine) + cl).append("\n") ; acceptLine = -1; continue; } } } if (o instanceof org.python.parser.ast.ImportFrom) { org.python.parser.ast.ImportFrom i = (org.python.parser.ast.ImportFrom) o; for (aliasType n : i.names) { importedNames.put(n.name, i.module); } } if (o instanceof org.python.parser.ast.If) { if (acceptLine > -1) { for (int i = acceptLine; i < beginLine; i++) { appendToResult(result, ss[i]).append("\n"); } } If iff = (If) o; boolean includeBlock; if (simplifyScriptToGetCompletionsCanResolve(iff.test, variableNames)) { for (int i = beginLine; i < iff.body[0].beginLine; i++) { result.append(ss[i]).append("\n"); } // write out the 'if' part includeBlock = true; } else { includeBlock = false; } int lastLine1; //lastLine1 is the last line of the "if" clause. (inclusive) int elseLine = -1; if (iff.orelse != null && iff.orelse.length > 0) { if (iff.orelse[0].beginLine > 0) { lastLine1 = iff.orelse[0].beginLine - 1; // -1 is for the "else:" part. if (ss[lastLine1].trim().startsWith("else")) { elseLine = lastLine1; lastLine1 = lastLine1 - 1; } else if (ss[lastLine1].trim().startsWith("elif")) { elseLine = lastLine1; lastLine1 = lastLine1 - 1; } } else { if (iff.orelse[0] instanceof If) { elseLine = ((If) iff.orelse[0]).test.beginLine; lastLine1 = elseLine - 1; } else { logger.warning("failure to deal with another day..."); throw new RuntimeException("this case needs to be dealt with..."); } } } else if ((istatement + 1) < stmts.length) { lastLine1 = stmts[istatement + 1].beginLine - 1; } else { lastLine1 = lastLine; } if (includeBlock) { HashSet variableNames1= new HashSet(variableNames); String ss1 = getIfBlock(ss, iff.body, variableNames1, Math.min(beginLine + 1, lastLine1), lastLine1, depth + 1); appendToResult(result, ss1); if (iff.orelse != null) { if ((istatement + 1) >= stmts.length) { lastLine1 = lastLine; } else { lastLine1 = stmts[istatement + 1].beginLine - 1; } if (iff.orelse[0].beginLine == 0) { result.append("\n"); } else { if (iff.orelse[0].beginLine > 0 && ss[iff.orelse[0].beginLine - 1].trim().startsWith("else:")) { result.append(ss[iff.orelse[0].beginLine - 1]).append("\n"); HashSet variableNames2= new HashSet(variableNames); ss1 = getIfBlock(ss, iff.orelse, variableNames2, iff.orelse[0].beginLine, lastLine1, depth + 1); appendToResult(result, ss1); //TODO: if variable is added to variableNames1 and variableNames2 then add it to variableNames. } else { result.append(ss[iff.orelse[0].beginLine]).append("\n"); } } } } currentLine = lastLine1; acceptLine = -1; } else if (o instanceof Assert) { String m = maybeModelAssert((Assert) o, variableNames); if (m != null) { result.append(m).append("\n"); currentLine = acceptLine; } } else if ( o instanceof Expr ) { if ( ((Expr)o).value instanceof Str ) { result.append(theLine).append("\n"); currentLine = o.beginLine; acceptLine= -1; } else { if (acceptLine > -1) { // always skip Expr. int thisLine = beginLine; for (int i = acceptLine; i <= thisLine; i++) { if (i < thisLine) { appendToResult(result, ss[i]).append("\n"); } else { if (ss[i].length() > 0 && Character.isWhitespace(ss[i].charAt(0))) { appendToResult(result, ss[i]).append("\n"); } } } appendToResult(result, "\n"); currentLine = thisLine; acceptLine = -1; } } } else { // Assign, etc if (simplifyScriptToGetCompletionsOkay(o, variableNames,importedNames)) { if (acceptLine < 0) { acceptLine = beginLine; for (int i = currentLine + 1; i < acceptLine; i++) { result.append("\n"); currentLine = acceptLine; } } } else { if (acceptLine > -1) { int thisLine = beginLine; for (int i = acceptLine; i <= thisLine; i++) { if (i < thisLine) { appendToResult(result, ss[i]).append("\n"); } else { if (ss[i].length() > 0 && Character.isWhitespace(ss[i].charAt(0))) { appendToResult(result, ss[i]).append("\n"); } } } appendToResult(result, "\n"); currentLine = thisLine; acceptLine = -1; } } } } if (acceptLine > -1) { lastLine = JythonUtil.handleContinue(ss, lastLine); int thisLine = lastLine; for (int i = acceptLine; i <= thisLine; i++) { appendToResult(result, ss[i]).append("\n"); } String slastLine = ss[thisLine].trim(); if (slastLine.endsWith(":")) { appendToResult(result, getIndent(slastLine) + " pass"); } } return result.toString(); } /** * can we resolve this node given the variable names we know? * * @param o * @param variableNames * @return true if the node can be resolved. */ private static boolean simplifyScriptToGetCompletionsCanResolve(SimpleNode o, HashSet variableNames) { //if ( o.beginLine>=617 && o.beginLine<619 ) { // System.err.println( "here at 617-ish"); //} if (o instanceof Name) { Name c = (Name) o; if (!variableNames.contains(c.id)) { logger.finest(String.format("%04d canResolve->false: %s", o.beginLine, o.toString())); return false; } } else if (o instanceof Attribute) { Attribute at = (Attribute) o; while (at.value instanceof Attribute || at.value instanceof Subscript) { if (at.value instanceof Attribute) { at = (Attribute) at.value; } else { Subscript s = (Subscript) at.value; if (s.value instanceof Attribute) { at = (Attribute) s.value; } else { return false; // oh just give up... } } } if ( !simplifyScriptToGetCompletionsCanResolve( at.value, variableNames ) ) { return false; } } else if ( o instanceof Compare ) { Compare c = (Compare)o; if ( simplifyScriptToGetCompletionsCanResolve( c.left, variableNames ) ) { for ( exprType e : c.comparators ) { if ( ! simplifyScriptToGetCompletionsCanResolve( e, variableNames ) ) { return false; } } return true; } else { return false; } } else if ( o instanceof Call ) { Call c = (Call)o; if ( !simplifyScriptToGetCompletionsCanResolve( c.func, variableNames ) ) { return false; } for ( exprType e : c.args ) { if ( ! simplifyScriptToGetCompletionsCanResolve( e, variableNames ) ) { return false; } } } else if ( o instanceof Subscript ) { return false; } MyVisitorBase vb = new MyVisitorBase(variableNames); try { o.traverse(vb); logger.finest(String.format(" %04d canResolve->%s: %s", o.beginLine, vb.visitNameFail, o)); return vb.looksOkay || !vb.visitNameFail; } catch (Exception ex) { logger.log(Level.SEVERE, ex.getMessage(), ex); } logger.finest(String.format("!! %04d canResolve->false: %s", o.beginLine, o)); return false; } /** * dumb kludge where no-arg constructor is called to get an instance for completions. This is really an experiment... * * @param a * @return */ private static String maybeModelAssert(Assert a, HashSet variableNames) { if (a.test instanceof Call) { org.python.parser.ast.Call cc = (org.python.parser.ast.Call) a.test; exprType f = cc.func; if (f instanceof Name) { if (((Name) f).id.equals("isinstance")) { if (cc.args.length == 2) { exprType a1 = cc.args[0]; if (a1 instanceof Name) { exprType a2 = cc.args[1]; if (a2 instanceof Name && variableNames.contains(((Name) a2).id)) { return String.format("%s" + JythonCompletionTask.__CLASSTYPE + "=%s # inserted by maybeModelAssert", ((Name) a1).id, ((Name) a2).id); } } } } } return null; } else { return null; } } /** * return true if we can include this in the script without a huge performance penalty. * This should be used for read access, not write access, so:
    *
  • p= dom.plots[0] is okay *
  • dom.plots[1].rowId= dom.plots[0].rowId is not okay. *
* @param o the statement, for example an import or an assignment * @param variableNames known symbol names * @param importedNames map of name to path (DatumRange → org.das2.datum) * @return true if we can include this in the script without a huge performance penalty. */ private static boolean simplifyScriptToGetCompletionsOkay(stmtType o, HashSet variableNames,Map importedNames) { logger.log(Level.FINEST, "simplify script line: {0}", o.beginLine); if ((o instanceof org.python.parser.ast.ImportFrom)) { org.python.parser.ast.ImportFrom importFrom = (org.python.parser.ast.ImportFrom) o; for (aliasType a : importFrom.names) { if (a.asname != null) { variableNames.add(a.asname); } else { variableNames.add(a.name); } } return true; } if ((o instanceof org.python.parser.ast.Import)) { org.python.parser.ast.Import imporrt = (org.python.parser.ast.Import) o; for (aliasType a : imporrt.names) { if (a.asname != null) { variableNames.add(a.asname); } else { variableNames.add(a.name); } } return true; } if ((o instanceof org.python.parser.ast.Expr)) { Expr e = (Expr) o; if ((e.value instanceof Call) ) { if ( trivialFunctionCall((Call) e.value) ) { return true; } else { Call c= (Call) e.value; if ( c.func instanceof Attribute ) { Attribute aa= (Attribute)c.func; if ( aa.value instanceof Name ) { Name naa= (Name)aa.value; if ( variableNames.contains(naa.id) ) { return true; } } } } } } if ((o instanceof org.python.parser.ast.ClassDef)) { return true; } if ((o instanceof org.python.parser.ast.FunctionDef)) { return true; } if ((o instanceof org.python.parser.ast.Assign)) { Assign a = (Assign) o; if (simplifyScriptToGetCompletionsOkayNoCalls(a.value, variableNames)) { if (!simplifyScriptToGetCompletionsCanResolve(a.value, variableNames)) { return false; } for (exprType target : a.targets) { exprType et = (exprType) target; if (et instanceof Name) { String id = ((Name) target).id; variableNames.add(id); logger.log(Level.FINEST, "assign to variable {0}", id); //TODO: can we identify type? Insert __CLASSTYPE=... for completions. if ( a.value instanceof Call ) { String type= maybeIdentifyReturnType( id, (Call)a.value, importedNames ); logger.log(Level.FINE, "id type: {0}__CLASSTYPE= {1}", new Object[]{id, type}); } } else if (et instanceof Attribute) { return false; // Attribute at = (Attribute) et; // while (at.value instanceof Attribute || at.value instanceof Subscript) { // if (at.value instanceof Attribute) { // at = (Attribute) at.value; // } else { // Subscript s = (Subscript) at.value; // if (s.value instanceof Attribute) { // at = (Attribute) s.value; // } else { // return false; // oh just give up... // } // } // } // if (at.value instanceof Name) { // Name n = (Name) at.value; // if (!variableNames.contains(n.id)) { // return false; // } // } } else if (et instanceof Subscript) { return false; //Subscript subscript = (Subscript) et; //exprType et2 = subscript.value; //if (et2 instanceof Name) { // Name n = (Name) et2; // if (variableNames.contains(n.id)) { // return true; // } //} //return false; } else { return false; } } return true; } else { return false; } } if ((o instanceof org.python.parser.ast.If)) { return simplifyScriptToGetCompletionsOkayNoCalls(o, variableNames); } if ((o instanceof org.python.parser.ast.Print)) { return false; } logger.log(Level.FINEST, "not okay to simplify: {0}", o); return false; } /** * inspect the node to look for function calls that are not to the function "getParam". This is awful code that will be * rewritten when we upgrade Python to 2.7. * * @param o * @param variableNames * @return */ private static boolean simplifyScriptToGetCompletionsOkayNoCalls(SimpleNode o, HashSet variableNames) { if (o instanceof Call) { Call c = (Call) o; if (!trivialFunctionCall(c)) { if (!trivialConstructorCall(c)) { logger.finest(String.format("%04d simplify->false: %s", o.beginLine, o.toString())); return false; } } } MyVisitorBase vb = new MyVisitorBase(variableNames); try { o.traverse(vb); logger.finest(String.format(" %04d simplify->%s: %s", o.beginLine, vb.looksOkay(), o)); return vb.looksOkay(); } catch (Exception ex) { logger.log(Level.SEVERE, ex.getMessage(), ex); } logger.finest(String.format("!! %04d simplify->false: %s", o.beginLine, o)); return false; } /** * there are a number of functions which take a trivial amount of time to execute and are needed for some scripts, such as the * string.upper() function. The commas are to guard against the id being a subset of another id ("lower," does not match * "lowercase"). TODO: update this after Python upgrade. * @see JythonUtil#okay */ private static final String[] okay = new String[]{ "range,", "xrange,", "irange,","map,","join,", "getParam,", "lower,", "upper,", "URI,", "URL,", "setScriptDescription", "setScriptTitle", "setScriptLabel", "setScriptIcon", "DatumRangeUtil,", "TimeParser,", "str,", "int,", "long,", "float,", "datum,", "datumRange,", "dataset,", "indgen,", "findgen,", "dindgen,", "ones,", "zeros,", "linspace,", "logspace,", "dblarr,", "fltarr,", "strarr,", "intarr,", "bytarr,", "ripples,",//"split,", // remove split because it is confused with the URISplit. "color,", "colorFromString,", "isinstance,"}; private static final Set okaySet = new HashSet<>(); static { for (String o : okay) { okaySet.add(o.substring(0, o.length() - 1)); } } private static String getFunctionName(exprType t) { if (t instanceof Name) { return ((Name) t).id; } else if (t instanceof Attribute) { Attribute a = (Attribute) t; return getFunctionName(a.value) + "." + a.attr; } else { return t.toString(); } } /** * return true if the function call is trivial to execute and can be evaluated within a few milliseconds. For example, findgen * can be called because no calculations are made in the call, but fft cannot. Typically these are Order 1 (a.k.a. constant * time) operations, but also many Order N operations are so fast they are allowed. * * @param sn an AST node pointed at a Call. * @return true if the function call is trivial to execute */ private static boolean trivialFunctionCall(SimpleNode sn) { if (sn instanceof Call) { Call c = (Call) sn; boolean klugdyOkay = false; String ss = c.func.toString(); // we just want "DatumRangeUtil" of the Attribute //String ss= getFunctionName(c.func); for (String s : okay) { if (ss.contains(s)) { klugdyOkay = true; } } if (klugdyOkay == false) { if (ss.contains("TimeUtil") && ss.contains("now")) { klugdyOkay = true; } } logger.log(Level.FINER, "trivialFunctionCall={0} for {1}", new Object[]{klugdyOkay, c.func.toString()}); return klugdyOkay; } else { return false; } } /** * return true if the function call is trivial to execute because it's a constructor, which presumably takes little time to * create. * * @param sn * @return true if it is a constructor call */ private static boolean trivialConstructorCall(SimpleNode sn) { if (sn instanceof Call) { Call c = (Call) sn; if (c.func instanceof Name) { String funcName = ((Name) c.func).id; return Character.isUpperCase(funcName.charAt(0)); } else if (c.func instanceof Attribute) { // Rectangle.Double String funcName = ((Attribute) c.func).attr; return Character.isUpperCase(funcName.charAt(0)); } else { return false; } } else { return false; } } /** * placeholder for code which identifies constructors. This is fragile and misguided code, which looks for the Java convention * of uppercase letter starting. * * @param name the name, like GrannyTextRenderer * @param importedNames map from name to path "GrannyTextRenderer" → "org.das2.util" * @return true if the name is known to be a constructor call. */ private static boolean isConstructor(String name, Map importedNames) { if (importedNames.containsKey(name)) { return name.length() > 2 && Character.isUpperCase(name.charAt(0)) && Character.isLowerCase(name.charAt(1)); } else { return false; } } /** * return the class for the name. * * @param name the name, like GrannyTextRenderer * @param importedNames map from name to path "GrannyTextRenderer" → "org.das2.util" * @return */ private static Class getClassFor(String name, Map importedNames) { String path = importedNames.get(name); try { return Class.forName(path + "." + name); } catch (ClassNotFoundException ex) { return null; } } private static String maybeIndentifyValue( exprType t ) { if ( t instanceof Num ) { return String.valueOf(((Num)t).n); } else if ( t instanceof Str ) { return "'"+ ((Str)t).toString() + "'"; } else if ( t instanceof Name ) { String n= ((Name)t).id; if ( n.equals("True") || n.equals("False") ) { return n; } else { return null; } } else { return null; } } /** * return the line of code needed to use the result, or null. * * @param id the identifier being assigned the value * @param c the function being called * @param importedNames map of name to path (DatumRange → org.das2.datum) * @return the assignment and possibly an additional import statement */ private static String maybeIdentifyReturnType(String id, Call c, Map importedNames) { if (c.func instanceof Name) { String funcName = ((Name) c.func).id; switch (funcName) { case "getApplication": return "from org.autoplot import AutoplotUI\n" + id + JythonCompletionTask.__CLASSTYPE + " = AutoplotUI\n"; case "getApplicationModel": return "from org.autoplot import ApplicationModel\n" + id + JythonCompletionTask.__CLASSTYPE + " = ApplicationModel\n"; case "getDataSource": return "from org.autoplot.datasource import DataSource\n" + id + JythonCompletionTask.__CLASSTYPE + " = DataSource\n"; case "getParam": // weird--there's a special getParam loaded, but this doesn't work for getParam( 'allowCache', True, 'allow storage of ephemeris to local file', [ True,False ] ) exprType e1= c.args[1]; String vv= maybeIndentifyValue(e1); if ( vv!=null ) { return id + " = " + vv; } else { if ( e1.getImage()==null ) { if ( e1 instanceof Subscript ) { Subscript s= (Subscript)e1; if ( s.slice instanceof Index && s.value instanceof Name ) { Index i= (Index)s.slice; String ss= maybeIndentifyValue(i.value); String nn= ((Name)s.value).id; return id + " = " + nn + "[" + ss +"]"; } else { return null; } } else { return null; } } else { return id + JythonCompletionTask.__CLASSTYPE + " = " + e1.getImage(); } } case "getDataSet": case "xtags": case "ytags": case "findgen": case "linspace": case "fftPower": case "magnitude": return id + JythonCompletionTask.__CLASSTYPE + " = QDataSet # return type from " + funcName + " (spot line789)\n"; case "datumRange": return id + JythonCompletionTask.__CLASSTYPE + " = DatumRange # return type from " + funcName + " (spot line896)\n"; case "datum": return id + JythonCompletionTask.__CLASSTYPE + " = Datum # return type from " + funcName + " (spot line898)\n"; default: break; } for ( Method m: Ops.class.getMethods() ) { if ( m.getName().equals(funcName) ) { if ( QDataSet.class.isAssignableFrom( m.getReturnType() ) ) { return id + JythonCompletionTask.__CLASSTYPE + " = QDataSet # ( spot line 898 )"; } } } if (isConstructor(funcName, importedNames)) { return id + JythonCompletionTask.__CLASSTYPE + " = " + funcName + " # isConstructor (line794)\n"; } else { return null; } } else if (c.func instanceof Attribute) { // p=PngWalkTool.start(...) Attribute at = (Attribute) c.func; if (at.value instanceof Name) { String attrName = ((Name) at.value).id; if (attrName.equals("PngWalkTool") && at.attr.equals("start")) { return "from org.autoplot.pngwalk import PngWalkTool\n" + id + JythonCompletionTask.__CLASSTYPE + "= PngWalkTool #(spot line802)\n"; } else if (importedNames.containsKey(attrName)) { Class claz = getClassFor(attrName, importedNames); if (claz == null) { return null; } Method[] mm = claz.getMethods(); String methodName = at.attr; for (Method m : mm) { if (m.getName().equals(methodName)) { //&& m.getGenericParameterTypes().length ) { Class rclz = m.getReturnType(); String rclzn = rclz.getSimpleName(); StringBuilder result= new StringBuilder(); if ( rclz.getPackage()!=null ) { result.append( String.format( "from %s import %s\n", rclz.getPackage().getName(), rclzn ) ); } result.append( String.format( "%s%s = %s # (line895)\n", id, JythonCompletionTask.__CLASSTYPE, rclzn ) ); return result.toString(); } } return null; } } else if (at.value instanceof Call) { String x = maybeIdentifyReturnType("x", (Call) at.value, importedNames); if (x != null) { int i = x.indexOf("="); try { String s= x.substring(i + 1); int i2= s.indexOf('#'); if ( i2>-1 ) { s= s.substring(0,i2); } s= s.trim(); if ( importedNames.containsKey(s) ) { String packg= importedNames.get(s); Class clz = Class.forName(packg+"."+s); try { Method m = clz.getMethod(at.attr); //TODO: this is only single-argument calls Class rclz = m.getReturnType(); String rclzn = rclz.getSimpleName(); return "from " + rclz.getPackage().getName() + " import " + rclzn + "\n" + id + JythonCompletionTask.__CLASSTYPE + " = " + rclzn + " # (spot line826)"; } catch (NoSuchMethodException | SecurityException ex) { logger.log(Level.SEVERE, null, ex); } } } catch (ClassNotFoundException ex) { logger.log(Level.SEVERE, null, ex); return null; // shouldn't happen } return null; } else { return null; } } return null; } else { return null; } } /** * if we recognize the function that is called, then go ahead and keep track of the type. This is a quick and cheesy * implementation that just looks for a few names. For example,
    *
  • x= getApplication() -- x is an AutoplotUI *
  • x= getApplicationModel() -- x is a ApplicationModel *
  • x= getDataSource() -- x is a DataSource *
  • x= PngWalkTool.start() -- x is a PngWalkTool *
* See bug https://sourceforge.net/p/autoplot/bugs/2319/ for some discussion on this. * * @param a the assignment * @return the Jython code to insert. */ private static String maybeIdentifyType(Assign a, Map importedNames) { if (a.targets.length == 1) { exprType target = a.targets[0]; exprType et = (exprType) target; if (et instanceof Name) { String id = ((Name) target).id; if (a.value instanceof Call) { Call c = (Call) a.value; return maybeIdentifyReturnType(id, c, importedNames); } else if ( a.value instanceof Subscript ) { Subscript s= ((Subscript)a.value); if ( s.value instanceof Attribute ) { Attribute att = (Attribute)s.value; if ( att.value instanceof Name ) { if ( ((Name)att.value).id.equals("dom") ) { if ( att.attr.equals("plots") ) { String rclzn = "org.autoplot.dom.Plot"; return "import org.autoplot.dom.Plot\n" + id + JythonCompletionTask.__CLASSTYPE + " = " + rclzn + " # (spot line955 a)\n"; } else if ( att.attr.equals("canvases") ) { String rclzn = "org.autoplot.dom.Canvas"; return "import org.autoplot.dom.Canvas\n" + id + JythonCompletionTask.__CLASSTYPE + " = " + rclzn + " # (spot line955 b)\n"; } else if ( att.attr.equals("plotElements") ) { String rclzn = "org.autoplot.dom.PlotElement"; return "import org.autoplot.dom.PlotElement\n" + id + JythonCompletionTask.__CLASSTYPE + " = " + rclzn + " # (spot line955 c)\n"; } } } } else if ( s.value instanceof Name ) { return id + JythonCompletionTask.__CLASSTYPE + " = QDataSet # (spot line1014 a)\n"; } } else if ( a.value instanceof BinOp ) { // just go ahead and assume it's a QDataSet return id + JythonCompletionTask.__CLASSTYPE + " = QDataSet # (spot line1014 b)\n"; } else if ( a.value instanceof Num ) { return id + " = " + ((Num)a.value).n; } else if ( a.value instanceof Str ) { return id + " = \"\"\"" + ((Str)a.value).s + "\"\"\""; } } else if ( et instanceof Tuple && a.value instanceof Tuple ) { // lowCut_toggle, highcut_toggle, filttype_toggle, order_toggle = 0.5, 20, 'Bandpass', 4 # filter toggles LOW FLYER Tuple targetTuple= (Tuple)et; Tuple valueTuple= (Tuple)a.value; if ( targetTuple.elts.length== valueTuple.elts.length ) { StringBuilder result= new StringBuilder(); for ( int i=0; i0 ) { return result.toString(); } } } } return null; } private static Object Name(exprType func) { throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. } private static class MyVisitorBase extends VisitorBase { boolean looksOkay = true; boolean visitNameFail = false; HashSet names; MyVisitorBase(HashSet names) { this.names = names; } @Override public Object visitName(Name node) throws Exception { logger.log(Level.FINER, "visitName({0})", node); if (!names.contains(node.id)) { visitNameFail = true; } return super.visitName(node); } @Override public Object visitCall(Call node) throws Exception { logger.log(Level.FINER, "visitCall({0})", node); return super.visitCall(node); } @Override protected Object unhandled_node(SimpleNode sn) throws Exception { return sn; } @Override public void traverse(SimpleNode sn) throws Exception { logger.log(Level.FINER, "traverse({0})", sn); if (sn instanceof Call) { looksOkay = trivialFunctionCall(sn) || trivialConstructorCall(sn); logger.log(Level.FINER, "looksOkay={0}", looksOkay); } else if (sn instanceof Assign) { Assign a = ((Assign) sn); exprType et = a.value; if (et instanceof Call) { looksOkay = trivialFunctionCall(et) || trivialConstructorCall(sn); logger.log(Level.FINER, "looksOkay={0}", looksOkay); } } else if (sn instanceof Name) { String t = ((Name) sn).id; if (t.length() > 1 && Character.isUpperCase(t.charAt(0))) { logger.log(Level.FINER, "name is assumed to be a constructor call name: {0}", t); return; } if (!names.contains(t) && !okaySet.contains(t)) { looksOkay = false; // TODO: why are there both looksOkay and visitNameFail? logger.log(Level.FINER, "looksOkay={0}", looksOkay); } } else if (sn instanceof Attribute) { traverse(((Attribute) sn).value); // DatumRangeUtil } else if (sn instanceof Subscript) { Subscript ss = (Subscript) sn; exprType et = ss.value; if (et instanceof Name) { traverse((Name) (et)); } //ss.value; //visitName((Name)) } else if (sn instanceof BinOp) { BinOp bo = (BinOp) sn; traverse(bo.left); traverse(bo.right); } else if (sn instanceof UnaryOp) { UnaryOp bo = (UnaryOp) sn; traverse(bo.operand); } else if (sn instanceof Num) { } else if ( sn instanceof Str ) { } else if (sn instanceof Index) { Index index = (Index) sn; traverse(index.value); } else if (sn instanceof Tuple) { Tuple tuple = (Tuple) sn; for ( exprType e: tuple.elts ) { traverse(e); } } else if (sn instanceof List) { List ll = (List) sn; for ( exprType e: ll.elts ) { traverse(e); } } else if (sn instanceof Dict) { Dict dict = (Dict) sn; for ( exprType e: dict.keys ) { traverse(e); } for ( exprType e: dict.values ) { traverse(e); } } else { logger.log(Level.FINE, "unchecked: {0}", sn); looksOkay= false; } } public boolean looksOkay() { return looksOkay; } /** * this contains a node whose name we can't resolve. * * @return */ public boolean visitNameFail() { return visitNameFail; } } }