package org.autoplot.jythonsupport;

import java.lang.reflect.Array;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.das2.jythoncompletion.JythonCompletionTask;
import org.das2.qds.QDataSet;
import org.das2.qds.ops.Ops;
import org.python.core.PySyntaxError;
import org.python.parser.ast.If;
import org.python.parser.ast.Module;
import org.python.parser.ast.stmtType;
import org.das2.util.LoggerManager;
import org.python.parser.SimpleNode;
import org.python.parser.ast.Assert;
import org.python.parser.ast.Assign;
import org.python.parser.ast.Attribute;
import org.python.parser.ast.BinOp;
import org.python.parser.ast.Call;
import org.python.parser.ast.Compare;
import org.python.parser.ast.Dict;
import org.python.parser.ast.Expr;
import org.python.parser.ast.Index;
import org.python.parser.ast.List;
import org.python.parser.ast.Name;
import org.python.parser.ast.Num;
import org.python.parser.ast.Str;
import org.python.parser.ast.Subscript;
import org.python.parser.ast.Tuple;
import org.python.parser.ast.UnaryOp;
import org.python.parser.ast.VisitorBase;
import org.python.parser.ast.aliasType;
import org.python.parser.ast.exprType;
import org.python.util.PythonInterpreter;

/**
 * AST support for Jython completions. This is not meant to be thorough, but instead should be helpful when working with scripts.
 *
 * @author jbf
 * @see JythonUtil#simplifyScriptToGetParams(java.lang.String, boolean)
 */
public class SimplifyScriptSupport {

    private static final Logger logger = LoggerManager.getLogger("jython.simplify");

    /**
     * eat away at the end of the script until it can be parsed
     *
     * @param script a Jython script.
     * @return the script with lines at the end removed such that the script can compile.
     * @see JythonCompletionTask#trimLinesToMakeValid(java.lang.String)
     */
    public static String alligatorParse(String script) {
        logger.entering("SimplifyScriptSupport", "alligatorParse");
        String[] ss = JythonUtil.splitCodeIntoLines(null, script);
        String scri = script;
        int lastLine = ss.length;
        boolean parseOkay= false;
        while (lastLine > 0) {
            scri = JythonUtil.join(Arrays.copyOfRange(ss, 0, lastLine), "\n");
            try {
                org.python.core.parser.parse(scri, "exec");
                parseOkay= true;
                break;
            } catch (Exception e) {
                logger.finest("fail to parse, no worries.");
            }
            lastLine--;
        }
        if ( parseOkay==false ) {
            scri= "";
        }
        logger.exiting("SimplifyScriptSupport", "alligatorParse");
        return scri;
    }

    /**
     * quick and dirty attempt to resolve tuple for format statement.
     * @param n
     * @param row
     * @param column
     * @param env
     * @return 
     */
    public static Object[] tryResolveTupleNode(  SimpleNode n, int row, int column, Map<String,Object> env ) {
        Tuple t= (Tuple)n;
        Object[] result= new Object[t.elts.length];
        for ( int i=0; i<result.length; i++ ) {
            result[i]= tryResolveStringNode( t.elts[i], row, column, env );
            if ( result[i]==null ) return null;
        }
        return result;
    }
    
    /**
     * given the node n, try to resolve its string value, maybe by implementing some of the addition (concatenation).
     * This was introduced to support URI completions within Jython codes, allowing the filename to be a variable and
     * thus shortening lines.
     * @param n node within an AST.
     * @param row the row of the caret
     * @param column the column of the caret
     * @param env any variables which have been identified as string values.
     * @return the string or null.
     */
    public static String tryResolveStringNode( SimpleNode n, int row, int column, Map<String,Object> env ) {
        if ( n.beginLine==row ) {
            if ( n instanceof Assign ) {
                Assign a= (Assign)n;
                return tryResolveStringNode( a.value, row, column, env );
            } else if ( n instanceof Str ) {
                return ((Str)n).s;
            } else if ( n instanceof Num ) {
                return String.valueOf(((Num)n).n);
            } else if ( n instanceof Expr ) {
                Expr e= (Expr)n;
                return tryResolveStringNode( e.value, row, column, env );
            } else if ( n instanceof Call ) {
                Call e= (Call)n;
                if ( e.func instanceof Name && ((Name)e.func).id.equals("getParam") ) {
                    if ( e.args.length>1 ) {
                        String s= tryResolveStringNode( e.args[1], row, column, env );
                        if ( s!=null ) return s;
                    }
                } else if ( e.func instanceof Name && ((Name)e.func).id.equals("str") ) {
                    if ( e.args.length==1 ) {
                        String s= tryResolveStringNode( e.args[0], row, column, env );
                        if ( s!=null ) return s;
                    }
                }
                return null;
            } else if ( n instanceof Name ) {
                Name na= (Name)n;
                Object o= env.get( na.id );
                if ( o instanceof String ) {
                    return (String)o;
                } else {
                    return null;
                }
            } else if ( n instanceof BinOp ) {
                BinOp e= (BinOp)n;
                String sleft= tryResolveStringNode( e.left, row, column, env );
                String sright= tryResolveStringNode( e.right, row, column, env );
                if ( sleft!=null && sright!=null && e.op==BinOp.Add ) {
                    return sleft + sright;
                } else if ( sleft!=null && e.right instanceof Tuple && e.op==BinOp.Mod ) {
                    //PythonInterpreter interp= new PythonInterpreter(null);
                    Object[] ss= tryResolveTupleNode( e.right, row, column, env );
                    if ( ss!=null ) {
                        sleft= sleft.replaceAll("\\%d", "%s"); // small cheat
                        try {
                            return String.format( sleft,ss );
                        } catch (Exception ex ) {
                            logger.log( Level.INFO, ex.getMessage(), ex );
                            return null;
                        }
                    } else {
                        return null;
                    }
                } else {
                    return null;
                }
            } else {
                return null;
            }
        } else {
            return null;
        }
    }
    
    /**
     * given the node n, try to resolve its string value, maybe by implementing some of the addition (concatenation).
     * This was introduced to support URI completions within Jython codes, allowing the filename to be a variable and
     * thus shortening lines.
     * @param n the AST
     * @param row the row of the caret
     * @param column the column of the caret
     * @param env any variables which have been identified as string values.
     * @return the string or null.
     */    
    public static String tryResolveStringNode( Module n, int row, int column, Map<String,Object> env ) {
        stmtType thet=null;
        for ( stmtType t : n.body ) {
            if ( t.beginLine>=row ) {
                thet= t;
                break;
            }
            if ( t instanceof Assign ) {
                Assign a= (Assign)t;
                if ( a.targets.length==1 && a.targets[0] instanceof Name ) {
                    if ( a.value instanceof Str ) {
                        env.put( ((Name)a.targets[0]).id, ((Str)a.value).s );
                    } else if ( a.value instanceof BinOp ) {
                        String s= tryResolveStringNode( a.value, a.beginLine, a.beginColumn, env );
                        if ( s!=null ) {
                            env.put( ((Name)a.targets[0]).id, s );
                        }
                    } else if ( a.value instanceof Call ) {
                        Call c= (Call)a.value;
                        if ( c.func instanceof Name && ((Name)c.func).id.equals("getParam") ) {
                            String s= tryResolveStringNode( a.value, a.beginLine, a.beginColumn, env );
                            if ( s!=null ) {
                                env.put( ((Name)a.targets[0]).id, s );
                            }
                        }
                    }
                }
            }
        }
        // find the node within thet containing the carot.
        if ( thet instanceof Assign ) {
            exprType t= ((Assign)thet).value;
            if ( t instanceof Call ) {
                Call c= (Call)t;
                if ( c.func instanceof Name && ((Name)c.func).id.equals("getDataSet") ) {
                    return tryResolveStringNode( c.args[0], row, column, env );
                }
            }
        }
        return null;
    }
    
    /**
     * Remove parts of the script which are expensive so that the script can be run and completions offered. TODO: What is the
     * difference between this and simplifyScriptToCompletions?
     *
     * @param script Jython script
     * @return simplified version of the script.
     * @see #simplifyScriptToCompletions(java.lang.String)
     * @see https://github.com/autoplot/dev/tree/master/bugs/sf/1687
     */
    public static String removeSideEffects(String script) {

        script = alligatorParse(script);

        String[] ss = JythonUtil.splitCodeIntoLines("# simplifyScriptToGetCompletions", script);

        Module n;
        try {
            n = (Module) org.python.core.parser.parse(script, "exec");
        } catch (Exception ex) {
            // do it again so we can debug.
            n = (Module) org.python.core.parser.parse(script, "exec");
        }

        if (false && n.body.length > 0 && n.body[0].beginLine > n.beginLine) {
            logger.fine("shifting line numbers!");
            int shift = n.body[0].beginLine - n.beginLine;
            // strange bug here.
            for (stmtType s : n.body) {
                s.beginLine -= shift;
            }
        }
        HashSet variableNames = new HashSet();
        int ilastLine = ss.length - 1;
        return simplifyScriptToGetCompletions(ss, n.body, variableNames, 1, ilastLine, 0);
    }

    /**
     * useful for debugging
     *
     * @param result
     * @param line
     * @return
     */
    private static StringBuilder appendToResult(StringBuilder result, String line) {
        result.append(line);
        return result;
    }

    /**
     * extracts the parts of the program that are quickly executed, generating a code which can be run and then queried for
     * completions. This uses a Jython syntax tree (AST), so the code must be free of syntax errors. This will remove lines from the
     * end of the code until the code compiles, in case the script has a partially defined def or class.
     *
     * @param script the entire python program
     * @return the python program with lengthy calls removed.
     * @see #removeSideEffects(java.lang.String)
     * @see JythonUtil#simplifyScriptToGetParams(java.lang.String, boolean)
     */
    public static String simplifyScriptToCompletions(String script) throws PySyntaxError {

        if (script.trim().length() == 0) {
            return script;
        }
        script = alligatorParse(script);

        String[] ss = JythonUtil.splitCodeIntoLines("# simplifyScriptToGetCompletions", script);

        int lastLine = ss.length - 1;

        HashSet variableNames = new HashSet();
        variableNames.add("getParam");  // this is what allows the getParam calls to be included.
        //variableNames.add("getDataSet"); // new class lookup is cleaner method for this.
        variableNames.add("str");  // include casts.
        variableNames.add("int");
        variableNames.add("long");
        variableNames.add("float");
        variableNames.add("datum");
        variableNames.add("datumRange");
        variableNames.add("dataset");
        variableNames.add("URI");
        variableNames.add("URL");
        variableNames.add("PWD");
        variableNames.add("dom"); // TODO: only true for .jy scripts.

        try {
            Module n = null;

            int count = 4;
            PySyntaxError ex0 = null;
            while (lastLine > 0 && count > 0) {
                try {
                    n = (Module) org.python.core.parser.parse(script, "exec");
                    break;
                } catch (PySyntaxError ex) { // pop off the last line and try again.
                    if (ex0 == null) {
                        ex0 = ex;
                    }
                    lastLine--;
                    script = JythonUtil.join(Arrays.copyOf(ss, lastLine), "\n");
                    count--;
                }
            }

            if (n == null) {
                throw ex0;
            }

            String s = simplifyScriptToGetCompletions(ss, n.body, variableNames, 1, lastLine, 0);
            //s= GETDATASET_CODE + s;
            //s= "PWD='file:/tmp/'\n"+s;
            return s;

        } catch (PySyntaxError ex) {
            throw ex;
        }
    }

    private static String getIfBlock(String[] ss, stmtType[] body, HashSet variableNames,
            int firstLine, int lastLine1, int depth) {
        StringBuilder result = new StringBuilder();
        String ss1 = simplifyScriptToGetCompletions(ss, body, variableNames, firstLine, lastLine1, depth + 1);
        if (ss1.length() == 0) {
//            String line;
//            if ( firstLine==0 && iff.beginLine>0 ) {
//                line= ss[body[0].beginLine-1]; 
//            } else {
//                line= ss[iff.beginLine];
//            }
            Pattern p = Pattern.compile("(\\s*)(\\S*).*");
            Matcher m = p.matcher(ss[firstLine]);
            String indent;
            if (m.matches()) {
                indent = m.group(1);
            } else {
                indent = "";
            }
            result.append(indent).append("pass  ## SimplifyScriptSupport.getIfBlock \n");
            logger.fine("things have probably gone wrong...");
        } else {
            appendToResult(result, ss1);
        }
        return result.toString();
    }

    /**
     * Using the stmtType get the line, or lines. If the last line contains a single triple-quote, we need to kludge a little and
     * look for the preceding triple-quote in previous lines.
     *
     * @param ss
     * @param o
     * @return
     */
    public static String getSourceForStatement(String[] ss, stmtType o) {
        if (o.beginLine == 0) {
            return "(bad line number)";
        }
        int endLine= o.beginLine;
        int beginLine= endLine;
        if ( o instanceof Expr ) {
            Expr e = (Expr)o;
            if ( e.value.beginLine<beginLine ) {
                beginLine= e.value.beginLine;
            }
        }
        StringBuilder s= new StringBuilder();
        for ( int i=beginLine; i<=endLine; i++ ) {
            s.append( ss[i] );
            if ( i<endLine ) s.append( "\n" );
        }
        return s.toString();
    }

    /**
     * return the indent for a line, for example the " " in " continue "
     *
     * @param line
     * @return the indent
     */
    public static String getIndent(String line) {
        String s = line.trim();
        return line.substring(0, line.indexOf(s));
    }

    /**
     * Extracts the parts of the program that get parameters or take a trivial amount of time to execute. This may call itself
     * recursively when if blocks are encountered. See test038.
     *
     * @param ss the entire script, with a null at index 0.
     * @param stmts statements being processed.
     * @param variableNames variable names that have been resolved.
     * @param beginLine first line of the script being processed, or -1 to use stmts[0].beginLine
     * @param lastLine INCLUSIVE last line of the script being processed.
     * @param depth recursion depth, for debugging.
     * @return the simplified script
     * @see JythonUtil#simplifyScriptToGetParams(java.lang.String[], org.python.parser.ast.stmtType[], java.util.HashSet, int, int,
     * int)
     */
    public static String simplifyScriptToGetCompletions(String[] ss, stmtType[] stmts,
            HashSet variableNames, int beginLine, int lastLine, int depth) {
        //String spaces= "                              "
        //        + "                              "
        //        + "                              ";
        if (lastLine >= ss.length) {
            throw new IllegalArgumentException("lastLine is >= number of lines");
        }
        if (!ss[0].equals("# simplifyScriptToGetCompletions")) {
            throw new IllegalArgumentException("first line must be '# simplifyScriptToGetCompletions'");
        }
        Map<String, String> importedNames = new LinkedHashMap<>();
        importedNames.put("Color", "java.awt");
        importedNames.put("DatumRange", "org.das2.datum" );
        importedNames.put("Units", "org.das2.datum" );
        importedNames.put("DatumRangeUtil","org.das2.datum");
        importedNames.put("TimeUtil","org.das2.datum");
        importedNames.put("URL", "java.net");
        importedNames.put("URI", "java.net");
        importedNames.put("TimeParser", "org.das2.datum" );
        int acceptLine = -1; // first line to accept
        int currentLine = beginLine; // current line we are writing.
        StringBuilder result = new StringBuilder();
        for (int istatement = 0; istatement < stmts.length; istatement++) {
            stmtType o = stmts[istatement];
            String theLine = getSourceForStatement(ss, o);
            int lineCount = theLine.split("\n", -2).length;

            if (depth == 0) {
                logger.finest(theLine); //breakpoint here.
                //System.err.println(theLine);
            }
            logger.log(Level.FINER, "line {0}: {1}", new Object[]{o.beginLine, theLine});
            if (o.beginLine > 0) {
                if (beginLine < 0 && istatement == 0) {
                    acceptLine = o.beginLine;
                }
                if (lineCount > 1) {
                    beginLine = o.beginLine - (lineCount - 1);
                } else {
                    beginLine = o.beginLine;
                }
            } else {
                acceptLine = beginLine; // elif clause in autoplot-test038/lastSuccessfulBuild/artifact/test038_demoParms1.jy
            }
            if (beginLine > lastLine) {
                continue;
            }
            if (o instanceof Assign ) {
                if ( !simplifyScriptToGetCompletionsOkay(o, variableNames,importedNames) ) {
                    // check for method calls where we know the type.
                    Assign a = (Assign) o;
                    String cl = maybeIdentifyType(a, importedNames);
                    if (cl != null) {
                        if (acceptLine > -1) {
                            for (int i = acceptLine; i < beginLine; i++) {
                                appendToResult(result, ss[i]).append("\n");
                            }
                        }
                        appendToResult(result, getIndent(theLine) + cl).append("\n") ;
                        acceptLine = -1;
                        continue;
                    }
                } else {
                    Assign a = (Assign) o;
                    String cl = maybeIdentifyType(a, importedNames);
                    if (cl != null) {
                        if (acceptLine > -1) {
                            for (int i = acceptLine; i < beginLine; i++) {
                                appendToResult(result, ss[i]).append("\n");
                            }
                        }
                        appendToResult(result, getIndent(theLine) + cl).append("\n") ;
                        acceptLine = -1;
                        continue;
                    }
                }
            }

            if (o instanceof org.python.parser.ast.ImportFrom) {
                org.python.parser.ast.ImportFrom i = (org.python.parser.ast.ImportFrom) o;
                for (aliasType n : i.names) {
                    importedNames.put(n.name, i.module);
                }
            }

            if (o instanceof org.python.parser.ast.If) {
                if (acceptLine > -1) {
                    for (int i = acceptLine; i < beginLine; i++) {
                        appendToResult(result, ss[i]).append("\n");
                    }
                }
                If iff = (If) o;
                boolean includeBlock;
                if (simplifyScriptToGetCompletionsCanResolve(iff.test, variableNames)) {
                    for (int i = beginLine; i < iff.body[0].beginLine; i++) {
                        result.append(ss[i]).append("\n");
                    } // write out the 'if' part
                    includeBlock = true;
                } else {
                    includeBlock = false;
                }
                int lastLine1;  //lastLine1 is the last line of the "if" clause. (inclusive)
                int elseLine = -1;
                if (iff.orelse != null && iff.orelse.length > 0) {
                    if (iff.orelse[0].beginLine > 0) {
                        lastLine1 = iff.orelse[0].beginLine - 1;  // -1 is for the "else:" part.
                        if (ss[lastLine1].trim().startsWith("else")) {
                            elseLine = lastLine1;
                            lastLine1 = lastLine1 - 1;
                        } else if (ss[lastLine1].trim().startsWith("elif")) {
                            elseLine = lastLine1;
                            lastLine1 = lastLine1 - 1;
                        }
                    } else {
                        if (iff.orelse[0] instanceof If) {
                            elseLine = ((If) iff.orelse[0]).test.beginLine;
                            lastLine1 = elseLine - 1;
                        } else {
                            logger.warning("failure to deal with another day...");
                            throw new RuntimeException("this case needs to be dealt with...");
                        }
                    }
                } else if ((istatement + 1) < stmts.length) {
                    lastLine1 = stmts[istatement + 1].beginLine - 1;
                } else {
                    lastLine1 = lastLine;
                }
                if (includeBlock) {
                    HashSet variableNames1= new HashSet(variableNames);
                    String ss1 = getIfBlock(ss, iff.body, variableNames1,
                            Math.min(beginLine + 1, lastLine1), lastLine1, depth + 1);
                    appendToResult(result, ss1);
                    if (iff.orelse != null) {
                        if ((istatement + 1) >= stmts.length) {
                            lastLine1 = lastLine;
                        } else {
                            lastLine1 = stmts[istatement + 1].beginLine - 1;
                        }
                        if (iff.orelse[0].beginLine == 0) {
                            result.append("\n");
                        } else {
                            if (iff.orelse[0].beginLine > 0 && ss[iff.orelse[0].beginLine - 1].trim().startsWith("else:")) {
                                result.append(ss[iff.orelse[0].beginLine - 1]).append("\n");
                                HashSet variableNames2= new HashSet(variableNames);
                                ss1 = getIfBlock(ss, iff.orelse, variableNames2, iff.orelse[0].beginLine, lastLine1, depth + 1);
                                appendToResult(result, ss1);
                                //TODO: if variable is added to variableNames1 and variableNames2 then add it to variableNames.
                            } else {
                                result.append(ss[iff.orelse[0].beginLine]).append("\n");
                            }
                        }

                    }
                }
                currentLine = lastLine1;
                acceptLine = -1;
            } else if (o instanceof Assert) {
                String m = maybeModelAssert((Assert) o, variableNames);
                if (m != null) {
                    result.append(m).append("\n");
                    currentLine = acceptLine;
                }
            } else if ( o instanceof Expr ) {
                if ( ((Expr)o).value instanceof Str ) {
                    result.append(theLine).append("\n");
                    currentLine = o.beginLine;
                    acceptLine= -1;
                } else {
                    if (acceptLine > -1) { // always skip Expr.
                        int thisLine = beginLine;
                        for (int i = acceptLine; i <= thisLine; i++) {
                            if (i < thisLine) {
                                appendToResult(result, ss[i]).append("\n");
                            } else {
                                if (ss[i].length() > 0 && Character.isWhitespace(ss[i].charAt(0))) {
                                    appendToResult(result, ss[i]).append("\n");
                                }
                            }
                        }
                        appendToResult(result, "\n");
                        currentLine = thisLine;
                        acceptLine = -1;
                    }       
                }
            } else { // Assign, etc
                if (simplifyScriptToGetCompletionsOkay(o, variableNames,importedNames)) {
                    if (acceptLine < 0) {
                        acceptLine = beginLine;
                        for (int i = currentLine + 1; i < acceptLine; i++) {
                            result.append("\n");
                            currentLine = acceptLine;
                        }
                    }
                } else {
                    if (acceptLine > -1) {
                        int thisLine = beginLine;
                        for (int i = acceptLine; i <= thisLine; i++) {
                            if (i < thisLine) {
                                appendToResult(result, ss[i]).append("\n");
                            } else {
                                if (ss[i].length() > 0 && Character.isWhitespace(ss[i].charAt(0))) {
                                    appendToResult(result, ss[i]).append("\n");
                                }
                            }
                        }
                        appendToResult(result, "\n");
                        currentLine = thisLine;
                        acceptLine = -1;
                    }
                }
            }
        }
        if (acceptLine > -1) {
            lastLine = JythonUtil.handleContinue(ss, lastLine);
            int thisLine = lastLine;
            for (int i = acceptLine; i <= thisLine; i++) {
                appendToResult(result, ss[i]).append("\n");
            }
            String slastLine = ss[thisLine].trim();
            if (slastLine.endsWith(":")) {
                appendToResult(result, getIndent(slastLine) + "    pass");
            }
        }
        return result.toString();
    }

    /**
     * can we resolve this node given the variable names we know?
     *
     * @param o
     * @param variableNames
     * @return true if the node can be resolved.
     */
    private static boolean simplifyScriptToGetCompletionsCanResolve(SimpleNode o, HashSet<String> variableNames) {
        //if ( o.beginLine>=617 && o.beginLine<619 ) {
        //    System.err.println( "here at 617-ish");
        //}
        if (o instanceof Name) {
            Name c = (Name) o;
            if (!variableNames.contains(c.id)) {
                logger.finest(String.format("%04d canResolve->false: %s", o.beginLine, o.toString()));
                return false;
            }
        } else if (o instanceof Attribute) {
            Attribute at = (Attribute) o;
            while (at.value instanceof Attribute || at.value instanceof Subscript) {
                if (at.value instanceof Attribute) {
                    at = (Attribute) at.value;
                } else {
                    Subscript s = (Subscript) at.value;
                    if (s.value instanceof Attribute) {
                        at = (Attribute) s.value;
                    } else {
                        return false; // oh just give up...
                    }
                }
            }   
            if ( !simplifyScriptToGetCompletionsCanResolve( at.value, variableNames ) ) {
                return false;
            }
        } else if ( o instanceof Compare ) {
            Compare c = (Compare)o;
            if ( simplifyScriptToGetCompletionsCanResolve( c.left, variableNames ) ) {
                for ( exprType e : c.comparators ) {
                    if ( ! simplifyScriptToGetCompletionsCanResolve( e, variableNames ) ) {
                        return false;
                    }
                }
                return true;
            } else {
                return false;
            }
                  
        } else if ( o instanceof Call ) {
            Call c = (Call)o;
            if ( !simplifyScriptToGetCompletionsCanResolve( c.func, variableNames ) ) {
                return false;
            }
            for ( exprType e : c.args ) {
                if ( ! simplifyScriptToGetCompletionsCanResolve( e, variableNames ) ) {
                    return false;
                }
            }
        } else if ( o instanceof Subscript ) {
            return false;
              
        }
        MyVisitorBase vb = new MyVisitorBase(variableNames);
        try {
            o.traverse(vb);
            logger.finest(String.format(" %04d canResolve->%s: %s", o.beginLine, vb.visitNameFail, o));
            return vb.looksOkay || !vb.visitNameFail;

        } catch (Exception ex) {
            logger.log(Level.SEVERE, ex.getMessage(), ex);
        }
        logger.finest(String.format("!! %04d canResolve->false: %s", o.beginLine, o));
        return false;
    }

    /**
     * dumb kludge where no-arg constructor is called to get an instance for completions. This is really an experiment...
     *
     * @param a
     * @return
     */
    private static String maybeModelAssert(Assert a, HashSet<String> variableNames) {
        if (a.test instanceof Call) {
            org.python.parser.ast.Call cc = (org.python.parser.ast.Call) a.test;
            exprType f = cc.func;
            if (f instanceof Name) {
                if (((Name) f).id.equals("isinstance")) {
                    if (cc.args.length == 2) {
                        exprType a1 = cc.args[0];
                        if (a1 instanceof Name) {
                            exprType a2 = cc.args[1];
                            if (a2 instanceof Name && variableNames.contains(((Name) a2).id)) {
                                return String.format("%s" + JythonCompletionTask.__CLASSTYPE + "=%s # inserted by maybeModelAssert", ((Name) a1).id, ((Name) a2).id);
                            }
                        }
                    }
                }
            }
            return null;
        } else {
            return null;
        }
    }

    /**
     * return true if we can include this in the script without a huge performance penalty.
     * This should be used for read access, not write access, so:<ul>
     * <li>p= dom.plots[0]  is okay
     * <li>dom.plots[1].rowId= dom.plots[0].rowId  is not okay.
     * </ul>
     * @param o the statement, for example an import or an assignment
     * @param variableNames known symbol names 
     * @param importedNames map of name to path (DatumRange &rarr; org.das2.datum)
     * @return true if we can include this in the script without a huge performance penalty.
     */
    private static boolean simplifyScriptToGetCompletionsOkay(stmtType o, HashSet<String> variableNames,Map<String,String> importedNames) {
        logger.log(Level.FINEST, "simplify script line: {0}", o.beginLine);
        if ((o instanceof org.python.parser.ast.ImportFrom)) {
            org.python.parser.ast.ImportFrom importFrom = (org.python.parser.ast.ImportFrom) o;
            for (aliasType a : importFrom.names) {
                if (a.asname != null) {
                    variableNames.add(a.asname);
                } else {
                    variableNames.add(a.name);
                }
            }
            return true;
        }
        if ((o instanceof org.python.parser.ast.Import)) {
            org.python.parser.ast.Import imporrt = (org.python.parser.ast.Import) o;
            for (aliasType a : imporrt.names) {
                if (a.asname != null) {
                    variableNames.add(a.asname);
                } else {
                    variableNames.add(a.name);
                }
            }
            return true;
        }
        if ((o instanceof org.python.parser.ast.Expr)) {
            Expr e = (Expr) o;
            if ((e.value instanceof Call) ) {
                if ( trivialFunctionCall((Call) e.value) ) {
                    return true;
                } else {
                    Call c= (Call) e.value;
                    if ( c.func instanceof Attribute ) {
                        Attribute aa= (Attribute)c.func;
                        if ( aa.value instanceof Name ) {
                            Name naa= (Name)aa.value;
                            if ( variableNames.contains(naa.id) ) {
                                return true;
                            }
                        }
                    }
                }
            }
        }
        if ((o instanceof org.python.parser.ast.ClassDef)) {
            return true;
        }
        if ((o instanceof org.python.parser.ast.FunctionDef)) {
            return true;
        }
        if ((o instanceof org.python.parser.ast.Assign)) {
            Assign a = (Assign) o;
            if (simplifyScriptToGetCompletionsOkayNoCalls(a.value, variableNames)) {
                if (!simplifyScriptToGetCompletionsCanResolve(a.value, variableNames)) {
                    return false;
                }
                for (exprType target : a.targets) {
                    exprType et = (exprType) target;
                    if (et instanceof Name) {
                        String id = ((Name) target).id;
                        variableNames.add(id);
                        logger.log(Level.FINEST, "assign to variable {0}", id);
                        //TODO: can we identify type?  Insert <id>__CLASSTYPE=... for completions.
                        if ( a.value instanceof Call ) {
                            String type= maybeIdentifyReturnType( id, (Call)a.value, importedNames );
                            logger.log(Level.INFO, "id type: {0}__CLASSTYPE= {1}", new Object[]{id, type});
                        }
                    } else if (et instanceof Attribute) {
                        return false;
//                        Attribute at = (Attribute) et;
//                        while (at.value instanceof Attribute || at.value instanceof Subscript) {
//                            if (at.value instanceof Attribute) {
//                                at = (Attribute) at.value;
//                            } else {
//                                Subscript s = (Subscript) at.value;
//                                if (s.value instanceof Attribute) {
//                                    at = (Attribute) s.value;
//                                } else {
//                                    return false; // oh just give up...
//                                }
//                            }
//                        }
//                        if (at.value instanceof Name) {
//                            Name n = (Name) at.value;
//                            if (!variableNames.contains(n.id)) {
//                                return false;
//                            }
//                        }
                    } else if (et instanceof Subscript) {
                        return false;
                        //Subscript subscript = (Subscript) et;
                        //exprType et2 = subscript.value;
                        //if (et2 instanceof Name) {
                        //    Name n = (Name) et2;
                        //    if (variableNames.contains(n.id)) {
                        //        return true;
                        //    }
                        //}
                        //return false;
                    } else {
                        return false;
                    }
                }
                return true;
            } else {
                return false;
            }
        }
        if ((o instanceof org.python.parser.ast.If)) {
            return simplifyScriptToGetCompletionsOkayNoCalls(o, variableNames);
        }
        if ((o instanceof org.python.parser.ast.Print)) {
            return false;
        }
        logger.log(Level.FINEST, "not okay to simplify: {0}", o);
        return false;
    }

    /**
     * inspect the node to look for function calls that are not to the function "getParam". This is awful code that will be
     * rewritten when we upgrade Python to 2.7.
     *
     * @param o
     * @param variableNames
     * @return
     */
    private static boolean simplifyScriptToGetCompletionsOkayNoCalls(SimpleNode o, HashSet<String> variableNames) {

        if (o instanceof Call) {
            Call c = (Call) o;

            if (!trivialFunctionCall(c)) {
                if (!trivialConstructorCall(c)) {
                    logger.finest(String.format("%04d simplify->false: %s", o.beginLine, o.toString()));
                    return false;
                }
            }
        }
        MyVisitorBase vb = new MyVisitorBase(variableNames);
        try {
            o.traverse(vb);
            logger.finest(String.format(" %04d simplify->%s: %s", o.beginLine, vb.looksOkay(), o));
            return vb.looksOkay();

        } catch (Exception ex) {
            logger.log(Level.SEVERE, ex.getMessage(), ex);
        }
        logger.finest(String.format("!! %04d simplify->false: %s", o.beginLine, o));
        return false;
    }

    /**
     * there are a number of functions which take a trivial amount of time to execute and are needed for some scripts, such as the
     * string.upper() function. The commas are to guard against the id being a subset of another id ("lower," does not match
     * "lowercase"). TODO: update this after Python upgrade.
     * @see JythonUtil#okay
     */
    private static final String[] okay = new String[]{
        "range,", "xrange,", "irange,","map,","join,",
        "getParam,", "lower,", "upper,", "URI,", "URL,",
        "setScriptDescription", "setScriptTitle", "setScriptLabel", "setScriptIcon",
        "DatumRangeUtil,", "TimeParser,",
        "str,", "int,", "long,", "float,", "datum,", "datumRange,", "dataset,",
        "indgen,", "findgen,", "dindgen,",
        "ones,", "zeros,",
        "linspace,", "logspace,",
        "dblarr,", "fltarr,", "strarr,", "intarr,", "bytarr,",
        "ripples,",//"split,", // remove split because it is confused with the URISplit.
        "color,", "colorFromString,", "isinstance,"};
 
    private static final Set<String> okaySet = new HashSet<>();

    static {
        for (String o : okay) {
            okaySet.add(o.substring(0, o.length() - 1));
        }
    }

    private static String getFunctionName(exprType t) {
        if (t instanceof Name) {
            return ((Name) t).id;
        } else if (t instanceof Attribute) {
            Attribute a = (Attribute) t;
            return getFunctionName(a.value) + "." + a.attr;
        } else {
            return t.toString();
        }
    }

    /**
     * return true if the function call is trivial to execute and can be evaluated within a few milliseconds. For example, findgen
     * can be called because no calculations are made in the call, but fft cannot. Typically these are Order 1 (a.k.a. constant
     * time) operations, but also many Order N operations are so fast they are allowed.
     *
     * @param sn an AST node pointed at a Call.
     * @return true if the function call is trivial to execute
     */
    private static boolean trivialFunctionCall(SimpleNode sn) {
        if (sn instanceof Call) {
            Call c = (Call) sn;
            boolean klugdyOkay = false;
            String ss = c.func.toString(); // we just want "DatumRangeUtil" of the Attribute
            //String ss= getFunctionName(c.func);
            for (String s : okay) {
                if (ss.contains(s)) {
                    klugdyOkay = true;
                }
            }
            if (klugdyOkay == false) {
                if (ss.contains("TimeUtil") && ss.contains("now")) {
                    klugdyOkay = true;
                }
            }
            logger.log(Level.FINER, "trivialFunctionCall={0} for {1}", new Object[]{klugdyOkay, c.func.toString()});
            return klugdyOkay;
        } else {
            return false;
        }
    }

    /**
     * return true if the function call is trivial to execute because it's a constructor, which presumably takes little time to
     * create.
     *
     * @param sn
     * @return true if it is a constructor call
     */
    private static boolean trivialConstructorCall(SimpleNode sn) {
        if (sn instanceof Call) {
            Call c = (Call) sn;
            if (c.func instanceof Name) {
                String funcName = ((Name) c.func).id;
                return Character.isUpperCase(funcName.charAt(0));
            } else if (c.func instanceof Attribute) {  // Rectangle.Double
                String funcName = ((Attribute) c.func).attr;
                return Character.isUpperCase(funcName.charAt(0));
            } else {
                return false;
            }
        } else {
            return false;
        }
    }

    /**
     * placeholder for code which identifies constructors. This is fragile and misguided code, which looks for the Java convention
     * of uppercase letter starting.
     *
     * @param name the name, like GrannyTextRenderer
     * @param importedNames map from name to path "GrannyTextRenderer" &rarr; "org.das2.util"
     * @return true if the name is known to be a constructor call.
     */
    private static boolean isConstructor(String name, Map<String, String> importedNames) {
        if (importedNames.containsKey(name)) {
            return name.length() > 2 && Character.isUpperCase(name.charAt(0)) && Character.isLowerCase(name.charAt(1));
        } else {
            return false;
        }

    }

    /**
     * return the class for the name.
     *
     * @param name the name, like GrannyTextRenderer
     * @param importedNames map from name to path "GrannyTextRenderer" &rarr; "org.das2.util"
     * @return
     */
    private static Class getClassFor(String name, Map<String, String> importedNames) {
        String path = importedNames.get(name);
        try {
            return Class.forName(path + "." + name);
        } catch (ClassNotFoundException ex) {
            return null;
        }
    }

    private static String maybeIndentifyValue( exprType t ) {
        if ( t instanceof Num ) {
            return String.valueOf(((Num)t).n);
        } else if ( t instanceof Str ) {
            return "'"+ ((Str)t).toString() + "'";
        } else if ( t instanceof Name ) {
            String n=  ((Name)t).id;
            if ( n.equals("True") || n.equals("False") ) {
                return n;
            } else {
                return null;
            }
        } else {
            return null;
        }
    }
    
    /**
     * return the line of code needed to use the result, or null.
     *
     * @param id the identifier being assigned the value
     * @param c the function being called
     * @param importedNames map of name to path (DatumRange &rarr; org.das2.datum)
     * @return the assignment and possibly an additional import statement
     */
    private static String maybeIdentifyReturnType(String id, Call c, Map<String, String> importedNames) {
        if (c.func instanceof Name) {
            String funcName = ((Name) c.func).id;
            switch (funcName) {
                case "getApplication":
                    return "from org.autoplot import AutoplotUI\n" + id + JythonCompletionTask.__CLASSTYPE + " = AutoplotUI\n";
                case "getApplicationModel":
                    return "from org.autoplot import ApplicationModel\n" + id + JythonCompletionTask.__CLASSTYPE + " = ApplicationModel\n";
                case "getDataSource":
                    return "from org.autoplot.datasource import DataSource\n" + id + JythonCompletionTask.__CLASSTYPE + " = DataSource\n";
                case "getParam": // weird--there's a special getParam loaded, but this doesn't work for  getParam( 'allowCache', True, 'allow storage of ephemeris to local file', [ True,False ] )
                    exprType e1= c.args[1];
                    String vv= maybeIndentifyValue(e1);
                    if ( vv!=null ) {
                        return id + " = " + vv;
                    } else {
                        if ( e1.getImage()==null ) {
                            if ( e1 instanceof Subscript ) {
                                Subscript s= (Subscript)e1;
                                if ( s.slice instanceof Index && s.value instanceof Name ) {
                                    Index i= (Index)s.slice;
                                    String ss= maybeIndentifyValue(i.value);
                                    String nn= ((Name)s.value).id;
                                    return id + " = " + nn + "[" + ss +"]";
                                } else {
                                    return null;
                                }
                            } else {
                                return null;
                            }
                        } else {
                            return id  + JythonCompletionTask.__CLASSTYPE + " = " + e1.getImage();
                        }
                    }
                case "getDataSet":
                case "xtags":
                case "ytags":
                case "findgen":
                case "linspace":
                case "fftPower":
                case "magnitude":
                    return id + JythonCompletionTask.__CLASSTYPE + " = QDataSet    # return type from " + funcName + " (spot line789)\n";
                case "datumRange":
                    return id + JythonCompletionTask.__CLASSTYPE + " = DatumRange    # return type from " + funcName + " (spot line896)\n";
                case "datum":
                    return id + JythonCompletionTask.__CLASSTYPE + " = Datum    # return type from " + funcName + " (spot line898)\n";
                default:
                    break;
            }
            for ( Method m: Ops.class.getMethods() ) {
                if ( m.getName().equals(funcName) ) {
                    if ( QDataSet.class.isAssignableFrom( m.getReturnType() ) ) {
                        return id + JythonCompletionTask.__CLASSTYPE + " = QDataSet    # ( spot line 898 )";
                    }
                }
            }
            if (isConstructor(funcName, importedNames)) {
                return id + JythonCompletionTask.__CLASSTYPE + " = " + funcName + "  # isConstructor (line794)\n";
            } else {
                return null;
            }
        } else if (c.func instanceof Attribute) {
            // p=PngWalkTool.start(...)
            Attribute at = (Attribute) c.func;
            if (at.value instanceof Name) {
                String attrName = ((Name) at.value).id;
                if (attrName.equals("PngWalkTool") && at.attr.equals("start")) {
                    return "from org.autoplot.pngwalk import PngWalkTool\n" + id + JythonCompletionTask.__CLASSTYPE + "= PngWalkTool #(spot line802)\n";
                } else if (importedNames.containsKey(attrName)) {
                    Class claz = getClassFor(attrName, importedNames);
                    if (claz == null) {
                        return null;
                    }
                    Method[] mm = claz.getMethods();
                    String methodName = at.attr;
                    for (Method m : mm) {
                        if (m.getName().equals(methodName)) { //&& m.getGenericParameterTypes().length ) {
                            Class rclz = m.getReturnType();
                            String rclzn = rclz.getSimpleName();
                            StringBuilder result= new StringBuilder();
                            if ( rclz.getPackage()!=null ) {
                                result.append( String.format( "from %s import %s\n", rclz.getPackage().getName(), rclzn ) );
                            }
                            result.append( String.format( "%s%s = %s # (line895)\n", id, JythonCompletionTask.__CLASSTYPE, rclzn ) );
                            return result.toString();
                        }
                    }
                    return null;
                }
            } else if (at.value instanceof Call) {
                String x = maybeIdentifyReturnType("x", (Call) at.value, importedNames);
                if (x != null) {
                    int i = x.indexOf("=");
                    try {
                        String s= x.substring(i + 1);
                        int i2= s.indexOf('#');
                        if ( i2>-1 ) {
                            s= s.substring(0,i2);
                        }
                        s= s.trim();
                        if ( importedNames.containsKey(s) ) {
                            String packg= importedNames.get(s);
                            Class clz = Class.forName(packg+"."+s);
                            try {
                                Method m = clz.getMethod(at.attr); //TODO: this is only single-argument calls
                                Class rclz = m.getReturnType();
                                String rclzn = rclz.getSimpleName();
                                return "from " + rclz.getPackage().getName() + " import " + rclzn + "\n"
                                        + id + JythonCompletionTask.__CLASSTYPE + " = " + rclzn + "   # (spot line826)";
                            } catch (NoSuchMethodException | SecurityException ex) {
                                logger.log(Level.SEVERE, null, ex);
                            }
                        }
                    } catch (ClassNotFoundException ex) {
                        logger.log(Level.SEVERE, null, ex);
                        return null; // shouldn't happen
                    }
                    return null;
                } else {
                    return null;
                }
            }
            return null;
        } else {
            return null;
        }
    }

    /**
     * if we recognize the function that is called, then go ahead and keep track of the type. This is a quick and cheesy
     * implementation that just looks for a few names. For example,<ul>
     * <li>x= getApplication() -- x is an AutoplotUI
     * <li>x= getApplicationModel() -- x is a ApplicationModel
     * <li>x= getDataSource() -- x is a DataSource
     * <li>x= PngWalkTool.start() -- x is a PngWalkTool
     * </ul>
     * See bug https://sourceforge.net/p/autoplot/bugs/2319/ for some discussion on this.
     *
     * @param a the assignment
     * @return the Jython code to insert.
     */
    private static String maybeIdentifyType(Assign a, Map<String, String> importedNames) {
        if (a.targets.length == 1) {
            exprType target = a.targets[0];
            exprType et = (exprType) target;
            if (et instanceof Name) {
                String id = ((Name) target).id;
                if (a.value instanceof Call) {
                    Call c = (Call) a.value;
                    return maybeIdentifyReturnType(id, c, importedNames);
                } else if ( a.value instanceof Subscript ) {
                    Subscript s= ((Subscript)a.value);
                    if ( s.value instanceof Attribute ) {
                        Attribute att = (Attribute)s.value;
                        if ( att.value instanceof Name ) {
                            if ( ((Name)att.value).id.equals("dom") ) {
                                if ( att.attr.equals("plots") ) {
                                    String rclzn = "org.autoplot.dom.Plot";
                                    return "import org.autoplot.dom.Plot\n"
                                    + id + JythonCompletionTask.__CLASSTYPE + " = " + rclzn + "  # (spot line955 a)\n";
                                } else if ( att.attr.equals("canvases") ) {
                                    String rclzn = "org.autoplot.dom.Canvas";
                                    return "import org.autoplot.dom.Canvas\n"
                                    + id + JythonCompletionTask.__CLASSTYPE + " = " + rclzn + "  # (spot line955 b)\n";
                                } else if ( att.attr.equals("plotElements") ) {
                                    String rclzn = "org.autoplot.dom.PlotElement";
                                    return "import org.autoplot.dom.PlotElement\n"
                                    + id + JythonCompletionTask.__CLASSTYPE + " = " + rclzn + "  # (spot line955 c)\n";
                                }
                            }
                        }
                    } else if ( s.value instanceof Name ) {
                        return id + JythonCompletionTask.__CLASSTYPE + " = QDataSet  # (spot line1014 a)\n";
                    }
                } else if ( a.value instanceof BinOp ) { // just go ahead and assume it's a QDataSet
                    return id + JythonCompletionTask.__CLASSTYPE + " = QDataSet  # (spot line1014 b)\n";
                }
            }
        }
        return null;
    }

    private static Object Name(exprType func) {
        throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
    }

    private static class MyVisitorBase<R> extends VisitorBase {

        boolean looksOkay = true;
        boolean visitNameFail = false;

        HashSet names;

        MyVisitorBase(HashSet names) {
            this.names = names;
        }

        @Override
        public Object visitName(Name node) throws Exception {
            logger.log(Level.FINER, "visitName({0})", node);
            if (!names.contains(node.id)) {
                visitNameFail = true;
            }
            return super.visitName(node);
        }

        @Override
        public Object visitCall(Call node) throws Exception {
            logger.log(Level.FINER, "visitCall({0})", node);
            return super.visitCall(node);
        }

        @Override
        protected Object unhandled_node(SimpleNode sn) throws Exception {
            return sn;
        }

        @Override
        public void traverse(SimpleNode sn) throws Exception {
            logger.log(Level.FINER, "traverse({0})", sn);
            if (sn instanceof Call) {
                looksOkay = trivialFunctionCall(sn) || trivialConstructorCall(sn);
                logger.log(Level.FINER, "looksOkay={0}", looksOkay);
            } else if (sn instanceof Assign) {
                Assign a = ((Assign) sn);
                exprType et = a.value;
                if (et instanceof Call) {
                    looksOkay = trivialFunctionCall(et) || trivialConstructorCall(sn);
                    logger.log(Level.FINER, "looksOkay={0}", looksOkay);
                }
            } else if (sn instanceof Name) {
                String t = ((Name) sn).id;
                if (t.length() > 1 && Character.isUpperCase(t.charAt(0))) {
                    logger.log(Level.FINER, "name is assumed to be a constructor call name: {0}", t);
                    return;
                }
                if (!names.contains(t)
                        && !okaySet.contains(t)) {
                    looksOkay = false; // TODO: why are there both looksOkay and visitNameFail?
                    logger.log(Level.FINER, "looksOkay={0}", looksOkay);
                }
            } else if (sn instanceof Attribute) {
                traverse(((Attribute) sn).value);  // DatumRangeUtil

            } else if (sn instanceof Subscript) {
                Subscript ss = (Subscript) sn;
                exprType et = ss.value;
                if (et instanceof Name) {
                    traverse((Name) (et));
                }
                //ss.value;
                //visitName((Name))
            } else if (sn instanceof BinOp) {
                BinOp bo = (BinOp) sn;
                traverse(bo.left);
                traverse(bo.right);
            } else if (sn instanceof UnaryOp) {
                UnaryOp bo = (UnaryOp) sn;
                traverse(bo.operand);
            } else if (sn instanceof Num) {

            } else if ( sn instanceof Str ) {
                
            } else if (sn instanceof Index) {
                Index index = (Index) sn;
                traverse(index.value);
                
            } else if (sn instanceof Tuple) {
                Tuple tuple = (Tuple) sn;
                for ( exprType e: tuple.elts ) {
                    traverse(e);
                }
            } else if (sn instanceof List) {
                List ll = (List) sn;
                for ( exprType e: ll.elts ) {
                    traverse(e);
                }   
            } else if (sn instanceof Dict) {
                Dict dict = (Dict) sn;
                for ( exprType e: dict.keys ) {
                    traverse(e);
                }   
                for ( exprType e: dict.values ) {
                    traverse(e);
                }   
            } else {
                logger.log(Level.FINE, "unchecked: {0}", sn);
                looksOkay= false;
            }
        }

        public boolean looksOkay() {
            return looksOkay;
        }

        /**
         * this contains a node whose name we can't resolve.
         *
         * @return
         */
        public boolean visitNameFail() {
            return visitNameFail;
        }
    }

}