package org.das2.qds;

import java.lang.reflect.Array;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.das2.qds.buffer.BufferDataSet;
import org.das2.qds.buffer.ByteDataSet;
import org.das2.qds.buffer.FloatDataSet;
import org.das2.qds.buffer.IntDataSet;
import org.das2.qds.buffer.ShortDataSet;
import org.das2.datum.CacheTag;
import org.das2.datum.EnumerationUnits;
import org.das2.datum.Units;
import org.das2.datum.UnitsConverter;
import org.das2.datum.UnitsUtil;
import org.das2.qds.buffer.LongDataSet;
import org.das2.util.LoggerManager;
import org.das2.qds.ops.Ops;

/**
 * ArrayDataSet is the abstract base class for QDataSets which are backed by
 * Java arrays  For example, DDataSet is a QDataSet which uses a double array
 * to store its data.  Data is stored in 1-D Java arrays for performance, 
 * inspired by https://www.cs.cmu.edu/~artigas/papers/cacm01.pdf.  Note for 
 * modern versions of Java, Arrays of Arrays are implemented with no performance
 * cost.
 * 
 * A number of static methods were initially defined in DDataSet, then
 * copied into FDataSet and others when they were made.  This super implementation
 * will parent all such datasets and provide common methods.
 * @author jbf
 */
public abstract class ArrayDataSet extends AbstractDataSet implements WritableDataSet {

    protected static final Logger logger= LoggerManager.getLogger("qdataset.array");

    int rank;

    int len0;
    int len1;
    int len2;
    int len3;

    float fill= Float.NaN;
    double dfill= Double.NaN;

    private static final boolean RANGE_CHECK = "true".equals( System.getProperty("rangeChecking","true") );

    Class componentType;
    
    protected ArrayDataSet( Class componentType ) {
        this.componentType= componentType;
    }
    
    protected static ArrayDataSet create( int rank, int len0, int len1, int len2, int len3, Object back ) {
        Class c= back.getClass().getComponentType();
        if ( c==double.class ) return new DDataSet( rank, len0, len1, len2, len3, (double[])back );
        if ( c==float.class ) return new FDataSet( rank, len0, len1, len2, len3, (float[])back );
        if ( c==long.class ) return new LDataSet( rank, len0, len1, len2, len3, (long[])back );
        if ( c==int.class ) return new IDataSet( rank, len0, len1, len2, len3, (int[])back );
        if ( c==short.class ) return new SDataSet( rank, len0, len1, len2, len3, (short[])back );
        if ( c==byte.class ) return new BDataSet( rank, len0, len1, len2, len3, (byte[])back );
        throw new IllegalArgumentException("class not supported: "+c);
    }
    
    /**
     * create a dataset with the backing type and the dimensions given.
     * The following component types are supported: double.class, float.class,
     * long.class, int.class, short.class, and byte.class.  
     * @param c the backing type, such as double.class, float.class, etc.
     * @param qube dimensions of the dataset
     * @return the dataset
     * @see #getComponentType() 
     * @see BufferDataSet which also supports unsigned types.
     */
    public static ArrayDataSet create( Class c, int[] qube ) {
        if ( c==double.class ) return DDataSet.create( qube );
        if ( c==float.class ) return FDataSet.create( qube );
        if ( c==long.class ) return LDataSet.create( qube );
        if ( c==int.class ) return IDataSet.create( qube );
        if ( c==short.class ) return SDataSet.create( qube );
        if ( c==byte.class ) return BDataSet.create( qube );
        throw new IllegalArgumentException("class not supported: "+c);      
    }

    /**
     * create a rank 0 dataset of the class, which can be 
     * double.class, float.class, long.class, int.class, short.class and byte.class
     * @param c the primitive class of each element.
     * @return the ArrayDataSet
     */
    public static ArrayDataSet createRank0( Class c ) {
        if ( c==double.class ) return new DDataSet( 0, 1, 1, 1, 1 );
        if ( c==float.class ) return new FDataSet( 0, 1, 1, 1, 1 );
        if ( c==long.class ) return new LDataSet( 0, 1, 1, 1, 1 );
        if ( c==int.class ) return new IDataSet( 0, 1, 1, 1, 1 );
        if ( c==short.class ) return new SDataSet( 0, 1, 1, 1, 1 );
        if ( c==byte.class ) return new BDataSet( 0, 1, 1, 1, 1 );
        throw new IllegalArgumentException("class not supported: "+c);
    }
    
    /**
     * create a rank 1 dataset of the class, which can be 
     * double.class, float.class, long.class, int.class, short.class and byte.class
     * @param c the primitive class of each element.
     * @param len0 the length of the dimension
     * @return the ArrayDataSet
     */
    public static ArrayDataSet createRank1( Class c, int len0 ) {
        if ( c==double.class ) return new DDataSet( 1, len0, 1, 1, 1 );
        if ( c==float.class ) return new FDataSet( 1, len0, 1, 1, 1 );
        if ( c==long.class ) return new LDataSet( 1, len0, 1, 1, 1 );
        if ( c==int.class ) return new IDataSet( 1, len0, 1, 1, 1 );
        if ( c==short.class ) return new SDataSet( 1, len0, 1, 1, 1 );
        if ( c==byte.class ) return new BDataSet( 1, len0, 1, 1, 1 );
        throw new IllegalArgumentException("class not supported: "+c);
    }

    /**
     * create a rank 2 dataset of the class, which can be 
     * double.class, float.class, long.class, int.class, short.class and byte.class
     * @param c the primitive class of each element.
     * @param len0 the length of the dimension
     * @param len1 the length of the dimension
     * @return the ArrayDataSet
     */
    public static ArrayDataSet createRank2( Class c, int len0, int len1 ) {
        if ( c==double.class ) return new DDataSet( 2, len0, len1, 1, 1 );
        if ( c==float.class ) return new FDataSet( 2, len0, len1, 1, 1 );
        if ( c==long.class ) return new LDataSet( 2, len0, len1, 1, 1 );
        if ( c==int.class ) return new IDataSet( 2, len0, len1, 1, 1 );
        if ( c==short.class ) return new SDataSet( 2, len0, len1, 1, 1 );
        if ( c==byte.class ) return new BDataSet( 2, len0, len1, 1, 1 );
        throw new IllegalArgumentException("class not supported: "+c);
    }

    /**
     * create a rank 3 dataset of the class, which can be 
     * double.class, float.class, long.class, int.class, short.class and byte.class
     * @param c the primitive class of each element.
     * @param len0 the length of the dimension
     * @param len1 the length of the dimension
     * @param len2 the length of the dimension
     * @return the ArrayDataSet
     */
    public static ArrayDataSet createRank3( Class c, int len0, int len1, int len2 ) {
        if ( c==double.class ) return new DDataSet( 3, len0, len1, len2, 1 );
        if ( c==float.class ) return new FDataSet( 3, len0, len1, len2, 1 );
        if ( c==long.class ) return new LDataSet( 3, len0, len1, len2, 1 );
        if ( c==int.class ) return new IDataSet( 3, len0, len1, len2, 1 );
        if ( c==short.class ) return new SDataSet( 3, len0, len1, len2, 1 );
        if ( c==byte.class ) return new BDataSet( 3, len0, len1, len2, 1 );
        throw new IllegalArgumentException("class not supported: "+c);
    }

    /**
     * create a rank 4 dataset of the class, which can be 
     * double.class, float.class, long.class, int.class, short.class and byte.class
     * @param c the primitive class of each element.
     * @param len0 the length of the dimension
     * @param len1 the length of the dimension
     * @param len2 the length of the dimension
     * @param len3 the length of the dimension
     * @return the ArrayDataSet
     */
    public static ArrayDataSet createRank4( Class c, int len0, int len1, int len2, int len3 ) {
        if ( c==double.class ) return new DDataSet( 4, len0, len1, len2, len3 );
        if ( c==float.class ) return new FDataSet( 4, len0, len1, len2, len3 );
        if ( c==long.class ) return new LDataSet( 4, len0, len1, len2, len3 );
        if ( c==int.class ) return new IDataSet( 4, len0, len1, len2, len3 );
        if ( c==short.class ) return new SDataSet( 4, len0, len1, len2, len3 );
        if ( c==byte.class ) return new BDataSet( 4, len0, len1, len2, len3 );
        throw new IllegalArgumentException("class not supported: "+c);
    }

    
    public static Object flattenArray( Object array, int[] qube ) {
        if ( qube==null ) qube= Ops.getQubeDimsForArray(array);
        if ( !array.getClass().isArray() ) throw new IllegalArgumentException("input must be an array");
        Class c= array.getClass().getComponentType();
        if ( c.isArray() && qube.length==1 ) {
            throw new IllegalArgumentException("input must be 1-D array");
        } else if ( qube.length==1 ) {
            return array; //trivial case
        } else {
            int[] ube= DataSetOps.removeElement( qube, 0 );
            int nelementsUbe= DataSetUtil.product(ube);
            int idx= 0;
            Object result= null;
            for ( int i=0; i<qube[0]; i++ ) {
                Object o= flattenArray( Array.get(array,i ), ube );
                if ( result==null ) { // we need a first instance to get the type.
                    result= Array.newInstance( o.getClass().getComponentType(), qube[0]*nelementsUbe );
                }
                System.arraycopy( o, 0, result, idx, nelementsUbe );
                idx+= nelementsUbe;
            }
            return result;
        }
    }
    
    /**
     * return the array as ArrayDataSet  The array must be a 1-D array and the
     * dimensions of the result are provided in qube.
     * @param array 1-D array
     * @param qube dimensions of the dataset
     * @param copy copy the data so that original data is not modified with putValue
     * @return ArrayDataSet
     * @see #create(java.lang.Class, int[]) 
     */
    public static ArrayDataSet wrap( Object array, int[] qube, boolean copy ) {
        Object arr;
        //check type
        if ( !array.getClass().isArray() ) throw new IllegalArgumentException("input must be an array");
        Class c= array.getClass().getComponentType();
        if ( c.isArray() ) {
            array= flattenArray( array, qube );
            c= array.getClass().getComponentType();
            if ( qube.length>1 ) copy=false; // because we're already copied
        }
        if ( copy ) {
            arr= Array.newInstance( c, Array.getLength(array) );
            System.arraycopy( array, 0, arr, 0, Array.getLength(array) );
        } else {
            arr= array;
        }
        if ( c==double.class ) return DDataSet.wrap( (double[])arr, qube );
        if ( c==float.class ) return FDataSet.wrap( (float[])arr, qube );
        if ( c==long.class ) return LDataSet.wrap( (long[])arr, qube );
        if ( c==int.class ) return IDataSet.wrap( (int[])arr, qube );
        if ( c==short.class ) return SDataSet.wrap( (short[])arr, qube );
        if ( c==byte.class ) return BDataSet.wrap( (byte[])arr, qube );

        throw new IllegalArgumentException("component type not supported: "+c );

    }
    
    /**
     * ensure that there are no non-monotonic or repeat records, by starting at the middle
     * of the dataset and finding the monotonic subsection.  The input need not be writable.
     * @param ds ArrayDataSet, which must be writable.
     * @return dataset, possibly with records removed.
     */
    public static ArrayDataSet monotonicSubset2( ArrayDataSet ds ) {
        
        QDataSet ssdep0= (QDataSet)ds.property(QDataSet.DEPEND_0);
        if ( ssdep0==null && UnitsUtil.isTimeLocation( SemanticOps.getUnits(ds) ) ) {
            ssdep0= ds;
        } else if ( ssdep0==null ) {
            return ds;
        }
        
        int mid= ssdep0.length()/2;
        int start=0;
        for ( int i=mid-1; i>0; i-- ) { // trim the left side
            if ( ssdep0.value(i)>=ssdep0.value(i+1) ) {
                start= i+1;
                break;
            }
        }
        int stop= ssdep0.length();
        for ( int i=mid+1; i<ssdep0.length(); i++ ) { // trim the right side
            if ( ssdep0.value(i-1)>=ssdep0.value(i) ) {
                stop= i-1;
                break;
            }
        }
        if ( start>0 || stop<ssdep0.length() ) {
            ds= (ArrayDataSet)ds.trim(start,stop);
            return ds;
        } else {
            return ds;
        }
        
    }
    
    /**
     * ensure that there are no non-monotonic or repeat records, by removing
     * the first N-1 records of N repeated records.  
     * @param ds ArrayDataSet, which must be writable.
     * @return dataset, possibly with records removed.
     */
    public static ArrayDataSet monotonicSubset( ArrayDataSet ds ) {
        
        if ( ds.isImmutable() ) {
            ds= ArrayDataSet.copy(ds);
            logger.warning("immutabilty forced copy.");
        }
               
        QDataSet sdep0= (QDataSet)ds.property(QDataSet.DEPEND_0);
        if ( sdep0==null && UnitsUtil.isTimeLocation( SemanticOps.getUnits(ds) ) ) {
            sdep0= ds;
        } else if ( sdep0==null ) {
            return ds;
        }
        ArrayDataSet dep0= ArrayDataSet.maybeCopy( sdep0 ); // I don't think this will copy.
        if ( !UnitsUtil.isTimeLocation( SemanticOps.getUnits(dep0) ) ) {
            return ds;
        }
        if ( dep0.length()<2 ) return ds;
        if ( dep0.rank()!=1 ) return ds;
        QDataSet vdep0= Ops.valid(dep0);
        
        int[] rback= new int[dep0.length()];
        rback[dep0.length()/2]= dep0.length()/2;
        int rindex=dep0.length()/2+1;
        double a= dep0.value(rindex-1);
        for ( int i=rindex; i<dep0.length(); i++ ) {
            if ( vdep0.value(i)>0 ) {
                double a1=dep0.value(i);
                if ( a1>a ) {
                    rback[rindex]= i;
                    a= a1;
                    rindex++;
                } else {
                    logger.log(Level.FINER, "data point breaks monotonic rule: {0}", i);
                }
            }
        }
        int lindex=dep0.length()/2;
        a= dep0.value(lindex+1);
        for ( int i=lindex; i>=0; i-- ) {
            if ( vdep0.value(i)>0 ) {
                double a1=dep0.value(i);
                if ( a1<a ) {
                    rback[lindex]= i;
                    a= a1;
                    lindex--;
                } else {
                    logger.log(Level.FINER, "data point breaks monotonic rule: {0}", i);
                }
            }
        }
        lindex+=1;
        
        int nrm= dep0.length() - ( rindex-lindex );
        if ( nrm>0 ) {
            if ( rindex==1 ) {
                logger.log(Level.FINE, "ensureMono removes all points, assume it's monotonic decreasing" );
                return ds;
            }
            logger.log(Level.FINE, "ensureMono removes {0} points", nrm);
            Class c= ds.getComponentType();
            int[] idx= new int[rindex-lindex];
            System.arraycopy( rback, lindex, idx, 0, ( rindex-lindex ) );
            ds.putProperty( QDataSet.DEPEND_0, null );
            ds= ArrayDataSet.copy( c, new SortDataSet( ds, Ops.dataset(idx) ) );
            Class depclass= dep0.getComponentType();
            ArrayDataSet dep0copy= ArrayDataSet.copy( depclass, new SortDataSet( dep0, Ops.dataset(idx) ) );
            dep0copy.putProperty( QDataSet.MONOTONIC, Boolean.TRUE );
            ds.putProperty( QDataSet.DEPEND_0, dep0copy );
        }
        return ds;
    }
    
    @Override
    public int rank() {
        return rank;
    }

    @Override
    public final int length() {
        return len0;
    }

    @Override
    public final int length(int i) {
        //if ( RANGE_CHECK && i>=len0 ) throw new IndexOutOfBoundsException("length("+i+") when dim 0 length="+len0); //TODO: allow disable with RANGE_CHECK for performance
        return len1;
    }

    @Override
    public final int length( int i0, int i1 ) {
        //if ( RANGE_CHECK ) {
        //    if ( i0>=len0 ) throw new IndexOutOfBoundsException("length("+i0+","+i1+") when dim 0 length="+len0);
        //    if ( i1>=len1 ) throw new IndexOutOfBoundsException("length("+i0+","+i1+") when dim 1 length="+len1);
        //}
        return len2;
    }

    @Override
    public final int length( int i0, int i1, int i2) {
        //if ( RANGE_CHECK ) {
        //    if ( i0>=len0 ) throw new IndexOutOfBoundsException("length("+i0+","+i1+","+i2+") when dim 0 length="+len0);
        //    if ( i1>=len1 ) throw new IndexOutOfBoundsException("length("+i0+","+i1+","+i2+") when dim 1 length="+len1);
        //    if ( i2>=len2 ) throw new IndexOutOfBoundsException("length("+i0+","+i1+","+i2+") when dim 2 length="+len2);
        //}
        return len3;
    }

    /**
     * return the component type of the backing array, such as double.class.
     * @return the component type of the backing array.
     * @see #create(java.lang.Class, int[]) 
     */
    public Class getComponentType() {
        return componentType;
    }

    /**
     * Shorten the dataset by changing it's dim 0 length parameter.  
     * The same backing array is used, so the element that remain will be the same.
     * This can only be used to shorten datasets, see grow to lengthen.
     * @param len new number of records
     * @see #grow(int) 
     */
    public void putLength( int len ) {
        int limit= Array.getLength( getBack() ) / ( len1*len2*len3 );
        if ( len>limit ) throw new IllegalArgumentException("dataset cannot be lengthened");
        len0= len;
    }

    /**
     * grow the internal store so that append may be used to resize the dataset.
     * @param newRecCount new number of records, which includes the old records.
     * @see #putLength(int) 
     */
    public void grow( int newRecCount ) {
        if ( newRecCount < len0 ) throw new IllegalArgumentException("new recsize for grow smaller than old");
        int newSize= newRecCount * len1 * len2 * len3;
        Object back= getBackReadOnly();
        int oldSize= Array.getLength(back);

        if ( newSize<oldSize ) { // it's possible that the dataset already has a backing that can support this.  Check for this.
            return;
        }

        Object newBack;
        if ( back instanceof double[] ) {
            newBack= new double[ newSize ];
        } else if ( back instanceof float[] ) {
            newBack= new float[ newSize ];
        } else if ( back instanceof long[] ) {
            newBack= new long[ newSize ];
        } else if ( back instanceof int[] ) {
            newBack= new int[ newSize ];
        } else if ( back instanceof short[] ) {
            newBack= new short[ newSize ];
        } else if ( back instanceof byte[] ) {
            newBack= new byte[ newSize ];
        } else {
            throw new IllegalArgumentException("shouldn't happen bad type");
        }
        System.arraycopy( back, 0, newBack, 0, Array.getLength(back) );
        setBack(newBack);
    }

    /**
     * append the dataset with the same geometry but different number of records (zeroth dim)
     * to this.  An IllegalArgumentException is thrown when there is not enough room.  
     * See grow(newRecCount).
     * Not thread safe--we need to go through and make it so...
     * @param ds
     */
    public synchronized void append( ArrayDataSet ds ) {
        if ( ds.rank()!=this.rank ) throw new IllegalArgumentException("rank mismatch");
        if ( ds.len1!=this.len1 ) throw new IllegalArgumentException("len1 mismatch");
        if ( ds.len2!=this.len2 ) throw new IllegalArgumentException("len2 mismatch");
        if ( ds.len3!=this.len3 ) throw new IllegalArgumentException("len3 mismatch");
        if ( this.getBackReadOnly().getClass()!=ds.getBackReadOnly().getClass() ) throw new IllegalArgumentException("backing type mismatch");

        int myLength= this.len0 * this.len1 * this.len2 * this.len3;
        int dsLength= ds.len0 * ds.len1 * ds.len2 * ds.len3;

        if ( Array.getLength(this.getBack()) < myLength + dsLength ) {
            throw new IllegalArgumentException("unable to append dataset, not enough room");
        }

        System.arraycopy( ds.getBackReadOnly(), 0, this.getBack(), myLength, dsLength );

        Units u1= SemanticOps.getUnits(this);
        Units u2= SemanticOps.getUnits(ds);
        if ( u1!=u2 ) {
            if ( u1 instanceof EnumerationUnits && u2 instanceof EnumerationUnits ) {
                for ( int i=myLength; i<myLength+dsLength; i++ ) {
                    double d= Array.getDouble( this.getBack(),i );
                    d= ((EnumerationUnits)u1).createDatum( u2.createDatum(d).toString() ).doubleValue(u1);
                    Array.set( this.getBack(), i, d );
                }
            } else {
                UnitsConverter uc= UnitsConverter.getConverter(u2,u1);
                Class backClass= this.getBack().getClass().getComponentType();
                for ( int i=myLength; i<myLength+dsLength; i++ ) {
                    Number nv=  uc.convert(Array.getDouble( this.getBack(),i) ) ;
                    if ( backClass==double.class ) {
                        Array.set( this.getBack(), i, nv.doubleValue() );
                    } else if ( backClass==float.class ) {
                        Array.set( this.getBack(), i, nv.floatValue() );
                    } else if ( backClass==long.class ) {
                        Array.set( this.getBack(), i, nv.longValue() );
                    } else if ( backClass==int.class ) {
                        Array.set( this.getBack(), i, nv.intValue() );
                    } else if ( backClass==short.class ) {
                        Array.set( this.getBack(), i, nv.shortValue() );
                    } else if ( backClass==byte.class ) {
                        Array.set( this.getBack(), i, nv.byteValue() );
                    } else {
                        throw new IllegalArgumentException("unsupported type: "+backClass );
                    }
                }
            }
        }
        
        this.len0= this.len0 + ds.len0;

        properties.putAll( joinProperties( this, ds ) ); 

    }

    /**
     * return true if the dataset can be appended.  Note this assumes that the
     * same length, etc.  This just checks that we have the number of spare records
     * in the backing store.
     * @param ds the dataset to test
     * @return true if the dataset can be appended.
     */
    public boolean canAppend( ArrayDataSet ds ) {
        if ( ds.rank()!=this.rank ) throw new IllegalArgumentException("rank mismatch");
        if ( ds.len1!=this.len1 ) throw new IllegalArgumentException("len1 mismatch");
        if ( ds.len2!=this.len2 ) throw new IllegalArgumentException("len2 mismatch");
        if ( ds.len3!=this.len3 ) throw new IllegalArgumentException("len3 mismatch");
        if ( this.getBackReadOnly().getClass()!=ds.getBackReadOnly().getClass() ) {
            Class a1= ds.getBackReadOnly().getClass();
            Class a2= this.getBack().getClass();
            String s1,s2;
            s1= "" + a1.getComponentType();
            s2= "" + a2.getComponentType();
            throw new IllegalArgumentException("backing type mismatch: "+ s2 + "["+ds.length()+",*] can't be appended to "+ s1 + "["+this.length()+",*]" );
        }
        int trec= Array.getLength(this.getBack()) / this.len1 / this.len2 / this.len3;
        return trec-this.len0 > ds.length();
    }

    /**
     * Copy the dataset to an ArrayDataSet only if the dataset is not 
     * already an ArrayDataSet that is mutable.
     * @param ds the dataset to copy, which may be an ArrayDataSet
     * @return an ArrayDataSet.
     */
    public static ArrayDataSet maybeCopy( QDataSet ds ) {
        if ( ds instanceof ArrayDataSet && !((ArrayDataSet)ds).isImmutable() ) {
            return (ArrayDataSet)ds;
        } else {
            return copy(ds);
        }
    }

    /**
     * Copy the dataset to a ArrayDataSet only if the dataset is not already a
     * particular instance of the given type of ArrayDataSet.
     * @param c component type, e.g. float.class double.class int.class, etc.
     * @param ds the dataset.
     * @return ArrayDayaSet of component type c, e.g. double.class, float.class, etc.
     */
    public static ArrayDataSet maybeCopy( Class c, QDataSet ds ) {
        if ( ds instanceof ArrayDataSet && ((ArrayDataSet)ds).getComponentType()==c ) {
            return (ArrayDataSet)ds;
        } else {
            return copy(c,ds);
        }
    }

    /**
     * provide access to the backing array.
     * @return the backing array for the data. 
     */
    protected abstract Object getBack();
    
    /**
     * return the size of the backing in JvmMemory.
     * @return the backing array for the data. 
     */
    protected abstract int getBackJvmMemory();
    
    /**
     * return a copy of the backing array, to support ArrayDataSet.copy.
     * @return a copy of the array for the data.
     */
    protected abstract Object getBackCopy();
    
    /**
     * return the backing array to the client, who promises not to modify the
     * memory.
     * @return the backing array for the data, which must not be modified.
     */
    protected abstract Object getBackReadOnly();
    
    /**
     * reset the back to this new backing array.
     * @param back the new backing array.
     */
    protected abstract void setBack(Object back);

    /**
     * make a copy of the array dataset.
     * @param ds the dataset to copy.
     * @return a copy of the dataset.
     */
    private static ArrayDataSet internalCopy(ArrayDataSet ds) {

        Object newback = ds.getBackCopy();

        ArrayDataSet result = ArrayDataSet.create(ds.rank, ds.len0, ds.len1, ds.len2, ds.len3, newback);
        result.properties.putAll( Ops.copyProperties(ds) ); // TODO: problems...
        result.checkFill();

        return result;
    }
    
    public static ArrayDataSet copy( QDataSet ds, LongReadAccess lra ) {
        int rank= ds.rank();
        ArrayDataSet result;
        
        Class c= long.class;
        
        switch (rank) {
            case 0:
                result= createRank0( c );
                LongWriteAccess lwa= (LongWriteAccess)result.capability(LongWriteAccess.class);
                lwa.putLValue( lra.lvalue() );
                break;
            case 1:
                result= createRank1( c, ds.length() );
                lwa= (LongWriteAccess)result.capability(LongWriteAccess.class);
                for ( int i=0; i<ds.length(); i++ ) {
                    lwa.putLValue( i, lra.lvalue(i) );
                }
                break;
            case 2:
                result= createRank2( c, ds.length(), ds.length(0) );
                lwa= (LongWriteAccess)result.capability(LongWriteAccess.class);
                int i0= ds.length()>0 ? ds.length(0) : -1;
                for ( int i=0; i<ds.length(); i++ ) {
                    if ( ds.length(i)!=i0 ) throw new IllegalArgumentException("Attempt to copy non-qube into ArrayDataSet which must be qube: "+ds );
                    for ( int j=0; j<ds.length(i); j++ ) {
                        lwa.putLValue( i, j, lra.lvalue(i,j) );
                    }
                }
                break;
            case 3:
                result= createRank3( c, ds.length(), ds.length(0), ds.length(0,0) );
                lwa= (LongWriteAccess)result.capability(LongWriteAccess.class);
                int i0_= ds.length()>0 ? ds.length(0) : -1;
                for ( int i=0; i<ds.length(); i++ ) {
                    if ( ds.length(i)!=i0_ ) throw new IllegalArgumentException("Attempt to copy non-qube into ArrayDataSet which must be qube: "+ds );
                    for ( int j=0; j<ds.length(i); j++ ) {
                        for ( int k=0; k<ds.length(i,j); k++ ) {
                            lwa.putLValue( i, j, k, lra.lvalue(i,j,k) );
                        }
                    }
                }
                break;
            case 4:
                result = createRank4( c, ds.length(), ds.length(0), ds.length(0,0), ds.length(0,0,0));
                lwa= (LongWriteAccess)result.capability(LongWriteAccess.class);
                for ( int i=0; i<ds.length(); i++ )
                    for ( int j=0; j<ds.length(i); j++ )
                        for ( int k=0; k<ds.length(i,j); k++ )
                            for ( int l=0; l<ds.length(i,j,k); l++ )
                                lwa.putLValue( i, j, k, l, lra.lvalue(i,j,k,l));
                break;

            default: 
                throw new IllegalArgumentException("bad rank");
        }

        result.properties.putAll( Ops.copyProperties(ds) );
        if ( result.properties.containsKey("NAME__0") ) {
            result.hasIndexedProperties= true;
        }
        result.checkFill();
        
        return result;
        
    }

    /**
     * Copy to array of specific type.  For example, copy( double.class, ds ) 
     * would return a copy in a DDataSet.
     * @param c  the primitive type to use (e.g. double.class, float.class, int.class).
     * @param ds the data to copy.
     * @return ArrayDataSet of specific type.
     */
    public static ArrayDataSet copy( Class c, QDataSet ds ) {

        if ( ds instanceof ArrayDataSet && ((ArrayDataSet)ds).getBackReadOnly().getClass().getComponentType()==c ) return internalCopy( (ArrayDataSet)ds );
        
        int rank= ds.rank();
        ArrayDataSet result;

        switch (rank) {
            case 0:
                result= createRank0( c );
                result.putValue( ds.value() );
                break;
            case 1:
                result= createRank1( c, ds.length() );
                for ( int i=0; i<ds.length(); i++ ) {
                    result.putValue( i, ds.value(i) );
                }
                break;
            case 2:
                result= createRank2( c, ds.length(), ds.length(0) );
                int i0= ds.length()>0 ? ds.length(0) : -1;
                for ( int i=0; i<ds.length(); i++ ) {
                    if ( ds.length(i)!=i0 ) throw new IllegalArgumentException("Attempt to copy non-qube into ArrayDataSet which must be qube: "+ds );
                    for ( int j=0; j<ds.length(i); j++ ) {
                        result.putValue( i, j, ds.value(i,j) );
                    }
                }
                break;
            case 3:
                result= createRank3( c, ds.length(), ds.length(0), ds.length(0,0) );
                int i0_= ds.length()>0 ? ds.length(0) : -1;
                for ( int i=0; i<ds.length(); i++ ) {
                    if ( ds.length(i)!=i0_ ) throw new IllegalArgumentException("Attempt to copy non-qube into ArrayDataSet which must be qube: "+ds );
                    for ( int j=0; j<ds.length(i); j++ ) {
                        for ( int k=0; k<ds.length(i,j); k++ ) {
                            result.putValue( i, j, k, ds.value(i,j,k) );
                        }
                    }
                }
                break;
            case 4:
                result = createRank4( c, ds.length(), ds.length(0), ds.length(0,0), ds.length(0,0,0));
                for ( int i=0; i<ds.length(); i++ )
                    for ( int j=0; j<ds.length(i); j++ )
                        for ( int k=0; k<ds.length(i,j); k++ )
                            for ( int l=0; l<ds.length(i,j,k); l++ )
                                result.putValue( i, j, k, l, ds.value(i,j,k,l));
                break;

            default: 
                throw new IllegalArgumentException("bad rank");
        }
        result.properties.putAll( Ops.copyProperties(ds) );
        if ( result.properties.containsKey("NAME__0") ) {
            result.hasIndexedProperties= true;
        }
        result.checkFill();

        return result;

    }

    /**
     * copies the dataset into a writable ArrayDataSet, and all of its depend 
     * datasets as well. Note this does not verify that the data is a qube!!!
     * @param ds the data to be copied.
     * @return a copy of the data.
     * @see #guessBackingStore(org.das2.qds.QDataSet) 
     */
    public static ArrayDataSet copy( QDataSet ds ) {
        if ( ds instanceof ArrayDataSet ) {
            return internalCopy( (ArrayDataSet)ds );
        } else if ( ds instanceof JoinDataSet && ds.length()>0 ) {
            QDataSet ds1= ds.slice(0);
            if ( ds1 instanceof ArrayDataSet ) { // Juno/Waves needed to save memory and avoid converting everything to doubles
                Class c= ((ArrayDataSet)ds1).getBackReadOnly().getClass().getComponentType();
                return copy( c, ds );
            } else {
                return copy( guessBackingStore(ds), ds );
            }
        } else {
            LongReadAccess lra= (LongReadAccess)ds.capability(LongReadAccess.class); // special support for TT2000.
            if ( lra!=null ) {
                return copy( ds, lra );
            } else {
                return copy( guessBackingStore(ds), ds ); // strange type does legacy behavior.
            }
        }
    }

    /**
     * guess the type of the backing store, returning double.class
     * if it cannot be determined.
     * @param ds the dataset
     * @return the backing store class, one of double.class, float.class, long.class, int.class, short.class, or byte.class.
     * @see #copy(org.das2.qds.QDataSet) copy
     */
    public static Class guessBackingStore( QDataSet ds ) {
        if ( ds instanceof BDataSet || ds instanceof ByteDataSet ) {
            return byte.class;
        } else if ( ds instanceof SDataSet || ds instanceof ShortDataSet ) {
            return short.class;
        } else if ( ds instanceof IDataSet || ds instanceof IntDataSet ) {
            return int.class;
        } else if ( ds instanceof LDataSet || ds instanceof LongDataSet ) {
            return long.class;
        } else if ( ds instanceof FDataSet || ds instanceof FloatDataSet ) {
            return float.class;
        } else {
            return double.class;
        }
    }
    
    
    /**
     * check for fill property and set local variable.
     */
    protected void checkFill() {
        Number f= (Number) properties.get(QDataSet.FILL_VALUE);
        if ( f!=null ) {
            fill= f.floatValue();
            dfill= f.doubleValue();
        } else {
            fill= Float.NaN;
            dfill= Double.NaN;
        }
    }
    
    /**
     * append the second dataset onto this dataset.  The two datasets need 
     * only have convertible units, so for example two time arrays may be appended 
     * even if their units don't have the same base, and the times will be
     * converted.  Only properties of the two datasets that do not change are preserved.
     * When two rank 0 datasets are appended, the result will be rank 1.
     * @param ds1 rank N dataset
     * @param ds2 rank N dataset of the same type and compatible geometry as ds1.
     * @return dataset with the data appended.
     */
    public static ArrayDataSet append( ArrayDataSet ds1, ArrayDataSet ds2 ) {
        if ( ds1==null ) {
            if ( ds2.rank>0 ) {
                return ds2;
            } else {
                ArrayDataSet result= ArrayDataSet.createRank1(ArrayDataSet.guessBackingStore(ds2),1);
                result.putValue( ds2.value() );
                DataSetUtil.copyDimensionProperties( ds2, result );
                QDataSet c= (QDataSet)ds2.property(QDataSet.CONTEXT_0);
                if ( c!=null && c.rank()==0 ) {
                    result.putProperty( QDataSet.DEPEND_0, Ops.append( null, c ) );
                }
                return result;
            }
        }
        if ( ds2==null ) throw new NullPointerException("ds is null");
        
        if ( ds1.rank()==ds2.rank()-1 ) {
            Units u= SemanticOps.getUnits(ds1);
            ds1= ArrayDataSet.create( ds1.rank()+1, 1, ds1.len0, ds1.len1, ds1.len2, ds1.getBack() );
            ds1.putProperty( QDataSet.UNITS,u);
        }
        if ( ds1.rank()-1==ds2.rank() ) {
            Units u= SemanticOps.getUnits(ds2);
            QDataSet c= (QDataSet)ds2.property(QDataSet.CONTEXT_0);
            ds2= ArrayDataSet.create( ds2.rank()+1, 1, ds2.len0, ds2.len1, ds2.len2, ds2.getBack() );
            ds2.putProperty( QDataSet.UNITS,u);
            if ( ds2.rank==1 ) {
                if ( c!=null && c.rank()==0 ) {
                    ds2.putProperty( QDataSet.DEPEND_0, Ops.append( null, c ) );
                }
            }
        }
        if ( ds2.rank()!=ds1.rank ) throw new IllegalArgumentException("rank mismatch");
        if ( ds2.len1!=ds1.len1 ) throw new IllegalArgumentException("len1 mismatch");
        if ( ds2.len2!=ds1.len2 ) throw new IllegalArgumentException("len2 mismatch");
        if ( ds2.len3!=ds1.len3 ) throw new IllegalArgumentException("len3 mismatch");
        if ( ds1.getBackReadOnly().getClass()!=ds2.getBackReadOnly().getClass() ) {
            throw new IllegalArgumentException("backing type mismatch");
        }

        int myLength= ds1.len0 * ds1.len1 * ds1.len2 * ds1.len3;
        int dsLength= ds2.len0 * ds2.len1 * ds2.len2 * ds2.len3;

        Object newback= Array.newInstance( ds1.getBackReadOnly().getClass().getComponentType(), myLength + dsLength );

        System.arraycopy( ds1.getBackReadOnly(), 0, newback, 0, myLength );
        System.arraycopy( ds2.getBackReadOnly(), 0, newback, myLength, dsLength );

        Class backClass= newback.getClass().getComponentType();
        
        QDataSet weights=null;
        if ( Ops.fillIsDifferent(ds1,ds2) ) {
            QDataSet v1= Ops.copy( Ops.valid(ds1) );
            QDataSet v2= Ops.copy( Ops.valid(ds2 ) );
            QDataSet valid1= v1.rank()==1 ? v1 : Ops.reform( v1, new int[] { myLength } );
            QDataSet valid2= v2.rank()==1 ? v2 : Ops.reform( v2, new int[] { dsLength } );
            if ( backClass==double.class ) {
                QDataSet r= Ops.where( Ops.eq( valid1,0 ) );
                for ( int i=0; i<r.length(); i++ ) {
                    Array.setDouble( newback, (int)r.value(i), Double.NaN );
                }
                r= Ops.where( Ops.eq( valid2,0 ) );
                for ( int i=0; i<r.length(); i++ ) {
                    Array.setDouble( newback, myLength + (int)r.value(i), Double.NaN );
                }
            } else if ( backClass==float.class ) {
                QDataSet r= Ops.where( Ops.eq( valid1,0 ) );
                for ( int i=0; i<r.length(); i++ ) {
                    Array.setFloat( newback, (int)r.value(i), Float.NaN );
                }
                r= Ops.where( Ops.eq( valid2,0 ) );
                for ( int i=0; i<r.length(); i++ ) {
                    Array.setFloat( newback, myLength + (int)r.value(i), Float.NaN );
                }
            } else {
                weights= Ops.append( Ops.valid(ds1), Ops.valid(ds2) );
            }
        }
        
        
        if ( SemanticOps.isBundle(ds1) && SemanticOps.isBundle(ds2) ) {
            QDataSet bds1= (QDataSet) ds1.property(QDataSet.BUNDLE_1);
            QDataSet bds2= (QDataSet) ds2.property(QDataSet.BUNDLE_1);
            for ( int j=0; j<ds1.length(0); j++ ) {
                Units u1= (Units) bds1.property(QDataSet.UNITS,j);
                if ( u1==null ) u1= Units.dimensionless;
                Units u2= (Units) bds2.property(QDataSet.UNITS,j);
                if ( u2==null ) u2= Units.dimensionless;
                if ( u1!=u2 ) { 
                    logger.log(Level.WARNING, "unable to properly append two bundle datasets with different units: \"{0}\" \"{1}\"", new Object[]{u1, u2});
                }
            }
        }
        
        Units u1= SemanticOps.getUnits(ds1);
        Units u2= SemanticOps.getUnits(ds2);
        if ( u1!=u2 ) {
            UnitsConverter uc= UnitsConverter.getConverter(u2,u1);
            for ( int i=myLength; i<myLength+dsLength; i++ ) { //TODO: this is going to be sub-optimal that its much slower than it needs to be because if statements.
                Number nv=  uc.convert(Array.getDouble( newback,i) ) ;
                if ( backClass==double.class ) {
                    Array.set( newback, i, nv.doubleValue() );
                } else if ( backClass==float.class ) {
                    Array.set( newback, i, nv.floatValue() );
                } else if ( backClass==long.class ) {
                    Array.set( newback, i, nv.longValue() );
                } else if ( backClass==int.class ) {
                    Array.set( newback, i, nv.intValue() );
                } else if ( backClass==short.class ) {
                    Array.set( newback, i, nv.shortValue() );
                } else if ( backClass==byte.class ) {
                    Array.set( newback, i, nv.byteValue() );
                } else {
                    throw new IllegalArgumentException("unsupported type: "+backClass );
                }
            }
        }

        int len0= ds1.len0 + ds2.len0;
        
        int rank= ( ds1.rank==0 ) ? 1 : ds1.rank;
        
        ArrayDataSet result= create( rank, len0, ds1.len1, ds1.len2, ds1.len3, newback );

        result.properties.putAll( joinProperties( ds1, ds2 ) );
        result.properties.put( QDataSet.UNITS, u1 ); // since we resolve units when they change (bug https://sourceforge.net/p/autoplot/bugs/825/)

        if ( weights!=null ) {
            result.putProperty( QDataSet.WEIGHTS, weights );
        }
        
        return result;
    }


    /**
     * join the properties of the two datasets.  This was introduced to
     * support append, but may be useful in other contexts.  Note there is special
     * handling of properties, like CACHE_TAG and TYPICAL_MIN, TYPICAL_MAX.
     * Note MONOTONIC assumes the ds2 will be added after ds1.
     * @param ds1 dataset, typically a time series.
     * @param ds2 dataset, typically a time series.
     * @return the two sets combined.
     */
    protected static Map joinProperties( ArrayDataSet ds1, ArrayDataSet ds2 ) {
        Map result= new HashMap();
        for ( int i=0; i<ds2.rank(); i++ ) {
            QDataSet thatDep= (QDataSet) ds2.property( "DEPEND_"+i );
            if ( thatDep!=null && ( i==0 || thatDep.rank()>1 ) ) {
                QDataSet thisDep= (QDataSet) ds1.property( "DEPEND_"+i );
                if ( thisDep==null ) {
                    logger.log(Level.FINE, "joinProperties is dropping DEPEND_{0} because only one dataset has this property.", i);
                } else {
                    ArrayDataSet djoin= copy( thisDep ); 
                    ArrayDataSet ddep1;
                    if ( thatDep instanceof ArrayDataSet ) {
                        ddep1= (ArrayDataSet) thatDep;
                        if ( ddep1.getComponentType()!=djoin.getComponentType() ) {
                            ddep1= ArrayDataSet.copy( djoin.getComponentType(), ddep1 );
                        }
                    } else {
                        ddep1= maybeCopy( thatDep );
                    }
                    // Ops.append will reconcile, but the result will be an ArrayDataSet.
                    djoin= (ArrayDataSet)Ops.append( djoin, ddep1 ); 
                    result.put( "DEPEND_"+i, djoin );
                }
            } else if ( thatDep!=null && thatDep.rank()==1 ) {
                //TODO: check properties equal.
                result.put( "DEPEND_"+i, thatDep );
            }
            QDataSet thatBundle= (QDataSet) ds2.property( "BUNDLE_"+i );
            QDataSet thisBundle= (QDataSet) ds1.property("BUNDLE_"+i );
            if ( i>0 && thatBundle!=null && thisBundle!=null ) {
                if ( thisBundle.length()!=thatBundle.length() ) {
                    throw new IllegalArgumentException("BUNDLE_"+i+" should be the same length to append, but they are not");
                }
                for ( int j=0; j<thatBundle.length(); j++ ) {
                    Units thatu= (Units)thatBundle.property( QDataSet.UNITS, j );
                    if ( thatu==null ) thatu= Units.dimensionless;
                    Units thisu= (Units)thisBundle.property( QDataSet.UNITS, j );
                    if ( thisu==null ) thisu= Units.dimensionless;
                    if ( thisu!=thatu ) {
                        throw new IllegalArgumentException("units in BUNDLE_"+i+" change...");
                    }
                }
                //TODO: other safety checks...
                result.put( "BUNDLE_"+i, thatBundle );
            }
        }
        String[] props;
        props= DataSetUtil.correlativeProperties();
        for ( int iprop= -1; iprop<props.length; iprop++ ) {
            String prop= iprop==-1 ? QDataSet.PLANE_0 : props[iprop];
            QDataSet w1= (QDataSet) ds2.property( prop );
            if ( w1!=null ) {
                QDataSet propds= (QDataSet) ds1.property( prop );
                if ( propds!=null ) {
                    ArrayDataSet djoin=  copy( propds );
                    ArrayDataSet dd1=  maybeCopy(w1);
                    if ( djoin.rank>0 || ds1.rank==0 ) {
                        djoin= (ArrayDataSet)Ops.append( djoin, dd1 );
                    }
                    result.put( prop, djoin );
                } else {
                    logger.log(Level.INFO, "dataset doesn''t have property \"{0}\" but other dataset does: {1}", new Object[]{prop, ds1});
                }
            }
        }

        // when the two datasets have the same values for a property, the property is kept.
        props= DataSetUtil.dimensionProperties();
        for (String prop : props) {
            Object value= ds1.property(prop);
            if ( value!=null && value.equals(ds2.property(prop) ) ) {
                result.put( prop, ds1.property(prop) );
            }
        }
        
        // special handling for QDataSet.CADENCE, and QDataSet.MONOTONIC
        props= new String[] { QDataSet.CADENCE, QDataSet.BINS_1 };
        for (String prop : props) {
            Object o = ds1.property(prop);
            if (o!=null && o.equals(ds2.property(prop))) {
                result.put(prop, o);
            }
        }

        // special handling for monotonic property.
        Boolean m= (Boolean) ds1.property( QDataSet.MONOTONIC );
        if ( m!=null && m.equals(Boolean.TRUE) && m.equals( ds2.property( QDataSet.MONOTONIC ) ) ) {
            // check to see that result would be monotonic
            try {
                int[] fl1= DataSetUtil.rangeOfMonotonic( ds1 );
                int[] fl2= DataSetUtil.rangeOfMonotonic( ds2 );
                Units u1= SemanticOps.getUnits(ds2);
                Units u2= SemanticOps.getUnits(ds1);
                UnitsConverter uc= u2.getConverter(u1);
                if ( ds2.value(fl2[0]) -  uc.convert( ds1.value(fl1[1]) ) >= 0 ) { 
                    result.put( QDataSet.MONOTONIC, Boolean.TRUE );
                }
            } catch ( IllegalArgumentException ex ) {
                logger.fine("rte_1282463981: can't show that result has monotonic timetags because each dataset is not monotonic.");
            }
        }

        // special handling for cacheTag property.
        org.das2.datum.CacheTag ct0= (CacheTag) ds1.property( QDataSet.CACHE_TAG );
        org.das2.datum.CacheTag ct1= (CacheTag) ds2.property( QDataSet.CACHE_TAG );
        if ( ct0!=null && ct1!=null ) {
            // If cache tags are not adjacent, the range between them is included in the new tag.
            CacheTag newTag= null;
            try {
                newTag= CacheTag.append(ct0, ct1);
            } catch ( IllegalArgumentException ex ) {
                logger.fine( "append of two datasets that have CACHE_TAGs and are not adjacent, dropping CACHE_TAG" );
            }
            if ( newTag!=null ) {
                result.put( QDataSet.CACHE_TAG, newTag );
            }
        }

        // special handling of TYPICAL_MIN _MAX properties
        Number dmin0= (Number) ds1.property(QDataSet.TYPICAL_MIN );
        Number dmax0= (Number) ds1.property(QDataSet.TYPICAL_MAX );
        Number dmin1= (Number) ds2.property(QDataSet.TYPICAL_MIN );
        Number dmax1= (Number) ds2.property(QDataSet.TYPICAL_MAX );
        if ( dmin0!=null && dmin1!=null ) result.put( QDataSet.TYPICAL_MIN, Math.min( dmin0.doubleValue(), dmin1.doubleValue() ) );
        if ( dmax0!=null && dmax1!=null ) result.put( QDataSet.TYPICAL_MAX, Math.max( dmax0.doubleValue(), dmax1.doubleValue() ) );

        return result;
    }

    /**
     * set the units for this dataset.  This is a convenience method
     * intended to encourage setting the data units.  For example in Jython:
     *<blockquote><pre>
     *ds= linspace(0.,10.,100).setUnits( Units.seconds )
     *</pre></blockquote>
     * @param units units object, like Units.seconds
     * @return this dataset.
     */
    public QDataSet setUnits( Units units ) {
        checkImmutable();        
        this.putProperty( QDataSet.UNITS, units );
        return this;
    }

    @Override
    public String toString( ) {
        return DataSetUtil.toString( this );
    }

    /**
     * returns the size of the dataset in bytes.
     * @return the size of the dataset in bytes.
     * @see org.das2.qds.buffer.BufferDataSet which stores data outside of the JVM.
     */
    public int jvmMemory() {
        return getBackJvmMemory();
    }
    
    
    /**
     * print some info about this ArrayDataSet.
     */
    public void about() {
        System.err.println("== "+this.toString() + "==");
        System.err.println("back is array of "+ this.getComponentType() );
        //QDataSet extent= Ops.extent(this);  // this is occasionally very slow. TODO: investigate
        //System.err.println("extent="+extent);
    }
    
}