package org.autoplot.idlsupport; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.io.UnsupportedEncodingException; import java.lang.reflect.Array; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.logging.ConsoleHandler; import java.util.logging.Handler; import java.util.logging.Level; import java.util.logging.Logger; /** * Read data from IDL Save Files. This was written using * http://www.physics.wisc.edu/~craigm/idl/savefmt/node20.html * https://cow.physics.wisc.edu/~craigm/idl/savefmt.pdf * and https://github.com/scipy/scipy/blob/master/scipy/io/idl.py * for reference, and with no involvement from individuals at * Harris Geospacial. No warrenties are implied and this must * be used at your own risk. * * @author jbf */ public class ReadIDLSav { private static final Logger logger= Logger.getLogger("autoplot.idlsav"); private static final int RECTYPE_VARIABLE = 2; private static final int RECTYPE_ENDMARKER = 6; private static final int RECTYPE_TIMESTAMP = 10; private static final int RECTYPE_VERSION = 14; private static final int RECTYPE_PROMOTE64 = 17; private static final int VARFLAG_ARRAY = 0x04; private static final int VARFLAG_STRUCT = 0x20; /** * return the next record buffer, or returns null at the end. * @param ch the bytebuffer * @param pos the position. * @return the record, including the twelve bytes at the beginning * @throws IOException */ private ByteBuffer readRecord( ByteBuffer ch, int pos ) throws IOException { ch.order( ByteOrder.BIG_ENDIAN ); int recType= ch.getInt(pos); int endpos= ch.getInt(pos+4); if ( recType==RECTYPE_ENDMARKER ) { return null; } else { return slice( ch, pos, endpos ); } } /** * somehow I didn't notice the length before other strings. In the Python * code they have "_read_string" and "_read_string_data" which has a * second length. * @param rec * @param pos * @return StringDesc to describe the string. */ private StringData readStringData( ByteBuffer rec, int pos ) { int len= rec.getInt(pos); byte[] mybytes= new byte[len]; rec.position(pos+4); rec.get(mybytes); StringData result= new StringData(); result.string= new String( mybytes ); result._lengthBytes= 4 + Math.max( 4, (int)( 4 * Math.ceil( ( len ) / 4.0 ) ) ); return result; } private StringData readString( ByteBuffer rec, int pos ) { int endPos= pos; while ( rec.get(endPos)!=0 ) { endPos++; } byte[] mybytes= new byte[endPos-pos]; rec.position(pos); rec.get(mybytes); StringData result= new StringData(); result.string= new String( mybytes ); result._lengthBytes= Math.max( 4, (int)( 4 * Math.ceil( ( result.string.length() ) / 4.0 ) ) ); return result; } private static final int TYPECODE_BYTE=1; private static final int TYPECODE_INT16=2; private static final int TYPECODE_INT32=3; private static final int TYPECODE_FLOAT=4; private static final int TYPECODE_DOUBLE=5; private static final int TYPECODE_COMPLEX_FLOAT=6; private static final int TYPECODE_STRING=7; private static final int TYPECODE_STRUCT=8; private static final int TYPECODE_COMPLEX_DOUBLE=9; private static final int TYPECODE_INT64=14; /** * return the size of the IDL data type in bytes. Note shorts are stored * in 4-bytes. * @param typeCode * @return */ private static int sizeOf( int typeCode ) { int[] sizes= new int[] { 0, 4, 4, 4, 4, 8, 8, 1, 0, 16, 0, 0, 0, 0, 8 }; return sizes[typeCode]; } /** * read the TypeDesc for the variable. * @param in * @param name * @return * @throws IOException */ private TypeDesc readTypeDesc( ByteBuffer in, String name ) throws IOException { int magic= in.getInt(0); if ( magic!=1397882884 ) { logger.warning("magic number is incorrect"); } int pos= 4; ByteBuffer rec= readRecord( in, pos ); while ( rec!=null ) { int type= rec.getInt(0); int nextPos= rec.getInt(4); logger.log(Level.CONFIG, "RecType: {0} Length: {1}", new Object[]{labelType(type), nextPos-pos}); switch ( type ) { case RECTYPE_VARIABLE: logger.config("variable"); StringData varName= readString( rec, 20 ); if ( name.startsWith(varName.string) ) { int nextField= 20 + varName._lengthBytes; ByteBuffer var= slice( rec, nextField, rec.limit() ); TypeDesc td= readTypeDesc(var); return td; // struct } else if ( varName.string.equals(name) ) { int nextField= 20 + varName._lengthBytes; ByteBuffer var= slice( rec, nextField, rec.limit() ); TypeDesc td= readTypeDesc(var); return td; } break; case RECTYPE_VERSION: logger.config("version"); break; case RECTYPE_TIMESTAMP: logger.config("timestamp"); break; default: logger.config("???"); break; } pos= nextPos; rec= readRecord( in, pos ); } throw new IllegalArgumentException("unable to find variable: "+name); } /** * return true if the name refers to an array * @param in ByteBuffer for the entire file * @param name the variable name * @return td.isStructure(); */ public boolean isArray(ByteBuffer in, String name) throws IOException { TypeDesc td= readTypeDesc(in, name); return isArray( td.varFlags ); } /** * return true if the name refers to a structure * @param in ByteBuffer for the entire file * @param name the variable name * @return true if the name refers to a structure */ public boolean isStructure(ByteBuffer in, String name) throws IOException { TypeDesc td= readTypeDesc(in, name); return isStructure( td.varFlags ); } private TagDesc findStructureTag(StructDesc structDesc, String s) { String[] ss= s.split("\\.",2); int istruct= 0; int iarray= 0; if ( ss.length==1 ) { int itagfind=-1; for ( int itag=0; itag1024 ) { throw new IllegalArgumentException("unbelievable len, something has gone wrong."); } byte[] bb= new byte[len]; for ( int i=0; i accumulator, Map rec, int j, int nj ) { if ( accumulator.entrySet().isEmpty() ) { for ( Entry e: rec.entrySet() ) { Object o; if ( e.getValue() instanceof ArrayData ) { ArrayData ad= (ArrayData)e.getValue(); // Java 14 is coming and we won't need a cast, so exciting. ArrayData ac= new ArrayData(); ac.dims= new int[ad.dims.length+1]; ac.dims[0]= nj; System.arraycopy( ad.dims, 0, ac.dims, 1, ad.dims.length ); ac.array= Array.newInstance( ad.array.getClass(), nj ); Array.set( ac.array, j, ad.array ); o= ac; } else if ( e.getValue() instanceof Map ) { Map accumulator1= new LinkedHashMap(); accumulate( accumulator1, (Map)e.getValue(), j, nj ); o= accumulator1; } else { Object d= e.getValue(); ArrayData ac= new ArrayData(); ac.dims= new int[] { nj }; Class t= getPrimativeClass( d.getClass() ); ac.array= Array.newInstance( t, nj ); Array.set( ac.array, j, d ); o= ac; } accumulator.put( e.getKey(), o ); } } for ( Entry e: accumulator.entrySet() ) { Object o= rec.get(e.getKey()); if ( o instanceof ArrayData ) { ArrayData ad= (ArrayData)o; ArrayData ac= (ArrayData)e.getValue(); Array.set( ac.array, j, ad.array ); } else if ( e.getValue() instanceof Map ) { accumulate( (Map)e.getValue(), (Map)o, j, nj); } else { ArrayData ac= (ArrayData)e.getValue(); Array.set( ac.array, j, o ); } } } private static class TypeDescStructure extends TypeDesc { ArrayDesc structArrayDesc; StructDesc structDesc; int offsetToData; boolean isSubstructure; // structure within a structure. /** * length of the data within the IDLSav file. */ int _lengthBytes; @Override Object readData(ByteBuffer data) { LinkedHashMap result; int nj= structArrayDesc.nelements; if ( nj>1 ) { result= new LinkedHashMap<>(); int iptr= offsetToData + ( isSubstructure ? 0 : 4 ); int iptr0= iptr; for ( int j=0; j(); int iptr= offsetToData + ( isSubstructure ? 0 : 4 ); int iptr0= iptr; int iarray= 0; int istructure= 0; for ( int i=0; i14 ) { throw new IllegalArgumentException("expected 0-14 for type code in readTypeDesc"); } int varFlags= typeDescBuf.getInt(4); if ( ( varFlags & VARFLAG_STRUCT ) == VARFLAG_STRUCT ) { return readTypeDescStructure(typeDescBuf); } else if ( ( varFlags & VARFLAG_ARRAY ) == VARFLAG_ARRAY ) { return readTypeDescArray(typeDescBuf); } else { return readTypeDescScalar(typeDescBuf); } } /** * read the scalar, array, or structure at this position. An * array is returned flattened, and readTypeDesc should be used * to unflatten it. Structures are returned as a LinkedHashMap. * @param rec * @param vars map containing read data. * @return the read data. */ private Object variable( ByteBuffer rec, Map vars) { int type= rec.getInt(0); if ( type!=RECTYPE_VARIABLE ) { throw new IllegalArgumentException("not a variable"); } //printBuffer(rec); StringData varName= readString( rec, 20 ); logger.log(Level.FINE, "variable name is {0}", varName ); int nextField= 20 + varName._lengthBytes; ByteBuffer var= slice( rec, nextField, rec.limit() ); TypeDesc typeDesc= readTypeDesc( var ); Object result= typeDesc.readData( var ); vars.put( varName.string, result ); return result; } private ByteBuffer slice( ByteBuffer src, int position, int limit ) { int position0= src.position(); int limit0= src.limit(); src.position(position); src.limit(limit); ByteBuffer r1= ByteBuffer.allocate(limit-position); r1.put(src.slice()); r1.flip(); src.limit(limit0); src.position(position0); return r1; } private String labelType( int type ) { switch (type) { case RECTYPE_TIMESTAMP: return "timeStamp"; case RECTYPE_VERSION: return "version"; case RECTYPE_VARIABLE: return "variable"; case RECTYPE_ENDMARKER: return "endmarker"; default: return ""; } } public static ByteBuffer readFileIntoByteBuffer( File f ) throws IOException { RandomAccessFile aFile = new RandomAccessFile(f,"r"); FileChannel inChannel = aFile.getChannel(); long fileSize = inChannel.size(); ByteBuffer buffer = ByteBuffer.allocate((int) fileSize); int bytesRead= 0; while ( bytesRead readVars( ByteBuffer in ) throws IOException { // 2 ch.write(getBytesStr("SR")); // 1 ch.write(getBytesByte((byte) 0)); // 1 ch.write(getBytesByte((byte) 4)); int magic= in.getInt(0); if ( magic!=1397882884 ) { logger.warning("magic number is incorrect"); } int pos= 4; Map result= new LinkedHashMap<>(); ByteBuffer rec= readRecord( in, pos ); while ( rec!=null ) { int type= rec.getInt(0); int nextPos= rec.getInt(4); if ( rec.getInt(8)!=0 ) { throw new IllegalArgumentException("records bigger than 2**32 bytes are not supported."); } logger.log(Level.CONFIG, "RecType: {0} Length: {1}", new Object[]{labelType(type), nextPos-pos}); switch ( type ) { case RECTYPE_VARIABLE: logger.config("variable"); variable(rec, result); break; case RECTYPE_VERSION: logger.config("version"); break; case RECTYPE_TIMESTAMP: logger.config("timestamp"); break; default: logger.config("???"); break; } pos= nextPos; rec= readRecord( in, pos ); } return result; } /** * list the names in the IDLSav file. This is only the supported * variable types. * @param in * @return the names found. * @throws IOException */ public String[] readVarNames( ByteBuffer in ) throws IOException { int magic= in.getInt(0); if ( magic!=1397882884 ) { logger.warning("magic number is incorrect"); } int pos= 4; List names= new ArrayList<>(); ByteBuffer rec= readRecord( in, pos ); while ( rec!=null ) { int type= rec.getInt(0); int nextPos= rec.getInt(4); logger.log(Level.CONFIG, "RecType: {0} Length: {1}", new Object[]{labelType(type), nextPos-pos}); switch ( type ) { case RECTYPE_VARIABLE: logger.config("variable"); StringData varName= readString( rec, 20 ); int nextField= varName._lengthBytes; ByteBuffer var= slice( rec, 20+nextField, rec.limit() ); names.add(varName.string); break; case RECTYPE_VERSION: logger.config("version"); break; case RECTYPE_TIMESTAMP: logger.config("timestamp"); break; default: logger.config("???"); break; } pos= nextPos; rec= readRecord( in, pos ); } return names.toArray( new String[names.size()] ); } /** * scan through the IDLSav and return just the one variable. * @param in * @param name * @return * @throws IOException */ public Object readVar( ByteBuffer in, String name ) throws IOException { int magic= in.getInt(0); if ( magic!=1397882884 ) { logger.warning("magic number is incorrect"); } int pos= 4; String name0= name; // keep name for reference. ByteBuffer rec= readRecord( in, pos ); while ( rec!=null ) { int type= rec.getInt(0); int nextPos= rec.getInt(4); logger.log(Level.CONFIG, "RecType: {0} Length: {1}", new Object[]{labelType(type), nextPos-pos}); switch ( type ) { case RECTYPE_VARIABLE: logger.config("variable"); StringData varName= readString( rec, 20 ); String rest= null; int i= name.indexOf("."); if ( i>-1 ) { rest= name.substring(i+1); name= name.substring(0,i); } if ( i==-1 ) { if ( varName.string.equals(name) ) { Map result= new HashMap<>(); variable(rec, result); return result.get(name); } } else { if ( varName.string.equals(name) ) { Map result= new HashMap<>(); variable(rec,result); Map res= (Map) result.get(name); assert rest!=null; i= rest.indexOf('.'); while ( i>-1 ) { res= (Map)res.get( rest.substring(0,i) ); rest= rest.substring(i+1); i= rest.indexOf('.'); } return res.get(rest); } } break; case RECTYPE_VERSION: logger.config("version"); break; case RECTYPE_TIMESTAMP: logger.config("timestamp"); break; case RECTYPE_PROMOTE64: logger.config("promote64"); throw new IllegalArgumentException("promote64 is not supported."); default: logger.config("???"); break; } pos= nextPos; rec= readRecord( in, pos ); } return null; } /** * scan through the IDLSav and retrieve information about the array. * @param in the idlsav loaded into a ByteBuffer. * @param name the name of the array * @return * @throws IOException */ public TagDesc readTagDesc( ByteBuffer in, String name ) throws IOException { int magic= in.getInt(0); if ( magic!=1397882884 ) { logger.warning("magic number is incorrect"); } int pos= 4; ByteBuffer rec= readRecord( in, pos ); while ( rec!=null ) { int type= rec.getInt(0); int nextPos= rec.getInt(4); logger.log(Level.CONFIG, "RecType: {0} Length: {1}", new Object[]{labelType(type), nextPos-pos}); switch ( type ) { case RECTYPE_VARIABLE: logger.config("variable"); StringData varName= readString( rec, 20 ); if ( name.startsWith(varName.string+".") || name.equals(varName.string) ) { int nextField= varName._lengthBytes; ByteBuffer var= slice( rec, 20+nextField, rec.limit() ); if ( var.getInt(0)==8 ) { // TODO: what is 8? if ( ( var.getInt(4) & VARFLAG_STRUCT ) == VARFLAG_STRUCT ) { TypeDescStructure typeDescStructure= readTypeDescStructure(var); if ( name.equals(varName.string) ) { return typeDescStructure.structDesc; } else { return findStructureTag( typeDescStructure.structDesc, name.substring(varName.string.length()+1) ); } } else { return readTypeDescArray(var).arrayDesc; } } else { if ( ( var.getInt(4) & VARFLAG_ARRAY ) == VARFLAG_ARRAY ) { return readTypeDescArray(var).arrayDesc; } else { return readTagDesc(var); } } } break; case RECTYPE_VERSION: logger.config("version"); break; case RECTYPE_TIMESTAMP: logger.config("timestamp"); break; default: logger.config("???"); break; } pos= nextPos; rec= readRecord( in, pos ); } return null; } private static void arrayToString( Object o, StringBuilder b ) { char delim=','; for ( int j=0; j<4; j++ ) { Object i= Array.get(o,j); if ( i.getClass().isArray() ) { delim=';'; if ( j>0 ) b.append(delim); arrayToString( i, b ); } else { if ( j>0 ) b.append(delim); b.append(i.toString()); } } if ( Array.getLength(o)>4 ) { b.append(delim); b.append("..."); } } // public static void main( String[] args ) throws IOException { // Logger logger= Logger.getLogger("autoplot.idlsav"); // //logger.setLevel( Level.FINE ); // Handler h= new ConsoleHandler(); // h.setLevel(Level.ALL); // logger.addHandler(h); // //// FileOutputStream fos = new FileOutputStream(new File("/tmp/test.autoplot.idlsav")); //// //// WriteIDLSav widls= new WriteIDLSav(); //// //widls.addVariable( "wxyz", new double[] { 120,100,120,45,46,47,48,49,120,100,120 } ); //// widls.addVariable( "abcd", 240 ); //// //widls.addVariable( "oneval", 19.95 ); //// widls.write(fos); //// //// fos.close(); // // //RandomAccessFile aFile = new RandomAccessFile( // // "/home/jbf/public_html/autoplot/data/sav/simple.idlsav","r"); // //RandomAccessFile aFile = new RandomAccessFile( // // "/home/jbf/public_html/autoplot/data/sav/vnames.idlsav","r"); // //RandomAccessFile aFile = new RandomAccessFile( // // "/home/jbf/public_html/autoplot/data/sav/scalars.idlsav","r"); // //RandomAccessFile aFile = new RandomAccessFile( // // "/home/jbf/public_html/autoplot/data/sav/arrayVsScalar.idlsav","r"); // //RandomAccessFile aFile = new RandomAccessFile( // // "/home/jbf/public_html/autoplot/data/sav/floats.idlsav","r"); // //RandomAccessFile aFile = new RandomAccessFile( // // /home/jbf/public_html/autoplot/data/sav/structureOfLonarr.idlsav "/home/jbf/public_html/autoplot/data/sav/doublearray.idlsav","r"); // //RandomAccessFile aFile = new RandomAccessFile( // // "/home/jbf/public_html/autoplot/data/sav/structureOfLonarr.idlsav","r"); // //RandomAccessFile aFile = new RandomAccessFile( // // "/home/jbf/public_html/autoplot/data/sav/arrayOfStruct.idlsav","r"); // //RandomAccessFile aFile = new RandomAccessFile( // // "/home/jbf/public_html/autoplot/data/sav/arrayOfStruct1Var.idlsav","r"); // //RandomAccessFile aFile = new RandomAccessFile( // // "/home/jbf/public_html/autoplot/data/sav/structure.idlsav","r"); // //RandomAccessFile aFile = new RandomAccessFile( // // "/home/jbf/public_html/autoplot/data/sav/structureWithinStructure.idlsav","r"); // //RandomAccessFile aFile = new RandomAccessFile( // // "/home/jbf/public_html/autoplot/data/sav/stuctOfStruct.idlsav","r"); // //RandomAccessFile aFile = new RandomAccessFile( // // "/home/jbf/public_html/autoplot/data/sav/stuctOfStructOfStruct.idlsav","r"); // //RandomAccessFile aFile = new RandomAccessFile( // // "/home/jbf/public_html/autoplot/data/sav/stuctOfStructOfStruct.idlsav","r"); // RandomAccessFile aFile = new RandomAccessFile( // "/home/jbf/ct/autoplot/data/sav/kristoff/test_fit.idlsav","r"); // // FileChannel inChannel = aFile.getChannel(); // long fileSize = inChannel.size(); // // ByteBuffer buffer = ByteBuffer.allocate((int) fileSize); // int bytesRead= 0; // while ( bytesRead vars= new ReadIDLSav().readVars(buffer); // // for ( Entry v : vars.entrySet() ) { // System.err.println( v ); // if ( v.getValue() instanceof Map ) { // Map m= (Map)v.getValue(); // for ( Entry j : m.entrySet() ) { // Object k= j.getValue(); // if ( k instanceof ArrayData ) { // System.err.print(j.getKey()+":"); // StringBuilder b= new StringBuilder(); // arrayToString( ((ArrayData)k).array, b); // System.err.println(b.toString()); // } else if ( k==null ) { // System.err.println("<>"); // } else { // System.err.println(k.toString()); // } // } // } else { // System.err.println(v.getValue()); // } // } // // } }