package org.autoplot.idlsupport; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.io.UnsupportedEncodingException; import java.lang.reflect.Array; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.logging.ConsoleHandler; import java.util.logging.Handler; import java.util.logging.Level; import java.util.logging.Logger; /** * Read data from IDL Save Files. This was written using * http://www.physics.wisc.edu/~craigm/idl/savefmt/node20.html * https://cow.physics.wisc.edu/~craigm/idl/savefmt.pdf * and https://github.com/scipy/scipy/blob/master/scipy/io/idl.py * for reference, and with no involvement from individuals at * Harris Geospacial. No warrenties are implied and this must * be used at your own risk. * * @author jbf */ public class ReadIDLSav { private static final Logger logger= Logger.getLogger("autoplot.idlsav"); private static final int RECTYPE_ENDMARKER = 6; private static final int RECTYPE_TIMESTAMP = 10; private static final int RECTYPE_VARIABLE = 2; private static final int RECTYPE_VERSION = 14; private static final int VARFLAG_ARRAY = 0x04; private static final int VARFLAG_STRUCT = 0x20; /** * return the next record buffer, or returns null at the end. * @param ch the bytebuffer * @param pos the position. * @return the record, including the twelve bytes at the beginning * @throws IOException */ private ByteBuffer readRecord( ByteBuffer ch, int pos ) throws IOException { ch.order( ByteOrder.BIG_ENDIAN ); int recType= ch.getInt(pos); int endpos= ch.getInt(pos+4); if ( recType==RECTYPE_ENDMARKER ) { return null; } else { return slice( ch, pos, endpos ); } } /** * somehow I didn't notice the length before other strings. In the Python * code they have "_read_string" and "_read_string_data" which has a * second length. * @param rec * @param pos * @return StringDesc to describe the string. */ private StringData readStringData( ByteBuffer rec, int pos ) { int len= rec.getInt(pos); byte[] mybytes= new byte[len]; rec.position(pos+4); rec.get(mybytes); StringData result= new StringData(); result.string= new String( mybytes ); result._lengthBytes= 4 + Math.max( 4, (int)( 4 * Math.ceil( ( len ) / 4.0 ) ) ); return result; } private StringData readString( ByteBuffer rec, int pos ) { int endPos= pos; while ( rec.get(endPos)!=0 ) { endPos++; } byte[] mybytes= new byte[endPos-pos]; rec.position(pos); rec.get(mybytes); StringData result= new StringData(); result.string= new String( mybytes ); result._lengthBytes= Math.max( 4, (int)( 4 * Math.ceil( ( result.string.length() ) / 4.0 ) ) ); return result; } private static void printBuffer( ByteBuffer rec ) { for ( int i=0; i32 && c<128 ) { System.err.println( String.format( "%05d %d (%c) I4: %d", i, c, c, theInt ) ); } else { System.err.println( String.format( "%05d %d I4: %d", i, c, theInt ) ); } } else { if ( c>32 && c<128 ) { System.err.println( String.format( "%05d %d (%c)", i, c, c ) ); } else { System.err.println( String.format( "%05d %d", i, c) ); } } } } private static final int TYPECODE_BYTE=1; private static final int TYPECODE_INT16=2; private static final int TYPECODE_INT32=3; private static final int TYPECODE_FLOAT=4; private static final int TYPECODE_DOUBLE=5; private static final int TYPECODE_COMPLEX_FLOAT=6; private static final int TYPECODE_STRING=7; private static final int TYPECODE_STRUCT=8; private static final int TYPECODE_COMPLEX_DOUBLE=9; private static final int TYPECODE_INT64=14; /** * return the size of the IDL data type in bytes. Note shorts are stored * in 4-bytes. * @param typeCode * @return */ private static int sizeOf( int typeCode ) { int[] sizes= new int[] { 0, 4, 4, 4, 4, 8, 8, 0, 0, 16, 0, 0, 0, 0, 8 }; return sizes[typeCode]; } /** * read the TypeDesc for the variable. * @param in * @param name * @return * @throws IOException */ private TypeDesc readTypeDesc( ByteBuffer in, String name ) throws IOException { int magic= in.getInt(0); if ( magic!=1397882884 ) { logger.warning("magic number is incorrect"); } int pos= 4; ByteBuffer rec= readRecord( in, pos ); while ( rec!=null ) { int type= rec.getInt(0); int nextPos= rec.getInt(4); logger.log(Level.CONFIG, "RecType: {0} Length: {1}", new Object[]{labelType(type), nextPos-pos}); switch ( type ) { case RECTYPE_VARIABLE: logger.config("variable"); StringData varName= readString( rec, 20 ); if ( varName.string.equals(name) ) { int nextField= 20 + varName._lengthBytes; ByteBuffer var= slice( rec, nextField, rec.limit() ); TypeDesc td= readTypeDesc(var); return td; } break; case RECTYPE_VERSION: logger.config("version"); break; case RECTYPE_TIMESTAMP: logger.config("timestamp"); break; default: logger.config("???"); break; } pos= nextPos; rec= readRecord( in, pos ); } throw new IllegalArgumentException("unable to find variable: "+name); } /** * return true if the name refers to an array * @param in ByteBuffer for the entire file * @param name the variable name * @return td.isStructure(); */ public boolean isArray(ByteBuffer in, String name) throws IOException { TypeDesc td= readTypeDesc(in, name); return isArray( td.varFlags ); } /** * return true if the name refers to a structure * @param in ByteBuffer for the entire file * @param name the variable name * @return true if the name refers to a structure */ public boolean isStructure(ByteBuffer in, String name) throws IOException { TypeDesc td= readTypeDesc(in, name); return isStructure( td.varFlags ); } private static class TypeDescScalar extends TypeDesc { int offs= 12; @Override Object readData( ByteBuffer buf ) { switch ( typeCode ) { case TYPECODE_INT16: return (short)buf.getInt(offs); case TYPECODE_INT32: return buf.getInt(offs); case TYPECODE_INT64: return buf.getLong(offs); case TYPECODE_FLOAT: return buf.getFloat(offs); case TYPECODE_DOUBLE: return buf.getDouble(offs); case TYPECODE_STRING: int len= buf.getInt(offs); byte[] bb= new byte[len]; for ( int i=0; i result; if ( structArrayDesc.nelements>1 ) { result= new LinkedHashMap<>(); int iptr= offsetToData + 4; int iptr0= iptr; for ( int j=0; j(); int iptr= offsetToData + 4; int iptr0= iptr; int iarray= 0; int istructure= 0; for ( int i=0; i14 ) { throw new IllegalArgumentException("expected 0-14 for type code in readTypeDesc"); } int varFlags= typeDescBuf.getInt(4); if ( ( varFlags & VARFLAG_STRUCT ) == VARFLAG_STRUCT ) { return readTypeDescStructure(typeDescBuf); } else if ( ( varFlags & VARFLAG_ARRAY ) == VARFLAG_ARRAY ) { return readTypeDescArray(typeDescBuf); } else { return readTypeDescScalar(typeDescBuf); } } /** * read the scalar, array, or structure at this position. An * array is returned flattened, and readTypeDesc should be used * to unflatten it. Structures are returned as a LinkedHashMap. * @param rec * @param vars * @return */ private Object variable( ByteBuffer rec, Map vars) { int type= rec.getInt(0); if ( type!=RECTYPE_VARIABLE ) { throw new IllegalArgumentException("not a variable"); } //printBuffer(rec); StringData varName= readString( rec, 20 ); logger.log(Level.FINE, "variable name is {0}", varName ); int nextField= 20 + varName._lengthBytes; ByteBuffer var= slice( rec, nextField, rec.limit() ); TypeDesc typeDesc= readTypeDesc( var ); Object result= typeDesc.readData( var ); vars.put( varName.string, result ); return result; } private ByteBuffer slice( ByteBuffer src, int position, int limit ) { int position0= src.position(); int limit0= src.limit(); src.position(position); src.limit(limit); ByteBuffer r1= ByteBuffer.allocate(limit-position); r1.put(src.slice()); r1.flip(); src.limit(limit0); src.position(position0); return r1; } private String labelType( int type ) { switch (type) { case RECTYPE_TIMESTAMP: return "timeStamp"; case RECTYPE_VERSION: return "version"; case RECTYPE_VARIABLE: return "variable"; case RECTYPE_ENDMARKER: return "endmarker"; default: return ""; } } public static ByteBuffer readFileIntoByteBuffer( File f ) throws IOException { RandomAccessFile aFile = new RandomAccessFile(f,"r"); FileChannel inChannel = aFile.getChannel(); long fileSize = inChannel.size(); ByteBuffer buffer = ByteBuffer.allocate((int) fileSize); int bytesRead= 0; while ( bytesRead readVars( ByteBuffer in ) throws IOException { // 2 ch.write(getBytesStr("SR")); // 1 ch.write(getBytesByte((byte) 0)); // 1 ch.write(getBytesByte((byte) 4)); int magic= in.getInt(0); if ( magic!=1397882884 ) { logger.warning("magic number is incorrect"); } int pos= 4; Map result= new LinkedHashMap<>(); ByteBuffer rec= readRecord( in, pos ); while ( rec!=null ) { int type= rec.getInt(0); int nextPos= rec.getInt(4); if ( rec.getInt(8)!=0 ) { throw new IllegalArgumentException("records bigger than 2**32 bytes are not supported."); } logger.log(Level.CONFIG, "RecType: {0} Length: {1}", new Object[]{labelType(type), nextPos-pos}); switch ( type ) { case RECTYPE_VARIABLE: logger.config("variable"); variable(rec, result); break; case RECTYPE_VERSION: logger.config("version"); break; case RECTYPE_TIMESTAMP: logger.config("timestamp"); break; default: logger.config("???"); break; } pos= nextPos; rec= readRecord( in, pos ); } return result; } /** * list the names in the IDLSav file. This is only the supported * variable types. * @param in * @return the names found. * @throws IOException */ public String[] readVarNames( ByteBuffer in ) throws IOException { int magic= in.getInt(0); if ( magic!=1397882884 ) { logger.warning("magic number is incorrect"); } int pos= 4; List names= new ArrayList<>(); ByteBuffer rec= readRecord( in, pos ); while ( rec!=null ) { int type= rec.getInt(0); int nextPos= rec.getInt(4); logger.log(Level.CONFIG, "RecType: {0} Length: {1}", new Object[]{labelType(type), nextPos-pos}); switch ( type ) { case RECTYPE_VARIABLE: logger.config("variable"); StringData varName= readString( rec, 20 ); int nextField= varName._lengthBytes; ByteBuffer var= slice( rec, 20+nextField, rec.limit() ); names.add(varName.string); break; case RECTYPE_VERSION: logger.config("version"); break; case RECTYPE_TIMESTAMP: logger.config("timestamp"); break; default: logger.config("???"); break; } pos= nextPos; rec= readRecord( in, pos ); } return names.toArray( new String[names.size()] ); } /** * scan through the IDLSav and return just the one variable. * @param in * @param name * @return * @throws IOException */ public Object readVar( ByteBuffer in, String name ) throws IOException { int magic= in.getInt(0); if ( magic!=1397882884 ) { logger.warning("magic number is incorrect"); } int pos= 4; ByteBuffer rec= readRecord( in, pos ); while ( rec!=null ) { int type= rec.getInt(0); int nextPos= rec.getInt(4); logger.log(Level.CONFIG, "RecType: {0} Length: {1}", new Object[]{labelType(type), nextPos-pos}); switch ( type ) { case RECTYPE_VARIABLE: logger.config("variable"); StringData varName= readString( rec, 20 ); if ( varName.string.equals(name) ) { Map result= new HashMap<>(); variable(rec, result); return result.get(name); } break; case RECTYPE_VERSION: logger.config("version"); break; case RECTYPE_TIMESTAMP: logger.config("timestamp"); break; default: logger.config("???"); break; } pos= nextPos; rec= readRecord( in, pos ); } return null; } /** * scan through the IDLSav and retrieve information about the array. * @param in the idlsav loaded into a ByteBuffer. * @param name the name of the array * @return * @throws IOException */ public ArrayDesc readArrayDesc( ByteBuffer in, String name ) throws IOException { int magic= in.getInt(0); if ( magic!=1397882884 ) { logger.warning("magic number is incorrect"); } int pos= 4; ByteBuffer rec= readRecord( in, pos ); while ( rec!=null ) { int type= rec.getInt(0); int nextPos= rec.getInt(4); logger.log(Level.CONFIG, "RecType: {0} Length: {1}", new Object[]{labelType(type), nextPos-pos}); switch ( type ) { case RECTYPE_VARIABLE: logger.config("variable"); StringData varName= readString( rec, 20 ); if ( varName.string.equals(name) ) { int nextField= varName._lengthBytes; ByteBuffer var= slice( rec, 20+nextField, rec.limit() ); return readTypeDescArray(var).arrayDesc; } break; case RECTYPE_VERSION: logger.config("version"); break; case RECTYPE_TIMESTAMP: logger.config("timestamp"); break; default: logger.config("???"); break; } pos= nextPos; rec= readRecord( in, pos ); } return null; } private static void arrayToString( Object o, StringBuilder b ) { char delim=','; for ( int j=0; j<4; j++ ) { Object i= Array.get(o,j); if ( i.getClass().isArray() ) { delim=';'; if ( j>0 ) b.append(delim); arrayToString( i, b ); } else { if ( j>0 ) b.append(delim); b.append(i.toString()); } } if ( Array.getLength(o)>4 ) { b.append(delim); b.append("..."); } } public static void main( String[] args ) throws IOException { Logger logger= Logger.getLogger("autoplot.idlsav"); //logger.setLevel( Level.FINE ); Handler h= new ConsoleHandler(); h.setLevel(Level.ALL); logger.addHandler(h); // FileOutputStream fos = new FileOutputStream(new File("/tmp/test.autoplot.idlsav")); // // WriteIDLSav widls= new WriteIDLSav(); // //widls.addVariable( "wxyz", new double[] { 120,100,120,45,46,47,48,49,120,100,120 } ); // widls.addVariable( "abcd", 240 ); // //widls.addVariable( "oneval", 19.95 ); // widls.write(fos); // // fos.close(); //RandomAccessFile aFile = new RandomAccessFile( // "/home/jbf/public_html/autoplot/data/sav/simple.idlsav","r"); //RandomAccessFile aFile = new RandomAccessFile( // "/home/jbf/public_html/autoplot/data/sav/vnames.idlsav","r"); //RandomAccessFile aFile = new RandomAccessFile( // "/home/jbf/public_html/autoplot/data/sav/scalars.idlsav","r"); //RandomAccessFile aFile = new RandomAccessFile( // "/home/jbf/public_html/autoplot/data/sav/arrayVsScalar.idlsav","r"); //RandomAccessFile aFile = new RandomAccessFile( // "/home/jbf/public_html/autoplot/data/sav/floats.idlsav","r"); //RandomAccessFile aFile = new RandomAccessFile( // /home/jbf/public_html/autoplot/data/sav/structureOfLonarr.idlsav "/home/jbf/public_html/autoplot/data/sav/doublearray.idlsav","r"); //RandomAccessFile aFile = new RandomAccessFile( // "/home/jbf/public_html/autoplot/data/sav/structureOfLonarr.idlsav","r"); //RandomAccessFile aFile = new RandomAccessFile( // "/home/jbf/public_html/autoplot/data/sav/arrayOfStruct.idlsav","r"); //RandomAccessFile aFile = new RandomAccessFile( // "/home/jbf/public_html/autoplot/data/sav/arrayOfStruct1Var.idlsav","r"); //RandomAccessFile aFile = new RandomAccessFile( // "/home/jbf/public_html/autoplot/data/sav/structure.idlsav","r"); RandomAccessFile aFile = new RandomAccessFile( "/home/jbf/public_html/autoplot/data/sav/structureWithinStructure.idlsav","r"); FileChannel inChannel = aFile.getChannel(); long fileSize = inChannel.size(); ByteBuffer buffer = ByteBuffer.allocate((int) fileSize); int bytesRead= 0; while ( bytesRead vars= new ReadIDLSav().readVars(buffer); for ( Entry v : vars.entrySet() ) { System.err.println( v ); if ( v.getValue() instanceof Map ) { Map m= (Map)v.getValue(); for ( Entry j : m.entrySet() ) { Object k= j.getValue(); if ( k instanceof ArrayData ) { System.err.print(j.getKey()+":"); StringBuilder b= new StringBuilder(); arrayToString( ((ArrayData)k).array, b); System.err.println(b.toString()); } else if ( k==null ) { System.err.println("<>"); } else { System.err.println(k.toString()); } } } else { System.err.println(v.getValue()); } } } }