/* This Java package, org.autoplot.das2Stream is part of the Autoplot application * * Autoplot is free software; you can redistribute it and/or modify it under the * terms of the GNU General Public License version 2 as published by the Free * Software Foundation, with the Classpath exception below. * * This program is distributed in the hope that it will be useful, but WITHOUT ANY * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A * PARTICULAR PURPOSE. See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License version 2 * containing the Classpath exception clause along with this library; if not, write * to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA * 02111-1307 USA * * Classpath Exception * ------------------- * The copyright holders designate this particular java package as subject to the * "Classpath" exception as provided here. * * Linking this package statically or dynamically with other modules is making a * combined work based on this package. Thus, the terms and conditions of the GNU * General Public License cover the whole combination. * * As a special exception, the copyright holders of this package give you * permission to link this package with independent modules to produce an * application, regardless of the license terms of these independent modules, and * to copy and distribute the resulting application under terms of your choice, * provided that you also meet, for each independent module, the terms and * conditions of the license of that module. An independent module is a module * which is not derived from or based on this package. If you modify this package, * you may extend this exception to your version of the package, but you are not * obligated to do so. If you do not wish to do so, delete this exception * statement from your version. */ package org.das2.qstream; import java.io.IOException; import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.logging.Logger; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import org.das2.datum.LoggerManager; import org.das2.datum.TimeLocationUnits; import org.das2.datum.Units; import org.das2.datum.UnitsUtil; import org.das2.qds.DataSetOps; import org.das2.qds.MutablePropertyDataSet; import org.das2.qds.QDataSet; import org.das2.qds.SemanticOps; import org.das2.qds.ops.Ops; import org.w3c.dom.DOMImplementation; import org.w3c.dom.Document; import org.w3c.dom.ls.DOMImplementationLS; import org.w3c.dom.ls.LSSerializer; /** Base class for QDataSet to das2 stream serializers * */ abstract public class QdsToD2sStream { // Private //////////////////////////////////////////////////////////////// private static final Logger log = LoggerManager.getLogger("qstream"); // Public interface ////////////////////////////////////////////////////// public static final String FORMAT_2_2 = "2.2"; public static final String FORMAT_2_3_BASIC = "2.3/basic"; public static final String FORMAT_2_4_GENERAL = "2.4/general"; public static final String[] formats = {FORMAT_2_2, FORMAT_2_3_BASIC}; public static final int DEFAUT_FRAC_SEC = 3; public static final int DEFAUT_SIG_DIGIT = 5; public static final int FIXED_PKT_TAGS = 0; // original tag format public static final int VAR_PKT_TAGS = 1; // new variable packet tags /** Initialize a binary QDataSet to das2 stream exporter */ public QdsToD2sStream(){ bBinary = true; } /** Initialize a text QDataSet to das2 stream exporter * * @param genSigDigits The number of significant digits used for general * text value data output. If you don't know what else to use, 5 is * typically a fine precision without being ridiculous. * * @param fracSecDigits The number of fractional seconds digits to * use for ISO-8601 date-time values. The number of fraction seconds * can be set as low as 0 and as high as 12 (picoseconds). If * successive time values vary by less than the specified precision * (but not 0, repeats are accepted) then stream writing fails. Use 3 (i.e. * microseconds) if you don't know what else to choose. */ public QdsToD2sStream(int genSigDigits, int fracSecDigits){ bBinary = false; if(genSigDigits > 1){ if(genSigDigits > 16){ throw new IllegalArgumentException(String.format( "Number of significant digits in the output must be between 2 " + "and 17, received %d", genSigDigits )); } nSigDigit = genSigDigits; } if(fracSecDigits >= 0){ if(fracSecDigits < 0 || fracSecDigits > 12){ throw new IllegalArgumentException(String.format( "Number of fractional seconds digits in the output must be " + "between 0 and 12 inclusive, received %d", fracSecDigits )); } nSecDigit = fracSecDigits; } } /** Determine if the given dataset can be written * * @param qds * @return */ public abstract boolean canWrite(QDataSet qds); /** Write the given dataset * * @param qds * @param os * @return * @throws java.io.IOException */ public abstract boolean write(QDataSet qds, OutputStream os) throws IOException; // protected ////////////////////////////////////////////////////////// /** Write a UTF-8 header string onto the stream, if packet ID == 0 then * write a stream header * * @param os The output stream * @param nTagType Either FIXED_PKT_TAGS or VAR_PKT_TAGS * @param nPktId * @param sHdr - The header string to write for this packet id * @throws UnsupportedEncodingException * @throws IOException */ static void writeHeader(OutputStream os, int nTagType, int nPktId, String sHdr) throws UnsupportedEncodingException, IOException{ byte[] aHdr = sHdr.getBytes(StandardCharsets.UTF_8); String sTag; if(nTagType == VAR_PKT_TAGS){ if(nPktId == 0) sTag = String.format("|Hs||%d|", aHdr.length); else sTag = String.format("|Hx|%d|%d|", nPktId, aHdr.length); } else{ sTag = String.format("[%02d]%06d", nPktId, aHdr.length); } byte[] aTag = sTag.getBytes(StandardCharsets.US_ASCII); os.write(aTag); os.write(aHdr); } // Handling for text output, throw this back to the user if they want text // output. The reason not to just do our best to figure it out on our own // is that it changes the packet header definitions, which we don't want to // do when writing cache files. protected int nSigDigit = DEFAUT_SIG_DIGIT; protected int nSecDigit = DEFAUT_FRAC_SEC; protected boolean bBinary; /** Determine and hold the information needed to transfer values out of a * given QDataSet into a byte buffer. The byte order is always native * endian */ static protected class QdsXferInfo { QDataSet qds; TransferType transtype; /** Figure out how to represent the values. In general everything is * output as a float unless it's an epoch time type. * * @param _qds The QDataset * @param bBinary True if user wanted binary data * @param nGenDigit Number of significant digits for general text values * @param nFracSec Number of fractional seconds for time text values */ QdsXferInfo( QDataSet _qds, boolean bBinary, int nGenSigDigit, int nFracSec ){ qds = _qds; if(bBinary){ nGenSigDigit = -1; nFracSec = -1; } Units units = (Units) qds.property(QDataSet.UNITS); if((units != null) && (units instanceof TimeLocationUnits)){ if(nFracSec < 0) transtype = new DoubleTransferType(); else transtype = new TransferTimeAtPrecision(units, nFracSec); } else{ //Non epoch stuff Class c = DataSetOps.getComponentType(qds); if((c == int.class)||(c == long.class)){ if(bBinary){ if(c == int.class) transtype = new IntegerTransferType(); else transtype = new LongTransferType(); } else{ // Yea, couldn't be something like QDataSet.max() QDataSet dsExt = Ops.extent(qds); double rMax = Math.abs(dsExt.value(0)); double rMin = Math.abs(dsExt.value(1)); if(rMin > rMax) rMax = rMin; int nChars = (int)(Math.ceil( Math.log10( rMax) ) + 2); transtype = new AsciiIntegerTransferType(nChars); } } else{ if(nGenSigDigit < 0) transtype = new FloatTransferType(); else transtype = new TransferSciNotation(nGenSigDigit); } } } /** Get a size independent type name * @return the type with any size information stripped off */ public String name(){ // Since float and double indicate size, use integer names // that indicate size as well for consistency. if(transtype.name().equals("int8")) return "long"; else return transtype.name().replaceAll("\\d",""); } /** @return the size in bytes of each output value */ public int size(){ return transtype.sizeBytes(); } /** Get the number items in a single X-axis slice of this dataset the * X-axis is synonymous with the QDataSet 0th axis for now. * @param i - The X point at which to get the items * @return */ public int xSliceItems(int i) throws IOException{ int nItems = 0; switch(qds.rank()){ case 1: return 1; case 2: return qds.length(i); case 3: for(int j=0; j < qds.length(i); ++j) nItems += qds.length(i,j); return nItems; case 4: // Das2 can't really do this, but calc anyway for(int j=0; j < qds.length(i); ++j){ for(int k=0; k < qds.length(i,j); ++k) nItems += qds.length(i,j,k); } return nItems; default: throw new IOException(String.format( "Can't stream rank %d data with this format.", qds.rank() )); } } /** Get the number of bytes needed to hold an x-slice of a single * * @param i * @return */ public int xSliceBytes(int i) throws IOException{ return transtype.sizeBytes() * xSliceItems(i); } } // Hold the information for serializing a single packet and it's dependencies static protected class PacketXferInfo { Document doc; // Overall header document List lDsXfer; PacketXferInfo(Document _doc, List _lDsXfer ){ doc = _doc; lDsXfer = _lDsXfer; } int datasets(){ return lDsXfer.size(); } // Calc length of buffer needed to hold a single slice in the 0th // dimension across all qdatasets. int xSliceBytes() throws IOException { int nLen = 0; for(QdsXferInfo qi: lDsXfer) nLen += qi.xSliceBytes(0); return nLen; } } // Send Bundle data protected void writeData( OutputStream out, int nTagType, int iPktId, PacketXferInfo pktXfer ) throws IOException { WritableByteChannel channel = Channels.newChannel(out); // Make the Xslice record tag. For now these all have the same length // if das2/general format is ever introduced will have to check the // length for each X-slice. String sRecTag; if(nTagType == VAR_PKT_TAGS){ sRecTag = String.format("|Dx|%d|%d|", iPktId, pktXfer.xSliceBytes()); } else{ sRecTag = String.format(":%02d:", iPktId); } byte[] aRecTag = sRecTag.getBytes(StandardCharsets.US_ASCII); int nBufLen = pktXfer.xSliceBytes() + aRecTag.length; byte[] aBuf = new byte[nBufLen]; ByteBuffer buffer = ByteBuffer.wrap(aBuf); buffer.order(ByteOrder.nativeOrder()); // Since we have a choice, use native buffer.put(aRecTag); // tag stays in buffer int nPkts = pktXfer.lDsXfer.get(0).qds.length(); for(int iPkt = 0; iPkt < nPkts; ++iPkt){ QdsXferInfo qi = null; for(int iDs = 0; iDs < pktXfer.datasets(); ++iDs){ qi = pktXfer.lDsXfer.get(iDs); switch(qi.qds.rank()){ case 1: qi.transtype.write(qi.qds.value(iPkt), buffer); break; case 2: // Does not assume cubic, if stream writers expect cubic, they // need to check for it before calling this function for(int iVal = 0; iVal < qi.qds.length(iPkt); ++iVal) qi.transtype.write(qi.qds.value(iPkt, iVal), buffer); break; case 3: // Does not assume cubic, if stream writers expect cubic, they // need to check for it before calling this function for(int iVal = 0; iVal < qi.qds.length(iPkt); ++iVal) for(int jVal = 0; jVal < qi.qds.length(iPkt, iVal); ++jVal) qi.transtype.write(qi.qds.value(iPkt, iVal, jVal), buffer); break; default: assert(false); } } if(qi != null && qi.transtype.isAscii()) buffer.put(nBufLen - 1, (byte)'\n'); buffer.flip(); channel.write(buffer); buffer.position(aRecTag.length); } } // Secondary helpers functions below here ////////////////////////////////// protected String makeNameFromUnits(Units units) { if(units == null) return ""; if(UnitsUtil.isTimeLocation(units)) return "time"; if(units.isConvertibleTo(Units.meters)) return "length"; if(units.isConvertibleTo(Units.hertz)) return "frequnecy"; if(units.isConvertibleTo(Units.eV)) return "energy"; if(units.isConvertibleTo(Units.degrees)) return "angle"; if(units.isConvertibleTo(Units.seconds)) return "interval"; if(units.isConvertibleTo(Units.bytes)) return "size"; if(units.isConvertibleTo(Units.cm_2s_1keV_1)) return "flux"; if(units.isConvertibleTo(Units.kelvin)) return "temperature"; String sUnits = units.toString(); // I wish there was some sort of units canonicalization function // Get frequency typo's if(sUnits.equalsIgnoreCase("mhz")) return "frequency"; if(sUnits.equalsIgnoreCase("khz")) return "frequency"; if(sUnits.equalsIgnoreCase("hz")) return "frequency"; // So where is spectral density? Try some wierd stuff if(sUnits.equalsIgnoreCase("V!a2!nm!a-2!nHz!a-1!n")) // yes, this happens return "e_spec_dens"; if(sUnits.equalsIgnoreCase("nT!a2!nHz!a-1!n")) // yes, this happens too return "b_spec_dens"; return ""; } protected boolean _stripDotProps(QDataSet qds){ // Ephemeris data (x-multi-y) ported in from das1 add many redundant // properties get rid of those if this is a bundle_1 dataset and each // sub-item is rank 1 if(qds.property(QDataSet.BUNDLE_1) != null){ for(int i = 0; i < qds.length(0); ++i){ QDataSet ds = DataSetOps.slice1(qds, i); if(ds.rank() != 1) return false; } } else{ return false; } return true; } static String xmlDocToStr(Document doc) { DOMImplementation imp = doc.getImplementation(); DOMImplementationLS ls = (DOMImplementationLS)imp.getFeature("LS", "3.0"); LSSerializer serializer = ls.createLSSerializer(); //DOMStringList props = serializer.getDomConfig().getParameterNames(); serializer.getDomConfig().setParameter("format-pretty-print", true); serializer.getDomConfig().setParameter("xml-declaration", false); String sDoc = serializer.writeToString(doc); return sDoc; } protected Document newXmlDoc() { try { DocumentBuilder bldr = DocumentBuilderFactory.newInstance().newDocumentBuilder(); return bldr.newDocument(); } catch (ParserConfigurationException pce) { throw new RuntimeException(pce); } } protected static class Sequence1D { String sMinval = null; String sInterval = null; } /** Determine if a rank 1 dataset is really just a sequence * * @param qds * @param rMaxJitter Recommend 1e-4 if no other values comes to mind * @return A Sequence1D object with minval and interval set. */ protected Sequence1D getSequenceRank1(QDataSet qds, double rMaxJitter) { if(qds.rank() != 1) return null; if(qds.length() < 2) return null; double rMin = qds.value(0); // Try to use sequence representation if we can, allow for jitter in // intervals of one part in the max jitter double rInterval = (qds.value( qds.length() - 1) - rMin)/(qds.length() - 1); double rNextInterval, rAvg, rDelta, rJitter; for(int i = 2; i < qds.length(); ++i){ rNextInterval = qds.value(i) - qds.value(i - 1); if(rNextInterval != rInterval){ rAvg = Math.abs(rNextInterval + rInterval)/2; rDelta = Math.abs(rNextInterval - rInterval); if(rDelta == 0.0) continue; if(rAvg == 0.0) return null; rJitter = rDelta/ rAvg; if(rJitter > rMaxJitter) return null; } } // Get my format string String sFmt = String.format("%%.%de", nSigDigit - 1); Sequence1D seq = new Sequence1D(); seq.sMinval = String.format(sFmt, rMin); seq.sInterval = String.format(sFmt, rInterval); return seq; } /////////////////////////////////////////////////////////////////////////// // Das2 specific dataset properties and functions that could just be an add // on for qdataset. These could be move somewhere else. /** Determine the name of the das2 axis on which values from a dataset * would typically be plotted. * * This is a duck-typing check. Which looks at the number of dependencies * and planes in a dataset. If the dataset has a PLANE_0 property, the axis * of the PLANE_0 values is returned instead of the axis of the primary * dataset. * * @param qds * @return one of "x", "y", "z","w" or null if we can't figure it out. */ public static String getQdsAxis(QDataSet qds){ //If join just look at the first element of the join if(SemanticOps.isJoin(qds)) qds = DataSetOps.slice0(qds, 0); //If see if the bundle is bigger than size 1. if(SemanticOps.isBundle(qds)){ if(qds.rank() == 1){ //trival bundle qds = DataSetOps.slice0(qds, 0); // now handle as single dataset below } else{ // I'm continuing the false assmption that paramenter space // dimensionality ~= rank which is the the core of the CDF/QDataSet // path dataset problem. We will have to face this head on someday. // The only indicator that a rank 1 item is actually a Z value is the // existance of a PLANE_0 property. int nMaxRank = 0; for(int i = 0; i < qds.length(); ++i){ QDataSet ds = DataSetOps.slice0(qds, i); int nRank = ds.rank(); if(ds.property(QDataSet.PLANE_0) != null) //throw new UnsupportedOperationException( // "Cannot determine the canonical axis for bundles which contain "+ // "datasets which have a PLANE_0 property." //); return null; if(nRank > nMaxRank) nMaxRank = ds.rank(); } if(nMaxRank == 1) return "y"; if(nMaxRank == 2) return "z"; return null; } } //Okay, not a bundle. To decide our axis just check our dependencies //rank, and planes. Here's the swiss cheese check: // rank 0 -> X // rank 1 with no depend_0 -> X // rank 1 with a depend_0 -> Y // rank 1 with a depend_0 and plane_0 -> Z (rem to pull out plane for Z's) // rank 2 -> Z switch(qds.rank()){ case 0: return "x"; case 2: return "z"; case 3: return "w"; case 1: // TODO: Read input to see how x/y/z data are handled if(qds.property(QDataSet.DEPEND_0) == null) return "x"; if(qds.property(QDataSet.PLANE_0) == null) return "y"; return "z"; } return null; } private static final List lSimpleKeys; static { lSimpleKeys = new ArrayList<>(); lSimpleKeys.add(QDataSet.NAME); lSimpleKeys.add(QDataSet.UNITS); lSimpleKeys.add(QDataSet.FORMAT); lSimpleKeys.add(QDataSet.SCALE_TYPE); lSimpleKeys.add(QDataSet.LABEL); lSimpleKeys.add(QDataSet.DESCRIPTION); lSimpleKeys.add(QDataSet.FILL_VALUE); lSimpleKeys.add(QDataSet.VALID_MIN); lSimpleKeys.add(QDataSet.VALID_MAX); lSimpleKeys.add(QDataSet.TYPICAL_MIN); lSimpleKeys.add(QDataSet.TYPICAL_MAX); lSimpleKeys.add(QDataSet.USER_PROPERTIES); } /** Only copy over the simple properties of a dataset, ignore the structural * items such as depend, offset, axis, reference, bundle etc. * * @param dsDest * @param dsSrc * @return */ protected int copySimpleProps(MutablePropertyDataSet dsDest, QDataSet dsSrc) { int nAdded = 0; Object oVal; for(String sKey: lSimpleKeys){ if( (oVal = dsSrc.property(sKey)) != null){ dsDest.putProperty(sKey, oVal); nAdded += 1; } } return nAdded; } }