Convert to write to a "shared ring buffer".HEADmaster

First steps, not actually fully functional yet (i.e. ring buffer is not
actually a ring buffer, and there is no log printer polling the other
side).

10 files changed, 789 insertions, 103 deletions

diff --git a/src/main/java/org/mamago/logging/Formattable.java b/src/main/java/org/mamago/logging/Formattable.java
new file mode 100644
index 0000000..2711ea1
--- /dev/null
+++ b/src/main/java/org/mamago/logging/Formattable.java
@@ -0,0 +1,13 @@
+package org.mamago.logging;
+
+import java.nio.ByteBuffer;
+
+public interface Formattable {
+    /**
+     * Format an argument to the logger buffer.
+     *
+     * @param buffer ByteBuffer to format to.
+     * @return length of bytes written to buffer.
+     */
+    int formatTo(ByteBuffer buffer);
+}
diff --git a/src/main/java/org/mamago/logging/Header.java b/src/main/java/org/mamago/logging/Header.java
new file mode 100644
index 0000000..392b328
--- /dev/null
+++ b/src/main/java/org/mamago/logging/Header.java
@@ -0,0 +1,82 @@
+package org.mamago.logging;
+
+import org.mamago.util.Bits;
+
+import java.nio.ByteBuffer;
+
+import static java.lang.Math.min;
+import static org.mamago.util.Cast.asByte;
+import static org.mamago.util.Cast.asShort;
+
+/**
+ * Header fields are as follows:
+ * <table>
+ *     <tr><th>Bytes</th><th>Field</th></tr>
+ *     <tr><td>8</td><td>Header size</td></tr>
+ *     <tr><td>8</td><td>Timestamp</td></tr>
+ *     <tr><td>8</td><td>Log level</td></tr>
+ *     <tr><td>8</td><td>Thread Id (tid)</td></tr>
+ *     <tr><td>130</td><td>Thread name (includes length field - 2 bytes)</td></tr>
+ *     <tr><td>130</td><td>Logger name (includes length field - 2 bytes)</td></tr>
+ *     <tr><td>?</td><td>Padding to align to long size</td></tr>
+ *     <tr><td>8</td><td>Body size</td></tr>
+ * </table>
+ */
+class Header {
+    static final int NAME_SIZE_LIMIT = 128;
+
+    static final int SIZE_OFFSET = 0;
+    static final int TS_OFFSET = SIZE_OFFSET + 8;
+    static final int LEVEL_OFFSET = TS_OFFSET + 8;
+    static final int TID_OFFSET = LEVEL_OFFSET + 8;
+    static final int TNAME_OFFSET = TID_OFFSET + 8;
+    static final int LOGGER_NAME_OFFSET = TNAME_OFFSET + NAME_SIZE_LIMIT + 2;
+    static final int PADDING_OFFSET = LOGGER_NAME_OFFSET + NAME_SIZE_LIMIT + 2;
+    static final int BODY_SIZE_OFFSET = Bits.roundUpToLongSize(PADDING_OFFSET);
+    static final int HEADER_SIZE = BODY_SIZE_OFFSET + 8;
+
+    private final ByteBuffer buffer;
+
+    Header(CharSequence loggerName, ByteBuffer buffer) {
+        this.buffer = buffer;
+        // Write header length at start
+        buffer.putLong(0, HEADER_SIZE);
+        put(LOGGER_NAME_OFFSET, loggerName);
+    }
+
+    Header timestamp(long ts) {
+        buffer.putLong(TS_OFFSET, ts);
+        return this;
+    }
+
+    Header level(Logger.Level level) {
+        buffer.putLong(LEVEL_OFFSET, asByte(level.ordinal()));
+        return this;
+    }
+
+    Header threadId(long tid) {
+        buffer.putLong(TID_OFFSET, tid);
+        return this;
+    }
+
+    Header threadName(CharSequence tName) {
+        put(TNAME_OFFSET, tName);
+        return this;
+    }
+
+    // Does not return this reference, as this is the terminator for building the header buffer
+    void bodySize(long size) {
+        buffer.putLong(BODY_SIZE_OFFSET, size);
+    }
+
+    private void put(int offset, CharSequence csq) {
+        // @Optimisation[MA]: CharSequence length as a short, as we limit header char sequence length
+        // (we don't expect long thread or logger names).
+        int length = min(csq.length(), NAME_SIZE_LIMIT);
+        buffer.putShort(offset++, asShort(length));
+        offset++;
+        for (int i = 0; i < length; i++) {
+            buffer.put(offset++, (byte) csq.charAt(i));
+        }
+    }
+}
diff --git a/src/main/java/org/mamago/logging/Logger.java b/src/main/java/org/mamago/logging/Logger.java
index 6610da8..0c6d06a 100644
--- a/src/main/java/org/mamago/logging/Logger.java
+++ b/src/main/java/org/mamago/logging/Logger.java
@@ -1,16 +1,22 @@
 package org.mamago.logging;
 
-import sun.misc.HexDumpEncoder;
-
 import java.nio.ByteBuffer;
 
-import static java.lang.Math.min;
+import static org.mamago.util.Cast.asByte;
+import static org.mamago.util.Cast.asShort;
 
+// @Speedup[MA]: Increased performance can be gained by providing our own buffer for CharSequence types that gives
+// direct access to the byte array for use with System.arrayCopy().
+//@NotThreadSafe @Fixme[MA]
 public class Logger {
     public enum Level {TRACE, DEBUG, INFO, WARN, ERROR}
-    private enum TypeMarker {BYTE, CHAR, SHORT, INT, LONG, FLOAT, DOUBLE, CHAR_SEQUENCE, FORMATTABLE}
+    enum TypeMarker {BYTE, CHAR, SHORT, INT, LONG, FLOAT, DOUBLE, CHAR_SEQUENCE, FORMATTABLE}
 
     private final LogBuilder builder;
+    // @Fixme[MA]: Extract to a ring buffer registry, or similar. Also allow custom ring buffer to be passed in to
+    //  Logger for complete flexible configuration.
+    final ByteBuffer logRingBuffer = ByteBuffer.allocateDirect(4 * 1024 * 1024); // 4 MiB
+
 
     // Factory methods  ------------------------------------
 
@@ -22,47 +28,44 @@ public class Logger {
         return new Logger(name);
     }
 
+
     // Instance methods  -----------------------------------
 
     private Logger(CharSequence name) {
-        builder = new LogBuilder(name.toString());
+        builder = new LogBuilder(name.toString(), logRingBuffer);
     }
 
     public LogBuilder log(Level level, CharSequence formatSpec) {
         return builder.reset(level, formatSpec);
     }
 
+
     // Helper methods --------------------------------------
 
     private static String getCallerClass() {
         return Thread.currentThread().getStackTrace()[3].getClassName();
     }
 
-    private static byte b(int i) {
-        // @Fixme[MA]: check whether casting give out-of-bounds check, otherwise add in for safety
-        return (byte) i;
-    }
 
     // Helper classes --------------------------------------
 
-    public interface Formattable {
-        void formatTo(ByteBuffer b);
-    }
-
     public static class LogBuilder implements Appendable {
-        // @Fixme[MA]: buffer needs to be auto-expandable
-        private final ByteBuffer headerBuf = ByteBuffer.allocate(256);
+        private static final short DUMMY = 0;
+
+        // @Fixme[MA]: handle log message overflowing message limit
+        private final ByteBuffer buffer = ByteBuffer.allocate(32 * 1024); // 32 KiB limit per log message
+        private final ThrowableFormatterFlyweight throwableFormatterFlyweight = new ThrowableFormatterFlyweight();
+        private final ByteBuffer logRingBuffer;
         private final Header header;
-        // @Fixme[MA]: buffer needs to be auto-expandable
-        private final ByteBuffer buffer = ByteBuffer.allocate(256);
         private Level level;
 
-        private LogBuilder(CharSequence loggerName) {
+        private LogBuilder(CharSequence loggerName, ByteBuffer logRingBuffer) {
+            this.logRingBuffer = logRingBuffer;
             // @Optimisation[MA]: Write logger name to the header buffer once. Should never have to be written again,
             // as LogBuilders are not shareable between loggers. In the event of seeing corruption of header buffer
             // data in the wild can remove this mini-optimisation and just write logger name every time in log() call.
             // (Though the real question would be: how if the buffer getting corrupted?)
-            header = new Header(loggerName);
+            header = new Header(loggerName, buffer);
         }
 
         @Override
@@ -73,141 +76,101 @@ public class Logger {
 
         @Override
         public LogBuilder append(CharSequence csq, int start, int end) {
-            buffer.put(b(TypeMarker.CHAR_SEQUENCE.ordinal()));
+            buffer.put(asByte(TypeMarker.CHAR_SEQUENCE.ordinal()));
             buffer.putInt(end - start); // length
             for (int i = start; i < end; i++) {
-                append(csq.charAt(i));
+                buffer.put(asByte(csq.charAt(i)));
             }
             return this;
         }
 
         @Override
         public LogBuilder append(char c) {
-            buffer.put(b(TypeMarker.CHAR.ordinal()));
-            buffer.put(b(c));
+            buffer.put(asByte(TypeMarker.CHAR.ordinal()));
+            buffer.put(asByte(c));
             return this;
         }
 
-        @Deprecated // @Fixme[MA]: really want an annotation to create a compiler warning.
+        @Deprecated // @Fixme[MA]: really want my own annotation to create a compiler warning.
         public LogBuilder append(String s) {
             append((CharSequence) s);
             return this;
         }
 
         public LogBuilder append(Formattable f) {
-            buffer.put(b(TypeMarker.FORMATTABLE.ordinal()));
-            // @Fixme[MA]: Implement this!
-            throw new RuntimeException("Not yet implemented");
-//            return this;
+            buffer.put(asByte(TypeMarker.FORMATTABLE.ordinal()));
+            int lengthPos = buffer.position();
+            // @Optimisation[MA]: would we ever add an argument whose length is longer than max short?
+            // @Fixme[MA]: Handle short length overflow by setting the high bit to indicate to the log writer that
+            // a value was truncated. The log writer can then log an appropriate warning.
+            buffer.putShort(DUMMY);
+            int length = f.formatTo(buffer);
+            // Set the position, as can't depend on formatter implementations to do so
+            buffer.position(lengthPos + length);
+            // Update the formatter length field
+            buffer.putShort(lengthPos, asShort(length));
+            return this;
         }
 
         public LogBuilder append(byte b) {
-            buffer.put(b(TypeMarker.BYTE.ordinal()));
+            buffer.put(asByte(TypeMarker.BYTE.ordinal()));
             buffer.put(b);
             return this;
         }
 
         public LogBuilder append(short s) {
-            buffer.put(b(TypeMarker.SHORT.ordinal()));
+            buffer.put(asByte(TypeMarker.SHORT.ordinal()));
             buffer.putShort(s);
             return this;
         }
 
         public LogBuilder append(int i) {
-            buffer.put(b(TypeMarker.INT.ordinal()));
+            buffer.put(asByte(TypeMarker.INT.ordinal()));
             buffer.putInt(i);
             return this;
         }
 
         public LogBuilder append(long l) {
-            buffer.put(b(TypeMarker.LONG.ordinal()));
+            buffer.put(asByte(TypeMarker.LONG.ordinal()));
             buffer.putLong(l);
             return this;
         }
 
         public LogBuilder append(float f) {
-            buffer.put(b(TypeMarker.FLOAT.ordinal()));
+            buffer.put(asByte(TypeMarker.FLOAT.ordinal()));
             buffer.putFloat(f);
             return this;
         }
 
         public LogBuilder append(double d) {
-            buffer.put(b(TypeMarker.DOUBLE.ordinal()));
+            buffer.put(asByte(TypeMarker.DOUBLE.ordinal()));
             buffer.putDouble(d);
             return this;
         }
 
-        // @Fixme[MA]: should generate a compiler warning if a log statement is missing the .log() call
+        public LogBuilder append(Throwable t) {
+            append(throwableFormatterFlyweight.wrap(t));
+            throwableFormatterFlyweight.unwrap();
+            return this;
+        }
+
+        // @Fixme[MA]: should generate a compiler warning if a client log statement is missing the .log() call
         public void log() {
             Thread ct = Thread.currentThread();
             header.timestamp(System.currentTimeMillis()) // @Fixme[MA] Use high precision wall clock
                     .level(level)
-                    .threadId(ct.getId()).threadName(ct.getName());
-//            logRingBuffer.write(headerBuf, buffer);
-            HexDumpEncoder encoder = new HexDumpEncoder();
-            System.out.println("Header:");
-            headerBuf.position(0).limit(headerBuf.get(0));
-            System.out.println(encoder.encodeBuffer(headerBuf));
-            System.out.println("Args:");
-            System.out.println(encoder.encodeBuffer((ByteBuffer) buffer.flip()));
+                    .threadId(ct.getId())
+                    .threadName(ct.getName())
+                    .bodySize(buffer.position() - Header.HEADER_SIZE);
+            buffer.flip();
+            logRingBuffer.put(buffer);
         }
 
         public LogBuilder reset(Level level, CharSequence formatSpec) {
             this.level = level;
-            buffer.clear();
+            buffer.clear().position(Header.HEADER_SIZE);
             append(formatSpec);
             return this;
         }
-
-        // Helper classes ----------------------------------
-
-        // @Speedup[MA]: this implementation just packs everything tight in the header.
-        // It is possible that alignment optimisations may make this faster.
-        private class Header {
-            private final int tsOffset;
-            private final int levelOffset;
-            private final int tidOffset;
-            private final int tNameOffset;
-
-            Header(CharSequence loggerName) {
-                tsOffset = put(1, loggerName);
-                levelOffset = tsOffset + 8;
-                tidOffset = levelOffset + 1;
-                tNameOffset = tidOffset + 8;
-            }
-
-            Header timestamp(long ts) {
-                System.out.println("ts = " + ts);
-                headerBuf.putLong(tsOffset, ts);
-                return this;
-            }
-
-            Header level(Level level) {
-                headerBuf.put(levelOffset, b(level.ordinal()));
-                return this;
-            }
-
-            Header threadId(long tid) {
-                headerBuf.putLong(tidOffset, tid);
-                return this;
-            }
-
-            // Does not return this reference, as this is the terminator for building the header buffer
-            void threadName(CharSequence tName) {
-                // Write header size to first byte
-                headerBuf.put(0, b(put(tNameOffset, tName)));
-            }
-
-            private int put(int offset, CharSequence csq) {
-                // @Optimisation[MA]: CharSequence length as a byte, as we chop header char sequences short (we don't
-                // expect long thread or logger names).
-                int length = min(csq.length(), 50);
-                headerBuf.put(offset++, b(length));
-                for (int i = 0; i < length; i++) {
-                    headerBuf.put(offset++, (byte) csq.charAt(i));
-                }
-                return offset;
-            }
-        }
     }
 }
diff --git a/src/main/java/org/mamago/logging/ThrowableFormatterFlyweight.java b/src/main/java/org/mamago/logging/ThrowableFormatterFlyweight.java
new file mode 100644
index 0000000..6761c8c
--- /dev/null
+++ b/src/main/java/org/mamago/logging/ThrowableFormatterFlyweight.java
@@ -0,0 +1,23 @@
+package org.mamago.logging;
+
+import java.nio.ByteBuffer;
+
+class ThrowableFormatterFlyweight implements Formattable {
+    private Throwable throwable;
+
+    ThrowableFormatterFlyweight wrap(Throwable t) {
+        throwable = t;
+        return this;
+    }
+
+    void unwrap() {
+        throwable = null;
+    }
+
+    @Override
+    public int formatTo(ByteBuffer b) {
+        int startPos = b.position();
+        // @Fixme[MA]: Implement this!
+        return b.position() - startPos;
+    }
+}
diff --git a/src/main/java/org/mamago/util/Bits.java b/src/main/java/org/mamago/util/Bits.java
new file mode 100644
index 0000000..4672ea9
--- /dev/null
+++ b/src/main/java/org/mamago/util/Bits.java
@@ -0,0 +1,16 @@
+package org.mamago.util;
+
+public class Bits {
+    // Prevent instantiation
+    private Bits() {}
+
+    public static int roundUpToLongSize(int size) {
+        return roundUpToSize(size, 8);
+    }
+
+    public static int roundUpToSize(int size, int width) {
+        // Formula:
+        //   (size + w) / w * w; // Adding w rounds up to the next whole width
+        return ((size + width) / width) * width;
+    }
+}
diff --git a/src/main/java/org/mamago/util/Cast.java b/src/main/java/org/mamago/util/Cast.java
new file mode 100644
index 0000000..4ed3d47
--- /dev/null
+++ b/src/main/java/org/mamago/util/Cast.java
@@ -0,0 +1,16 @@
+package org.mamago.util;
+
+public class Cast {
+    // Prevent instantiation
+    private Cast() {}
+
+    public static byte asByte(int i) {
+        // @Fixme[MA]: check whether casting give out-of-bounds check, otherwise add in for safety
+        return (byte) i;
+    }
+
+    public static short asShort(int i) {
+        // @Fixme[MA]: check whether casting give out-of-bounds check, otherwise add in for safety
+        return (byte) i;
+    }
+}
diff --git a/src/test/java/org/mamago/logging/LoggerTest.java b/src/test/java/org/mamago/logging/LoggerTest.java
index 3a1efbd..ef1e765 100644
--- a/src/test/java/org/mamago/logging/LoggerTest.java
+++ b/src/test/java/org/mamago/logging/LoggerTest.java
@@ -1,17 +1,102 @@
 package org.mamago.logging;
 
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.mamago.util.HexDumpEncoder;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class LoggerTest {
-    final static Logger LOG = Logger.createLogger();
+    private final static Logger LOG = Logger.createLogger();
+
+    @BeforeEach
+    public void clearBuffer() {
+        LOG.logRingBuffer.clear();
+    }
 
     @Test
-    public void testLogger() {
-        System.out.println("header length = " +
-                ( 1 // Header length
-                + 1 // Thread name length
-                + LoggerTest.class.getName().length()
-                + 8 + 8 + 1 + "main".length() + 1));
-        LOG.log(Logger.Level.INFO, "Log this {}").append(3).log();
+    public void logSimpleMessage() {
+        try {
+            LOG.log(Logger.Level.INFO, "Log this {}").append(3).log();
+            long now = System.currentTimeMillis();
+            ByteBuffer buffer = LOG.logRingBuffer;
+
+            // Assert Header fields
+            assertEquals(304, buffer.getLong(Header.SIZE_OFFSET), "Header size");
+            assertTrue(within(TimeUnit.MILLISECONDS.toMillis(20), now, buffer.getLong(Header.TS_OFFSET)),
+                    "Timestamp - now: " + now + " got: " + buffer.getLong(Header.TS_OFFSET));
+            assertEquals(Logger.Level.INFO.ordinal(), buffer.getLong(Header.LEVEL_OFFSET), "Level");
+            assertEquals(Thread.currentThread().getId(), buffer.getLong(Header.TID_OFFSET), "TID");
+            assertHeaderCharSequence(Thread.currentThread().getName(), buffer, Header.TNAME_OFFSET, "Thread name");
+            assertHeaderCharSequence(getClass().getName(), buffer, Header.LOGGER_NAME_OFFSET, "Logger name");
+            assertEquals(21, buffer.getLong(Header.BODY_SIZE_OFFSET), "Body size");
+
+            // Assert Message fields
+            assertMessageField("Log this {}", buffer, Header.HEADER_SIZE, "Format string");
+            assertMessageField(3, buffer, "Arg 1 = 3");
+        } catch (Throwable t) {
+            dumpHex("Buffer", LOG.logRingBuffer);
+            throw t;
+        }
+    }
+
+    @Test
+    public void logException() {
+        LOG.log(Logger.Level.ERROR, "Argh! {}").append(new RuntimeException("Darn it!")).log();
+        ByteBuffer buffer = LOG.logRingBuffer;
+
+        // Assert Header fields
+        assertEquals(Logger.Level.ERROR.ordinal(), buffer.getLong(Header.LEVEL_OFFSET), "Level");
+        assertEquals(Thread.currentThread().getId(), buffer.getLong(Header.TID_OFFSET), "TID");
+        assertHeaderCharSequence(Thread.currentThread().getName(), buffer, Header.TNAME_OFFSET, "Thread name");
+        assertHeaderCharSequence(getClass().getName(), buffer, Header.LOGGER_NAME_OFFSET, "Logger name");
+        assertEquals(14, buffer.getLong(Header.BODY_SIZE_OFFSET), "Body size");
+
+        // Assert Message fields
+        assertMessageField("Argh! {}", buffer, Header.HEADER_SIZE, "Format string");
+        // @Fixme[MA]: test exceptions properly
+        // Consider making stack traces an explicit type, rather than a generic Formattable
+        assertEquals(Logger.TypeMarker.FORMATTABLE.ordinal(), buffer.get(), "Formattable type");
+    }
+
+    void assertHeaderCharSequence(String expected, ByteBuffer buffer, int pos, String message) {
+        short length = buffer.getShort(pos);
+        assertEquals(expected.length(), length, message + " length");
+        byte[] name = new byte[length];
+        buffer.position(pos + 2);
+        buffer.get(name, 0, length);
+        assertEquals(expected, new String(name), message);
+    }
+
+    void assertMessageField(String expected, ByteBuffer buffer, int pos, String message) {
+        buffer.position(pos);
+        assertEquals(Logger.TypeMarker.CHAR_SEQUENCE.ordinal(), buffer.get(), message + " type");
+        int length = buffer.getInt();
+        assertEquals(expected.length(), length, message + " length");
+        byte[] name = new byte[length];
+        buffer.get(name, 0, length);
+        assertEquals(expected, new String(name), message);
+    }
+
+    void assertMessageField(int expected, ByteBuffer buffer, String message) {
+        assertEquals(Logger.TypeMarker.INT.ordinal(), buffer.get(), message + " type");
+        assertEquals(expected, buffer.getInt(), message);
+    }
+
+    boolean within(long epsilon, long a, long b) {
+        return Math.abs(a - b) < epsilon;
+    }
+
+    void dumpHex(CharSequence title, ByteBuffer buffer) {
+        HexDumpEncoder encoder = new HexDumpEncoder();
+        System.out.println(title + " [first 512 bytes]:");
+        buffer.position(0).limit(512);
+        System.out.println(encoder.encodeBuffer(buffer));
+        // Reset position & limit
+        buffer.flip();
     }
 }
diff --git a/src/test/java/org/mamago/util/CharacterEncoder.java b/src/test/java/org/mamago/util/CharacterEncoder.java
new file mode 100644
index 0000000..6d83cc6
--- /dev/null
+++ b/src/test/java/org/mamago/util/CharacterEncoder.java
@@ -0,0 +1,358 @@
+/*
+ * Copyright (c) 1995, 2005, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package org.mamago.util;
+
+import java.io.InputStream;
+import java.io.ByteArrayInputStream;
+import java.io.OutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+// ---------------------------------------------------------
+// Copied from sun.misc.HexDumpEncoder, as I got sick of
+// "internal API, may be removed" warnings.
+// ---------------------------------------------------------
+
+/**
+ * This class defines the encoding half of character encoders.
+ * A character encoder is an algorithim for transforming 8 bit binary
+ * data into text (generally 7 bit ASCII or 8 bit ISO-Latin-1 text)
+ * for transmition over text channels such as e-mail and network news.
+ *
+ * The character encoders have been structured around a central theme
+ * that, in general, the encoded text has the form:
+ *
+ * <pre>
+ *      [Buffer Prefix]
+ *      [Line Prefix][encoded data atoms][Line Suffix]
+ *      [Buffer Suffix]
+ * </pre>
+ *
+ * In the CharacterEncoder and CharacterDecoder classes, one complete
+ * chunk of data is referred to as a <i>buffer</i>. Encoded buffers
+ * are all text, and decoded buffers (sometimes just referred to as
+ * buffers) are binary octets.
+ *
+ * To create a custom encoder, you must, at a minimum,  overide three
+ * abstract methods in this class.
+ * <DL>
+ * <DD>bytesPerAtom which tells the encoder how many bytes to
+ * send to encodeAtom
+ * <DD>encodeAtom which encodes the bytes sent to it as text.
+ * <DD>bytesPerLine which tells the encoder the maximum number of
+ * bytes per line.
+ * </DL>
+ *
+ * Several useful encoders have already been written and are
+ * referenced in the See Also list below.
+ *
+ * @author      Chuck McManis
+ * @see         CharacterDecoder;
+ * @see         UCEncoder
+ * @see         UUEncoder
+ * @see         BASE64Encoder
+ */
+public abstract class CharacterEncoder {
+
+    /** Stream that understands "printing" */
+    protected PrintStream pStream;
+
+    /** Return the number of bytes per atom of encoding */
+    abstract protected int bytesPerAtom();
+
+    /** Return the number of bytes that can be encoded per line */
+    abstract protected int bytesPerLine();
+
+    /**
+     * Encode the prefix for the entire buffer. By default is simply
+     * opens the PrintStream for use by the other functions.
+     */
+    protected void encodeBufferPrefix(OutputStream aStream) throws IOException {
+        pStream = new PrintStream(aStream);
+    }
+
+    /**
+     * Encode the suffix for the entire buffer.
+     */
+    protected void encodeBufferSuffix(OutputStream aStream) throws IOException {
+    }
+
+    /**
+     * Encode the prefix that starts every output line.
+     */
+    protected void encodeLinePrefix(OutputStream aStream, int aLength)
+            throws IOException {
+    }
+
+    /**
+     * Encode the suffix that ends every output line. By default
+     * this method just prints a <newline> into the output stream.
+     */
+    protected void encodeLineSuffix(OutputStream aStream) throws IOException {
+        pStream.println();
+    }
+
+    /** Encode one "atom" of information into characters. */
+    abstract protected void encodeAtom(OutputStream aStream, byte someBytes[],
+                                       int anOffset, int aLength) throws IOException;
+
+    /**
+     * This method works around the bizarre semantics of BufferedInputStream's
+     * read method.
+     */
+    protected int readFully(InputStream in, byte buffer[])
+            throws java.io.IOException {
+        for (int i = 0; i < buffer.length; i++) {
+            int q = in.read();
+            if (q == -1)
+                return i;
+            buffer[i] = (byte)q;
+        }
+        return buffer.length;
+    }
+
+    /**
+     * Encode bytes from the input stream, and write them as text characters
+     * to the output stream. This method will run until it exhausts the
+     * input stream, but does not print the line suffix for a final
+     * line that is shorter than bytesPerLine().
+     */
+    public void encode(InputStream inStream, OutputStream outStream)
+            throws IOException {
+        int     j;
+        int     numBytes;
+        byte    tmpbuffer[] = new byte[bytesPerLine()];
+
+        encodeBufferPrefix(outStream);
+
+        while (true) {
+            numBytes = readFully(inStream, tmpbuffer);
+            if (numBytes == 0) {
+                break;
+            }
+            encodeLinePrefix(outStream, numBytes);
+            for (j = 0; j < numBytes; j += bytesPerAtom()) {
+
+                if ((j + bytesPerAtom()) <= numBytes) {
+                    encodeAtom(outStream, tmpbuffer, j, bytesPerAtom());
+                } else {
+                    encodeAtom(outStream, tmpbuffer, j, (numBytes)- j);
+                }
+            }
+            if (numBytes < bytesPerLine()) {
+                break;
+            } else {
+                encodeLineSuffix(outStream);
+            }
+        }
+        encodeBufferSuffix(outStream);
+    }
+
+    /**
+     * Encode the buffer in <i>aBuffer</i> and write the encoded
+     * result to the OutputStream <i>aStream</i>.
+     */
+    public void encode(byte aBuffer[], OutputStream aStream)
+            throws IOException {
+        ByteArrayInputStream inStream = new ByteArrayInputStream(aBuffer);
+        encode(inStream, aStream);
+    }
+
+    /**
+     * A 'streamless' version of encode that simply takes a buffer of
+     * bytes and returns a string containing the encoded buffer.
+     */
+    public String encode(byte aBuffer[]) {
+        ByteArrayOutputStream   outStream = new ByteArrayOutputStream();
+        ByteArrayInputStream    inStream = new ByteArrayInputStream(aBuffer);
+        String retVal = null;
+        try {
+            encode(inStream, outStream);
+            // explicit ascii->unicode conversion
+            retVal = outStream.toString("8859_1");
+        } catch (Exception IOException) {
+            // This should never happen.
+            throw new Error("CharacterEncoder.encode internal error");
+        }
+        return (retVal);
+    }
+
+    /**
+     * Return a byte array from the remaining bytes in this ByteBuffer.
+     * <P>
+     * The ByteBuffer's position will be advanced to ByteBuffer's limit.
+     * <P>
+     * To avoid an extra copy, the implementation will attempt to return the
+     * byte array backing the ByteBuffer.  If this is not possible, a
+     * new byte array will be created.
+     */
+    private byte [] getBytes(ByteBuffer bb) {
+        /*
+         * This should never return a BufferOverflowException, as we're
+         * careful to allocate just the right amount.
+         */
+        byte [] buf = null;
+
+        /*
+         * If it has a usable backing byte buffer, use it.  Use only
+         * if the array exactly represents the current ByteBuffer.
+         */
+        if (bb.hasArray()) {
+            byte [] tmp = bb.array();
+            if ((tmp.length == bb.capacity()) &&
+                    (tmp.length == bb.remaining())) {
+                buf = tmp;
+                bb.position(bb.limit());
+            }
+        }
+
+        if (buf == null) {
+            /*
+             * This class doesn't have a concept of encode(buf, len, off),
+             * so if we have a partial buffer, we must reallocate
+             * space.
+             */
+            buf = new byte[bb.remaining()];
+
+            /*
+             * position() automatically updated
+             */
+            bb.get(buf);
+        }
+
+        return buf;
+    }
+
+    /**
+     * Encode the <i>aBuffer</i> ByteBuffer and write the encoded
+     * result to the OutputStream <i>aStream</i>.
+     * <P>
+     * The ByteBuffer's position will be advanced to ByteBuffer's limit.
+     */
+    public void encode(ByteBuffer aBuffer, OutputStream aStream)
+            throws IOException {
+        byte [] buf = getBytes(aBuffer);
+        encode(buf, aStream);
+    }
+
+    /**
+     * A 'streamless' version of encode that simply takes a ByteBuffer
+     * and returns a string containing the encoded buffer.
+     * <P>
+     * The ByteBuffer's position will be advanced to ByteBuffer's limit.
+     */
+    public String encode(ByteBuffer aBuffer) {
+        byte [] buf = getBytes(aBuffer);
+        return encode(buf);
+    }
+
+    /**
+     * Encode bytes from the input stream, and write them as text characters
+     * to the output stream. This method will run until it exhausts the
+     * input stream. It differs from encode in that it will add the
+     * line at the end of a final line that is shorter than bytesPerLine().
+     */
+    public void encodeBuffer(InputStream inStream, OutputStream outStream)
+            throws IOException {
+        int     j;
+        int     numBytes;
+        byte    tmpbuffer[] = new byte[bytesPerLine()];
+
+        encodeBufferPrefix(outStream);
+
+        while (true) {
+            numBytes = readFully(inStream, tmpbuffer);
+            if (numBytes == 0) {
+                break;
+            }
+            encodeLinePrefix(outStream, numBytes);
+            for (j = 0; j < numBytes; j += bytesPerAtom()) {
+                if ((j + bytesPerAtom()) <= numBytes) {
+                    encodeAtom(outStream, tmpbuffer, j, bytesPerAtom());
+                } else {
+                    encodeAtom(outStream, tmpbuffer, j, (numBytes)- j);
+                }
+            }
+            encodeLineSuffix(outStream);
+            if (numBytes < bytesPerLine()) {
+                break;
+            }
+        }
+        encodeBufferSuffix(outStream);
+    }
+
+    /**
+     * Encode the buffer in <i>aBuffer</i> and write the encoded
+     * result to the OutputStream <i>aStream</i>.
+     */
+    public void encodeBuffer(byte aBuffer[], OutputStream aStream)
+            throws IOException {
+        ByteArrayInputStream inStream = new ByteArrayInputStream(aBuffer);
+        encodeBuffer(inStream, aStream);
+    }
+
+    /**
+     * A 'streamless' version of encode that simply takes a buffer of
+     * bytes and returns a string containing the encoded buffer.
+     */
+    public String encodeBuffer(byte aBuffer[]) {
+        ByteArrayOutputStream   outStream = new ByteArrayOutputStream();
+        ByteArrayInputStream    inStream = new ByteArrayInputStream(aBuffer);
+        try {
+            encodeBuffer(inStream, outStream);
+        } catch (Exception IOException) {
+            // This should never happen.
+            throw new Error("CharacterEncoder.encodeBuffer internal error");
+        }
+        return (outStream.toString());
+    }
+
+    /**
+     * Encode the <i>aBuffer</i> ByteBuffer and write the encoded
+     * result to the OutputStream <i>aStream</i>.
+     * <P>
+     * The ByteBuffer's position will be advanced to ByteBuffer's limit.
+     */
+    public void encodeBuffer(ByteBuffer aBuffer, OutputStream aStream)
+            throws IOException {
+        byte [] buf = getBytes(aBuffer);
+        encodeBuffer(buf, aStream);
+    }
+
+    /**
+     * A 'streamless' version of encode that simply takes a ByteBuffer
+     * and returns a string containing the encoded buffer.
+     * <P>
+     * The ByteBuffer's position will be advanced to ByteBuffer's limit.
+     */
+    public String encodeBuffer(ByteBuffer aBuffer) {
+        byte [] buf = getBytes(aBuffer);
+        return encodeBuffer(buf);
+    }
+
+}
diff --git a/src/test/java/org/mamago/util/HexDumpEncoder.java b/src/test/java/org/mamago/util/HexDumpEncoder.java
new file mode 100644
index 0000000..7e105d6
--- /dev/null
+++ b/src/test/java/org/mamago/util/HexDumpEncoder.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 1995, 1997, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.mamago.util;
+
+import java.io.PrintStream;
+import java.io.OutputStream;
+import java.io.IOException;
+
+// ---------------------------------------------------------
+// Copied from sun.misc.HexDumpEncoder, as I got sick of
+// "internal API, may be removed" warnings.
+// ---------------------------------------------------------
+
+/**
+ * This class encodes a buffer into the classic: "Hexadecimal Dump" format of
+ * the past. It is useful for analyzing the contents of binary buffers.
+ * The format produced is as follows:
+ * <pre>
+ * xxxx: 00 11 22 33 44 55 66 77   88 99 aa bb cc dd ee ff ................
+ * </pre>
+ * Where xxxx is the offset into the buffer in 16 byte chunks, followed
+ * by ascii coded hexadecimal bytes followed by the ASCII representation of
+ * the bytes or '.' if they are not valid bytes.
+ *
+ * @author      Chuck McManis
+ */
+
+public class HexDumpEncoder extends CharacterEncoder {
+
+    private int offset;
+    private int thisLineLength;
+    private int currentByte;
+    private byte thisLine[] = new byte[16];
+
+    static void hexDigit(PrintStream p, byte x) {
+        char c;
+
+        c = (char) ((x >> 4) & 0xf);
+        if (c > 9)
+            c = (char) ((c-10) + 'A');
+        else
+            c = (char)(c + '0');
+        p.write(c);
+        c = (char) (x & 0xf);
+        if (c > 9)
+            c = (char)((c-10) + 'A');
+        else
+            c = (char)(c + '0');
+        p.write(c);
+    }
+
+    protected int bytesPerAtom() {
+        return (1);
+    }
+
+    protected int bytesPerLine() {
+        return (16);
+    }
+
+    protected void encodeBufferPrefix(OutputStream o) throws IOException {
+        offset = 0;
+        super.encodeBufferPrefix(o);
+    }
+
+    protected void encodeLinePrefix(OutputStream o, int len) throws IOException {
+        hexDigit(pStream, (byte)((offset >>> 8) & 0xff));
+        hexDigit(pStream, (byte)(offset & 0xff));
+        pStream.print(": ");
+        currentByte = 0;
+        thisLineLength = len;
+    }
+
+    protected void encodeAtom(OutputStream o, byte buf[], int off, int len) throws IOException {
+        thisLine[currentByte] = buf[off];
+        hexDigit(pStream, buf[off]);
+        pStream.print(" ");
+        currentByte++;
+        if (currentByte == 8)
+            pStream.print("  ");
+    }
+
+    protected void encodeLineSuffix(OutputStream o) throws IOException {
+        if (thisLineLength < 16) {
+            for (int i = thisLineLength; i < 16; i++) {
+                pStream.print("   ");
+                if (i == 7)
+                    pStream.print("  ");
+            }
+        }
+        pStream.print(" ");
+        for (int i = 0; i < thisLineLength; i++) {
+            if ((thisLine[i] < ' ') || (thisLine[i] > 'z')) {
+                pStream.print(".");
+            } else {
+                pStream.write(thisLine[i]);
+            }
+        }
+        pStream.println();
+        offset += thisLineLength;
+    }
+
+}
diff --git a/todo.md b/todo.md
new file mode 100644
index 0000000..2b632da
--- /dev/null
+++ b/todo.md
@@ -0,0 +1,4 @@
+TODO
+====
+
+* Externalise size limits for configurability
\ No newline at end of file