Let a base64 decoder be told where its input ends

doc/dataflow.html

@@ -126,10 +126,29 @@ <h4>Dereferencing</h4>
 <h4>Comparison</h4>
 The default implementation of iterator equality of <code style="white-space: normal">iterator_adaptor</code> just
 invokes the equality operator on the base iterators.  Generally this is satisfactory.
-However, this implies that other operations (E. G. dereference) do not prematurely 
+However, this implies that other operations (E. G. dereference) do not prematurely
 increment the base iterator.  Avoiding this can be surprisingly tricky in some cases.
 (E.G. transform_width)
 
+<p>
+An iterator whose output elements do not line up with its input ones cannot
+always avoid it.  <code style="white-space: normal">transform_width</code>
+learns that the input is over only by reading past the end of it, which is
+undefined behaviour, and it does so whenever the length of the input is not a
+common multiple of the two widths.  Decoding base64, any length of the form
+4n + 1 ends in six bits which do not make up a byte, and those six bits send
+the iterator looking for the two it is missing.
+
+<p>
+Passing the end of the input as a second constructor argument avoids this.
+The iterator then stops there, discarding whatever is left over rather than
+reading on for it.  Both
+<code style="white-space: normal">transform_width</code> and
+<code style="white-space: normal">remove_whitespace</code> accept it.  The one
+argument form behaves as it always has, and remains the one to use for a
+composed sequence whose length divides evenly, which is how the archives
+themselves use these iterators.
+
 <p>
 Iterators which fulfill the above requirements should be composable and the above sample
 code should implement our binary to base64 conversion.

include/boost/archive/iterators/transform_width.hpp

@@ -72,8 +72,19 @@ class transform_width :
     }
 
     bool equal_impl(const this_t & rhs){
-        if(BitsIn < BitsOut) // discard any left over bits
+        if(BitsIn < BitsOut){ // discard any left over bits
+            if(m_bounded){
+                // Fill here rather than on dereference.  The bits left when
+                // the input runs out part way through an output value do not
+                // make a whole one, so the sequence has to end before that
+                // value is handed out.
+                if(! m_buffer_out_full && ! m_exhausted){
+                    fill();
+                }
+                return m_exhausted;
+            }
             return this->base_reference() == rhs.base_reference();
+        }
         else{
             // BitsIn > BitsOut  // zero fill
             if(this->base_reference() == rhs.base_reference()){
@@ -105,6 +116,12 @@ class transform_width :
     // flag to indicate we've reached end of data.
     bool m_end_of_sequence;
 
+    // end of the input, when one has been given
+    Base m_end;
+    bool m_bounded;
+    // set once the input can yield no further whole output value
+    bool m_exhausted;
+
 public:
     // make composable by using templated constructor
     template<class T>
@@ -117,7 +134,28 @@ class transform_width :
 	    //used because m_remaining_bits == 0)
         m_buffer_in(0),
         m_remaining_bits(0),
-        m_end_of_sequence(false)
+        m_end_of_sequence(false),
+        // m_end means nothing unless bounded.  It is built from start only
+        // because Base need not be default constructible: it is usually
+        // another adaptor of this family, and those have just the templated
+        // constructor below.
+        m_end(Base(static_cast< T >(start))),
+        m_bounded(false),
+        m_exhausted(false)
+    {}
+    // Knowing where the input ends is what lets the iterator stop before
+    // reading past it, which the form above cannot do.
+    template<class T>
+    transform_width(T start, T end) :
+        super_t(Base(static_cast< T >(start))),
+        m_buffer_out_full(false),
+        m_buffer_out(0),
+        m_buffer_in(0),
+        m_remaining_bits(0),
+        m_end_of_sequence(false),
+        m_end(Base(static_cast< T >(end))),
+        m_bounded(true),
+        m_exhausted(false)
     {}
 };
 
@@ -136,6 +174,12 @@ void transform_width<Base, BitsOut, BitsIn, CharType>::fill() {
                 m_buffer_in = 0;
                 m_remaining_bits = missing_bits;
             }
+            else if(m_bounded && this->base_reference() == m_end){
+                // The input ended part way through an output value.  What is
+                // left cannot make a whole one, so drop it.
+                m_exhausted = true;
+                return;
+            }
             else{
                 m_buffer_in = * this->base_reference()++;
                 m_remaining_bits = BitsIn;

test/test_iterators_base64.cpp

@@ -7,7 +7,9 @@
 // http://www.boost.org/LICENSE_1_0.txt)
 
 #include <algorithm>
+#include <iterator>
 #include <list>
+#include <string>
 
 #if (defined _MSC_VER) && (_MSC_VER == 1200)
 #  pragma warning (disable : 4786) // too long name, harmless warning
@@ -86,9 +88,38 @@ void test_base64(unsigned int size){
 
 }
 
+// A base64 sequence whose length is not a whole number of four character
+// groups carries some bits which do not make up a byte.  Decoding must drop
+// them and stop, rather than look for the byte's remaining bits beyond the
+// end of the input.  Reported by ROCKFAL1 in
+// https://github.com/boostorg/serialization/issues/324.  Thanks!
+void test_base64_partial_group(){
+    typedef boost::archive::iterators::transform_width<
+        boost::archive::iterators::binary_from_base64<const char *>, 8, 6
+    > decoder;
+
+    // "1234567890" encoded, less the padding
+    const std::string encoded("MTIzNDU2Nzg5MA");
+    const std::string decoded("1234567890");
+
+    for(std::size_t n = 0; n <= encoded.size(); ++n){
+        const char * const first = encoded.data();
+        const char * const last = first + n;
+        std::string result;
+        std::copy(
+            decoder(first, last),
+            decoder(last),
+            std::back_inserter(result)
+        );
+        // six bits in, eight bits out, and no partial byte at the end
+        BOOST_CHECK(result == decoded.substr(0, n * 6 / 8));
+    }
+}
+
 int
 test_main( int /*argc*/, char* /*argv*/[] )
 {
+    test_base64_partial_group();
     test_base64<char>(1);
     test_base64<char>(2);
     test_base64<char>(3);