diff --git a/include/libbase64.h b/include/libbase64.h index c5908973..14459064 100644 --- a/include/libbase64.h +++ b/include/libbase64.h @@ -55,6 +55,11 @@ extern "C" { #define BASE64_FORCE_AVX (1 << 7) #define BASE64_FORCE_AVX512 (1 << 8) +#define BASE64_CPU_MASK (0x1FF) + +#define BASE64_NO_PADDING (1 << 13) /* encoding: do not output padding bytes, decoding: enforce no padding bytes */ +#define BASE64_CANONICAL (1 << 14) /* decoding: enforce there are no padding bits (i.e. carry == 0) */ + struct base64_state { int eof; int bytes; @@ -139,6 +144,11 @@ int BASE64_EXPORT base64_stream_decode , size_t *outlen ) ; +/* Finalizes checks begun by previous successful calls to `base64_stream_decode()`. */ +int BASE64_EXPORT base64_stream_decode_final + ( struct base64_state *state + ) ; + #ifdef __cplusplus } #endif diff --git a/lib/arch/generic/dec_tail.c b/lib/arch/generic/dec_tail.c index e64f7247..e2b63251 100644 --- a/lib/arch/generic/dec_tail.c +++ b/lib/arch/generic/dec_tail.c @@ -35,10 +35,11 @@ break; } if ((q = base64_table_dec_8bit[*s++]) >= 254) { + int const pad_check = (state->flags & BASE64_NO_PADDING) ? 0 : 254; st.bytes++; // When q == 254, the input char is '='. // Check if next byte is also '=': - if (q == 254) { + if (q == pad_check) { if (slen-- != 0) { st.bytes = 0; // EOF: @@ -70,11 +71,12 @@ break; } if ((q = base64_table_dec_8bit[*s++]) >= 254) { + int const pad_check = (state->flags & BASE64_NO_PADDING) ? 0 : 254; st.bytes = 0; st.eof = BASE64_EOF; // When q == 254, the input char is '='. Return 1 and EOF. // When q == 255, the input char is invalid. Return 0 and EOF. - ret = ((q == 254) && (slen == 0)) ? 1 : 0; + ret = ((q == pad_check) && (slen == 0)) ? 1 : 0; break; } *o++ = st.carry | q; diff --git a/lib/codec_choose.c b/lib/codec_choose.c index 8b5938dd..ae0adf74 100644 --- a/lib/codec_choose.c +++ b/lib/codec_choose.c @@ -108,7 +108,7 @@ codec_choose_forced (struct codec *codec, int flags) // always allow it, even if the codec is a no-op. // For testing purposes. - if (!(flags & 0xFFFF)) { + if (!(flags & BASE64_CPU_MASK)) { return false; } diff --git a/lib/env.h b/lib/env.h index 08370650..ad8aeb53 100644 --- a/lib/env.h +++ b/lib/env.h @@ -62,6 +62,10 @@ // End-of-file when stream end has been reached or invalid input provided: #define BASE64_EOF 2 +// Due to the overhead of initializing OpenMP and creating a team of +// threads, we require the data length to be larger than a threshold: +#define BASE64_OMP_THRESHOLD 20000 + // GCC 7 defaults to issuing a warning for fallthrough in switch statements, // unless the fallthrough cases are marked with an attribute. As we use // fallthrough deliberately, define an alias for the attribute: diff --git a/lib/exports.txt b/lib/exports.txt index 67d45588..e3795d10 100644 --- a/lib/exports.txt +++ b/lib/exports.txt @@ -5,3 +5,4 @@ base64_stream_encode_final base64_decode base64_stream_decode base64_stream_decode_init +base64_stream_decode_final diff --git a/lib/lib.c b/lib/lib.c index 053931a9..968c39f0 100644 --- a/lib/lib.c +++ b/lib/lib.c @@ -18,8 +18,8 @@ void base64_stream_encode_init (struct base64_state *state, int flags) { // If any of the codec flags are set, redo choice: - if (codec.enc == NULL || flags & 0xFF) { - codec_choose(&codec, flags); + if (codec.enc == NULL || flags & BASE64_CPU_MASK) { + codec_choose(&codec, flags & BASE64_CPU_MASK); } state->eof = 0; state->bytes = 0; @@ -50,6 +50,10 @@ base64_stream_encode_final if (state->bytes == 1) { *o++ = base64_table_enc_6bit[state->carry]; + if (state->flags & BASE64_NO_PADDING) { + *outlen = 1; + return; + } *o++ = '='; *o++ = '='; *outlen = 3; @@ -57,6 +61,10 @@ base64_stream_encode_final } if (state->bytes == 2) { *o++ = base64_table_enc_6bit[state->carry]; + if (state->flags & BASE64_NO_PADDING) { + *outlen = 1; + return; + } *o++ = '='; *outlen = 2; return; @@ -68,8 +76,8 @@ void base64_stream_decode_init (struct base64_state *state, int flags) { // If any of the codec flags are set, redo choice: - if (codec.dec == NULL || flags & 0xFFFF) { - codec_choose(&codec, flags); + if (codec.dec == NULL || flags & BASE64_CPU_MASK) { + codec_choose(&codec, flags & BASE64_CPU_MASK); } state->eof = 0; state->bytes = 0; @@ -89,12 +97,30 @@ base64_stream_decode return codec.dec(state, src, srclen, out, outlen); } -#ifdef _OPENMP - - // Due to the overhead of initializing OpenMP and creating a team of - // threads, we require the data length to be larger than a threshold: - #define OMP_THRESHOLD 20000 +int +base64_stream_decode_final + ( struct base64_state *state + ) +{ + if ((state->flags & BASE64_CANONICAL) && state->carry) { + return 0; + } + if (state->bytes == 0) { + return 1; + } + if (state->flags & BASE64_NO_PADDING) { + switch (state->bytes) { + case 2: + case 3: + return 1; + default: + break; + } + } + return 0; +} +#ifdef _OPENMP // Conditionally include OpenMP-accelerated codec implementations: #include "lib_openmp.c" #endif @@ -113,7 +139,7 @@ base64_encode struct base64_state state; #ifdef _OPENMP - if (srclen >= OMP_THRESHOLD) { + if (srclen >= BASE64_OMP_THRESHOLD) { base64_encode_openmp(src, srclen, out, outlen, flags); return; } @@ -145,7 +171,7 @@ base64_decode struct base64_state state; #ifdef _OPENMP - if (srclen >= OMP_THRESHOLD) { + if (srclen >= BASE64_OMP_THRESHOLD) { return base64_decode_openmp(src, srclen, out, outlen, flags); } #endif @@ -157,8 +183,8 @@ base64_decode ret = base64_stream_decode(&state, src, srclen, out, outlen); // If when decoding a whole block, we're still waiting for input then fail: - if (ret && (state.bytes == 0)) { - return ret; + if (ret > 0) { + ret = base64_stream_decode_final(&state); } - return 0; + return ret; } diff --git a/lib/lib_openmp.c b/lib/lib_openmp.c index 6b87c524..fcf4abfe 100644 --- a/lib/lib_openmp.c +++ b/lib/lib_openmp.c @@ -97,8 +97,9 @@ base64_decode_openmp // Split the input string into num_threads parts, each // part a multiple of 4 bytes. The remaining bytes will - // be done later: - len = srclen / (num_threads * 4); + // be done later, always including the last 4 bytes to + // process padding correctly: + len = (srclen - 4) / (num_threads * 4); len *= 4; last_len = srclen - num_threads * len; @@ -106,6 +107,7 @@ base64_decode_openmp base64_stream_decode_init(&state, flags); initial_state = state; + state.flags |= BASE64_NO_PADDING; } // Single has an implicit barrier to wait here for the above to @@ -141,9 +143,9 @@ base64_decode_openmp sum += s; *outlen = sum; - // If when decoding a whole block, we're still waiting for input then fail: - if (result && (state.bytes == 0)) { - return result; + // Final check: + if (result) { + result = base64_stream_decode_final(&state); } - return 0; + return result; } diff --git a/test/test_base64.c b/test/test_base64.c index ae5e6635..a84ee532 100644 --- a/test/test_base64.c +++ b/test/test_base64.c @@ -3,8 +3,12 @@ #include #include #include "../include/libbase64.h" +#include "../lib/env.h" #include "codec_supported.h" #include "moby_dick.h" +#ifdef _OPENMP +#include +#endif static char out[2000]; static size_t outlen; @@ -58,6 +62,102 @@ assert_dec (int flags, const char *src, const char *dst) return false; } +static bool +assert_dec_full (int expected, int flags, const char *src, const char *dst) +{ + size_t srclen = strlen(src); + size_t dstlen = strlen(dst); + + int ret = base64_decode(src, srclen, out, &outlen, flags); + + if (expected) + { + if (ret <= 0) { + printf("FAIL: decoding of '%s': decoding error\n", src); + return true; + } + if (outlen != dstlen) { + printf("FAIL: decoding of '%s': " + "length expected %lu, got %lu\n", src, + (unsigned long)dstlen, + (unsigned long)outlen + ); + return true; + } + if (strncmp(dst, out, outlen) != 0) { + out[outlen] = '\0'; + printf("FAIL: decoding of '%s': expected output '%s', got '%s'\n", src, dst, out); + return true; + } + for (size_t bs = 1; bs <= srclen; ++bs) { + struct base64_state state; + char const* tmpsrc = src; + size_t tmpsrclen = srclen; + char *tmpout = out; + size_t tmpoutlen; + outlen = 0; + + base64_stream_decode_init(&state, flags); + for (size_t b = 0; b < ((srclen + (bs - 1)) / bs); ++b, tmpsrc += bs, tmpsrclen -= bs) { + size_t tmpbs = (tmpsrclen > bs) ? bs : tmpsrclen; + ret = base64_stream_decode(&state, tmpsrc, tmpbs, tmpout, &tmpoutlen); + if (ret <= 0) { + printf("FAIL: decoding of '%s': decoding by %lu error\n", src, (unsigned long)bs); + return true; + } + tmpout += tmpoutlen; + outlen += tmpoutlen; + } + ret = base64_stream_decode_final(&state); + if (ret <= 0) { + printf("FAIL: decoding of '%s': decoding by %lu error\n", src, (unsigned long)bs); + return true; + } + if (outlen != dstlen) { + printf("FAIL: decoding of '%s': " + "length expected %lu, got %lu\n", src, + (unsigned long)dstlen, + (unsigned long)outlen + ); + return true; + } + if (strncmp(dst, out, outlen) != 0) { + out[outlen] = '\0'; + printf("FAIL: decoding of '%s': expected output '%s', got '%s'\n", src, dst, out); + return true; + } + } + } + else { + if (ret > 0) { + printf("FAIL: decoding of '%s': decoding succeeded\n", src); + return true; + } + for (size_t bs = 1; bs <= srclen; ++bs) { + struct base64_state state; + char const* tmpsrc = src; + size_t tmpsrclen = srclen; + + base64_stream_decode_init(&state, flags); + for (size_t b = 0; b < ((srclen + (bs - 1)) / bs); ++b, tmpsrc += bs, tmpsrclen -= bs) { + size_t tmpbs = (tmpsrclen > bs) ? bs : tmpsrclen; + ret = base64_stream_decode(&state, tmpsrc, tmpbs, out, &outlen); + if (ret <= 0) { + break; + } + } + if (ret > 0) { + ret = base64_stream_decode_final(&state); + } + if (ret > 0) { + printf("FAIL: decoding of '%s': bad no decoding error\n", src); + return true; + } + } + } + return false; +} + static int assert_roundtrip (int flags, const char *src) { @@ -150,6 +250,7 @@ test_char_table (int flags, bool use_malloc) return fail; } + static int test_streaming (int flags) { @@ -157,7 +258,6 @@ test_streaming (int flags) char chr[256]; char ref[400], enc[400]; size_t reflen; - struct base64_state state; // Fill array with all characters 0..255: for (int i = 0; i < 256; i++) @@ -172,6 +272,7 @@ test_streaming (int flags) size_t inpos = 0; size_t partlen = 0; size_t enclen = 0; + struct base64_state state; base64_stream_encode_init(&state, flags); memset(enc, 0, 400); @@ -209,17 +310,28 @@ test_streaming (int flags) size_t inpos = 0; size_t partlen = 0; size_t enclen = 0; + struct base64_state state; base64_stream_decode_init(&state, flags); memset(enc, 0, 400); - while (base64_stream_decode(&state, &ref[inpos], (inpos + bs > reflen) ? reflen - inpos : bs, &enc[enclen], &partlen)) { - enclen += partlen; - inpos += bs; - - // Has the entire buffer been consumed? - if (inpos >= 400) { + for (size_t b = 0; b < ((reflen + (bs - 1)) / bs); ++b, inpos += bs) { + size_t tmpbs = ((reflen - inpos) > bs) ? bs : reflen - inpos; + if (base64_stream_decode(&state, &ref[inpos], tmpbs, &enc[enclen], &partlen) <=0) { + printf( + "FAIL: stream decoding with blocksize %lu failed at block start %lu\n", + (unsigned long)bs, + (unsigned long)inpos + ); + fail |= true; break; } + enclen += partlen; + } + if (base64_stream_decode_final(&state) <= 0) { + printf( + "FAIL: final stream decoding with blocksize %lu failed\n", + (unsigned long)bs ); + fail |= true; } if (enclen != 256) { printf("FAIL: stream decoding gave incorrect size: " @@ -312,6 +424,146 @@ test_invalid_dec_input (int flags) return fail; } +static int +test_canonical (int flags) +{ + bool fail = false; + + // Test vectors: + struct { + const char *in; + const char *out; + int success; + } vec[] = { + {"", "", 1}, + {"Zg==", "f", 1}, + {"Zm8=", "fo", 1}, + {"Zm9v", "foo", 1}, + {"Zh==", "f", 0}, + {"Zm9=", "fo", 0}, + }; + + for (size_t i = 0; i < sizeof(vec) / sizeof(vec[0]); i++) { + fail |= assert_dec_full(1, flags, vec[i].in, vec[i].out); + fail |= assert_dec_full(vec[i].success, flags | BASE64_CANONICAL, vec[i].in, vec[i].out); + } + return fail; +} + + +static int +test_padded_option(int flags) +{ + bool fail = false; + + // Test vectors: + struct { + const char *in; + const char *out; + } vec[] = { + {"", ""}, + {"YQ==", "a"}, + {"YQ", "a"}, + {"YWI=", "ab"}, + {"YWI", "ab"}, + {"YWJj", "abc"} + }; + + for (size_t i = 0; i < sizeof(vec) / sizeof(vec[0]); i++) { + /* test with padded = false & validation */ + if (strchr(vec[i].in, '=') != NULL) { + fail |= assert_dec_full(0, flags | BASE64_NO_PADDING, vec[i].in, vec[i].out); + fail |= assert_dec_full(1, flags, vec[i].in, vec[i].out); + fail |= assert_enc(flags, vec[i].out, vec[i].in); + } + else { + fail |= assert_dec_full(1, flags | BASE64_NO_PADDING, vec[i].in, vec[i].out); + fail |= assert_enc(flags | BASE64_NO_PADDING, vec[i].out, vec[i].in); + } + } + + return fail; +} + +static int +test_openmp_limit(int flags) +{ + bool fail = false; + size_t const encodedLen = BASE64_OMP_THRESHOLD + 2U * 4U; + size_t decodedLen = (encodedLen / 4) * 3; + size_t const mobyDickLen = strlen(moby_dick_base64) - 4; + char* encoded = malloc(encodedLen); + char* decoded = malloc(decodedLen); + if ((encoded == NULL) || (decoded == NULL)) { + puts("allocation failed"); + free(encoded); + free(decoded); + return true; + } + for (size_t i = 0; i < ((encodedLen + mobyDickLen - 1) / mobyDickLen); ++i) { + size_t const len = (((i + 1) * mobyDickLen) < encodedLen) ? mobyDickLen : encodedLen - (i * mobyDickLen); + memcpy(encoded + i * mobyDickLen, moby_dick_base64, len); + } + + /* fully valid stream */ + encoded[(encodedLen/2) - 4] = 'Z'; + encoded[(encodedLen/2) - 3] = 'g'; + encoded[(encodedLen/2) - 2] = 'g'; + encoded[(encodedLen/2) - 1] = 'g'; + encoded[encodedLen - 4] = 'Z'; + encoded[encodedLen - 3] = 'g'; + encoded[encodedLen - 2] = '='; + encoded[encodedLen - 1] = '='; + + if (base64_decode(encoded, encodedLen, decoded, &decodedLen, flags | BASE64_CANONICAL) <= 0) { + puts("test_openmp_limit failed test 1"); + fail = true; + } + + encoded[(encodedLen/2) - 2] = '='; + encoded[(encodedLen/2) - 1] = '='; + + if (base64_decode(encoded, encodedLen, decoded, &decodedLen, flags) > 0) { + puts("test_openmp_limit failed test 2"); + fail = true; + } + + /* restore fully valid stream */ + encoded[(encodedLen/2) - 2] = 'g'; /* valid */ + encoded[(encodedLen/2) - 1] = 'g'; /* valid */ + if (base64_decode(encoded, encodedLen - 2, decoded, &decodedLen, flags | BASE64_NO_PADDING | BASE64_CANONICAL) <= 0) { + puts("test_openmp_limit failed test 3"); + fail = true; + } + + /* not canonical */ + encoded[encodedLen - 3] = 'h'; + if (base64_decode(encoded, encodedLen, decoded, &decodedLen, flags) <= 0) { + puts("test_openmp_limit failed test 4"); + fail = true; + } + if (base64_decode(encoded, encodedLen, decoded, &decodedLen, flags | BASE64_CANONICAL) > 0) { + puts("test_openmp_limit failed test 5"); + fail = true; + } + if (base64_decode(encoded, encodedLen, decoded, &decodedLen, flags | BASE64_NO_PADDING) > 0) { + puts("test_openmp_limit failed test 6"); + fail = true; + } + if (base64_decode(encoded, encodedLen - 2, decoded, &decodedLen, flags | BASE64_NO_PADDING) <= 0) { + puts("test_openmp_limit failed test 7"); + fail = true; + } + if (base64_decode(encoded, encodedLen - 2, decoded, &decodedLen, flags | BASE64_NO_PADDING | BASE64_CANONICAL) > 0) { + puts("test_openmp_limit failed test 8"); + fail = true; + } + + free(encoded); + free(decoded); + return fail; +} + static int test_one_codec (size_t codec_index) { @@ -364,6 +616,9 @@ test_one_codec (size_t codec_index) fail |= test_char_table(flags, true); /* test for out-of-bound input read */ fail |= test_streaming(flags); fail |= test_invalid_dec_input(flags); + fail |= test_canonical(flags); + fail |= test_padded_option(flags); + fail |= test_openmp_limit(flags); if (!fail) puts(" all tests passed."); @@ -376,6 +631,12 @@ main () { bool fail = false; +#ifdef _OPENMP + if (omp_get_max_threads() >= 2) { + omp_set_num_threads(2); + } +#endif + // Loop over all codecs: for (size_t i = 0; codecs[i]; i++) { // Test this codec, merge the results: