9 #include <glog/logging.h> 10 #include <gtest/gtest.h> 36 template<
class T,
class U>
38 std::string expected(expected_.begin(), expected_.end());
39 std::string actual(actual_.begin(), actual_.end());
41 ASSERT_EQ(expected, actual);
45 template<
class T,
class U>
47 std::vector<uint64_t> expected(expected_.begin(), expected_.end());
48 std::vector<uint64_t> actual(actual_.begin(), actual_.end());
50 ASSERT_EQ(expected, actual);
54 template<
class T,
class U>
56 ASSERT_EQ(expected, actual);
60 template<
class T,
class U>
62 ASSERT_EQ(expected.size(), actual.size()) <<
"assert_eq_sequence: sizes differ";
63 for (
size_t i = 0; i < expected.size(); i++)
64 ASSERT_EQ(expected[i], actual[i]) <<
"assert_eq_sequence: failed at i=" << i;
74 template<
class Lambda>
77 std::ostream& os = ss;
90 template<
class Lambda>
93 return std::vector<uint8_t>(s.begin(), s.end());
107 f(
"abcdefgh#defgh_abcde"_v); \
109 f(
"abcdebcdeabcd"_v);
118 "asdfasctjkcbweasbebvtiwetwcnbwbbqnqxernqzezwuqwezuet" 119 "qcrnzxbneqebwcbqwicbqcbtnqweqxcbwuexcbzqwezcqbwecqbw" 120 "dassdasdfzdfgfsdfsdgfducezctzqwebctuiqwiiqcbnzcebzqc"_v);
122 f(
"ประเทศไทย中华Việt Nam"_v);
125 "Lorem ipsum dolor sit amet, sea ut etiam solet salut" 126 "andi, sint complectitur et his, ad salutandi imperdi" 127 "et gubergren per mei."_v);
130 "Лорэм атоморюм ут хаж, эа граэки емпыдит ёудёкабет " 131 "мэль, декам дежпютатионй про ты. Нэ ёужто жэмпэр" 132 " жкрибэнтур векж, незл коррюмпит."_v);
135 "報チ申猛あち涙境ワセ周兵いわ郵入せすをだ漏告されて話巡わッき" 136 "や間紙あいきり諤止テヘエラ鳥提フ健2銀稿97傷エ映田ヒマ役請多" 137 "暫械ゅにうて。関国ヘフヲオ場三をおか小都供セクヲ前俳著ゅ向深" 138 "まも月10言スひす胆集ヌヱナ賀提63劇とやぽ生牟56詰ひめつそ総愛" 139 "ス院攻せいまて報当アラノ日府ラのがし。"_v);
142 "Εαμ ανσιλλαε περισυλα συαφιθαθε εξ, δυο ιδ ρεβυμ σομ" 143 "μοδο. Φυγιθ ηομερω ιυς ατ, ει αυδιρε ινθελλεγαμ νες." 144 " Ρεκυε ωμνιυμ μανδαμυς κυο εα. Αδμοδυμ σωνσεκυαθ υθ " 145 "φιξ, εσθ ετ πρωβατυς συαφιθαθε ραθιονιβυς, ταντας αυ" 146 "διαμ ινστρυσθιορ ει σεα."_v);
148 f(
"struct Foo { uint8_t bar }"_v);
154 f(
"abbbbbbbbbbcbbbbbbbbbb"_v);
158 std::vector<uint8_t> all_bytes {
159 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
160 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
161 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
162 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
163 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
164 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95,
165 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111,
166 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127,
167 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143,
168 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
169 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175,
170 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191,
171 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207,
172 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223,
173 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,
174 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255,
182 VLOG(1) <<
"fibonacci_word ...";
183 for(
size_t i = 0; i < n; ++i) {
188 VLOG(1) <<
"thue_morse_word ...";
189 for(
size_t i = 0; i < n; ++i) {
194 VLOG(1) <<
"rich ...";
195 for(
size_t i = 0; i < n; ++i) {
200 VLOG(1) <<
"random ...";
201 for(
size_t i = 2; i < n; ++i) {
202 for(
size_t j = 0; j < 2+50/(i+1); ++j) {
212 return TEST_FILE_PATH +
"/" + filename;
219 return (stat(test_file_name.c_str(), &buf) == 0);
223 std::ostringstream sout;
226 std::ifstream fin(test_file_name);
232 std::string msg =
"Could not open test file \"";
233 msg += test_file_name;
235 throw std::runtime_error(msg);
244 mkdir(TEST_FILE_PATH.c_str(), 0777);
262 CHECK(ints.size() % 2 == 0);
263 std::vector<uint8_t> bits;
266 for (
size_t i = 0; i < ints.size(); i += 2) {
267 uint64_t val = ints[i];
268 uint64_t val_bits = ints[i + 1];
269 for (uint64_t bit = 0; bit < val_bits; bit++) {
275 uint8_t& b = bits[bits.size() - 1];
276 if (val & (uint64_t(1) << (val_bits - bit - 1))) {
277 b |= (1 << (7 - bit_pos));
287 std::string
format_diff(
const std::string& a,
const std::string& b) {
289 for(
size_t i = 0; i < std::max(a.size(), b.size()); i++) {
290 if (i < std::min(a.size(), b.size())
303 for(
size_t i = 0; i < std::max(a.size(), b.size()); i++) {
304 if (i < std::min(a.size(), b.size())
305 && a[i] ==
' ' && b[i] ==
' ' 308 }
else if (i < std::min(a.size(), b.size())
323 auto print_bits = [&expected](
string_ref s,
bool byte_units =
false) -> std::string {
324 std::stringstream ss;
329 size_t last_packed_i = 0;
331 for (uint64_t i = 0; i < s.
size() * 8; i++) {
332 if (!byte_units && packed_i < expected.size() && i == last_packed_i + expected[packed_i + 1]) {
336 }
else if (byte_units && i > 0 && i % 8 == 0) {
340 uint8_t c = s[i / 8];
341 ss << int((c >> (8 - (i % 8) - 1)) & 1);
347 auto p_is = print_bits(actual);
348 auto p_should = print_bits(out);
351 auto p_is_b = print_bits(actual,
true);
352 auto p_should_b = print_bits(out,
true);
356 <<
"Should Be: " << p_should <<
"\n" 357 <<
" Is: " << p_is <<
"\n" 358 <<
" Diff: " << p_diff <<
"\n" 359 <<
"As Bytes:" <<
"\n" 360 <<
"Should Be: " << p_should_b <<
"\n" 361 <<
" Is: " << p_is_b <<
"\n" 362 <<
" Diff: " << p_diff_b <<
"\n";
377 std::vector<uint8_t>&& p_bytes,
379 std::string&& p_original,
380 std::string&& p_options):
381 m_registry(registry),
382 bytes(
std::move(p_bytes)),
383 str(
std::move(p_str)),
384 orginal_text(
std::move(p_original)),
385 options(
std::move(p_options)) {}
388 std::vector<uint8_t> decoded_buffer;
393 auto compressor = create_algo_with_registry<C>(
options, m_registry);
395 if (C::meta().textds_flags().has_restrictions()) {
396 decoded_out =
Output(decoded_out, C::meta().textds_flags());
399 compressor.decompress(text_in, decoded_out);
401 std::string decompressed_text {
402 decoded_buffer.begin(),
403 decoded_buffer.end(),
405 ASSERT_EQ(orginal_text, decompressed_text);
409 std::vector<uint8_t> decompressed_bytes;
414 auto compressor = create_algo_with_registry<C>(
options, m_registry);
416 if (C::meta().textds_flags().has_restrictions()) {
417 decoded_out =
Output(decoded_out, C::meta().textds_flags());
420 compressor.decompress(text_in, decoded_out);
422 std::vector<uint8_t> orginal_bytes {
423 orginal_text.begin(),
426 ASSERT_EQ(orginal_bytes, decompressed_bytes);
432 std::string m_options;
443 std::vector<uint8_t> encoded_buffer;
448 auto compressor = create_algo_with_registry<C>(m_options, m_registry);
450 if (C::meta().textds_flags().has_restrictions()) {
451 text_in =
Input(text_in, C::meta().textds_flags());
453 compressor.compress(text_in, encoded_out);
455 std::string s(encoded_buffer.begin(), encoded_buffer.end());
458 std::move(encoded_buffer),
461 std::string(m_options),
468 const std::string&
options =
"",
476 const std::string&
options =
"",
479 auto& compressed_text = e.str;
481 if(expected_compressed_text.
size() > 0) {
482 ASSERT_EQ(std::string(expected_compressed_text), compressed_text);
485 e.assert_decompress();
490 roundtrip_ex<T>(original_text,
"");
495 const std::vector<uint64_t>& expected_compressed_text_packed_ints = {},
496 const std::string&
options =
"",
499 auto& compressed_text = e.bytes;
501 if(expected_compressed_text_packed_ints.size() > 0)
504 e.assert_decompress_bytes();
524 Output(static_cast<
std::vector<uint8_t>&>(*this))
577 template<
typename Coder>
584 auto env = tdc::builder<Coder>().env();
585 std::shared_ptr<BitOStream> bo = std::make_shared<BitOStream>(o);
586 typename Coder::Encoder coder(std::move(env), bo,
ViewLiterals(v));
588 bool was_zero =
true;
590 if (was_zero && interleave) {
TestOutput decompress_output()
Creates an instance of an tdc::Output to be used with Compressor::decompress().
Contains the text compression and encoding framework.
std::string ostream_to_string(Lambda f)
Temporary provides a ostream to write into, and returns it as a string.
CompressResult< C > compress(string_ref text)
virtual std::string generate() override
Generates a string based on the environment settings.
io::Input Input
Convenience shortcut to io::Input.
void roundtrip_binary(string_ref original_text, const std::vector< uint64_t > &expected_compressed_text_packed_ints={}, const std::string &options="", const Registry< Compressor > ®istry=Registry< Compressor >("compressor"))
A const view into a slice of memory.
void assert_eq_integers(const T &expected_, const U &actual_)
Error diagnostic optimized for binary data.
void remove_test_file(const std::string &filename)
uint8_t uliteral_t
Type to represent signed single literals.
void roundtrip_batch(F f)
Call the given function with a number of different strings testing common corner cases and unicode in...
void assert_eq_hybrid_strings(const T &expected, const U &actual)
Error diagnostic optimized for mixed binary/ascii data.
void on_string_generators(F func, size_t n)
bool test_file_exists(const std::string &filename)
virtual std::string generate() override
Generates a string based on the environment settings.
void assert_decompress_bytes()
void create_test_directory()
TestInput decompress_input_file(string_ref path)
Creates an instance of an tdc::Input to be used with Compressor::decompress().
void test_binary_out(string_ref in, std::vector< uint64_t > packed_ints_out, bool interleave=false)
CompressResult< T > compress(string_ref text, const std::string &options="", const Registry< Compressor > ®istry=Registry< Compressor >("compressor"))
TestInput compress_input_file(string_ref path)
Creates an instance of an tdc::Input to be used with Compressor::compress().
size_type size() const
Returns size of the View.
A literal iterator that yields every character from a View.
std::vector< uint8_t > ostream_to_bytes(Lambda f)
Temporary provides a ostream to write into, and returns it as a byte vector.
void assert_eq_sequence(const T &expected, const U &actual)
Error diagnostic optimized for arbitrary data.
A registry for algorithms to be made available in the driver application.
void roundtrip_ex(string_ref original_text, string_ref expected_compressed_text, const std::string &options="", const Registry< Compressor > ®istry=Registry< Compressor >("compressor"))
void assert_eq_binary(string_ref actual, PacketIntegers expected)
CompressResult(const Registry< Compressor > ®istry, std::vector< uint8_t > &&p_bytes, std::string &&p_str, std::string &&p_original, std::string &&p_options)
std::string read_test_file(const std::string &filename)
const std::string TEST_FILE_PATH
static Output from_memory(std::vector< uint8_t > &buf)
Constructs an output to a byte buffer.
TestInput decompress_input(string_ref text)
Creates an instance of an tdc::Input to be used with Compressor::decompress().
constexpr auto literal_r
Global predefined reange for literals.
An abstraction layer for algorithm output.
TestInput compress_input(string_ref text)
Creates an instance of an tdc::Input to be used with Compressor::compress().
std::string format_diff(const std::string &a, const std::string &b)
RoundTrip(const std::string &options="", const Registry< Compressor > ®istry=Registry< Compressor >("compressor"))
std::vector< uint8_t > bytes
void roundtrip(string_ref original_text)
void write_test_file(const std::string &filename, string_ref text)
std::string format_diff_bin(const std::string &a, const std::string &b)
void assert_eq_strings(const T &expected_, const U &actual_)
Error diagnostic optimized for string data.
TestOutput compress_output()
Creates an instance of an tdc::Output to be used with Compressor::compress().
std::string test_file_path(const std::string &filename)
std::vector< uint64_t > PacketIntegers
std::vector< uint8_t > pack_integers(std::vector< uint64_t > ints)
TestOutput(bool sentinel)
virtual std::string generate() override
Generates a string based on the environment settings.
io::Output Output
Convenience shortcut to io::Output.