Line data Source code
1 : // Copyright (c) 2006-2022 Made to Order Software Corp. All Rights Reserved
2 : //
3 : // https://snapwebsites.org/project/advgetopt
4 : // contact@m2osw.com
5 : //
6 : // This program is free software; you can redistribute it and/or modify
7 : // it under the terms of the GNU General Public License as published by
8 : // the Free Software Foundation; either version 2 of the License, or
9 : // (at your option) any later version.
10 : //
11 : // This program is distributed in the hope that it will be useful,
12 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 : // GNU General Public License for more details.
15 : //
16 : // You should have received a copy of the GNU General Public License along
17 : // with this program; if not, write to the Free Software Foundation, Inc.,
18 : // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19 :
20 : // murmur3
21 : //
22 : #include <murmur3/stream.h>
23 :
24 : #include <murmur3/murmur3.h>
25 :
26 :
27 : // self
28 : //
29 : #include "catch_main.h"
30 :
31 :
32 : // snapdev
33 : //
34 : #include <snapdev/file_contents.h>
35 : #include <snapdev/glob_to_list.h>
36 :
37 :
38 : // C
39 : //
40 : #include <sys/random.h>
41 :
42 :
43 : // last include
44 : //
45 : #include <snapdev/poison.h>
46 :
47 :
48 :
49 5 : CATCH_TEST_CASE("stream_basic", "[stream][valid]")
50 : {
51 6 : CATCH_START_SECTION("Verify small stream (Hello World!)")
52 : {
53 1 : murmur3::stream sum(123, 123);
54 :
55 1 : murmur3::seed_t seed1(0);
56 1 : murmur3::seed_t seed2(0);
57 1 : sum.get_seeds(seed1, seed2);
58 1 : CATCH_CHECK(seed1 == 123);
59 1 : CATCH_CHECK(seed2 == 123);
60 :
61 2 : std::string const hello_world("Hello, world!");
62 1 : sum.add_data(hello_world.c_str(), hello_world.length());
63 1 : CATCH_CHECK(sum.flush().to_string() == "8743acad421c8c73d373c3f5f19732fd");
64 :
65 1 : CATCH_CHECK(murmur3::sum(hello_world.c_str(), hello_world.length(), 123).to_string() == "8743acad421c8c73d373c3f5f19732fd");
66 :
67 : // verify the reset()
68 : //
69 1 : sum.reset();
70 :
71 1 : sum.add_data(hello_world.c_str(), hello_world.length());
72 1 : CATCH_CHECK(sum.flush().to_string() == "8743acad421c8c73d373c3f5f19732fd");
73 :
74 : // verify different seed
75 : //
76 1 : sum.reset(321);
77 :
78 1 : sum.add_data(hello_world.c_str(), hello_world.length());
79 1 : CATCH_CHECK(sum.flush().to_string() == "f86d4004ca47f42bb9546c7979200aee");
80 :
81 1 : CATCH_CHECK(murmur3::sum(hello_world.c_str(), hello_world.length(), 321).to_string() == "f86d4004ca47f42bb9546c7979200aee");
82 : }
83 : CATCH_END_SECTION()
84 :
85 6 : CATCH_START_SECTION("X28 stream")
86 : {
87 1 : murmur3::stream sum(123, 123);
88 :
89 1 : murmur3::seed_t seed1(0);
90 1 : murmur3::seed_t seed2(0);
91 1 : sum.get_seeds(seed1, seed2);
92 1 : CATCH_CHECK(seed1 == 123);
93 1 : CATCH_CHECK(seed2 == 123);
94 :
95 2 : std::string const x28("xxxxxxxxxxxxxxxxxxxxxxxxxxxx");
96 1 : sum.add_data(x28.c_str(), x28.length());
97 1 : CATCH_CHECK(sum.flush().to_string() == "becf7e04dbcf74637751664ef66e73e0");
98 :
99 1 : CATCH_CHECK(murmur3::sum(x28.c_str(), x28.length(), 123).to_string() == "becf7e04dbcf74637751664ef66e73e0");
100 : }
101 : CATCH_END_SECTION()
102 :
103 6 : CATCH_START_SECTION("Empty file stream")
104 : {
105 1 : murmur3::stream sum(123, 123);
106 :
107 1 : murmur3::seed_t seed1(0);
108 1 : murmur3::seed_t seed2(0);
109 1 : sum.get_seeds(seed1, seed2);
110 1 : CATCH_CHECK(seed1 == 123);
111 1 : CATCH_CHECK(seed2 == 123);
112 :
113 2 : std::string const empty("");
114 1 : sum.add_data(empty.c_str(), empty.length());
115 1 : CATCH_CHECK(sum.flush().to_string() == "4cd9597081679d1abd92f8784bace33d");
116 :
117 1 : CATCH_CHECK(murmur3::sum(empty.c_str(), empty.length(), 123).to_string() == "4cd9597081679d1abd92f8784bace33d");
118 : }
119 : CATCH_END_SECTION()
120 3 : }
121 :
122 :
123 5 : CATCH_TEST_CASE("stream_file", "[stream][valid]")
124 : {
125 6 : CATCH_START_SECTION("Stream files")
126 : {
127 2 : snapdev::glob_to_list<std::list<snapdev::file>> glob;
128 1 : CATCH_REQUIRE(glob.read_path<
129 : snapdev::glob_to_list_flag_t::GLOB_FLAG_IGNORE_ERRORS,
130 : snapdev::glob_to_list_flag_t::GLOB_FLAG_PERIOD>("..."));
131 :
132 208 : for(auto const & f : glob)
133 : {
134 207 : if(!f.is_regular_file())
135 : {
136 : // skip directories and special files
137 : //
138 85 : continue;
139 : }
140 122 : if(SNAP_CATCH2_NAMESPACE::g_verbose())
141 : {
142 0 : std::cout << "-- testing with file \"" << f.filename() << "\".\n";
143 : }
144 :
145 244 : snapdev::file_contents file(f.filename());
146 122 : CATCH_REQUIRE(file.read_all());
147 122 : std::string const & contents(file.contents());
148 :
149 : // first compute the hash with the basic C function
150 : //
151 122 : std::uint32_t hash[4];
152 122 : MurmurHash3_x64_128(contents.c_str(), contents.length(), 0, hash);
153 244 : std::string c_hash(SNAP_CATCH2_NAMESPACE::hex128(hash));
154 :
155 : // second compute the same hash with the stream function 4Kb at
156 : // a time so we should hit all possible cases
157 : //
158 122 : constexpr std::size_t four_kb(1024 * 4);
159 122 : murmur3::stream sum(0);
160 122 : std::size_t const nblocks(contents.length() / four_kb);
161 122 : std::size_t const left_over(contents.length() % four_kb);
162 3937 : for(std::size_t b(0); b < nblocks; ++b)
163 : {
164 3815 : sum.add_data(contents.c_str() + b * four_kb, four_kb);
165 : }
166 122 : if(left_over != 0)
167 : {
168 122 : sum.add_data(contents.c_str() + nblocks * four_kb, left_over);
169 : }
170 :
171 122 : CATCH_CHECK(sum.flush().to_string() == c_hash);
172 :
173 : // test the helper function that computes the murmur3 of a file
174 : // using the streaming mechanism (to avoid loading the file in
175 : // memory all at once)
176 : //
177 122 : CATCH_CHECK(murmur3::sum(f.filename()).to_string() == c_hash);
178 : }
179 : }
180 : CATCH_END_SECTION()
181 :
182 6 : CATCH_START_SECTION("Stream 4Kb exactly (special case)")
183 : {
184 1 : constexpr std::size_t four_kb(1024 * 4);
185 1 : char buf[four_kb];
186 1 : getrandom(buf, sizeof(buf), 0);
187 :
188 : // first compute the hash with the basic C function
189 : //
190 1 : std::uint32_t hash[4];
191 1 : MurmurHash3_x64_128(buf, sizeof(buf), 0, hash);
192 2 : std::string c_hash(SNAP_CATCH2_NAMESPACE::hex128(hash));
193 :
194 : // second compute the same hash with the stream function 4Kb at
195 : // a time so we should hit all possible cases
196 : //
197 1 : murmur3::stream sum(0);
198 1 : sum.add_data(buf, sizeof(buf));
199 1 : CATCH_CHECK(sum.flush().to_string() == c_hash);
200 :
201 : // test the helper function that computes the murmur3 of a file
202 : // using the streaming mechanism (to avoid loading the file in
203 : // memory all at once)
204 : //
205 2 : std::string const tmp_dir(SNAP_CATCH2_NAMESPACE::g_tmp_dir());
206 2 : std::string const test_filename(tmp_dir + "/4kb.bin");
207 2 : std::ofstream out(test_filename);
208 1 : out.write(buf, sizeof(buf));
209 1 : CATCH_CHECK(murmur3::sum(test_filename).to_string() == c_hash);
210 : }
211 : CATCH_END_SECTION()
212 :
213 6 : CATCH_START_SECTION("Stream 0Kb to 4Kb (many special cases)")
214 : {
215 1 : constexpr std::size_t four_kb(1024 * 4);
216 4097 : for(std::size_t size(0); size < four_kb; ++size)
217 : {
218 4096 : char buf[size];
219 4096 : getrandom(buf, sizeof(buf), 0);
220 :
221 : // first compute the hash with the basic C function
222 : //
223 4096 : std::uint32_t hash[4];
224 4096 : MurmurHash3_x64_128(buf, sizeof(buf), 0, hash);
225 8192 : std::string c_hash(SNAP_CATCH2_NAMESPACE::hex128(hash));
226 :
227 : // second compute the same hash with the stream function 4Kb at
228 : // a time so we should hit all possible cases
229 : //
230 4096 : murmur3::stream sum(0);
231 4096 : sum.add_data(buf, sizeof(buf));
232 4096 : CATCH_CHECK(sum.flush().to_string() == c_hash);
233 :
234 : // test the helper function that computes the murmur3 of a file
235 : // using the streaming mechanism (to avoid loading the file in
236 : // memory all at once)
237 : //
238 8192 : std::string const tmp_dir(SNAP_CATCH2_NAMESPACE::g_tmp_dir());
239 8192 : std::string const test_filename(tmp_dir + "/" + std::to_string(size) + "_bytes.bin");
240 : {
241 8192 : std::ofstream out(test_filename);
242 4096 : out.write(buf, sizeof(buf));
243 : }
244 : //std::cerr << "-- test with file [" << test_filename << "] -> " << murmur3::sum(test_filename).to_string() << "\n";
245 4096 : CATCH_CHECK(murmur3::sum(test_filename).to_string() == c_hash);
246 4096 : }
247 : }
248 : CATCH_END_SECTION()
249 9 : }
250 :
251 :
252 : // vim: ts=4 sw=4 et
|
