Line data Source code
1 : // Copyright (c) 2019 Made to Order Software Corp. All Rights Reserved
2 : //
3 : // https://snapwebsites.org/project/snapdatabase
4 : // contact@m2osw.com
5 : //
6 : // This program is free software; you can redistribute it and/or modify
7 : // it under the terms of the GNU General Public License as published by
8 : // the Free Software Foundation; either version 2 of the License, or
9 : // (at your option) any later version.
10 : //
11 : // This program is distributed in the hope that it will be useful,
12 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 : // GNU General Public License for more details.
15 : //
16 : // You should have received a copy of the GNU General Public License along
17 : // with this program; if not, write to the Free Software Foundation, Inc.,
18 : // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19 :
20 :
21 : /** \file
22 : * \brief Database file implementation.
23 : *
24 : * Each table uses one or more files. Each file is handled by a dbfile
25 : * object and a corresponding set of blocks.
26 : */
27 :
28 : // self
29 : //
30 : #include "snapdatabase/database/table.h"
31 :
32 : #include "snapdatabase/database/context.h"
33 :
34 : // all the blocks since we create them here
35 : //
36 : #include "snapdatabase/block/block_blob.h"
37 : #include "snapdatabase/block/block_data.h"
38 : #include "snapdatabase/block/block_entry_index.h"
39 : #include "snapdatabase/block/block_free_block.h"
40 : #include "snapdatabase/block/block_free_space.h"
41 : #include "snapdatabase/block/block_header.h"
42 : #include "snapdatabase/block/block_index_pointers.h"
43 : #include "snapdatabase/block/block_indirect_index.h"
44 : #include "snapdatabase/block/block_secondary_index.h"
45 : #include "snapdatabase/block/block_schema.h"
46 : #include "snapdatabase/block/block_top_index.h"
47 : #include "snapdatabase/file/file_bloom_filter.h"
48 : #include "snapdatabase/file/file_external_index.h"
49 : #include "snapdatabase/file/file_snap_database_table.h"
50 :
51 :
52 : // C++ lib
53 : //
54 : #include <iostream>
55 :
56 :
57 : // last include
58 : //
59 : #include <snapdev/poison.h>
60 :
61 :
62 :
63 : namespace snapdatabase
64 : {
65 :
66 :
67 :
68 : namespace detail
69 : {
70 :
71 :
72 :
73 :
74 :
75 :
76 :
77 :
78 0 : class table_impl
79 : {
80 : public:
81 : table_impl(
82 : context * c
83 : , table * t
84 : , xml_node::pointer_t x
85 : , xml_node::map_t complex_types);
86 : table_impl(table_impl const & rhs) = delete;
87 :
88 : table_impl operator = (table_impl const & rhs) = delete;
89 :
90 : void load_extension(xml_node::pointer_t e);
91 :
92 : dbfile::pointer_t get_dbfile() const;
93 : version_t version() const;
94 : bool is_secure() const;
95 : bool is_sparse() const;
96 : std::string name() const;
97 : model_t model() const;
98 : column_ids_t row_key() const;
99 : schema_column::pointer_t column(std::string const & name) const;
100 : schema_column::pointer_t column(column_id_t id) const;
101 : schema_column::map_by_id_t columns_by_id() const;
102 : schema_column::map_by_name_t columns_by_name() const;
103 : std::string description() const;
104 : size_t get_size() const;
105 : size_t get_page_size() const;
106 : block::pointer_t get_block(reference_t offset);
107 : block::pointer_t allocate_new_block(dbtype_t type);
108 : void free_block(block::pointer_t block, bool clear_block);
109 : bool verify_schema();
110 :
111 : private:
112 : block::pointer_t allocate_block(dbtype_t type, reference_t offset);
113 :
114 : context * f_context = nullptr;
115 : table * f_table = nullptr;
116 : schema_table::pointer_t f_schema_table = schema_table::pointer_t();
117 : dbfile::pointer_t f_dbfile = dbfile::pointer_t();
118 : xml_node::map_t f_complex_types = xml_node::map_t();
119 : block::map_t f_blocks = block::map_t();
120 : };
121 :
122 :
123 1 : table_impl::table_impl(
124 : context * c
125 : , table * t
126 : , xml_node::pointer_t x
127 1 : , xml_node::map_t complex_types)
128 : : f_context(c)
129 : , f_table(t)
130 : , f_schema_table(std::make_shared<schema_table>())
131 1 : , f_complex_types(complex_types)
132 : {
133 1 : f_schema_table->from_xml(x);
134 1 : f_dbfile = std::make_shared<dbfile>(c->get_path(), f_schema_table->name(), "main");
135 1 : f_dbfile->set_page_size(f_schema_table->block_size());
136 1 : }
137 :
138 :
139 0 : void table_impl::load_extension(xml_node::pointer_t e)
140 : {
141 0 : f_schema_table->load_extension(e);
142 0 : }
143 :
144 :
145 3 : dbfile::pointer_t table_impl::get_dbfile() const
146 : {
147 3 : return f_dbfile;
148 : }
149 :
150 :
151 1 : bool table_impl::verify_schema()
152 : {
153 : // load schema from dbfile (if present) and then compare against schema
154 : // from XML; if different increase version, save new version in file and
155 : // start process to update the table existing data; once the update is
156 : // done, we can remove the old schema from the file (i.e. any new writes
157 : // always use the newest version of the schema; the rows that use an
158 : // older version, use that specific schema until they get updated to the
159 : // new format.)
160 : //
161 : file_snap_database_table::pointer_t sdbt(
162 : std::static_pointer_cast<file_snap_database_table>(
163 2 : get_block(0)));
164 :
165 1 : reference_t const schema_offset(sdbt->get_table_definition());
166 1 : if(schema_offset == 0)
167 : {
168 : // no schema defined yet, just save ours and we're all good
169 : //
170 1 : f_schema_table->assign_column_ids();
171 : block_schema::pointer_t schm(
172 : std::static_pointer_cast<block_schema>(
173 2 : allocate_new_block(dbtype_t::BLOCK_TYPE_SCHEMA)));
174 2 : virtual_buffer::pointer_t bin_schema(f_schema_table->to_binary());
175 1 : schm->set_schema(bin_schema);
176 : }
177 : else
178 : {
179 : // load the binary schema (it may reside on multiple blocks and we
180 : // have to read the entire schema at once)
181 : //
182 0 : std::cerr << "table: TODO verify schema...\n";
183 : }
184 :
185 2 : return true;
186 : }
187 :
188 :
189 0 : version_t table_impl::version() const
190 : {
191 0 : return f_schema_table->version();
192 : }
193 :
194 :
195 0 : bool table_impl::is_secure() const
196 : {
197 0 : return f_schema_table->is_secure();
198 : }
199 :
200 :
201 1 : bool table_impl::is_sparse() const
202 : {
203 1 : return f_schema_table->is_sparse();
204 : }
205 :
206 :
207 1 : std::string table_impl::name() const
208 : {
209 1 : return f_schema_table->name();
210 : }
211 :
212 :
213 0 : model_t table_impl::model() const
214 : {
215 0 : return f_schema_table->model();
216 : }
217 :
218 :
219 0 : column_ids_t table_impl::row_key() const
220 : {
221 0 : return f_schema_table->row_key();
222 : }
223 :
224 :
225 0 : schema_column::pointer_t table_impl::column(std::string const & name) const
226 : {
227 0 : return f_schema_table->column(name);
228 : }
229 :
230 :
231 0 : schema_column::pointer_t table_impl::column(column_id_t id) const
232 : {
233 0 : return f_schema_table->column(id);
234 : }
235 :
236 :
237 0 : schema_column::map_by_id_t table_impl::columns_by_id() const
238 : {
239 0 : return f_schema_table->columns_by_id();
240 : }
241 :
242 :
243 0 : schema_column::map_by_name_t table_impl::columns_by_name() const
244 : {
245 0 : return f_schema_table->columns_by_name();
246 : }
247 :
248 :
249 0 : std::string table_impl::description() const
250 : {
251 0 : return f_schema_table->description();
252 : }
253 :
254 :
255 0 : size_t table_impl::get_size() const
256 : {
257 0 : return f_dbfile->get_size();
258 : }
259 :
260 :
261 26 : size_t table_impl::get_page_size() const
262 : {
263 26 : return f_dbfile->get_page_size();
264 : }
265 :
266 :
267 5 : block::pointer_t table_impl::allocate_block(dbtype_t type, reference_t offset)
268 : {
269 5 : auto it(f_blocks.find(offset));
270 5 : if(it != f_blocks.end())
271 : {
272 3 : if(type == it->second->get_dbtype())
273 : {
274 2 : return it->second;
275 : }
276 : // TBD: I think only FREE blocks can be replaced by something else
277 : // and vice versa or we've got a bug on our hands
278 : //
279 1 : if(type != dbtype_t::BLOCK_TYPE_FREE_BLOCK
280 1 : && it->second->get_dbtype() != dbtype_t::BLOCK_TYPE_FREE_BLOCK)
281 : {
282 : throw snapdatabase_logic_error(
283 : "allocate_block() called a non-free block type trying to allocate a non-free block ("
284 0 : + std::to_string(static_cast<int>(type))
285 0 : + "). You can go from a free to non-free and non-free to free, but not the opposite.");
286 : }
287 : //it->second->replacing(); -- this won't work right at this time TODO...
288 1 : f_blocks.erase(it);
289 : }
290 :
291 6 : block::pointer_t b;
292 3 : switch(type)
293 : {
294 1 : case dbtype_t::FILE_TYPE_SNAP_DATABASE_TABLE:
295 1 : b = std::make_shared<file_snap_database_table>(f_dbfile, offset);
296 1 : break;
297 :
298 0 : case dbtype_t::FILE_TYPE_EXTERNAL_INDEX:
299 0 : b = std::make_shared<file_external_index>(f_dbfile, offset);
300 0 : break;
301 :
302 0 : case dbtype_t::FILE_TYPE_BLOOM_FILTER:
303 0 : b = std::make_shared<file_bloom_filter>(f_dbfile, offset);
304 0 : break;
305 :
306 0 : case dbtype_t::BLOCK_TYPE_BLOB:
307 0 : b = std::make_shared<block_blob>(f_dbfile, offset);
308 0 : break;
309 :
310 0 : case dbtype_t::BLOCK_TYPE_DATA:
311 0 : b = std::make_shared<block_data>(f_dbfile, offset);
312 0 : break;
313 :
314 0 : case dbtype_t::BLOCK_TYPE_ENTRY_INDEX:
315 0 : b = std::make_shared<block_entry_index>(f_dbfile, offset);
316 0 : break;
317 :
318 1 : case dbtype_t::BLOCK_TYPE_FREE_BLOCK:
319 : //throw snapdatabase_logic_error("You can't allocate a Free Block with allocate_block()");
320 1 : b = std::make_shared<block_free_block>(f_dbfile, offset);
321 1 : break;
322 :
323 0 : case dbtype_t::BLOCK_TYPE_FREE_SPACE:
324 0 : b = std::make_shared<block_free_space>(f_dbfile, offset);
325 0 : break;
326 :
327 0 : case dbtype_t::BLOCK_TYPE_INDEX_POINTERS:
328 0 : b = std::make_shared<block_index_pointers>(f_dbfile, offset);
329 0 : break;
330 :
331 0 : case dbtype_t::BLOCK_TYPE_INDIRECT_INDEX:
332 0 : b = std::make_shared<block_indirect_index>(f_dbfile, offset);
333 0 : break;
334 :
335 0 : case dbtype_t::BLOCK_TYPE_SECONDARY_INDEX:
336 0 : b = std::make_shared<block_secondary_index>(f_dbfile, offset);
337 0 : break;
338 :
339 1 : case dbtype_t::BLOCK_TYPE_SCHEMA:
340 1 : b = std::make_shared<block_schema>(f_dbfile, offset);
341 1 : break;
342 :
343 0 : case dbtype_t::BLOCK_TYPE_TOP_INDEX:
344 0 : b = std::make_shared<block_top_index>(f_dbfile, offset);
345 0 : break;
346 :
347 0 : default:
348 : throw snapdatabase_logic_error(
349 : "allocate_block() called with an unknown dbtype_t value ("
350 0 : + std::to_string(static_cast<int>(type))
351 0 : + ").");
352 :
353 : }
354 :
355 3 : b->set_table(f_table->get_pointer());
356 3 : b->set_data(f_dbfile->data(offset));
357 3 : b->get_structure()->set_block(b, 0, f_dbfile->get_page_size());
358 3 : b->set_dbtype(type);
359 :
360 3 : f_context->limit_allocated_memory();
361 :
362 : // we add this block to the list of blocks only after the call to
363 : // limit the allocated memory
364 : //
365 3 : f_blocks[offset] = b;
366 :
367 3 : return b;
368 : }
369 :
370 :
371 3 : block::pointer_t table_impl::get_block(reference_t offset)
372 : {
373 3 : if(offset != 0
374 3 : && offset >= f_dbfile->get_size())
375 : {
376 0 : throw snapdatabase_logic_error("Requested a block with an offset >= to the existing file size.");
377 : }
378 :
379 6 : structure::pointer_t s(std::make_shared<structure>(g_block_header));
380 3 : data_t d(f_dbfile->data(offset));
381 6 : virtual_buffer::pointer_t header(std::make_shared<virtual_buffer>());
382 : #ifdef _DEBUG
383 3 : if(s->get_size() != BLOCK_HEADER_SIZE)
384 : {
385 0 : throw snapdatabase_logic_error("sizeof(g_block_header) != BLOCK_HEADER_SIZE");
386 : }
387 : #endif
388 3 : header->pwrite(d, s->get_size(), 0, true);
389 3 : s->set_virtual_buffer(header, 0);
390 3 : dbtype_t const type(static_cast<dbtype_t>(s->get_uinteger("magic")));
391 : //version_t const version(s->get_uinteger("version"));
392 :
393 3 : block::pointer_t b(allocate_block(type, offset));
394 :
395 : // this last call is used to convert the binary data from the
396 : // file version to the latest running version; the result will
397 : // be saved back in the block so that way the conversion doesn't
398 : // happen over and over again; if the version is already up to
399 : // date, then nothing happens
400 : //
401 3 : b->from_current_file_version();
402 :
403 6 : return b;
404 :
405 : }
406 :
407 :
408 2 : block::pointer_t table_impl::allocate_new_block(dbtype_t type)
409 : {
410 2 : if(type == dbtype_t::BLOCK_TYPE_FREE_BLOCK)
411 : {
412 0 : throw snapdatabase_logic_error("You can't allocate a Free Block with allocate_new_block().");
413 : }
414 :
415 2 : reference_t offset(0);
416 2 : if(f_dbfile->get_size() == 0)
417 : {
418 1 : switch(type)
419 : {
420 1 : case dbtype_t::FILE_TYPE_SNAP_DATABASE_TABLE:
421 : case dbtype_t::FILE_TYPE_EXTERNAL_INDEX:
422 : case dbtype_t::FILE_TYPE_BLOOM_FILTER:
423 1 : break;
424 :
425 0 : default:
426 : throw snapdatabase_logic_error(
427 : "a new file can't be created with type \""
428 0 : + to_string(type)
429 0 : + "\".");
430 :
431 : }
432 :
433 : // this is a new file, create 16 `FREE` blocks
434 : //
435 1 : f_dbfile->append_free_block(NULL_FILE_ADDR);
436 1 : size_t const page_size(f_dbfile->get_page_size());
437 1 : reference_t next(page_size * 2);
438 15 : for(int idx(0); idx < 14; ++idx, next += page_size)
439 : {
440 14 : f_dbfile->append_free_block(next);
441 : }
442 1 : f_dbfile->append_free_block(NULL_FILE_ADDR);
443 :
444 : // offset is already 0
445 : }
446 : else
447 : {
448 1 : switch(type)
449 : {
450 0 : case dbtype_t::FILE_TYPE_SNAP_DATABASE_TABLE:
451 : case dbtype_t::FILE_TYPE_EXTERNAL_INDEX:
452 : case dbtype_t::FILE_TYPE_BLOOM_FILTER:
453 : throw snapdatabase_logic_error(
454 : "a file type such as \""
455 0 : + to_string(type)
456 0 : + "\" is only for when you create a file.");
457 :
458 1 : default:
459 1 : break;
460 :
461 : }
462 :
463 : // get next free block from the header
464 : //
465 2 : file_snap_database_table::pointer_t header(std::static_pointer_cast<file_snap_database_table>(get_block(0)));
466 1 : offset = header->get_first_free_block();
467 1 : if(offset == NULL_FILE_ADDR)
468 : {
469 0 : offset = f_dbfile->append_free_block(NULL_FILE_ADDR);
470 :
471 0 : size_t const page_size(f_dbfile->get_page_size());
472 0 : reference_t next(offset + page_size * 2);
473 0 : for(int idx(0); idx < 14; ++idx, next += page_size)
474 : {
475 0 : f_dbfile->append_free_block(next);
476 : }
477 0 : f_dbfile->append_free_block(NULL_FILE_ADDR);
478 :
479 0 : header->set_first_free_block(offset + page_size);
480 : }
481 : else
482 : {
483 2 : block_free_block::pointer_t p(std::static_pointer_cast<block_free_block>(get_block(offset)));
484 1 : header->set_first_free_block(p->get_next_free_block());
485 : }
486 : }
487 :
488 : // this should probably use a factory for better extendability
489 : // but at this time we don't need such at all
490 : //
491 2 : block::pointer_t b(allocate_block(type, offset));
492 2 : b->set_structure_version();
493 2 : return b;
494 : }
495 :
496 :
497 0 : void table_impl::free_block(block::pointer_t block, bool clear_block)
498 : {
499 0 : if(block == nullptr)
500 : {
501 0 : return;
502 : }
503 :
504 0 : reference_t const offset(block->get_offset());
505 : block_free_block::pointer_t p(std::static_pointer_cast<block_free_block>(
506 0 : allocate_block(dbtype_t::BLOCK_TYPE_FREE_BLOCK, offset)));
507 :
508 0 : if(clear_block)
509 : {
510 0 : p->clear_block();
511 : }
512 :
513 0 : file_snap_database_table::pointer_t header(std::static_pointer_cast<file_snap_database_table>(get_block(0)));
514 0 : reference_t const next_offset(header->get_first_free_block());
515 0 : p->set_next_free_block(next_offset);
516 0 : header->set_first_free_block(offset);
517 : }
518 :
519 :
520 :
521 :
522 :
523 : } // namespace detail
524 :
525 :
526 :
527 1 : table::table(
528 : context * c
529 : , xml_node::pointer_t x
530 1 : , xml_node::map_t complex_types)
531 1 : : f_impl(std::make_shared<detail::table_impl>(c, this, x, complex_types))
532 : {
533 1 : }
534 :
535 :
536 3 : table::pointer_t table::get_pointer()
537 : {
538 3 : return shared_from_this();
539 : }
540 :
541 :
542 0 : void table::load_extension(xml_node::pointer_t e)
543 : {
544 0 : f_impl->load_extension(e);
545 0 : }
546 :
547 :
548 3 : dbfile::pointer_t table::get_dbfile() const
549 : {
550 3 : return f_impl->get_dbfile();
551 : }
552 :
553 :
554 0 : version_t table::version() const
555 : {
556 0 : return f_impl->version();
557 : }
558 :
559 :
560 1 : std::string table::name() const
561 : {
562 1 : return f_impl->name();
563 : }
564 :
565 :
566 0 : model_t table::model() const
567 : {
568 0 : return f_impl->model();
569 : }
570 :
571 :
572 0 : column_ids_t table::row_key() const
573 : {
574 0 : return f_impl->row_key();
575 : }
576 :
577 :
578 0 : schema_column::pointer_t table::column(std::string const & name) const
579 : {
580 0 : return f_impl->column(name);
581 : }
582 :
583 :
584 0 : schema_column::pointer_t table::column(column_id_t id) const
585 : {
586 0 : return f_impl->column(id);
587 : }
588 :
589 :
590 0 : schema_column::map_by_id_t table::columns_by_id() const
591 : {
592 0 : return f_impl->columns_by_id();
593 : }
594 :
595 :
596 0 : schema_column::map_by_name_t table::columns_by_name() const
597 : {
598 0 : return f_impl->columns_by_name();
599 : }
600 :
601 :
602 0 : bool table::is_secure() const
603 : {
604 0 : return f_impl->is_secure();
605 : }
606 :
607 :
608 1 : bool table::is_sparse() const
609 : {
610 1 : return f_impl->is_sparse();
611 : }
612 :
613 :
614 0 : std::string table::description() const
615 : {
616 0 : return f_impl->description();
617 : }
618 :
619 :
620 0 : size_t table::get_size() const
621 : {
622 0 : return f_impl->get_size();
623 : }
624 :
625 :
626 26 : size_t table::get_page_size() const
627 : {
628 26 : return f_impl->get_page_size();
629 : }
630 :
631 :
632 0 : block::pointer_t table::get_block(reference_t offset)
633 : {
634 0 : return f_impl->get_block(offset);
635 : }
636 :
637 :
638 1 : block::pointer_t table::allocate_new_block(dbtype_t type)
639 : {
640 1 : return f_impl->allocate_new_block(type);
641 : }
642 :
643 :
644 0 : void table::free_block(block::pointer_t block, bool clear_block)
645 : {
646 0 : return f_impl->free_block(block, clear_block);
647 : }
648 :
649 :
650 1 : bool table::verify_schema()
651 : {
652 1 : return f_impl->verify_schema();
653 : }
654 :
655 :
656 6 : } // namespace snapdatabase
657 : // vim: ts=4 sw=4 et
|
