Line data Source code
1 : // Copyright (c) 2019 Made to Order Software Corp. All Rights Reserved
2 : //
3 : // https://snapwebsites.org/project/snapdatabase
4 : // contact@m2osw.com
5 : //
6 : // This program is free software; you can redistribute it and/or modify
7 : // it under the terms of the GNU General Public License as published by
8 : // the Free Software Foundation; either version 2 of the License, or
9 : // (at your option) any later version.
10 : //
11 : // This program is distributed in the hope that it will be useful,
12 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 : // GNU General Public License for more details.
15 : //
16 : // You should have received a copy of the GNU General Public License along
17 : // with this program; if not, write to the Free Software Foundation, Inc.,
18 : // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19 :
20 :
21 : /** \file
22 : * \brief Database file implementation.
23 : *
24 : * Each table uses one or more files. Each file is handled by a dbfile
25 : * object and a corresponding set of blocks.
26 : */
27 :
28 : #define NOQT
29 :
30 : // self
31 : //
32 : #include "snapdatabase/database/context.h"
33 :
34 :
35 : // snapwebsites lib
36 : //
37 : #include <snapwebsites/mkdir_p.h>
38 :
39 :
40 : // snaplogger lib
41 : //
42 : #include <snaplogger/message.h>
43 :
44 :
45 : // snapdev lib
46 : //
47 : #include <snapdev/glob_to_list.h>
48 :
49 :
50 : // C++ lib
51 : //
52 : #include <deque>
53 :
54 :
55 : // last include
56 : //
57 : #include <snapdev/poison.h>
58 :
59 :
60 :
61 : namespace snapdatabase
62 : {
63 :
64 :
65 :
66 : namespace detail
67 : {
68 :
69 :
70 :
71 : //#pragma GCC diagnostic push
72 : //#pragma GCC diagnostic ignored "-Weffc++"
73 : class context_impl
74 : {
75 : public:
76 : context_impl(context *c, advgetopt::getopt::pointer_t opts);
77 : context_impl(context_impl const & rhs) = delete;
78 : ~context_impl();
79 :
80 : context_impl & operator = (context_impl const & rhs) = delete;
81 :
82 : void initialize();
83 : table::pointer_t get_table(std::string const & name) const;
84 : table::map_t list_tables() const;
85 : std::string get_path() const;
86 :
87 : private:
88 : context * f_context = nullptr;
89 : advgetopt::getopt::pointer_t f_opts = advgetopt::getopt::pointer_t();
90 : std::string f_path = std::string();
91 : int f_lock = -1; // TODO: lock the context so only one snapdatabasedaemon can run against it
92 : table::map_t f_tables = table::map_t();
93 : };
94 : //#pragma GCC diagnostic pop
95 :
96 :
97 1 : context_impl::context_impl(context * c, advgetopt::getopt::pointer_t opts)
98 : : f_context(c)
99 1 : , f_opts(opts)
100 : {
101 1 : }
102 :
103 :
104 1 : context_impl::~context_impl()
105 : {
106 1 : }
107 :
108 :
109 1 : void context_impl::initialize()
110 : {
111 1 : f_path = f_opts->get_string("context");
112 :
113 2 : SNAP_LOG_NOTICE
114 1 : << "Initialize context \""
115 : << f_path
116 : << "\"."
117 : << SNAP_LOG_SEND;
118 :
119 1 : if(f_path.empty())
120 : {
121 0 : f_path = "/var/lib/snapwebsites/database";
122 : }
123 1 : if(snap::mkdir_p(f_path, false, 0700, "snapwebsites", "snapwebsites") != 0)
124 : {
125 : throw io_error(
126 : "Could not create or access the context directory \""
127 0 : + f_path
128 0 : + "\".");
129 : }
130 :
131 2 : xml_node::deque_t table_extensions;
132 :
133 1 : size_t const max(f_opts->size("table_schema_path"));
134 :
135 2 : SNAP_LOG_NOTICE
136 1 : << "Reading context "
137 1 : << max
138 : << " XML schemata."
139 : << SNAP_LOG_SEND;
140 :
141 2 : for(size_t idx(0); idx < max; ++idx)
142 : {
143 2 : std::string const path(f_opts->get_string("table_schema_path", idx));
144 :
145 2 : snap::glob_to_list<std::deque<std::string>> list;
146 2 : if(!list.read_path<
147 : snap::glob_to_list_flag_t::GLOB_FLAG_ONLY_DIRECTORIES
148 2 : , snap::glob_to_list_flag_t::GLOB_FLAG_TILDE>(path + "/*.xml"))
149 : {
150 0 : SNAP_LOG_WARNING
151 0 : << "Could not read directory \""
152 : << path
153 : << "\" for XML table declarations."
154 : << SNAP_LOG_SEND;
155 0 : continue;
156 : }
157 :
158 1 : if(list.empty())
159 : {
160 0 : SNAP_LOG_DEBUG
161 0 : << "Directory \""
162 : << path
163 : << "\" is empty."
164 : << SNAP_LOG_SEND;
165 0 : continue;
166 : }
167 :
168 : // TODO: this is perfect for workers to distribute the load on many
169 : // threads (and then the creation/loading of each table)
170 : //
171 2 : for(auto const & filename : list)
172 : {
173 2 : xml x(filename);
174 :
175 2 : xml_node::pointer_t root(x.root());
176 1 : if(root == nullptr)
177 : {
178 0 : SNAP_LOG_WARNING
179 0 : << "Problem reading table schema \""
180 : << filename
181 : << "\" is not acceptable."
182 : << SNAP_LOG_SEND;
183 0 : continue;
184 : }
185 :
186 2 : if(root->tag_name() != "keyspaces"
187 1 : && root->tag_name() != "context")
188 : {
189 0 : SNAP_LOG_WARNING
190 0 : << "Table \""
191 : << filename
192 : << "\" schema must be a \"keyspaces\" or \"context\". \""
193 0 : << root->tag_name()
194 : << "\" is not acceptable."
195 : << SNAP_LOG_SEND;
196 0 : continue;
197 : }
198 :
199 2 : xml_node::map_t complex_types;
200 2 : for(auto child(root->first_child()); child != nullptr; child = child->next())
201 : {
202 1 : if(child->tag_name() == "complex-type")
203 : {
204 0 : std::string const name(child->attribute("name"));
205 0 : if(name_to_struct_type(name) != INVALID_STRUCT_TYPE)
206 : {
207 0 : SNAP_LOG_WARNING
208 0 : << "The name of a complex type cannot be the name of a system type. \""
209 : << name
210 : << "\" is not acceptable."
211 : << SNAP_LOG_SEND;
212 0 : continue;
213 : }
214 0 : if(complex_types.find(name) != complex_types.end())
215 : {
216 0 : SNAP_LOG_WARNING
217 0 : << "The complex type named \""
218 : << name
219 : << "\" is defined twice. Only the very first intance is used."
220 : << SNAP_LOG_SEND;
221 0 : continue;
222 : }
223 0 : complex_types[name] = child;
224 : }
225 : }
226 :
227 2 : for(auto child(root->first_child()); child != nullptr; child = child->next())
228 : {
229 1 : if(child->tag_name() == "table")
230 : {
231 2 : table::pointer_t t(std::make_shared<table>(f_context, child, complex_types));
232 1 : f_tables[t->name()] = t;
233 :
234 2 : dbfile::pointer_t dbfile(t->get_dbfile());
235 1 : dbfile->set_table(t);
236 1 : dbfile->set_sparse(t->is_sparse());
237 : }
238 0 : else if(child->tag_name() == "table-extension")
239 : {
240 : // we collect those and process them in a second
241 : // pass after we loaded all the XML files because
242 : // otherwise the table may still be missing
243 : //
244 0 : table_extensions.push_back(child);
245 : }
246 0 : else if(child->tag_name() == "complex-type")
247 : {
248 : // already worked on; ignore in this loop
249 : }
250 : else
251 : {
252 : // generate an error for unknown tags or ignore?
253 : //
254 0 : SNAP_LOG_WARNING
255 0 : << "Unknown tag \""
256 0 : << child->tag_name()
257 : << "\" within a <context> tag ignored."
258 : << SNAP_LOG_SEND;
259 : }
260 : }
261 : }
262 : }
263 :
264 2 : SNAP_LOG_NOTICE
265 1 : << "Adding "
266 1 : << table_extensions.size()
267 : << " XML schema extensions."
268 : << SNAP_LOG_SEND;
269 :
270 1 : for(auto const & e : table_extensions)
271 : {
272 0 : std::string const name(e->attribute("name"));
273 0 : table::pointer_t t(get_table(name));
274 0 : if(t == nullptr)
275 : {
276 0 : SNAP_LOG_WARNING
277 0 : << "Unknown table \""
278 0 : << e->tag_name()
279 : << "\" within a <table-extension>, tag ignored."
280 : << SNAP_LOG_SEND;
281 0 : continue;
282 : }
283 0 : t->load_extension(e);
284 : }
285 :
286 2 : SNAP_LOG_NOTICE
287 1 : << "Verify "
288 1 : << f_tables.size()
289 : << " table schemata."
290 : << SNAP_LOG_SEND;
291 :
292 2 : for(auto & t : f_tables)
293 : {
294 1 : t.second->verify_schema();
295 : }
296 :
297 2 : SNAP_LOG_INFORMATION
298 1 : << "Context \""
299 : << f_path
300 : << "\" ready."
301 : << SNAP_LOG_SEND;
302 1 : }
303 :
304 :
305 0 : table::pointer_t context_impl::get_table(std::string const & name) const
306 : {
307 0 : auto it(f_tables.find(name));
308 0 : if(it == f_tables.end())
309 : {
310 0 : return table::pointer_t();
311 : }
312 :
313 0 : return it->second;
314 : }
315 :
316 :
317 0 : table::map_t context_impl::list_tables() const
318 : {
319 0 : return f_tables;
320 : }
321 :
322 :
323 1 : std::string context_impl::get_path() const
324 : {
325 1 : return f_path;
326 : }
327 :
328 :
329 :
330 : } // namespace detail
331 :
332 :
333 :
334 1 : context::context(advgetopt::getopt::pointer_t opts)
335 1 : : f_impl(std::make_unique<detail::context_impl>(this, opts))
336 : {
337 1 : }
338 :
339 :
340 : // required for the std::unique_ptr<>() of the impl
341 1 : context::~context()
342 : {
343 1 : }
344 :
345 :
346 1 : context::pointer_t context::get_context(advgetopt::getopt::pointer_t opts)
347 : {
348 1 : std::cerr << "context: create\n";
349 1 : pointer_t c(new context(opts));
350 1 : std::cerr << "context: initialize\n";
351 1 : c->initialize();
352 1 : std::cerr << "context: ready to return\n";
353 1 : return c;
354 : }
355 :
356 :
357 1 : void context::initialize()
358 : {
359 1 : f_impl->initialize();
360 1 : }
361 :
362 :
363 0 : table::pointer_t context::get_table(std::string const & name) const
364 : {
365 0 : return f_impl->get_table(name);
366 : }
367 :
368 :
369 0 : table::map_t context::list_tables() const
370 : {
371 0 : return f_impl->list_tables();
372 : }
373 :
374 :
375 1 : std::string context::get_path() const
376 : {
377 1 : return f_impl->get_path();
378 : }
379 :
380 :
381 : /** \brief Signal that a new allocation was made.
382 : *
383 : * This function is just a signal sent to the memory manager so it knows
384 : * it should check and see whether too much memory is currently used and
385 : * attempt to release some memory.
386 : *
387 : * \note
388 : * The memory manager runs in a separate thread.
389 : */
390 3 : void context::limit_allocated_memory()
391 : {
392 3 : }
393 :
394 :
395 :
396 6 : } // namespace snapdatabase
397 : // vim: ts=4 sw=4 et
|
