# # # add_file "threads.hh" # content [80006806f27cc64ff57db6193ac7ffd97016ce56] # # add_file "unix/threads.cc" # content [ad31b34f7d40748854fdd212a97c3b4fead03985] # # patch "Makefile.am" # from [f38db1b6891b304f75527eb30853aa570ff15b4d] # to [98ab414b4971b02706d3b6437220c3a6d3c8d637] # # patch "automate.cc" # from [b23eced8b6c237fd768369cf7d32ba6119f19fe4] # to [f725eb6f6b9a1ef11b3caa5602fed3ec313e1304] # # patch "base.hh" # from [0d3a9bd364e4f1d791384e11074a86e29a24080c] # to [fcc11f30cf6be39c5eb04d1907259572cdfbbb14] # # patch "file_io.cc" # from [e18d07679c2d210a40f18b67ed0b4b96f5a2c21a] # to [b754b2fa192ca610b61f8e7e6533024be0bcf454] # # patch "file_io.hh" # from [e477a87af2f1d3326d5f07f9c9ed6ab322809fe6] # to [5f5e79a1c9190ef94d51c05910a67972478ef8bd] # # patch "work.cc" # from [0261cdf7dbc3baed0cdfc0e75a3a858f64eefa97] # to [f341028f669933e21aa7cfeb2de00b639f2ae472] # ============================================================ --- threads.hh 80006806f27cc64ff57db6193ac7ffd97016ce56 +++ threads.hh 80006806f27cc64ff57db6193ac7ffd97016ce56 @@ -0,0 +1,61 @@ +#ifndef __THREADS_HH__ +#define __THREADS_HH__ + +// Copyright (C) 2008 Markus Schiltknecht +// +// This program is made available under the GNU GPL version 2.0 or +// greater. See the accompanying file COPYING for details. +// +// This program is distributed WITHOUT ANY WARRANTY; without even the +// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +// PURPOSE. + +#include + +class thread_functor +{ +public: + virtual void operator()() = 0; +}; + +struct +thread_context +{ + thread_functor * func; +}; + +extern void create_thread_for(thread_functor * func); + +template +class worker_pool +{ +public: + worker_pool() + { }; + + void add_job(boost::shared_ptr p1, boost::shared_ptr p2) + { + I(p1); + I(p2); + thread_functor *func(new TASK(p1, p2)); + func->operator()(); + //create_thread_for(func); + delete func; + } + + void wait(void) + { + // + // sleep(10); + } +}; + +// Local Variables: +// mode: C++ +// fill-column: 76 +// c-file-style: "gnu" +// indent-tabs-mode: nil +// End: +// vim: et:sw=2:sts=2:ts=2:cino=>2s,{s,\:s,+s,t0,g0,^-2,e-2,n-2,p2s,(0,=s: + +#endif // __THREADS_HH__ ============================================================ --- unix/threads.cc ad31b34f7d40748854fdd212a97c3b4fead03985 +++ unix/threads.cc ad31b34f7d40748854fdd212a97c3b4fead03985 @@ -0,0 +1,64 @@ +// Copyright (C) 2008 Markus Schiltknecht +// +// This program is made available under the GNU GPL version 2.0 or +// greater. See the accompanying file COPYING for details. +// +// This program is distributed WITHOUT ANY WARRANTY; without even the +// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +// PURPOSE. + + +// This file provides very basic threading support, just enough to allow +// multiple worker threads to process small, enclosed jobs concurrently. +// These jobs need to be separated very well. They get an input and +// should provide some output, but may not interfere with the rest of +// monotone in any other way, because we don't want to buy into complex +// locking problems. +// +// When writing jobs for a threaded task, please be aware, that these +// tasks MUST NOT throw exceptions. Additionally, you have to take care +// and make sure all objects used by that thread are valid until the job +// is done. And of course you need to avoid concurrent access to objects. + +#include "base.hh" +#include + +#include "sanity.hh" +#include "threads.hh" + +void *threaded_call(void *c) +{ + thread_context * ctx = (thread_context*) c; + + (*ctx->func)(); + + pthread_exit(NULL); +} + +void +create_thread_for(thread_functor * func) +{ + int rc; + void *status; + pthread_t thread; + + thread_context * ctx = new thread_context(); + ctx->func = func; + + rc = pthread_create(&thread, NULL, threaded_call, (void*) &ctx); + I(!rc); + + rc = pthread_join(thread, &status); + I(!rc); + + delete ctx; + delete func; +} + +// Local Variables: +// mode: C++ +// fill-column: 76 +// c-file-style: "gnu" +// indent-tabs-mode: nil +// End: +// vim: et:sw=2:sts=2:ts=2:cino=>2s,{s,\:s,+s,t0,g0,^-2,e-2,n-2,p2s,(0,=s: ============================================================ --- Makefile.am f38db1b6891b304f75527eb30853aa570ff15b4d +++ Makefile.am 98ab414b4971b02706d3b6437220c3a6d3c8d637 @@ -293,7 +293,7 @@ UNIX_PLATFORM_SOURCES = \ unix/process.cc unix/terminal.cc unix/inodeprint.cc \ unix/fs.cc unix/make_io_binary.cc unix/os_strerror.cc \ unix/cputime.cc unix/ssh_agent_platform.cc \ - unix/ssh_agent_platform.hh + unix/ssh_agent_platform.hh unix/threads.cc WIN32_PLATFORM_SOURCES = \ win32/read_password.cc win32/get_system_flavour.cc \ @@ -415,21 +415,21 @@ mtn_LDFLAGS = lib3rdparty_a_CXXFLAGS = $(AM_CXXFLAGS) $(LIB3RDPARTY_CXXFLAGS) mtn_LDFLAGS = -mtn_CPPFLAGS = -I$(top_srcdir)/lua +mtn_CPPFLAGS = -D_REENTRANT -I$(top_srcdir)/lua mtn_CFLAGS = $(AM_CFLAGS) $(MTN_CFLAGS) mtn_CXXFLAGS = $(AM_CXXFLAGS) $(PCH_FLAGS) $(MTN_CXXFLAGS) -mtn_LDADD = libplatform.a lib3rdparty.a $(BOOSTLIBS) $(LIBICONV) $(LIBINTL) +mtn_LDADD = libplatform.a lib3rdparty.a -lpthread $(BOOSTLIBS) $(LIBICONV) $(LIBINTL) unit_tester_LDFLAGS = -unit_tester_CPPFLAGS = -DBUILD_UNIT_TESTS -I$(top_srcdir)/lua +unit_tester_CPPFLAGS = -DBUILD_UNIT_TESTS -D_REENTRANT -I$(top_srcdir)/lua unit_tester_CXXFLAGS = $(AM_CXXFLAGS) $(PCH_FLAGS) -unit_tester_LDADD += libplatform.a lib3rdparty.a \ +unit_tester_LDADD += libplatform.a lib3rdparty.a -lpthread \ $(BOOSTLIBS) $(LIBICONV) $(LIBINTL) tester_LDFLAGS = -tester_CPPFLAGS = -I$(top_srcdir)/lua +tester_CPPFLAGS = -D_REENTRANT -I$(top_srcdir)/lua tester_CXXFLAGS = $(AM_CXXFLAGS) -tester_LDADD += libplatform.a lib3rdparty.a $(BOOSTLIBS) $(LIBICONV) $(LIBINTL) +tester_LDADD += libplatform.a lib3rdparty.a -lpthread $(BOOSTLIBS) $(LIBICONV) $(LIBINTL) if INCLUDED_PCRE lib3rdparty_a_CPPFLAGS += -I$(top_srcdir)/pcre ============================================================ --- automate.cc b23eced8b6c237fd768369cf7d32ba6119f19fe4 +++ automate.cc f725eb6f6b9a1ef11b3caa5602fed3ec313e1304 @@ -19,6 +19,7 @@ #include #include #include +#include #include "app_state.hh" #include "project.hh" @@ -62,7 +63,9 @@ using std::vector; using std::string; using std::vector; +using boost::shared_ptr; + // Name: heads // Arguments: // 1: branch name (optional, default branch is used if non-existant) @@ -755,11 +758,13 @@ struct inventory_itemizer : public tree_ inventory_map & inventory; inodeprint_map ipm; workspace & work; + file_ident_pool & pool; inventory_itemizer(workspace & work, + file_ident_pool & pool, path_restriction const & m, inventory_map & i) - : mask(m), inventory(i), work(work) + : mask(m), inventory(i), work(work), pool(pool) { if (work.in_inodeprints_mode()) { @@ -797,7 +802,14 @@ inventory_itemizer::visit_file(file_path if (inodeprint_unchanged(ipm, path)) item.fs_ident = item.old_node.ident; else - ident_existing_file(path, item.fs_ident); + { + // FIXME: pool?? + shared_ptr fpath(new file_path(path)); + shared_ptr fid(new file_id()); + ident_existing_file(pool, fpath, fid); + pool.wait(); + item.fs_ident = *fid; + } } } } @@ -807,7 +819,8 @@ inventory_filesystem(workspace & work, path_restriction const & mask, inventory_map & inventory) { - inventory_itemizer itemizer(work, mask, inventory); + file_ident_pool pool; + inventory_itemizer itemizer(work, pool, mask, inventory); file_path const root; // The constructor file_path() returns ""; the root directory. walk_tree // does not visit that node, so set fs_type now, if it meets the @@ -817,6 +830,7 @@ inventory_filesystem(workspace & work, inventory[root].fs_type = path::directory; } walk_tree(root, itemizer); + pool.wait(); } namespace ============================================================ --- base.hh 0d3a9bd364e4f1d791384e11074a86e29a24080c +++ base.hh fcc11f30cf6be39c5eb04d1907259572cdfbbb14 @@ -16,8 +16,6 @@ // Configuration directives #include "config.h" -#define BOOST_DISABLE_THREADS -#define BOOST_SP_DISABLE_THREADS #define BOOST_MULTI_INDEX_DISABLE_SERIALIZATION #include ============================================================ --- file_io.cc e18d07679c2d210a40f18b67ed0b4b96f5a2c21a +++ file_io.cc b754b2fa192ca610b61f8e7e6533024be0bcf454 @@ -11,6 +11,8 @@ #include #include +#include + #include "botan/botan.h" #include "botan_pipe_cache.hh" @@ -32,6 +34,8 @@ using std::vector; using std::string; using std::vector; +using boost::shared_ptr; + void assert_path_is_nonexistent(any_path const & path) { @@ -529,14 +533,36 @@ walk_tree(file_path const & path, tree_w } } +class file_hash_calc_task + : public thread_functor +{ + shared_ptr path; + shared_ptr ident; + +public: + file_hash_calc_task(shared_ptr path, shared_ptr ident) + : path(path), + ident(ident) + { } + + virtual void operator()() + { + calculate_ident(*path, *ident); + } +}; + +template class worker_pool; + bool -ident_existing_file(file_path const & p, file_id & ident) +ident_existing_file(file_ident_pool & pool, shared_ptr p, + shared_ptr ident) { - return ident_existing_file(p, ident, get_path_status(p)); + return ident_existing_file(pool, p, ident, get_path_status(*p)); } bool -ident_existing_file(file_path const & p, file_id & ident, path::status status) +ident_existing_file(file_ident_pool & pool, shared_ptr p, + shared_ptr ident, path::status status) { switch (status) { @@ -549,7 +575,7 @@ ident_existing_file(file_path const & p, return false; } - calculate_ident(p, ident); + pool.add_job(p, ident); return true; } ============================================================ --- file_io.hh e477a87af2f1d3326d5f07f9c9ed6ab322809fe6 +++ file_io.hh 5f5e79a1c9190ef94d51c05910a67972478ef8bd @@ -15,7 +15,12 @@ #include "sanity.hh" #include "platform-wrapped.hh" #include "vector.hh" +#include "threads.hh" +#include + +using std::pair; + // this layer deals with talking to the filesystem, loading and saving // files, walking trees, etc. @@ -116,9 +121,19 @@ void walk_tree(file_path const & path, void walk_tree(file_path const & path, tree_walker & walker); +// define a thread pool for file identifiers, where the workers take +// a file_path as an input, and output a success indicator as well +// as a file_id, in case of success. +struct file_hash_calc_task; +typedef worker_pool file_ident_pool; -bool ident_existing_file(file_path const & p, file_id & ident); -bool ident_existing_file(file_path const & p, file_id & ident, path::status status); +bool ident_existing_file(file_ident_pool & pool, + boost::shared_ptr p, + boost::shared_ptr ident); +bool ident_existing_file(file_ident_pool & pool, + boost::shared_ptr p, + boost::shared_ptr ident, + path::status status); void calculate_ident(file_path const & file, file_id & ident); ============================================================ --- work.cc 0261cdf7dbc3baed0cdfc0e75a3a858f64eefa97 +++ work.cc f341028f669933e21aa7cfeb2de00b639f2ae472 @@ -16,6 +16,8 @@ #include #include +#include + #include "lexical_cast.hh" #include "basic_io.hh" #include "cset.hh" @@ -45,6 +47,7 @@ using boost::lexical_cast; using std::vector; using boost::lexical_cast; +using boost::shared_ptr; // workspace / book-keeping file code @@ -803,7 +806,12 @@ addition_builder::add_nodes_for(file_pat case path::file: { file_id ident; - I(ident_existing_file(path, ident)); + file_ident_pool pool; // FIXME! + shared_ptr fpath(new file_path(path)); + shared_ptr fid(new file_id()); + I(ident_existing_file(pool, fpath, fid)); + pool.wait(); + ident = *fid; nid = er.create_file_node(ident); } break; @@ -1309,8 +1317,18 @@ workspace::update_current_roster_from_fi missing_items++; } - file_t file = downcast_to_file_t(node); - ident_existing_file(fp, file->content, status); + { + file_ident_pool pool; + shared_ptr fid(new file_id()); + shared_ptr fpath(new file_path(fp)); + ident_existing_file(pool, fpath, fid, status); + + // wait until all jobs in the pool are done. + pool.wait(); + + file_t file = downcast_to_file_t(node); + file->content = *fid; + } } } @@ -1508,9 +1526,12 @@ workspace::perform_deletions(database & else { file_t file = downcast_to_file_t(n); - file_id fid; - I(ident_existing_file(name, fid)); - if (file->content == fid) + shared_ptr fpath(new file_path(name)); + shared_ptr fid(new file_id()); + file_ident_pool pool; // FIXME: speed this up! + I(ident_existing_file(pool, fpath, fid)); + pool.wait(); + if (file->content == *fid) delete_file_or_dir_shallow(name); else W(F("file %s changed - "