# # patch "commands.cc" # from [2a7bdadba12f9ce7d848b26444d99e54af32d066] # to [e786428e36bd7fc5df44bd4d98eb400e5dfc4239] # # patch "cset.hh" # from [876dbf82988387b752c4df4af310bad90e74e17d] # to [138e97c86d28f67b6acea60fec121f71f6ce7705] # # patch "database.cc" # from [f685efd798be6574f23f31e8040923211d8175a9] # to [4ee08f7847908cc5b369a804a246bffdbf7aaa7e] # # patch "hash_map.hh" # from [70ec673ffc3af0f40bd3379dbb93b52296cff714] # to [b703a6be06d5a17c3808574028b185071b579d92] # # patch "merkle_tree.cc" # from [618189095f626048644ebf41a3054d62d52c9775] # to [a5cbb7d8f6886a4ad7f538bd81bd4cabda269652] # # patch "merkle_tree.hh" # from [aa4d5bed0d7dee09fd72c2068c626061c30a6937] # to [e3f9a996ca4157fd1bf7cd6ad2b1fa6388480bbb] # # patch "netcmd.cc" # from [75a0d51313f112ec54f83d29eca1d1077d2547bb] # to [a7cfac95c23b6f581873c46a4d2c81f8b0c74a0e] # # patch "netsync.cc" # from [50c6e680f917a805743bdbfec8d0235823a31e3d] # to [2726e956d173e0d3fc6e378c9c8c0b5bf3515856] # # patch "packet.cc" # from [7571b82d3dd14b2b562fec9de767ba069c9e801a] # to [0156fd6ccc47c2a0b3c7066cbd55f3bc682435a1] # # patch "packet.hh" # from [ec7178c2332473305c0aa7d00c727e338fc7810d] # to [c10d66b39bd6aa88cebc4993187fa710f7e7d277] # ======================================================================== --- commands.cc 2a7bdadba12f9ce7d848b26444d99e54af32d066 +++ commands.cc e786428e36bd7fc5df44bd4d98eb400e5dfc4239 @@ -1768,32 +1768,6 @@ ALIAS(ls, list) -CMD(mdelta, N_("packet i/o"), N_("OLDID NEWID"), - N_("write manifest delta packet to stdout"), - OPT_NONE) -{ - if (args.size() != 2) - throw usage(name); - - packet_writer pw(cout); - - manifest_id m_old_id, m_new_id; - manifest_map m_old, m_new; - - complete(app, idx(args, 0)(), m_old_id); - complete(app, idx(args, 1)(), m_new_id); - - N(app.db.manifest_version_exists(m_old_id), F("no such manifest '%s'") % m_old_id); - app.db.get_manifest(m_old_id, m_old); - N(app.db.manifest_version_exists(m_new_id), F("no such manifest '%s'") % m_new_id); - app.db.get_manifest(m_new_id, m_new); - - delta del; - diff(m_old, m_new, del); - pw.consume_manifest_delta(m_old_id, m_new_id, - manifest_delta(del)); -} - CMD(fdelta, N_("packet i/o"), N_("OLDID NEWID"), N_("write file delta packet to stdout"), OPT_NONE) @@ -1836,25 +1810,7 @@ pw.consume_revision_data(r_id, r_data); } -CMD(mdata, N_("packet i/o"), N_("ID"), N_("write manifest data packet to stdout"), - OPT_NONE) -{ - if (args.size() != 1) - throw usage(name); - packet_writer pw(cout); - - manifest_id m_id; - manifest_data m_data; - - complete(app, idx(args, 0)(), m_id); - - N(app.db.manifest_version_exists(m_id), F("no such manifest '%s'") % m_id); - app.db.get_manifest_version(m_id, m_data); - pw.consume_manifest_data(m_id, m_data); -} - - CMD(fdata, N_("packet i/o"), N_("ID"), N_("write file data packet to stdout"), OPT_NONE) { ======================================================================== --- cset.hh 876dbf82988387b752c4df4af310bad90e74e17d +++ cset.hh 138e97c86d28f67b6acea60fec121f71f6ce7705 @@ -90,6 +90,25 @@ void clear(); }; +inline split_path const & +delta_entry_path(std::map >::const_iterator i) +{ + return i->first; +} + +inline file_id const & +delta_entry_src(std::map >::const_iterator i) +{ + return i->second.first; +} + +inline file_id const & +delta_entry_dst(std::map >::const_iterator i) +{ + return i->second.second; +} + + namespace basic_io { struct printer; struct parser; } void ======================================================================== --- database.cc f685efd798be6574f23f31e8040923211d8175a9 +++ database.cc 4ee08f7847908cc5b369a804a246bffdbf7aaa7e @@ -1,9 +1,11 @@ // -*- mode: C++; c-file-style: "gnu"; indent-tabs-mode: nil -*- // copyright (C) 2002, 2003 graydon hoare // all rights reserved. // licensed to the public under the terms of the GNU GPL (>= 2) // see the file COPYING for details +#include +#include #include #include #include @@ -23,6 +25,7 @@ #include "cleanup.hh" #include "constants.hh" #include "database.hh" +#include "hash_map.hh" #include "keys.hh" #include "sanity.hh" #include "schema_migration.hh" @@ -2454,22 +2457,111 @@ I(false); } + +typedef hashmap::hash_multimap ancestry_map; + +static void +transitive_closure(string const & x, + ancestry_map const & m, + set & results) +{ + results.clear(); + + deque work; + work.push_back(x); + while (!work.empty()) + { + string c = work.front(); + work.pop_front(); + revision_id curr(c); + if (results.find(curr) == results.end()) + { + results.insert(curr); + pair range; + range = m.equal_range(c); + for (ancestry_map::const_iterator i = range.first; i != range.second; ++i) + { + if (i->first == c) + work.push_back(i->second); + } + } + } +} + void database::get_uncommon_ancestors(revision_id const & a, revision_id const & b, - std::set & a_uncommon_ancs, - std::set & b_uncommon_ancs) + set & a_uncommon_ancs, + set & b_uncommon_ancs) { - // FIXME: implement - I(false); + // FIXME: This is a somewhat ugly, and possibly unaccepably slow way + // to do it. Another approach involves maintaining frontier sets for + // each and slowly deepening them into history; would need to + // benchmark to know which is actually faster on real datasets. + + a_uncommon_ancs.clear(); + b_uncommon_ancs.clear(); + + results res; + a_uncommon_ancs.clear(); + b_uncommon_ancs.clear(); + + fetch(res, 2, any_rows, + "SELECT parent,child FROM revision_ancestry"); + + set a_ancs, b_ancs; + + ancestry_map child_to_parent_map; + for (size_t i = 0; i < res.size(); ++i) + child_to_parent_map.insert(make_pair(res[i][1], res[i][0])); + + transitive_closure(a.inner()(), child_to_parent_map, a_ancs); + transitive_closure(b.inner()(), child_to_parent_map, b_ancs); + + set_difference(a_ancs.begin(), a_ancs.end(), + b_ancs.begin(), b_ancs.end(), + inserter(a_uncommon_ancs, a_uncommon_ancs.begin())); + + set_difference(b_ancs.begin(), b_ancs.end(), + a_ancs.begin(), a_ancs.end(), + inserter(b_uncommon_ancs, b_uncommon_ancs.begin())); } node_id database::next_node_id() { - // FIXME: implement - I(false); - return 0; + // If you acquire a new ID, we require that you write the new roster + // that uses the ID atomically with the counter update. A + // half-measure to meet this requirement is to require that we're at + // least inside a transaction when called. + I(transaction_level > 0); + results res; + + // We implement this as a fixed db var. + + string domain = "system"; + string name = "next_roster_id"; + + fetch(res, 1, any_rows, + "SELECT value FROM db_vars WHERE domain = ? AND name = ?", + domain.c_str(), name.c_str()); + + node_id n = 0; + if (res.empty()) + { + n = 1; + execute ("INSERT INTO db_vars VALUES(?, ?, ?)", + domain.c_str(), name.c_str(), lexical_cast(n).c_str()); + } + else + { + n = lexical_cast(res[0][0]); + ++n; + execute ("UPDATE db_vars SET value = ? WHERE domain = ? and name = ?", + lexical_cast(n).c_str(), domain.c_str(), name.c_str()); + + } + return n; } ======================================================================== --- hash_map.hh 70ec673ffc3af0f40bd3379dbb93b52296cff714 +++ hash_map.hh b703a6be06d5a17c3808574028b185071b579d92 @@ -10,6 +10,7 @@ namespace hashmap { using __gnu_cxx::hash_map; + using __gnu_cxx::hash_multimap; struct string_hash { ======================================================================== --- merkle_tree.cc 618189095f626048644ebf41a3054d62d52c9775 +++ merkle_tree.cc a5cbb7d8f6886a4ad7f538bd81bd4cabda269652 @@ -32,9 +32,6 @@ case revision_item: typestr = "revision"; break; - case manifest_item: - typestr = "manifest"; - break; case file_item: typestr = "file"; break; @@ -73,7 +70,7 @@ total_num_leaves(0), bitmap(constants::merkle_bitmap_length_in_bits), slots(constants::merkle_num_slots), - type(manifest_item) + type(revision_item) {} bool ======================================================================== --- merkle_tree.hh aa4d5bed0d7dee09fd72c2068c626061c30a6937 +++ merkle_tree.hh e3f9a996ca4157fd1bf7cd6ad2b1fa6388480bbb @@ -33,7 +33,6 @@ typedef enum { - manifest_item = 1, file_item = 2, key_item = 3, revision_item = 4, ======================================================================== --- netcmd.cc 75a0d51313f112ec54f83d29eca1d1077d2547bb +++ netcmd.cc a7cfac95c23b6f581873c46a4d2c81f8b0c74a0e @@ -28,8 +28,6 @@ { case static_cast(revision_item): return revision_item; - case static_cast(manifest_item): - return manifest_item; case static_cast(file_item): return file_item; case static_cast(cert_item): @@ -747,7 +745,7 @@ L(boost::format("checking i/o round trip on done_cmd\n")); netcmd out_cmd, in_cmd; size_t out_level(12), in_level; - netcmd_item_type out_type(key_item), in_type(manifest_item); + netcmd_item_type out_type(key_item), in_type(revision_item); string buf; out_cmd.write_done_cmd(out_level, out_type); ======================================================================== --- netsync.cc 50c6e680f917a805743bdbfec8d0235823a31e3d +++ netsync.cc 2726e956d173e0d3fc6e378c9c8c0b5bf3515856 @@ -365,7 +365,6 @@ map attached, set & visited); void analyze_ancestry_graph(); - void analyze_manifest(manifest_map const & man); Netxx::Probe::ready_type which_events() const; bool read_some(); @@ -538,14 +537,12 @@ requested_items.insert(make_pair(cert_item, boost::shared_ptr< set >(new set()))); requested_items.insert(make_pair(key_item, boost::shared_ptr< set >(new set()))); requested_items.insert(make_pair(revision_item, boost::shared_ptr< set >(new set()))); - requested_items.insert(make_pair(manifest_item, boost::shared_ptr< set >(new set()))); requested_items.insert(make_pair(file_item, boost::shared_ptr< set >(new set()))); requested_items.insert(make_pair(epoch_item, boost::shared_ptr< set >(new set()))); received_items.insert(make_pair(cert_item, boost::shared_ptr< set >(new set()))); received_items.insert(make_pair(key_item, boost::shared_ptr< set >(new set()))); received_items.insert(make_pair(revision_item, boost::shared_ptr< set >(new set()))); - received_items.insert(make_pair(manifest_item, boost::shared_ptr< set >(new set()))); received_items.insert(make_pair(file_item, boost::shared_ptr< set >(new set()))); received_items.insert(make_pair(epoch_item, boost::shared_ptr< set >(new set()))); } @@ -860,21 +857,6 @@ encountered_error = true; } -void -session::analyze_manifest(manifest_map const & man) -{ - L(F("analyzing %d entries in manifest\n") % man.size()); - for (manifest_map::const_iterator i = man.begin(); - i != man.end(); ++i) - { - if (! this->app.db.file_version_exists(manifest_entry_id(i))) - { - id tmp; - decode_hexenc(manifest_entry_id(i).inner(), tmp); - queue_send_data_cmd(file_item, tmp); - } - } -} static bool is_attached(revision_id const & i, @@ -960,11 +942,8 @@ map attached, set visited) { -/* -// FIXME_ROSTERS: disabled until rewritten to use rosters typedef map > > ancestryT; - set seen_manifests; set seen_files; set frontier; @@ -993,51 +972,33 @@ next_frontier.insert(edge_old_revision(k)); - // check out the manifest delta edge - manifest_id parent_manifest = edge_old_manifest(k); - manifest_id child_manifest = j->second->second.new_manifest; + // check out each file delta edge + cset const & cs = edge_changes(k); - // first, if we have a child we've never seen before we will need - // to request it in its entrety. - if (seen_manifests.find(child_manifest) == seen_manifests.end()) - { - if (this->app.db.manifest_version_exists(child_manifest)) - L(F("not requesting (in reverse) initial manifest %s as we already have it\n") % child_manifest); - else + for (std::map::const_iterator a = cs.files_added.begin(); + a != cs.files_added.end(); ++a) + { + file_id child_file = a->second; + if (seen_files.find(child_file) == seen_files.end()) { - L(F("requesting (in reverse) initial manifest data %s\n") % child_manifest); - queue_send_data_cmd(manifest_item, plain_id(child_manifest)); + if (this->app.db.file_version_exists(child_file)) + L(F("not requesting (in reverse) initial file %s as we already have it\n") % child_file); + else + { + L(F("requesting (in reverse) initial file data %s\n") % child_file); + queue_send_data_cmd(file_item, plain_id(child_file)); + } + seen_files.insert(child_file); } - seen_manifests.insert(child_manifest); - } + } - // second, if the parent is nonempty, we want to ask for an edge to it - if (!parent_manifest.inner()().empty()) + for (std::map >::const_iterator d + = cs.deltas_applied.begin(); + d != cs.deltas_applied.end(); ++d) { - if (this->app.db.manifest_version_exists(parent_manifest)) - L(F("not requesting (in reverse) manifest delta to %s as we already have it\n") % parent_manifest); - else - { - L(F("requesting (in reverse) manifest delta %s -> %s\n") - % child_manifest % parent_manifest); - reverse_delta_requests.insert(make_pair(plain_id(child_manifest), - plain_id(parent_manifest))); - queue_send_delta_cmd(manifest_item, - plain_id(child_manifest), - plain_id(parent_manifest)); - } - seen_manifests.insert(parent_manifest); - } - - // check out each file delta edge - change_set const & cset = edge_changes(k); - for (change_set::delta_map::const_iterator d = cset.deltas.begin(); - d != cset.deltas.end(); ++d) - { file_id parent_file (delta_entry_src(d)); file_id child_file (delta_entry_dst(d)); - // first, if we have a child we've never seen before we will need // to request it in its entrety. if (seen_files.find(child_file) == seen_files.end()) @@ -1059,8 +1020,8 @@ L(F("not requesting (in reverse) file delta to %s as we already have it\n") % parent_file); else { - L(F("requesting (in reverse) file delta %s -> %s on %s\n") - % child_file % parent_file % delta_entry_path(d)); + L(F("requesting (in reverse) file delta %s -> %s\n") + % child_file % parent_file); reverse_delta_requests.insert(make_pair(plain_id(child_file), plain_id(parent_file))); queue_send_delta_cmd(file_item, @@ -1079,7 +1040,6 @@ } frontier = next_frontier; } -*/ } void @@ -1087,8 +1047,6 @@ map attached, set & visited) { -/* -// FIXME_ROSTERS: disabled until rewritten to use rosters if (visited.find(i) != visited.end()) return; @@ -1118,37 +1076,31 @@ I(an_attached_edge != j->second->second.edges.end()); - // check out the manifest delta edge - manifest_id parent_manifest = edge_old_manifest(an_attached_edge); - manifest_id child_manifest = j->second->second.new_manifest; - if (this->app.db.manifest_version_exists(child_manifest)) - L(F("not requesting forward manifest delta to '%s' as we already have it\n") - % child_manifest); - else - { - if (parent_manifest.inner()().empty()) - { - L(F("requesting full manifest data %s\n") % child_manifest); - queue_send_data_cmd(manifest_item, plain_id(child_manifest)); - } + // check out each file add and file delta edge + + cset const & an_attached_cset = edge_changes(an_attached_edge); + + for (std::map::const_iterator a = an_attached_cset.files_added.begin(); + a != an_attached_cset.files_added.end(); ++a) + { + file_id child_id = a->second; + if (this->app.db.file_version_exists(child_id)) + L(F("not requesting added file %s as we already have it\n") + % child_id); else { - L(F("requesting forward manifest delta %s -> %s\n") - % parent_manifest % child_manifest); - queue_send_delta_cmd(manifest_item, - plain_id(parent_manifest), - plain_id(child_manifest)); - } - } - - // check out each file delta edge - change_set const & an_attached_cset = edge_changes(an_attached_edge); - for (change_set::delta_map::const_iterator k = an_attached_cset.deltas.begin(); - k != an_attached_cset.deltas.end(); ++k) + L(F("requesting full file data %s\n") % child_id); + queue_send_data_cmd(file_item, plain_id(child_id)); + } + } + + for (std::map >::const_iterator k + = an_attached_cset.deltas_applied.begin(); + k != an_attached_cset.deltas_applied.end(); ++k) { if (this->app.db.file_version_exists(delta_entry_dst(k))) - L(F("not requesting forward delta %s -> %s on file %s as we already have it\n") - % delta_entry_src(k) % delta_entry_dst(k) % delta_entry_path(k)); + L(F("not requesting forward delta %s -> %s as we already have it\n") + % delta_entry_src(k) % delta_entry_dst(k)); else { if (delta_entry_src(k).inner()().empty()) @@ -1159,8 +1111,8 @@ else { - L(F("requesting forward delta %s -> %s on file %s\n") - % delta_entry_src(k) % delta_entry_dst(k) % delta_entry_path(k)); + L(F("requesting forward delta %s -> %s\n") + % delta_entry_src(k) % delta_entry_dst(k)); queue_send_delta_cmd(file_item, plain_id(delta_entry_src(k)), plain_id(delta_entry_dst(k))); @@ -1171,7 +1123,6 @@ // arrival of its prerequisites in the packet_db_writer this->dbw.consume_revision_data(j->first, j->second->first); } -*/ } void @@ -1566,7 +1517,7 @@ id const & base, id const & ident) { - I(type == manifest_item || type == file_item); + I(type == file_item); string typestr; netcmd_item_type_to_string(type, typestr); @@ -1637,7 +1588,7 @@ id const & ident, delta const & del) { - I(type == manifest_item || type == file_item); + I(type == file_item); I(! del().empty() || ident == base); string typestr; netcmd_item_type_to_string(type, typestr); @@ -2144,8 +2095,6 @@ { case key_item: return app.db.public_key_exists(hitem); - case manifest_item: - return app.db.manifest_version_exists(manifest_id(hitem)); case file_item: return app.db.file_version_exists(file_id(hitem)); case revision_item: @@ -2213,20 +2162,6 @@ } break; - case manifest_item: - if (app.db.manifest_version_exists(manifest_id(hitem))) - { - manifest_data mdat; - data dat; - app.db.get_manifest_version(manifest_id(hitem), mdat); - out = mdat.inner()(); - } - else - { - throw bad_decode(F("manifest '%s' does not exist in our database") % hitem); - } - break; - case file_item: if (app.db.file_version_exists(file_id(hitem))) { @@ -2717,30 +2652,6 @@ } } break; - - case manifest_item: - { - manifest_id mbase(hbase), mident(hident); - manifest_delta mdel; - if (this->app.db.manifest_version_exists(mbase) - && this->app.db.manifest_version_exists(mident)) - { - manifest_data base_mdat, ident_mdat; - data base_dat, ident_dat; - this->app.db.get_manifest_version(mbase, base_mdat); - this->app.db.get_manifest_version(mident, ident_mdat); - string tmp; - base_dat = base_mdat.inner(); - ident_dat = ident_mdat.inner(); - compute_delta(base_dat(), ident_dat(), tmp); - del = delta(tmp); - } - else - { - return process_send_data_cmd(type, ident); - } - } - break; default: throw bad_decode(F("delta requested for item type %s\n") % typestr); @@ -2868,21 +2779,6 @@ } break; - case manifest_item: - { - manifest_id mid(hitem); - if (this->app.db.manifest_version_exists(mid)) - L(F("manifest version '%s' already exists in our database\n") % hitem); - else - { - this->dbw.consume_manifest_data(mid, manifest_data(dat)); - manifest_map man; - read_manifest_map(data(dat), man); - analyze_manifest(man); - } - } - break; - case file_item: { file_id fid(hitem); @@ -2919,25 +2815,6 @@ switch (type) { - case manifest_item: - { - manifest_id src_manifest(hbase), dst_manifest(hident); - if (reverse_delta_requests.find(id_pair) - != reverse_delta_requests.end()) - { - reverse_delta_requests.erase(id_pair); - this->dbw.consume_manifest_reverse_delta(src_manifest, - dst_manifest, - manifest_delta(del)); - } - else - this->dbw.consume_manifest_delta(src_manifest, - dst_manifest, - manifest_delta(del)); - - } - break; - case file_item: { file_id src_file(hbase), dst_file(hident); ======================================================================== --- packet.cc 7571b82d3dd14b2b562fec9de767ba069c9e801a +++ packet.cc 0156fd6ccc47c2a0b3c7066cbd55f3bc682435a1 @@ -95,7 +95,6 @@ typedef enum { prereq_revision, - prereq_manifest, prereq_file } prereq_type; @@ -208,20 +207,6 @@ virtual ~delayed_revision_data_packet(); }; -class -delayed_manifest_data_packet - : public delayed_packet -{ - manifest_id ident; - manifest_data dat; -public: - delayed_manifest_data_packet(manifest_id const & i, - manifest_data const & md) - : ident(i), dat(md) - {} - virtual void apply_delayed_packet(packet_db_writer & pw); - virtual ~delayed_manifest_data_packet(); -}; class delayed_file_data_packet @@ -257,24 +242,6 @@ virtual ~delayed_file_delta_packet(); }; -class -delayed_manifest_delta_packet - : public delayed_packet -{ - manifest_id old_id; - manifest_id new_id; - manifest_delta del; - bool forward_delta; -public: - delayed_manifest_delta_packet(manifest_id const & oi, - manifest_id const & ni, - manifest_delta const & md, - bool fwd) - : old_id(oi), new_id(ni), del(md), forward_delta(fwd) - {} - virtual void apply_delayed_packet(packet_db_writer & pw); - virtual ~delayed_manifest_delta_packet(); -}; class delayed_revision_cert_packet @@ -332,19 +299,7 @@ W(F("discarding revision data packet %s with unmet dependencies\n") % ident); } -void -delayed_manifest_data_packet::apply_delayed_packet(packet_db_writer & pw) -{ - L(F("writing delayed manifest data packet for %s\n") % ident); - pw.consume_manifest_data(ident, dat); -} -delayed_manifest_data_packet::~delayed_manifest_data_packet() -{ - if (!all_prerequisites_satisfied()) - W(F("discarding manifest data packet %s with unmet dependencies\n") % ident); -} - void delayed_file_data_packet::apply_delayed_packet(packet_db_writer & pw) { @@ -358,26 +313,7 @@ I(all_prerequisites_satisfied()); } -void -delayed_manifest_delta_packet::apply_delayed_packet(packet_db_writer & pw) -{ - L(F("writing delayed manifest %s packet for %s -> %s\n") - % (forward_delta ? "delta" : "reverse delta") - % (forward_delta ? old_id : new_id) - % (forward_delta ? new_id : old_id)); - if (forward_delta) - pw.consume_manifest_delta(old_id, new_id, del); - else - pw.consume_manifest_reverse_delta(new_id, old_id, del); -} -delayed_manifest_delta_packet::~delayed_manifest_delta_packet() -{ - if (!all_prerequisites_satisfied()) - W(F("discarding manifest delta packet %s -> %s with unmet dependencies\n") - % old_id % new_id); -} - void delayed_file_delta_packet::apply_delayed_packet(packet_db_writer & pw) { @@ -475,7 +411,6 @@ size_t count; map > revision_prereqs; - map > manifest_prereqs; map > file_prereqs; // ticker cert; @@ -484,15 +419,12 @@ // ticker filec; bool revision_exists_in_db(revision_id const & r); - bool manifest_version_exists_in_db(manifest_id const & m); bool file_version_exists_in_db(file_id const & f); void get_revision_prereq(revision_id const & revision, shared_ptr & p); - void get_manifest_prereq(manifest_id const & manifest, shared_ptr & p); void get_file_prereq(file_id const & file, shared_ptr & p); void accepted_revision(revision_id const & r, packet_db_writer & dbw); - void accepted_manifest(manifest_id const & m, packet_db_writer & dbw); void accepted_file(file_id const & f, packet_db_writer & dbw); impl(app_state & app, bool take_keys) @@ -519,11 +451,6 @@ { i->second->cleanup(); } - for (map >::const_iterator i = - manifest_prereqs.begin(); i != manifest_prereqs.end(); i++) - { - i->second->cleanup(); - } for (map >::const_iterator i = file_prereqs.begin(); i != file_prereqs.end(); i++) { @@ -537,11 +464,6 @@ return app.db.revision_exists(r); } -bool -packet_db_writer::impl::manifest_version_exists_in_db(manifest_id const & m) -{ - return app.db.manifest_version_exists(m); -} bool packet_db_writer::impl::file_version_exists_in_db(file_id const & f) @@ -564,20 +486,6 @@ } } -void -packet_db_writer::impl::get_manifest_prereq(manifest_id const & man, - shared_ptr & p) -{ - map >::const_iterator i; - i = manifest_prereqs.find(man); - if (i != manifest_prereqs.end()) - p = i->second; - else - { - p = shared_ptr(new prerequisite(man.inner(), prereq_manifest)); - manifest_prereqs.insert(make_pair(man, p)); - } -} void packet_db_writer::impl::get_revision_prereq(revision_id const & rev, @@ -608,18 +516,6 @@ } } -void -packet_db_writer::impl::accepted_manifest(manifest_id const & m, packet_db_writer & dbw) -{ - L(F("noting acceptence of manifest %s\n") % m); - map >::iterator i = manifest_prereqs.find(m); - if (i != manifest_prereqs.end()) - { - shared_ptr prereq = i->second; - manifest_prereqs.erase(i); - prereq->satisfy(prereq, dbw); - } -} void packet_db_writer::impl::accepted_file(file_id const & f, packet_db_writer & dbw) @@ -741,111 +637,6 @@ void -packet_db_writer::consume_manifest_data(manifest_id const & ident, - manifest_data const & dat) -{ - transaction_guard guard(pimpl->app.db); - if (! pimpl->manifest_version_exists_in_db(ident)) - { - pimpl->app.db.put_manifest(ident, dat); - pimpl->accepted_manifest(ident, *this); - } - else - L(F("skipping existing manifest version %s\n") % ident); - ++(pimpl->count); - guard.commit(); -} - -void -packet_db_writer::consume_manifest_delta(manifest_id const & old_id, - manifest_id const & new_id, - manifest_delta const & del) -{ - transaction_guard guard(pimpl->app.db); - if (! pimpl->manifest_version_exists_in_db(new_id)) - { - if (pimpl->manifest_version_exists_in_db(old_id)) - { - manifest_id confirm; - manifest_data old_dat; - data new_dat; - pimpl->app.db.get_manifest_version(old_id, old_dat); - patch(old_dat.inner(), del.inner(), new_dat); - calculate_ident(manifest_data(new_dat), confirm); - if (confirm == new_id) - { - pimpl->app.db.put_manifest_version(old_id, new_id, del); - pimpl->accepted_manifest(new_id, *this); - } - else - { - W(F("reconstructed manifest from delta '%s' -> '%s' has wrong id '%s'\n") - % old_id % new_id % confirm); - } - } - else - { - L(F("delaying manifest delta %s -> %s for preimage\n") % old_id % new_id); - shared_ptr dp; - dp = shared_ptr(new delayed_manifest_delta_packet(old_id, new_id, del, true)); - shared_ptr fp; - pimpl->get_manifest_prereq(old_id, fp); - dp->add_prerequisite(fp); - fp->add_dependent(dp); - } - } - else - L(F("skipping delta to existing manifest version %s\n") % new_id); - ++(pimpl->count); - guard.commit(); -} - -void -packet_db_writer::consume_manifest_reverse_delta(manifest_id const & new_id, - manifest_id const & old_id, - manifest_delta const & del) -{ - transaction_guard guard(pimpl->app.db); - if (! pimpl->manifest_version_exists_in_db(old_id)) - { - if (pimpl->manifest_version_exists_in_db(new_id)) - { - manifest_id confirm; - manifest_data new_dat; - data old_dat; - pimpl->app.db.get_manifest_version(new_id, new_dat); - patch(new_dat.inner(), del.inner(), old_dat); - calculate_ident(manifest_data(old_dat), confirm); - if (confirm == old_id) - { - pimpl->app.db.put_manifest_reverse_version(new_id, old_id, del); - pimpl->accepted_manifest(old_id, *this); - } - else - { - W(F("reconstructed manifest from reverse delta '%s' -> '%s' has wrong id '%s'\n") - % new_id % old_id % confirm); - } - } - else - { - L(F("delaying manifest reverse delta %s -> %s for preimage\n") % new_id % old_id); - shared_ptr dp; - dp = shared_ptr(new delayed_manifest_delta_packet(old_id, new_id, del, false)); - shared_ptr fp; - pimpl->get_manifest_prereq(new_id, fp); - dp->add_prerequisite(fp); - fp->add_dependent(dp); - } - } - else - L(F("skipping reverse delta to existing manifest version %s\n") % old_id); - ++(pimpl->count); - guard.commit(); -} - - -void packet_db_writer::consume_revision_data(revision_id const & ident, revision_data const & dat) { @@ -858,30 +649,10 @@ revision_set rev; read_revision_set(dat, rev); - - if (! pimpl->manifest_version_exists_in_db(rev.new_manifest)) - { - L(F("delaying revision %s for new manifest %s\n") - % ident % rev.new_manifest); - shared_ptr fp; - pimpl->get_manifest_prereq(rev.new_manifest, fp); - dp->add_prerequisite(fp); - fp->add_dependent(dp); - } for (edge_map::const_iterator i = rev.edges.begin(); i != rev.edges.end(); ++i) { - if (! (edge_old_manifest(i).inner()().empty() - || pimpl->manifest_version_exists_in_db(edge_old_manifest(i)))) - { - L(F("delaying revision %s for old manifest %s\n") - % ident % edge_old_manifest(i)); - shared_ptr fp; - pimpl->get_manifest_prereq(edge_old_manifest(i), fp); - dp->add_prerequisite(fp); - fp->add_dependent(dp); - } if (! (edge_old_revision(i).inner()().empty() || pimpl->revision_exists_in_db(edge_old_revision(i)))) { @@ -892,14 +663,29 @@ dp->add_prerequisite(fp); fp->add_dependent(dp); } -/* -// FIXME_ROSTERS: disabled until rewritten to use rosters - for (change_set::delta_map::const_iterator d = edge_changes(i).deltas.begin(); - d != edge_changes(i).deltas.end(); ++d) + for (std::map::const_iterator a + = edge_changes(i).files_added.begin(); + a != edge_changes(i).files_added.end(); ++a) + { + if (! pimpl->file_version_exists_in_db(a->second)) + { + L(F("delaying revision %s for added file %s\n") + % ident % a->second); + shared_ptr fp; + pimpl->get_file_prereq(a->second, fp); + dp->add_prerequisite(fp); + fp->add_dependent(dp); + } + } + + for (std::map >::const_iterator d + = edge_changes(i).deltas_applied.begin(); + d != edge_changes(i).deltas_applied.end(); ++d) { - if (! (delta_entry_src(d).inner()().empty() - || pimpl->file_version_exists_in_db(delta_entry_src(d)))) + I(!delta_entry_src(d).inner()().empty()); + I(!delta_entry_dst(d).inner()().empty()); + if (! pimpl->file_version_exists_in_db(delta_entry_src(d))) { L(F("delaying revision %s for old file %s\n") % ident % delta_entry_src(d)); @@ -908,7 +694,6 @@ dp->add_prerequisite(fp); fp->add_dependent(dp); } - I(!delta_entry_dst(d).inner()().empty()); if (! pimpl->file_version_exists_in_db(delta_entry_dst(d))) { L(F("delaying revision %s for new file %s\n") @@ -919,7 +704,6 @@ fp->add_dependent(dp); } } -*/ } if (dp->all_prerequisites_satisfied()) @@ -1120,29 +904,6 @@ } void -packet_db_valve::consume_manifest_data(manifest_id const & ident, - manifest_data const & dat) -{ - DOIT(delayed_manifest_data_packet(ident, dat)); -} - -void -packet_db_valve::consume_manifest_delta(manifest_id const & id_old, - manifest_id const & id_new, - manifest_delta const & del) -{ - DOIT(delayed_manifest_delta_packet(id_old, id_new, del, true)); -} - -void -packet_db_valve::consume_manifest_reverse_delta(manifest_id const & id_new, - manifest_id const & id_old, - manifest_delta const & del) -{ - DOIT(delayed_manifest_delta_packet(id_old, id_new, del, false)); -} - -void packet_db_valve::consume_revision_data(revision_id const & ident, revision_data const & dat) { @@ -1213,17 +974,6 @@ } void -packet_writer::consume_manifest_data(manifest_id const & ident, - manifest_data const & dat) -{ - base64 > packed; - pack(dat.inner(), packed); - ost << "[mdata " << ident.inner()() << "]" << endl - << trim_ws(packed()) << endl - << "[end]" << endl; -} - -void packet_writer::consume_revision_data(revision_id const & ident, revision_data const & dat) { @@ -1235,32 +985,6 @@ } void -packet_writer::consume_manifest_delta(manifest_id const & old_id, - manifest_id const & new_id, - manifest_delta const & del) -{ - base64 > packed; - pack(del.inner(), packed); - ost << "[mdelta " << old_id.inner()() << endl - << " " << new_id.inner()() << "]" << endl - << trim_ws(packed()) << endl - << "[end]" << endl; -} - -void -packet_writer::consume_manifest_reverse_delta(manifest_id const & new_id, - manifest_id const & old_id, - manifest_delta const & del) -{ - base64 > packed; - pack(del.inner(), packed); - ost << "[mrdelta " << new_id.inner()() << endl - << " " << old_id.inner()() << "]" << endl - << trim_ws(packed()) << endl - << "[end]" << endl; -} - -void packet_writer::consume_revision_cert(revision const & t) { ost << "[rcert " << t.inner().ident() << endl @@ -1319,9 +1043,6 @@ if (head == "rdata") cons.consume_revision_data(revision_id(hexenc(ident)), revision_data(body)); - else if (head == "mdata") - cons.consume_manifest_data(manifest_id(hexenc(ident)), - manifest_data(body)); else if (head == "fdata") cons.consume_file_data(file_id(hexenc(ident)), file_data(body)); @@ -1340,18 +1061,10 @@ base64 > body_packed(trim_ws(string(res[7].first, res[7].second))); delta body; unpack(body_packed, body); - if (head == "mdelta") - cons.consume_manifest_delta(manifest_id(hexenc(src_id)), - manifest_id(hexenc(dst_id)), - manifest_delta(body)); - else if (head == "fdelta") + if (head == "fdelta") cons.consume_file_delta(file_id(hexenc(src_id)), file_id(hexenc(dst_id)), file_delta(body)); - else if (head == "mrdelta") - cons.consume_manifest_reverse_delta(manifest_id(hexenc(src_id)), - manifest_id(hexenc(dst_id)), - manifest_delta(body)); else if (head == "frdelta") cons.consume_file_reverse_delta(file_id(hexenc(src_id)), file_id(hexenc(dst_id)), @@ -1417,8 +1130,8 @@ string const bra("\\["); string const ket("\\]"); string const certhead("(rcert)"); - string const datahead("([mfr]data)"); - string const deltahead("([mf]r?delta)"); + string const datahead("([fr]data)"); + string const deltahead("([f]r?delta)"); string const keyhead("(pubkey|privkey)"); string const key("(address@hidden)"); string const certname("([-a-zA-Z0-9]+)"); @@ -1471,7 +1184,6 @@ #ifdef BUILD_UNIT_TESTS #include "unit_tests.hh" #include "transforms.hh" -#include "manifest.hh" static void packet_roundabout_test() @@ -1506,32 +1218,7 @@ cert c(fid.inner(), cert_name("smell"), val, rsa_keypair_id("address@hidden"), sig); pw.consume_revision_cert(revision(c)); - - // a manifest data packet - manifest_map mm; - manifest_data mdata; - manifest_id mid; - mm.insert(make_pair(file_path_internal("foo/bar.txt"), - file_id(hexenc("cfb81b30ab3133a31b52eb50bd1c86df67eddec4")))); - write_manifest_map(mm, mdata); - calculate_ident(mdata, mid); - pw.consume_manifest_data(mid, mdata); - // a manifest delta packet - manifest_map mm2; - manifest_data mdata2; - manifest_id mid2; - manifest_delta mdelta; - mm2.insert(make_pair(file_path_internal("foo/bar.txt"), - file_id(hexenc("5b20eb5e5bdd9cd674337fc95498f468d80ef7bc")))); - mm2.insert(make_pair(file_path_internal("bunk.txt"), - file_id(hexenc("54f373ed07b4c5a88eaa93370e1bbac02dc432a8")))); - write_manifest_map(mm2, mdata2); - calculate_ident(mdata2, mid2); - delta del2; - diff(mdata.inner(), mdata2.inner(), del2); - pw.consume_manifest_delta(mid, mid2, manifest_delta(del)); - // a public key packet base64 puk; encode_base64(rsa_pub_key("this is not a real rsa key"), puk); ======================================================================== --- packet.hh ec7178c2332473305c0aa7d00c727e338fc7810d +++ packet.hh c10d66b39bd6aa88cebc4993187fa710f7e7d277 @@ -63,16 +63,6 @@ file_delta const & del) = 0; - virtual void consume_manifest_data(manifest_id const & ident, - manifest_data const & dat) = 0; - virtual void consume_manifest_delta(manifest_id const & id_old, - manifest_id const & id_new, - manifest_delta const & del) = 0; - virtual void consume_manifest_reverse_delta(manifest_id const & id_new, - manifest_id const & id_old, - manifest_delta const & del) = 0; - - virtual void consume_revision_data(revision_id const & ident, revision_data const & dat) = 0; virtual void consume_revision_cert(revision const & t) = 0; @@ -100,15 +90,6 @@ file_id const & id_old, file_delta const & del); - virtual void consume_manifest_data(manifest_id const & ident, - manifest_data const & dat); - virtual void consume_manifest_delta(manifest_id const & id_old, - manifest_id const & id_new, - manifest_delta const & del); - virtual void consume_manifest_reverse_delta(manifest_id const & id_new, - manifest_id const & id_old, - manifest_delta const & del); - virtual void consume_revision_data(revision_id const & ident, revision_data const & dat); virtual void consume_revision_cert(revision const & t); @@ -140,15 +121,6 @@ file_id const & id_old, file_delta const & del); - virtual void consume_manifest_data(manifest_id const & ident, - manifest_data const & dat); - virtual void consume_manifest_delta(manifest_id const & id_old, - manifest_id const & id_new, - manifest_delta const & del); - virtual void consume_manifest_reverse_delta(manifest_id const & id_new, - manifest_id const & id_old, - manifest_delta const & del); - virtual void consume_revision_data(revision_id const & ident, revision_data const & dat); virtual void consume_revision_cert(revision const & t); @@ -189,15 +161,6 @@ file_id const & id_old, file_delta const & del); - virtual void consume_manifest_data(manifest_id const & ident, - manifest_data const & dat); - virtual void consume_manifest_delta(manifest_id const & id_old, - manifest_id const & id_new, - manifest_delta const & del); - virtual void consume_manifest_reverse_delta(manifest_id const & id_new, - manifest_id const & id_old, - manifest_delta const & del); - virtual void consume_revision_data(revision_id const & ident, revision_data const & dat); virtual void consume_revision_cert(revision const & t);