# # patch "database.cc" # from [9579f5f073c33da80d05c37b2a490feb9a6e2bba] # to [f43a456ebcb0b54a63a50dce5125ea8e6d61e28d] # # patch "database.hh" # from [de2fb54bfcbd307beb283f48e7db26454d6b7509] # to [a6d58350ae742e2cdefceab07b7598f88749f8cb] # # patch "database_check.cc" # from [91484f8c51ca3e6010adbd800401f3ec9763222c] # to [ddecb300ef2802d08947a15707ac1f14337253b8] # # patch "vocab.hh" # from [efc08a5853fc7de32fb181f3e5508709bd7b47c4] # to [596953ad6326b68846702a85c22977f434e760e9] # ======================================================================== --- database.cc 9579f5f073c33da80d05c37b2a490feb9a6e2bba +++ database.cc f43a456ebcb0b54a63a50dce5125ea8e6d61e28d @@ -1178,6 +1178,16 @@ } void +database::get_roster_ids(set< hexenc > & ids) +{ + ids.clear(); + set< hexenc > tmp; + get_ids("rosters", tmp); + get_ids("roster_deltas", tmp); + ids.insert(tmp.begin(), tmp.end()); +} + +void database::get_file_version(file_id const & id, file_data & dat) { @@ -2534,6 +2544,16 @@ } void +database::get_roster(hexenc const & ros_id, + data & dat) +{ + string data_table = "rosters"; + string delta_table = "roster_deltas"; + + get_version(ros_id, dat, data_table, delta_table); +} + +void database::get_roster(revision_id const & rev_id, roster_t & roster, marking_map & marks) @@ -2545,13 +2565,11 @@ return; } - string data_table = "rosters"; - string delta_table = "roster_deltas"; data dat; hexenc ident; get_roster_id_for_revision(rev_id, ident); - get_version(ident, dat, data_table, delta_table); + get_roster(ident, dat); read_roster_and_marking(dat, roster, marks); } ======================================================================== --- database.hh de2fb54bfcbd307beb283f48e7db26454d6b7509 +++ database.hh a6d58350ae742e2cdefceab07b7598f88749f8cb @@ -202,9 +202,6 @@ delta const & del, database & db); - void get_roster_id_for_revision(revision_id const & rev_id, - hexenc & roster_id); - void put_roster(revision_id const & rev_id, roster_t & roster, marking_map & marks); @@ -236,6 +233,7 @@ void get_file_ids(std::set & ids); void get_manifest_ids(std::set & ids); void get_revision_ids(std::set & ids); + void get_roster_ids(std::set< hexenc > & ids) ; void set_app(app_state * app); @@ -422,14 +420,19 @@ void get_branches(std::vector & names); // roster and node_id stuff + void get_roster_id_for_revision(revision_id const & rev_id, + hexenc & roster_id); void get_roster(revision_id const & rid, roster_t & roster); - + void get_roster(revision_id const & rid, roster_t & roster, marking_map & marks); + void get_roster(hexenc const & roster_id, + data & dat); + void get_uncommon_ancestors(revision_id const & a, revision_id const & b, std::set & a_uncommon_ancs, ======================================================================== --- database_check.cc 91484f8c51ca3e6010adbd800401f3ec9763222c +++ database_check.cc ddecb300ef2802d08947a15707ac1f14337253b8 @@ -28,6 +28,10 @@ // files // +using std::set; +using std::map; +using std::vector; + struct checked_cert { revision rcert; bool found_key; @@ -47,22 +51,26 @@ struct checked_file { bool found; // found in db, retrieved and verified sha1 hash - size_t manifest_refs; // number of manifest references to this file + size_t roster_refs; // number of roster references to this file - checked_file(): found(false), manifest_refs(0) {} + checked_file(): found(false), roster_refs(0) {} }; -struct checked_manifest { +struct checked_roster { bool found; // found in db, retrieved and verified sha1 hash - size_t revision_refs; // number of revision references to this manifest - size_t missing_files; // number of missing files referenced by this manifest + size_t revision_refs; // number of revision references to this roster + size_t missing_files; // number of missing files referenced by this roster + size_t missing_mark_revs; // number of missing revisions referenced in node markings by this roster - bool parseable; // read_manifest_map does not throw - bool normalized; // write_manifest_map( read_manifest_map(dat) ) == dat + bool parseable; // read_roster_and_marking does not throw + bool normalized; // write_roster_and_marking( read_roster_and_marking(dat) ) == dat - checked_manifest(): + manifest_id man_id; // manifest id of this roster's public part + + checked_roster(): found(false), revision_refs(0), - missing_files(0), parseable(false), normalized(false) {} + missing_files(0), missing_mark_revs(0), + parseable(false), normalized(false), man_id() {} }; // the number of times a revision is referenced (revision_refs) @@ -79,9 +87,11 @@ size_t ancestry_parent_refs; // number of references to this revision by ancestry parent size_t ancestry_child_refs; // number of references to this revision by ancestry child + bool found_roster; // the roster for this revision exists + bool manifest_mismatch; // manifest doesn't match the roster for this revision + bool incomplete_roster; // the roster for this revision is missing files size_t missing_manifests; // number of missing manifests referenced by this revision size_t missing_revisions; // number of missing revisions referenced by this revision - size_t incomplete_manifests; // number of manifests missing files referenced by this revision size_t cert_refs; // number of references to this revision by revision certs; @@ -96,7 +106,8 @@ checked_revision(): found(false), revision_refs(0), ancestry_parent_refs(0), ancestry_child_refs(0), - missing_manifests(0), missing_revisions(0), incomplete_manifests(0), + found_roster(false), manifest_mismatch(false), incomplete_roster(false), + missing_manifests(0), missing_revisions(0), cert_refs(0), parseable(false), normalized(false) {} }; @@ -123,70 +134,138 @@ I(checked_files.size() == files.size()); } -/* FIXME_ROSTERS - +// first phase of roster checking, checks manifest-related parts of the +// roster, and general parsability/normalisation static void -check_manifests(app_state & app, - std::map & checked_manifests, - std::map & checked_files) +check_rosters_manifest(app_state & app, + std::map, checked_roster> & checked_rosters, + std::map & checked_revisions, + std::set & found_manifests, + std::map & checked_files) { - std::set manifests; + set< hexenc > rosters; - app.db.get_manifest_ids(manifests); - L(F("checking %d manifests\n") % manifests.size()); + app.db.get_roster_ids(rosters); + L(F("checking %d rosters, manifest pass\n") % rosters.size()); - ticker ticks(_("manifests"), "m", manifests.size()/70+1); + ticker ticks(_("rosters"), "r", rosters.size()/70+1); - for (std::set::const_iterator i = manifests.begin(); - i != manifests.end(); ++i) + for (set >::const_iterator i = rosters.begin(); + i != rosters.end(); ++i) { - L(F("checking manifest %s\n") % *i); - manifest_data data; - app.db.get_manifest_version(*i, data); - checked_manifests[*i].found = true; - manifest_map man; + L(F("checking roster %s\n") % *i); + data dat; + app.db.get_roster(*i, dat); + checked_rosters[*i].found = true; + + roster_t ros; + marking_map mm; try { - read_manifest_map(data, man); + read_roster_and_marking(dat, ros, mm); } catch (std::logic_error & e) { - L(F("error parsing manifest %s: %s") % *i % e.what()); - checked_manifests[*i].parseable = false; + L(F("error parsing roster %s: %s") % *i % e.what()); + checked_rosters[*i].parseable = false; continue; } - checked_manifests[*i].parseable = true; + checked_rosters[*i].parseable = true; + + // normalisation check + { + hexenc norm_ident; + data norm_data; + write_roster_and_marking(ros, mm, norm_data); + calculate_ident(norm_data, norm_ident); + if (norm_ident == *i) + checked_rosters[*i].normalized = true; + } - // normalisation check - manifest_id norm_ident; - manifest_data norm_data; - write_manifest_map(man, norm_data); - calculate_ident(norm_data, norm_ident); - if (norm_ident == *i) - checked_manifests[*i].normalized = true; + manifest_id man_id; + calculate_ident(ros, man_id); + checked_rosters[*i].man_id = man_id; + found_manifests.insert(man_id); - for (manifest_map::const_iterator entry = man.begin(); entry != man.end(); - ++entry) + for (node_map::const_iterator n = ros.all_nodes().begin(); + n != ros.all_nodes().end(); n++) { - checked_files[manifest_entry_id(entry)].manifest_refs++; - if (!checked_files[manifest_entry_id(entry)].found) - checked_manifests[*i].missing_files++; + if (is_file_t(n->second)) + { + file_id fid = downcast_to_file_t(n->second)->content; + checked_files[fid].roster_refs++; + if (!checked_files[fid].found) + checked_rosters[*i].missing_files++; + } } ++ticks; } + I(checked_rosters.size() == rosters.size()); +} - I(checked_manifests.size() == manifests.size()); -} +// second phase of roster checking. examine the marking of a roster, checking +// that the referenced revisions exist. +static void +check_rosters_marking(app_state & app, + std::map, checked_roster> & checked_rosters, + std::map & checked_revisions) +{ + L(F("checking %d rosters, marking pass\n") % checked_rosters.size()); -*/ + ticker ticks(_("markings"), "m", checked_rosters.size()/70+1); + for (std::map, checked_roster>::const_iterator i + = checked_rosters.begin(); i != checked_rosters.end(); i++) + { + hexenc ros_id = i->first; + L(F("checking roster %s\n") % i->first); + if (!i->second.parseable) + continue; + + data dat; + app.db.get_roster(ros_id, dat); + + roster_t ros; + marking_map mm; + read_roster_and_marking(dat, ros, mm); + + for (node_map::const_iterator n = ros.all_nodes().begin(); + n != ros.all_nodes().end(); n++) + { + // lots of revisions that must exist + marking_t mark = mm[n->first]; + if (!checked_revisions[mark.birth_revision].found) + checked_rosters[ros_id].missing_mark_revs++; + + for (set::const_iterator r = mark.parent_name.begin(); + r != mark.parent_name.end(); r++) + if (!checked_revisions[*r].found) + checked_rosters[ros_id].missing_mark_revs++; + + for (set::const_iterator r = mark.file_content.begin(); + r != mark.file_content.end(); r++) + if (!checked_revisions[*r].found) + checked_rosters[ros_id].missing_mark_revs++; + + for (map >::const_iterator attr = + mark.attrs.begin(); attr != mark.attrs.end(); attr++) + for (set::const_iterator r = attr->second.begin(); + r != attr->second.end(); r++) + if (!checked_revisions[*r].found) + checked_rosters[ros_id].missing_mark_revs++; + } + ticks++; + } +} + static void check_revisions(app_state & app, std::map & checked_revisions, - std::map & checked_manifests) + std::map, checked_roster> & checked_rosters, + std::set const & found_manifests) { std::set revisions; @@ -224,28 +303,41 @@ if (norm_ident == *i) checked_revisions[*i].normalized = true; - checked_manifests[rev.new_manifest].revision_refs++; + // roster checks + hexenc roster_id; + app.db.get_roster_id_for_revision(*i, roster_id); + if (!null_id(roster_id)) + { + checked_revisions[*i].found_roster = true; + checked_rosters[roster_id].revision_refs++; + if (!(rev.new_manifest == checked_rosters[roster_id].man_id)) + checked_revisions[*i].manifest_mismatch = true; + if (checked_rosters[roster_id].missing_files > 0) + checked_revisions[*i].incomplete_roster = true; + } - if (!checked_manifests[rev.new_manifest].found) + if (found_manifests.find(rev.new_manifest) == found_manifests.end()) checked_revisions[*i].missing_manifests++; - if (checked_manifests[rev.new_manifest].missing_files > 0) - checked_revisions[*i].incomplete_manifests++; - for (edge_map::const_iterator edge = rev.edges.begin(); edge != rev.edges.end(); ++edge) { // ignore [] -> [...] manifests - if (!null_id(edge_old_manifest(edge))) + manifest_id old = edge_old_manifest(edge); + if (!null_id(old)) { + if (found_manifests.find(old) == found_manifests.end()) + checked_revisions[*i].missing_manifests++; + + /* TODO ROSTER: doesn't make sense with rosters checked_manifests[edge_old_manifest(edge)].revision_refs++; if (!checked_manifests[edge_old_manifest(edge)].found) checked_revisions[*i].missing_manifests++; if (checked_manifests[edge_old_manifest(edge)].missing_files > 0) - checked_revisions[*i].incomplete_manifests++; + checked_revisions[*i].incomplete_manifests++; */ } // ignore [] -> [...] revisions @@ -418,10 +510,10 @@ { missing_files++; P(F("file %s missing (%d manifest references)\n") - % i->first % file.manifest_refs); + % i->first % file.roster_refs); } - if (file.manifest_refs == 0) + if (file.roster_refs == 0) { unreferenced_files++; P(F("file %s unreferenced\n") % i->first); @@ -431,49 +523,56 @@ } static void -report_manifests(std::map const & checked_manifests, - size_t & missing_manifests, - size_t & unreferenced_manifests, - size_t & incomplete_manifests, - size_t & non_parseable_manifests, - size_t & non_normalized_manifests) +report_rosters(std::map, checked_roster> const & checked_rosters, + size_t & missing_rosters, + size_t & unreferenced_rosters, + size_t & incomplete_rosters, + size_t & non_parseable_rosters, + size_t & non_normalized_rosters) { - for (std::map::const_iterator - i = checked_manifests.begin(); i != checked_manifests.end(); ++i) + for (std::map, checked_roster>::const_iterator + i = checked_rosters.begin(); i != checked_rosters.end(); ++i) { - checked_manifest manifest = i->second; + checked_roster roster = i->second; - if (!manifest.found) + if (!roster.found) { - missing_manifests++; - P(F("manifest %s missing (%d revision references)\n") - % i->first % manifest.revision_refs); + missing_rosters++; + P(F("roster %s missing (%d revision references)\n") + % i->first % roster.revision_refs); } - if (manifest.revision_refs == 0) + if (roster.revision_refs == 0) { - unreferenced_manifests++; - P(F("manifest %s unreferenced\n") % i->first); + unreferenced_rosters++; + P(F("roster %s unreferenced\n") % i->first); } - if (manifest.missing_files > 0) + if (roster.missing_files > 0) { - incomplete_manifests++; - P(F("manifest %s incomplete (%d missing files)\n") - % i->first % manifest.missing_files); + incomplete_rosters++; + P(F("roster %s incomplete (%d missing files)\n") + % i->first % roster.missing_files); } - if (!manifest.parseable) + if (roster.missing_mark_revs > 0) { - non_parseable_manifests++; - P(F("manifest %s is not parseable (perhaps with unnormalized paths?)\n") + incomplete_rosters++; + P(F("roster %s incomplete (%d missing revisions)\n") + % i->first % roster.missing_mark_revs); + } + + if (!roster.parseable) + { + non_parseable_rosters++; + P(F("roster %s is not parseable (perhaps with unnormalized paths?)\n") % i->first); } - if (manifest.parseable && !manifest.normalized) + if (roster.parseable && !roster.normalized) { - non_normalized_manifests++; - P(F("manifest %s is not in normalized form\n") + non_normalized_rosters++; + P(F("roster %s is not in normalized form\n") % i->first); } } @@ -485,6 +584,7 @@ size_t & incomplete_revisions, size_t & mismatched_parents, size_t & mismatched_children, + size_t & manifest_mismatch, size_t & bad_history, size_t & non_parseable_revisions, size_t & non_normalized_revisions) @@ -515,13 +615,24 @@ % i->first % revision.missing_revisions); } - if (revision.incomplete_manifests > 0) + if (!revision.found_roster) { incomplete_revisions++; - P(F("revision %s incomplete (%d incomplete manifests)\n") - % i->first % revision.incomplete_manifests); + P(F("revision %s incomplete (missing roster)\n") % i->first); } + if (revision.manifest_mismatch) + { + manifest_mismatch++; + P(F("revision %s mismatched roster and manifest\n") % i->first); + } + + if (revision.incomplete_roster) + { + incomplete_revisions++; + P(F("revision %s incomplete (incomplete roster)\n") % i->first); + } + if (revision.ancestry_parent_refs != revision.revision_refs) { mismatched_parents++; @@ -663,18 +774,19 @@ check_db(app_state & app) { std::map checked_files; - std::map checked_manifests; + std::set found_manifests; + std::map, checked_roster> checked_rosters; std::map checked_revisions; std::map checked_keys; size_t missing_files = 0; size_t unreferenced_files = 0; - size_t missing_manifests = 0; - size_t unreferenced_manifests = 0; - size_t incomplete_manifests = 0; - size_t non_parseable_manifests = 0; - size_t non_normalized_manifests = 0; + size_t missing_rosters = 0; + size_t unreferenced_rosters = 0; + size_t incomplete_rosters = 0; + size_t non_parseable_rosters = 0; + size_t non_normalized_rosters = 0; size_t missing_revisions = 0; size_t incomplete_revisions = 0; @@ -689,13 +801,15 @@ size_t total_certs = 0; size_t missing_certs = 0; size_t mismatched_certs = 0; + size_t manifest_mismatch = 0; size_t unchecked_sigs = 0; size_t bad_sigs = 0; check_files(app, checked_files); - // FIXME_ROSTERS - //check_manifests(app, checked_manifests, checked_files); - check_revisions(app, checked_revisions, checked_manifests); + check_rosters_manifest(app, checked_rosters, checked_revisions, + found_manifests, checked_files); + check_revisions(app, checked_revisions, checked_rosters, found_manifests); + check_rosters_marking(app, checked_rosters, checked_revisions); check_ancestry(app, checked_revisions); check_sane(app, checked_revisions); check_keys(app, checked_keys); @@ -703,15 +817,16 @@ report_files(checked_files, missing_files, unreferenced_files); - report_manifests(checked_manifests, - missing_manifests, unreferenced_manifests, - incomplete_manifests, - non_parseable_manifests, - non_normalized_manifests); + report_rosters(checked_rosters, + missing_rosters, unreferenced_rosters, + incomplete_rosters, + non_parseable_rosters, + non_normalized_rosters); report_revisions(checked_revisions, missing_revisions, incomplete_revisions, mismatched_parents, mismatched_children, + manifest_mismatch, bad_history, non_parseable_revisions, non_normalized_revisions); @@ -726,17 +841,17 @@ if (unreferenced_files > 0) W(F("%d unreferenced files\n") % unreferenced_files); - if (missing_manifests > 0) - W(F("%d missing manifests\n") % missing_manifests); - if (unreferenced_manifests > 0) - W(F("%d unreferenced manifests\n") % unreferenced_manifests); - if (incomplete_manifests > 0) - W(F("%d incomplete manifests\n") % incomplete_manifests); - if (non_parseable_manifests > 0) - W(F("%d manifests not parseable (perhaps with invalid paths)\n") - % non_parseable_manifests); - if (non_normalized_manifests > 0) - W(F("%d manifests not in normalized form\n") % non_normalized_manifests); + if (missing_rosters > 0) + W(F("%d missing rosters\n") % missing_rosters); + if (unreferenced_rosters > 0) + W(F("%d unreferenced rosters\n") % unreferenced_rosters); + if (incomplete_rosters > 0) + W(F("%d incomplete rosters\n") % incomplete_rosters); + if (non_parseable_rosters > 0) + W(F("%d rosters not parseable (perhaps with invalid paths)\n") + % non_parseable_rosters); + if (non_normalized_rosters > 0) + W(F("%d rosters not in normalized form\n") % non_normalized_rosters); if (missing_revisions > 0) W(F("%d missing revisions\n") % missing_revisions); @@ -767,8 +882,8 @@ W(F("%d bad signatures\n") % bad_sigs); size_t total = missing_files + unreferenced_files + - missing_manifests + unreferenced_manifests + incomplete_manifests + - non_parseable_manifests + non_normalized_manifests + + missing_rosters + unreferenced_rosters + incomplete_rosters + + non_parseable_rosters + non_normalized_rosters + missing_revisions + incomplete_revisions + non_parseable_revisions + non_normalized_revisions + mismatched_parents + mismatched_children + @@ -776,22 +891,22 @@ missing_certs + mismatched_certs + unchecked_sigs + bad_sigs + missing_keys; - // unreferenced files and manifests and mismatched certs are not actually + // unreferenced files and rosters and mismatched certs are not actually // serious errors; odd, but nothing will break. size_t serious = missing_files + - missing_manifests + incomplete_manifests + - non_parseable_manifests + non_normalized_manifests + + missing_rosters + incomplete_rosters + + non_parseable_rosters + non_normalized_rosters + missing_revisions + incomplete_revisions + non_parseable_revisions + non_normalized_revisions + - mismatched_parents + mismatched_children + + mismatched_parents + mismatched_children + manifest_mismatch + bad_history + missing_certs + unchecked_sigs + bad_sigs + missing_keys; - P(F("check complete: %d files; %d manifests; %d revisions; %d keys; %d certs\n") + P(F("check complete: %d files; %d rosters; %d revisions; %d keys; %d certs\n") % checked_files.size() - % checked_manifests.size() + % checked_rosters.size() % checked_revisions.size() % checked_keys.size() % total_certs); ======================================================================== --- vocab.hh efc08a5853fc7de32fb181f3e5508709bd7b47c4 +++ vocab.hh 596953ad6326b68846702a85c22977f434e760e9 @@ -188,6 +188,12 @@ // do these belong here? inline bool +null_id(hexenc const & i) +{ + return i().empty(); +} + +inline bool null_id(file_id const & i) { return i.inner()().empty();