# # # patch "automate.cc" # from [8fa450b0e98f21120ab9d205c99e6f34247b7f88] # to [32cf436afc6f692ffb1383fa316fa66b04a92ddd] # # patch "inodeprint.hh" # from [56326dab006b882f5e2593740da4742bd28bbc35] # to [1beb487da6296d348636126bf64a75896c1cd4da] # # patch "roster.cc" # from [b9454f9f181d981ea2154209f2bd50ffcf759cb1] # to [8f15edb939b7f1d63d3efb37b8c83df393ecddb8] # # patch "roster.hh" # from [06d636445795608feb6a03f008fb52c665afe1b1] # to [c3e4fce7062cd1e8f27ccaa62365705d55aae2e1] # # patch "tests/t_automate_inventory.at" # from [c178579a16b3ef84d0dc8d1395e3dc9fe28e0595] # to [30ccf1380206c8df60b2259f7e0f3c592c58b7a7] # ============================================================ --- automate.cc 8fa450b0e98f21120ab9d205c99e6f34247b7f88 +++ automate.cc 32cf436afc6f692ffb1383fa316fa66b04a92ddd @@ -14,12 +14,14 @@ #include #include +#include #include #include "app_state.hh" #include "basic_io.hh" #include "commands.hh" #include "constants.hh" +#include "inodeprint.hh" #include "restrictions.hh" #include "revision.hh" #include "transforms.hh" @@ -27,6 +29,8 @@ #include "keys.hh" #include "packet.hh" +using boost::lexical_cast; + static std::string const interface_version = "2.0"; // Name: interface_version @@ -514,128 +518,128 @@ output << *i << std::endl; } -// consider a changeset with the following -// -// deletions -// renames from to -// additions -// -// pre-state corresponds to deletions and the "from" side of renames -// post-state corresponds to the "to" side of renames and additions -// node-state corresponds to the state of the node with the given name -// -// pre/post state are related to the path rearrangement in _MTN/work -// node state is related to the details of the resulting path +struct node_info +{ + bool exists; + node_id id; + path::status type; + file_id ident; + full_attr_map_t attrs; -struct inventory_item -{ - // pre/post rearrangement state - enum pstate - { UNCHANGED_PATH, ADDED_PATH, DROPPED_PATH, RENAMED_PATH } - pre_state, post_state; - - enum nstate - { UNCHANGED_NODE, PATCHED_NODE, MISSING_NODE, UNKNOWN_NODE, IGNORED_NODE } - node_state; - - size_t pre_id, post_id; - - inventory_item(): - pre_state(UNCHANGED_PATH), post_state(UNCHANGED_PATH), - node_state(UNCHANGED_NODE), - pre_id(0), post_id(0) {} + node_info() : exists(false), type(path::nonexistent) {} }; -typedef std::map inventory_map; -typedef std::map rename_map; // this might be good in cset.hh -typedef std::map addition_map; // ditto - static void -inventory_pre_state(inventory_map & inventory, - path_set const & paths, - inventory_item::pstate pre_state, - size_t rename_id) +get_node_info(roster_t const & roster, split_path const & path, node_info & info) { - for (path_set::const_iterator i = paths.begin(); i != paths.end(); i++) + if (roster.has_node(path)) { - L(FL("%d %d %s\n") % inventory[*i].pre_state % pre_state % file_path(*i)); - I(inventory[*i].pre_state == inventory_item::UNCHANGED_PATH); - inventory[*i].pre_state = pre_state; - if (rename_id != 0) + node_t node = roster.get_node(path); + info.exists = true; + info.id = node->self; + info.attrs = node->attrs; + if (is_file_t(node)) { - I(inventory[*i].pre_id == 0); - inventory[*i].pre_id = rename_id; + info.type = path::file; + info.ident = downcast_to_file_t(node)->content; } + else if (is_dir_t(node)) + info.type = path::directory; + else + I(false); } } +struct inventory_item +{ + node_info old_node; + node_info new_node; + + path::status fs_type; + file_id fs_ident; + + inventory_item() : fs_type(path::nonexistent) {} +}; + +typedef std::map inventory_map; + static void -inventory_post_state(inventory_map & inventory, - path_set const & paths, - inventory_item::pstate post_state, - size_t rename_id) +inventory_rosters(roster_t const & old_roster, + roster_t const & new_roster, + inventory_map & inventory) { - for (path_set::const_iterator i = paths.begin(); i != paths.end(); i++) + node_map const & old_nodes = old_roster.all_nodes(); + for (node_map::const_iterator i = old_nodes.begin(); i != old_nodes.end(); ++i) { - L(FL("%d %d %s\n") % inventory[*i].post_state % post_state % file_path(*i)); - I(inventory[*i].post_state == inventory_item::UNCHANGED_PATH); - inventory[*i].post_state = post_state; - if (rename_id != 0) - { - I(inventory[*i].post_id == 0); - inventory[*i].post_id = rename_id; - } + split_path sp; + old_roster.get_name(i->first, sp); + get_node_info(old_roster, sp, inventory[sp].old_node); } -} -static void -inventory_node_state(inventory_map & inventory, - path_set const & paths, - inventory_item::nstate node_state) -{ - for (path_set::const_iterator i = paths.begin(); i != paths.end(); i++) + node_map const & new_nodes = new_roster.all_nodes(); + for (node_map::const_iterator i = new_nodes.begin(); i != new_nodes.end(); ++i) { - L(FL("%d %d %s\n") % inventory[*i].node_state % node_state % file_path(*i)); - I(inventory[*i].node_state == inventory_item::UNCHANGED_NODE); - inventory[*i].node_state = node_state; + split_path sp; + new_roster.get_name(i->first, sp); + get_node_info(new_roster, sp, inventory[sp].new_node); } + } -static void -inventory_renames(inventory_map & inventory, - rename_map const & renames) +struct inventory_itemizer : public tree_walker { - path_set old_name; - path_set new_name; + app_state & app; + inventory_map & inventory; + inodeprint_map ipm; - static size_t rename_id = 1; + inventory_itemizer(app_state & a, inventory_map & i) : + app(a), inventory(i) + { + if (in_inodeprints_mode()) + { + data dat; + read_inodeprints(dat); + read_inodeprint_map(dat, ipm); + } + } + virtual void visit_dir(file_path const & path); + virtual void visit_file(file_path const & path); +}; - for (rename_map::const_iterator i = renames.begin(); - i != renames.end(); i++) - { - old_name.clear(); - new_name.clear(); +void +inventory_itemizer::visit_dir(file_path const & path) +{ + split_path sp; + path.split(sp); + inventory[sp].fs_type = path::directory; +} - old_name.insert(i->first); - new_name.insert(i->second); +void +inventory_itemizer::visit_file(file_path const & path) +{ + split_path sp; + path.split(sp); - inventory_pre_state(inventory, old_name, inventory_item::RENAMED_PATH, rename_id); - inventory_post_state(inventory, new_name, inventory_item::RENAMED_PATH, rename_id); + inventory_item & item = inventory[sp]; - rename_id++; + item.fs_type = path::file; + + if (item.new_node.exists) + { + if (inodeprint_unchanged(ipm, path)) + item.fs_ident = item.old_node.ident; + else + ident_existing_file(path, item.fs_ident, app.lua); } } static void -extract_added_file_paths(addition_map const & additions, path_set & paths) +inventory_filesystem(app_state & app, inventory_map & inventory) { - for (addition_map::const_iterator i = additions.begin(); i != additions.end(); ++i) - { - paths.insert(i->first); - } + inventory_itemizer itemizer(app, inventory); + walk_tree(file_path(), itemizer); } - // Name: inventory // Arguments: none // Added in: 1.0 @@ -681,102 +685,137 @@ app.require_workspace(); temp_node_id_source nis; - roster_t base, curr; - inventory_map inventory; - cset cs; MM(cs); - path_set unchanged, changed, missing, known, unknown, ignored; + roster_t old_roster, new_roster; - get_base_and_current_roster_shape(base, curr, nis, app); - make_cset(base, curr, cs); + get_base_and_current_roster_shape(old_roster, new_roster, nis, app); - I(cs.deltas_applied.empty()); + inventory_map inventory; - // the current roster (curr) has the complete set of registered nodes - // conveniently with unchanged sha1 hash values + inventory_rosters(old_roster, new_roster, inventory); + inventory_filesystem(app, inventory); - // the cset (cs) has the list of drops/renames/adds that have occurred between - // the two rosters along with an empty list of deltas. this list is empty - // only because the current roster used to generate the cset does not have - // current hash values as recorded on the filesystem (because get_..._shape - // was used to build it) + basic_io::printer pr; - path_set nodes_added(cs.dirs_added); - extract_added_file_paths(cs.files_added, nodes_added); - - inventory_pre_state(inventory, cs.nodes_deleted, inventory_item::DROPPED_PATH, 0); - inventory_renames(inventory, cs.nodes_renamed); - inventory_post_state(inventory, nodes_added, inventory_item::ADDED_PATH, 0); - - classify_roster_paths(curr, unchanged, changed, missing, app); - curr.extract_path_set(known); - - file_itemizer u(app, known, unknown, ignored); - walk_tree(file_path(), u); - - inventory_node_state(inventory, unchanged, inventory_item::UNCHANGED_NODE); - inventory_node_state(inventory, changed, inventory_item::PATCHED_NODE); - inventory_node_state(inventory, missing, inventory_item::MISSING_NODE); - inventory_node_state(inventory, unknown, inventory_item::UNKNOWN_NODE); - inventory_node_state(inventory, ignored, inventory_item::IGNORED_NODE); - - // FIXME: do we want to report on attribute changes here?!? - - for (inventory_map::const_iterator i = inventory.begin(); i != inventory.end(); ++i) + for (inventory_map::const_iterator i = inventory.begin(); i != inventory.end(); + ++i) { + basic_io::stanza st; + inventory_item const & item = i->second; - std::string path_suffix; + st.push_file_pair("path", i->first); - if (curr.has_node(i->first)) + if (item.old_node.exists) { - // explicitly skip the root dir for now... - // the trailing / dir format isn't going to work here - node_t n = curr.get_node(i->first); - if (is_root_dir_t(n)) continue; - if (is_dir_t(n)) path_suffix = "/"; +// std::string id = lexical_cast(item.old_node.id); + st.push_str_pair("old_id", lexical_cast(item.old_node.id)); + switch (item.old_node.type) + { +// case path::file: st.push_str_triple("old", id, "file"); break; +// case path::directory: st.push_str_triple("old", id, "directory"); break; + case path::file: st.push_str_pair("old_type", "file"); break; + case path::directory: st.push_str_pair("old_type", "directory"); break; + case path::nonexistent: I(false); + } } - else if (directory_exists(file_path(i->first))) + + if (item.new_node.exists) { - path_suffix = "/"; +// std::string id = lexical_cast(item.new_node.id); + st.push_str_pair("new_id", lexical_cast(item.new_node.id)); + switch (item.new_node.type) + { +// case path::file: st.push_str_triple("new", id, "file"); break; +// case path::directory: st.push_str_triple("new", id, "directory"); break; + case path::file: st.push_str_pair("new_type", "file"); break; + case path::directory: st.push_str_pair("new_type", "directory"); break; + case path::nonexistent: I(false); + } } - switch (i->second.pre_state) + switch (item.fs_type) { - case inventory_item::UNCHANGED_PATH: output << " "; break; - case inventory_item::DROPPED_PATH: output << "D"; break; - case inventory_item::RENAMED_PATH: output << "R"; break; - default: I(false); // invalid pre_state + case path::file: st.push_str_pair("fs_type", "file"); break; + case path::directory: st.push_str_pair("fs_type", "directory"); break; + case path::nonexistent: st.push_str_pair("fs_type", "none"); break; } - switch (i->second.post_state) + if (item.fs_type == path::nonexistent) { - case inventory_item::UNCHANGED_PATH: output << " "; break; - case inventory_item::RENAMED_PATH: output << "R"; break; - case inventory_item::ADDED_PATH: output << "A"; break; - default: I(false); // invalid post_state + if (item.new_node.exists) + st.push_str_pair("status", "missing"); } - - switch (i->second.node_state) + else // exists on filesystem { - case inventory_item::UNCHANGED_NODE: output << " "; break; - case inventory_item::PATCHED_NODE: output << "P"; break; - case inventory_item::UNKNOWN_NODE: output << "U"; break; - case inventory_item::IGNORED_NODE: output << "I"; break; - case inventory_item::MISSING_NODE: output << "M"; break; - default: I(false); // invalid node_state + if (!item.new_node.exists) + { + if (app.lua.hook_ignore_file(i->first)) + st.push_str_pair("status", "ignored"); + else + st.push_str_pair("status", "unknown"); + } + else if (item.new_node.type != item.fs_type) + st.push_str_pair("status", "invalid"); + // TODO: would an ls_invalid command be good for listing these paths? + else + st.push_str_pair("status", "known"); } - output << " " << i->second.pre_id - << " " << i->second.post_id - << " " << i->first; + // note that we have three sources of information here + // + // the old roster + // the new roster + // the filesystem + // + // the new roster is synthesised from the old roster and the contents of + // _MTN/work and has *not* been updated with content hashes from the + // filesystem. + // + // one path can represent different nodes in the old and new rosters and + // the two different nodes can potentially be different types (file vs dir). + // + // we're interested in comparing the content and attributes of the current + // path in the new roster against their corresponding values in the old + // roster. + // + // the new content hash comes from the filesystem since the new roster has + // not been updated. the new attributes can come directly from the new + // roster. + // + // the old content hash and attributes both come from the old roster but + // we must use the node id of the path in the new roster to get the node + // from the old roster to compare against. - // FIXME: it's possible that a directory was deleted and a file was added - // in it's place (or vice-versa) so we need something like pre/post node - // type indicators rather than a simple path suffix! ugh. - - output << path_suffix; + if (item.new_node.exists) + { + if (item.new_node.type == path::file) + { + if (item.fs_type == path::nonexistent) + st.push_str_pair("content", "unknown"); + else if (old_roster.has_node(item.new_node.id)) + { + file_t old_file = downcast_to_file_t(old_roster.get_node(item.new_node.id)); + old_file->content; + if (item.fs_ident == old_file->content) + st.push_str_pair("content", "unchanged"); + else + st.push_str_pair("content", "changed"); + } + } - output << std::endl; + if (old_roster.has_node(item.new_node.id)) + { + node_t old_node = old_roster.get_node(item.new_node.id); + if (old_node->attrs == item.new_node.attrs) + st.push_str_pair("attrs", "unchanged"); + else + st.push_str_pair("attrs", "changed"); + } + } + + pr.print_stanza(st); } + + output.write(pr.buf.data(), pr.buf.size()); } // Name: certs ============================================================ --- inodeprint.hh 56326dab006b882f5e2593740da4742bd28bbc35 +++ inodeprint.hh 1beb487da6296d348636126bf64a75896c1cd4da @@ -25,5 +25,21 @@ void write_inodeprint_map(inodeprint_map const & ipm, data & dat); +inline bool +inodeprint_unchanged(inodeprint_map const & ipm, file_path const & path) +{ + inodeprint_map::const_iterator old_ip = ipm.find(path); + if (old_ip != ipm.end()) + { + hexenc ip; + if (inodeprint_file(path, ip) && ip == old_ip->second) + return true; // unchanged + else + return false; // changed or unavailable + } + else + return false; // unavailable +} + #endif ============================================================ --- roster.cc b9454f9f181d981ea2154209f2bd50ffcf759cb1 +++ roster.cc 8f15edb939b7f1d63d3efb37b8c83df393ecddb8 @@ -1979,89 +1979,7 @@ // getting rosters from the workspace //////////////////////////////////////////////////////////////////// -inline static bool -inodeprint_unchanged(inodeprint_map const & ipm, file_path const & path) -{ - inodeprint_map::const_iterator old_ip = ipm.find(path); - if (old_ip != ipm.end()) - { - hexenc ip; - if (inodeprint_file(path, ip) && ip == old_ip->second) - return true; // unchanged - else - return false; // changed or unavailable - } - else - return false; // unavailable -} - void -classify_roster_paths(roster_t const & ros, - path_set & unchanged, - path_set & changed, - path_set & missing, - app_state & app) -{ - temp_node_id_source nis; - inodeprint_map ipm; - - if (in_inodeprints_mode()) - { - data dat; - read_inodeprints(dat); - read_inodeprint_map(dat, ipm); - } - - // this code is speed critical, hence the use of inode fingerprints so be - // careful when making changes in here and preferably do some timing tests - - if (!ros.has_root()) - return; - - node_map const & nodes = ros.all_nodes(); - for (node_map::const_iterator i = nodes.begin(); i != nodes.end(); ++i) - { - node_id nid = i->first; - node_t node = i->second; - - split_path sp; - ros.get_name(nid, sp); - file_path fp(sp); - - // Only analyze restriction-included files. - if (app.restriction_includes(sp)) - { - if (is_dir_t(node) || inodeprint_unchanged(ipm, fp)) - { - // dirs don't have content changes - unchanged.insert(sp); - } - else - { - file_t file = downcast_to_file_t(node); - file_id fid; - if (ident_existing_file(fp, fid, app.lua)) - { - if (file->content == fid) - unchanged.insert(sp); - else - changed.insert(sp); - } - else - { - missing.insert(sp); - } - } - } - else - { - // changes to excluded files are ignored - unchanged.insert(sp); - } - } -} - -void update_restricted_roster_from_filesystem(roster_t & ros, app_state & app) { ============================================================ --- roster.hh 06d636445795608feb6a03f008fb52c665afe1b1 +++ roster.hh c3e4fce7062cd1e8f27ccaa62365705d55aae2e1 @@ -339,13 +339,6 @@ std::set & nodes_born); void -classify_roster_paths(roster_t const & ros, - path_set & unchanged, - path_set & changed, - path_set & missing, - app_state & app); - -void update_restricted_roster_from_filesystem(roster_t & ros, app_state & app); ============================================================ --- tests/t_automate_inventory.at c178579a16b3ef84d0dc8d1395e3dc9fe28e0595 +++ tests/t_automate_inventory.at 30ccf1380206c8df60b2259f7e0f3c592c58b7a7 @@ -46,8 +46,8 @@ AT_CHECK(grep '^ M 0 0 missing$' stdout, [], [ignore], [ignore]) AT_CHECK(grep '^ A 0 0 added$' stdout, [], [ignore], [ignore]) AT_CHECK(grep '^D 0 0 dropped$' stdout, [], [ignore], [ignore]) -AT_CHECK(grep '^R 1 0 original$' stdout, [], [ignore], [ignore]) -AT_CHECK(grep '^ R 0 1 renamed$' stdout, [], [ignore], [ignore]) +AT_CHECK(grep '^R 4 0 original$' stdout, [], [ignore], [ignore]) +AT_CHECK(grep '^ R 0 4 renamed$' stdout, [], [ignore], [ignore]) AT_CHECK(grep '^ P 0 0 patched$' stdout, [], [ignore], [ignore]) AT_CHECK(grep '^ 0 0 unchanged$' stdout, [], [ignore], [ignore]) AT_CHECK(grep '^ U 0 0 unknown$' stdout, [], [ignore], [ignore]) @@ -62,8 +62,8 @@ AT_CHECK(MTN rename temporary original, [], [ignore], [ignore]) AT_CHECK(MTN automate inventory --rcfile=inventory_hooks.lua, [], [stdout], [ignore]) -AT_CHECK(grep '^RRP 1 2 original$' stdout, [], [ignore], [ignore]) -AT_CHECK(grep '^RRP 2 1 unchanged$' stdout, [], [ignore], [ignore]) +AT_CHECK(grep '^RRP 4 6 original$' stdout, [], [ignore], [ignore]) +AT_CHECK(grep '^RRP 6 4 unchanged$' stdout, [], [ignore], [ignore]) # swapped and moved @@ -72,8 +72,8 @@ AT_CHECK(mv temporary original) AT_CHECK(MTN automate inventory --rcfile=inventory_hooks.lua, [], [stdout], [ignore]) -AT_CHECK(grep '^RR 1 2 original$' stdout, [], [ignore], [ignore]) -AT_CHECK(grep '^RR 2 1 unchanged$' stdout, [], [ignore], [ignore]) +AT_CHECK(grep '^RR 4 6 original$' stdout, [], [ignore], [ignore]) +AT_CHECK(grep '^RR 6 4 unchanged$' stdout, [], [ignore], [ignore]) # rename foo bar; add foo @@ -83,8 +83,8 @@ AT_CHECK(MTN add original, [], [ignore], [ignore]) AT_CHECK(MTN automate inventory --rcfile=inventory_hooks.lua, [], [stdout], [ignore]) -AT_CHECK(grep '^RA 1 0 original$' stdout, [], [ignore], [ignore]) -AT_CHECK(grep '^ R 0 1 renamed$' stdout, [], [ignore], [ignore]) +AT_CHECK(grep '^RA 4 0 original$' stdout, [], [ignore], [ignore]) +AT_CHECK(grep '^ R 0 4 renamed$' stdout, [], [ignore], [ignore]) # rotated but not moved # - note that things are listed and numbered in path collating order @@ -98,9 +98,9 @@ AT_CHECK(MTN rename temporary dropped, [], [ignore], [ignore]) AT_CHECK(MTN automate inventory --rcfile=inventory_hooks.lua, [], [stdout], [ignore]) -AT_CHECK(grep '^RRP 1 3 dropped$' stdout, [], [ignore], [ignore]) -AT_CHECK(grep '^RRP 2 1 missing$' stdout, [], [ignore], [ignore]) -AT_CHECK(grep '^RRP 3 2 original$' stdout, [], [ignore], [ignore]) +AT_CHECK(grep '^RRP 2 4 dropped$' stdout, [], [ignore], [ignore]) +AT_CHECK(grep '^RRP 3 2 missing$' stdout, [], [ignore], [ignore]) +AT_CHECK(grep '^RRP 4 3 original$' stdout, [], [ignore], [ignore]) # rotated and moved @@ -110,9 +110,9 @@ AT_CHECK(mv temporary dropped) AT_CHECK(MTN automate inventory --rcfile=inventory_hooks.lua, [], [stdout], [ignore]) -AT_CHECK(grep '^RR 1 3 dropped$' stdout, [], [ignore], [ignore]) -AT_CHECK(grep '^RR 2 1 missing$' stdout, [], [ignore], [ignore]) -AT_CHECK(grep '^RR 3 2 original$' stdout, [], [ignore], [ignore]) +AT_CHECK(grep '^RR 2 4 dropped$' stdout, [], [ignore], [ignore]) +AT_CHECK(grep '^RR 3 2 missing$' stdout, [], [ignore], [ignore]) +AT_CHECK(grep '^RR 4 3 original$' stdout, [], [ignore], [ignore]) # dropped but not removed and thus unknown @@ -141,8 +141,8 @@ AT_CHECK(MTN rename original renamed, [], [ignore], [ignore]) AT_CHECK(MTN automate inventory --rcfile=inventory_hooks.lua, [], [stdout], [ignore]) -AT_CHECK(grep '^R U 1 0 original$' stdout, [], [ignore], [ignore]) -AT_CHECK(grep '^ RM 0 1 renamed$' stdout, [], [ignore], [ignore]) +AT_CHECK(grep '^R U 4 0 original$' stdout, [], [ignore], [ignore]) +AT_CHECK(grep '^ RM 0 4 renamed$' stdout, [], [ignore], [ignore]) # moved but not renamed and thus missing source and unknown target @@ -165,8 +165,8 @@ AT_CHECK(MTN rename original renamed, [], [ignore], [ignore]) AT_CHECK(MTN automate inventory --rcfile=inventory_hooks.lua, [], [stdout], [ignore]) -AT_CHECK(grep '^R 1 0 original$' stdout, [], [ignore], [ignore]) -AT_CHECK(grep '^ RP 0 1 renamed$' stdout, [], [ignore], [ignore]) +AT_CHECK(grep '^R 4 0 original$' stdout, [], [ignore], [ignore]) +AT_CHECK(grep '^ RP 0 4 renamed$' stdout, [], [ignore], [ignore]) # need tests for deleted and renamed directories, once these actually work!