# # # patch "README.nuskool" # from [c44d99d78566da43f00fd36f23d262a7918f6289] # to [37607bce4fb4f81331a1878c8739294d3bbabf8f] # # patch "cmd_netsync.cc" # from [8afdb52702eb46694f476e910e538dca9dc2b0d6] # to [002313331d05e4ccbc3c582055f3fbb05d857503] # # patch "gsync.hh" # from [ceb06a9cee5d328e8c087e633e63e730b96ef6cc] # to [75ff928adbc2d121ce148b0e5e24d3e23ceede12] # # patch "http_client.cc" # from [9701fd83cece30f2d98cef3cea8f8dd59e6ae5cb] # to [8426be887870d861c82ca8be3bdcdfdb94d843eb] # # patch "http_client.hh" # from [aac41129c4ac4c3ca02e7b01168ce2c7ed202c27] # to [0419c936506dca108023fd5f9528d33ba735b696] # # patch "json_msgs.cc" # from [01a1ceced5984fcb428b110d8ca54c3707b0f148] # to [e5675a4dd2f25383886856de7e108b33c197dff8] # # patch "json_msgs.hh" # from [49912f31573a7568ff4b112db180da479735be03] # to [a1e594d833128a3f9c60d81591b3b856c84cf923] # ============================================================ --- README.nuskool c44d99d78566da43f00fd36f23d262a7918f6289 +++ README.nuskool 37607bce4fb4f81331a1878c8739294d3bbabf8f @@ -1,4 +1,4 @@ -Here's what you need to do to try out the nuskool sync protocol +Here's what you need to do to try out the nuskool sync protocol using SCGI 1. install lighttpd (apache should work too but the config below is for lighty) @@ -18,10 +18,23 @@ 3. start lighttpd $ /etc/init.d/lighttpd start -4. start an scgi server for lighttpd to proxy to +4. start the gsync server for lighttpd to proxy to - $ mtn scgi --db server.db + $ mtn gserve --db server.db 5. run the gsync client $ mtn --db client.db gsync http://localhost/monotone '*' --debug + + + +Here's what you need to do to try out the nuskool sync protocol using raw HTTP + +1. start the gsync server listening for direct http connections on port 8008 + + $ mtn gserve --db server.db --http + +2. run the gsync client and specify the alternate http port 8008 + + $ mtn --db client.db gsync http://localhost:8008 '*' --debug + ============================================================ --- cmd_netsync.cc 8afdb52702eb46694f476e910e538dca9dc2b0d6 +++ cmd_netsync.cc 002313331d05e4ccbc3c582055f3fbb05d857503 @@ -534,8 +534,10 @@ CMD(gsync, "gsync", "", CMD_REF(network) netsync_connection_info info; extract_client_connection_info(app.opts, app.lua, db, keys, args, info); + // how do we select between json and basic_io? + http_client h(app.opts, app.lua, info); - run_gsync_protocol(app.lua, db, http_channel(h), + run_gsync_protocol(app.lua, db, json_channel(h), app.opts.dryrun); } ============================================================ --- gsync.hh ceb06a9cee5d328e8c087e633e63e730b96ef6cc +++ gsync.hh 75ff928adbc2d121ce148b0e5e24d3e23ceede12 @@ -18,9 +18,24 @@ struct globish; struct uri; struct globish; -struct file_data_record; -struct file_delta_record; +struct file_data_record +{ + file_id id; + file_data dat; + file_data_record(file_id id, file_data dat) : + id(id), dat(dat) {} +}; + +struct file_delta_record +{ + file_id src_id; + file_id dst_id; + file_delta del; + file_delta_record(file_id src_id, file_id dst_id, file_delta del) : + src_id(src_id), dst_id(dst_id), del(del) {} +}; + class lua_hooks; class database; class revision_t; ============================================================ --- http_client.cc 9701fd83cece30f2d98cef3cea8f8dd59e6ae5cb +++ http_client.cc 8426be887870d861c82ca8be3bdcdfdb94d843eb @@ -62,8 +62,8 @@ http_client::http_client(options & opts, open(true) {} -json_value_t -http_client::transact_json(json_value_t v) +void +http_client::execute(string const & request, string & response) { if (!open) { @@ -83,45 +83,36 @@ http_client::transact_json(json_value_t I(io); I(open); - json_io::printer out; - v->write(out); string header = (F("POST %s HTTP/1.1\r\n" "Host: %s\r\n" "Content-Length: %s\r\n" - "Content-Type: application/jsonrequest\r\n" - "Accept: application/jsonrequest\r\n" + "Content-Type: application/octet-stream\r\n" + "Accept: application/octet-stream\r\n" "Accept-Encoding: identity\r\n" "\r\n") % (info.client.u.path.empty() ? "/" : info.client.u.path) % info.client.u.host - % lexical_cast(out.buf.size())).str(); + % lexical_cast(request.size())).str(); L(FL("http_client: sending request [[POST %s HTTP/1.1]]") % (info.client.u.path.empty() ? "/" : info.client.u.path)); L(FL("http_client: to [[Host: %s]]") % info.client.u.host); - L(FL("http_client: sending %d-byte body") % out.buf.size()); + L(FL("http_client: sending %d-byte body") % request.size()); io->write(header.data(), header.size()); - io->write(out.buf.data(), out.buf.length()); + io->write(request.data(), request.size()); io->flush(); - L(FL("http_client: sent %d-byte body") % out.buf.size()); + L(FL("http_client: sent %d-byte body") % request.size()); -// std::cerr << "json request" << std::endl +// std::cerr << "http request" << std::endl // << out.buf.data() << std::endl; // Now read back the result - string data; - parse_http_response(data); + parse_http_response(response); -// std::cerr << "json response" << std::endl +// std::cerr << "http response" << std::endl // << data << std::endl; - - json_io::input_source in(data, "scgi"); - json_io::tokenizer tok(in); - json_io::parser p(tok); - return p.parse_object(); } - void http_client::parse_http_status_line() { @@ -232,34 +223,50 @@ http_client::parse_http_response(std::st ///////////////////////////////////////////////////////////////////// -// http_channel adaptor +// json_channel adaptor ///////////////////////////////////////////////////////////////////// +json_value_t +json_channel::transact(json_value_t v) const +{ + json_io::printer out; + v->write(out); + + string request(out.buf); + string response; + client.execute(request, response); + + json_io::input_source in(response, "json"); + json_io::tokenizer tok(in); + json_io::parser p(tok); + return p.parse_object(); +} + void -http_channel::inquire_about_revs(set const & query_set, +json_channel::inquire_about_revs(set const & query_set, set & theirs) const { theirs.clear(); json_value_t request = encode_msg_inquire_request(query_set); - json_value_t response = client.transact_json(request); + json_value_t response = transact(request); E(decode_msg_inquire_response(response, theirs), F("received unexpected reply to 'inquire_request' message")); } void -http_channel::get_descendants(set const & common_revs, +json_channel::get_descendants(set const & common_revs, vector & inbound_revs) const { inbound_revs.clear(); json_value_t request = encode_msg_descendants_request(common_revs); - json_value_t response = client.transact_json(request); + json_value_t response = transact(request); E(decode_msg_descendants_response(response, inbound_revs), F("received unexpected reply to 'descendants_request' message")); } void -http_channel::push_full_rev(revision_id const & rid, +json_channel::push_full_rev(revision_id const & rid, revision_t const & rev, vector const & data_records, vector const & delta_records) const @@ -267,82 +274,82 @@ http_channel::push_full_rev(revision_id json_value_t request = encode_msg_put_full_rev_request(rid, rev, data_records, delta_records); - json_value_t response = client.transact_json(request); + json_value_t response = transact(request); E(decode_msg_put_full_rev_response(response), F("received unexpected reply to 'put_full_rev_request' message")); } void -http_channel::pull_full_rev(revision_id const & rid, +json_channel::pull_full_rev(revision_id const & rid, revision_t & rev, vector & data_records, vector & delta_records) const { json_value_t request = encode_msg_get_full_rev_request(rid); - json_value_t response = client.transact_json(request); + json_value_t response = transact(request); E(decode_msg_get_full_rev_response(response, rev, data_records, delta_records), F("received unexpected reply to 'get_full_rev_request' message")); } void -http_channel::push_file_data(file_id const & id, +json_channel::push_file_data(file_id const & id, file_data const & data) const { json_value_t request = encode_msg_put_file_data_request(id, data); - json_value_t response = client.transact_json(request); + json_value_t response = transact(request); E(decode_msg_put_file_data_response(response), F("received unexpected reply to 'put_file_data_request' message")); } void -http_channel::push_file_delta(file_id const & old_id, +json_channel::push_file_delta(file_id const & old_id, file_id const & new_id, file_delta const & delta) const { json_value_t request = encode_msg_put_file_delta_request( old_id, new_id, delta); - json_value_t response = client.transact_json(request); + json_value_t response = transact(request); E(decode_msg_put_file_delta_response(response), F("received unexpected reply to 'put_file_delta_request' message")); } void -http_channel::push_rev(revision_id const & rid, +json_channel::push_rev(revision_id const & rid, revision_t const & rev) const { json_value_t request = encode_msg_put_rev_request(rid, rev); - json_value_t response = client.transact_json(request); + json_value_t response = transact(request); E(decode_msg_put_rev_response(response), F("received unexpected reply to 'put_rev_request' message")); } void -http_channel::pull_rev(revision_id const & rid, revision_t & rev) const +json_channel::pull_rev(revision_id const & rid, revision_t & rev) const { json_value_t request = encode_msg_get_rev_request(rid); - json_value_t response = client.transact_json(request); + json_value_t response = transact(request); E(decode_msg_get_rev_response(response, rev), F("received unexpected reply to 'get_rev_request' message")); } void -http_channel::pull_file_data(file_id const & id, +json_channel::pull_file_data(file_id const & id, file_data & data) const { json_value_t request = encode_msg_get_file_data_request(id); - json_value_t response = client.transact_json(request); + json_value_t response = transact(request); E(decode_msg_get_file_data_response(response, data), F("received unexpected reply to 'get_file_data_request' message")); } void -http_channel::pull_file_delta(file_id const & old_id, +json_channel::pull_file_delta(file_id const & old_id, file_id const & new_id, file_delta & delta) const { json_value_t request = encode_msg_get_file_delta_request(old_id, new_id); - json_value_t response = client.transact_json(request); + json_value_t response = transact(request); E(decode_msg_get_file_delta_response(response, delta), F("received unexpected reply to 'get_file_delta_request' message")); } ============================================================ --- http_client.hh aac41129c4ac4c3ca02e7b01168ce2c7ed202c27 +++ http_client.hh 0419c936506dca108023fd5f9528d33ba735b696 @@ -26,9 +26,9 @@ class lua_hooks; struct netsync_connection_info; class lua_hooks; -struct -http_client +struct http_client { + options & opts; lua_hooks & lua; netsync_connection_info const & info; @@ -41,7 +41,8 @@ http_client http_client(options & opts, lua_hooks & lua, netsync_connection_info const & info); - json_io::json_value_t transact_json(json_io::json_value_t v); + void execute(std::string const & request, std::string & response); + void parse_http_status_line(); void parse_http_header_line(size_t & content_length, bool & keepalive); @@ -49,14 +50,17 @@ http_client void crlf(); }; -class http_channel +class json_channel : public channel { http_client & client; public: - http_channel(http_client & c) + json_channel(http_client & c) : client(c) { }; + + json_io::json_value_t transact(json_io::json_value_t v) const; + virtual void inquire_about_revs(std::set const & query_set, std::set & theirs) const; virtual void get_descendants(std::set const & common_revs, ============================================================ --- json_msgs.cc 01a1ceced5984fcb428b110d8ca54c3707b0f148 +++ json_msgs.cc e5675a4dd2f25383886856de7e108b33c197dff8 @@ -9,9 +9,10 @@ #include "base.hh" +#include "cset.hh" +#include "gsync.hh" #include "json_io.hh" #include "json_msgs.hh" -#include "cset.hh" #include "transforms.hh" #include ============================================================ --- json_msgs.hh 49912f31573a7568ff4b112db180da479735be03 +++ json_msgs.hh a1e594d833128a3f9c60d81591b3b856c84cf923 @@ -22,6 +22,9 @@ #include "revision.hh" #include "vocab.hh" +struct file_data_record; +struct file_delta_record; + json_io::json_value_t encode_msg_error(std::string const & note); bool decode_msg_error(json_io::json_value_t val, std::string & note); @@ -66,23 +69,6 @@ bool decode_msg_put_rev_response(json_io // full revs with all file data and deltas in one request/response -struct file_data_record -{ - file_id id; - file_data dat; - file_data_record(file_id id, file_data dat) : - id(id), dat(dat) {} -}; - -struct file_delta_record -{ - file_id src_id; - file_id dst_id; - file_delta del; - file_delta_record(file_id src_id, file_id dst_id, file_delta del) : - src_id(src_id), dst_id(dst_id), del(del) {} -}; - json_io::json_value_t encode_msg_get_full_rev_request(revision_id const & rid); bool decode_msg_get_full_rev_request(json_io::json_value_t val, revision_id & rid);