[Wesnoth-cvs-commits] wesnoth/src tstring.cpp tstring.hpp serializati...

[Philippe Plantier]

CVSROOT:        /cvsroot/wesnoth
Module name:    wesnoth
Branch:         
Changes by:     Philippe Plantier <address@hidden>      05/03/29 20:19:33

Modified files:
        src            : tstring.cpp tstring.hpp 
        src/serialization: binary_wml.cpp tokenizer.cpp tokenizer.hpp 

Log message:
        Indexing in-memory textdomains to reduce memory usage. Bind textdomain 
when
        they are first indexed, to fix a bug where textdomains where not bound 
when
        loading binary WML saves.

CVSWeb URLs:
http://savannah.gnu.org/cgi-bin/viewcvs/wesnoth/wesnoth/src/tstring.cpp.diff?tr1=1.1&tr2=1.2&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/wesnoth/wesnoth/src/tstring.hpp.diff?tr1=1.2&tr2=1.3&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/wesnoth/wesnoth/src/serialization/binary_wml.cpp.diff?tr1=1.10&tr2=1.11&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/wesnoth/wesnoth/src/serialization/tokenizer.cpp.diff?tr1=1.4&tr2=1.5&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/wesnoth/wesnoth/src/serialization/tokenizer.hpp.diff?tr1=1.1&tr2=1.2&r1=text&r2=text

Patches:
Index: wesnoth/src/serialization/binary_wml.cpp
diff -u wesnoth/src/serialization/binary_wml.cpp:1.10 
wesnoth/src/serialization/binary_wml.cpp:1.11
--- wesnoth/src/serialization/binary_wml.cpp:1.10       Sun Mar 27 23:06:17 2005
+++ wesnoth/src/serialization/binary_wml.cpp    Tue Mar 29 20:19:33 2005
@@ -1,4 +1,4 @@
-/* $Id: binary_wml.cpp,v 1.10 2005/03/27 23:06:17 gruikya Exp $ */
+/* $Id: binary_wml.cpp,v 1.11 2005/03/29 20:19:33 gruikya Exp $ */
 /*
    Copyright (C) 2003 by David White <address@hidden>
    Copyright (C) 2005 by Guillaume Melquiond <address@hidden>
@@ -131,7 +131,7 @@
                        compress_emit_word(out, i->first, schema);
 
                        //output the value, with no compression
-                       compress_output_literal_word(out, i->second.value());
+                       compress_output_literal_word(out, 
i->second.to_serialized());
                }
        }
 
Index: wesnoth/src/serialization/tokenizer.cpp
diff -u wesnoth/src/serialization/tokenizer.cpp:1.4 
wesnoth/src/serialization/tokenizer.cpp:1.5
--- wesnoth/src/serialization/tokenizer.cpp:1.4 Mon Mar 28 14:07:31 2005
+++ wesnoth/src/serialization/tokenizer.cpp     Tue Mar 29 20:19:33 2005
@@ -1,4 +1,4 @@
-/* $Id: tokenizer.cpp,v 1.4 2005/03/28 14:07:31 gruikya Exp $ */
+/* $Id: tokenizer.cpp,v 1.5 2005/03/29 20:19:33 gruikya Exp $ */
 /*
    Copyright (C) 2004 by Philippe Plantier <address@hidden>
    Part of the Battle for Wesnoth Project http://www.wesnoth.org
@@ -13,8 +13,6 @@
 
 #include "tokenizer.hpp"
 #include "string_utils.hpp"
-#include "filesystem.hpp"
-#include "gettext.hpp"
 
 #include <iostream>
 #include <sstream>
@@ -55,10 +53,12 @@
                if ((comment_line.size() == 2 || comment_line.size() == 3)
                                && comment_line[0] == "#textdomain") {
                        textdomain_ = comment_line[1];
+#if 0
                        std::string path;
                        if (comment_line.size() == 3)
                                path = comment_line[2];
                        textdomain_init(textdomain_, path);
+#endif
                }
        } 
 
@@ -163,8 +163,3 @@
        return textdomain_;
 }
 
-void tokenizer::textdomain_init(const std::string& domain, const std::string& 
path)
-{
-       bindtextdomain(domain.c_str(), path.empty() ? get_intl_dir().c_str() : 
path.c_str());
-       bind_textdomain_codeset(domain.c_str(), "UTF-8");
-}
Index: wesnoth/src/serialization/tokenizer.hpp
diff -u wesnoth/src/serialization/tokenizer.hpp:1.1 
wesnoth/src/serialization/tokenizer.hpp:1.2
--- wesnoth/src/serialization/tokenizer.hpp:1.1 Sun Mar 27 23:06:17 2005
+++ wesnoth/src/serialization/tokenizer.hpp     Tue Mar 29 20:19:33 2005
@@ -1,4 +1,4 @@
-/* $Id: tokenizer.hpp,v 1.1 2005/03/27 23:06:17 gruikya Exp $ */
+/* $Id: tokenizer.hpp,v 1.2 2005/03/29 20:19:33 gruikya Exp $ */
 /*
    Copyright (C) 2004 by Philippe Plantier <address@hidden>
    Part of the Battle for Wesnoth Project http://www.wesnoth.org
@@ -58,7 +58,7 @@
        int peek_char();
        bool is_space(int c);
        bool is_alnum(int c);
-       void textdomain_init(const std::string& domain, const std::string& 
path);
+//     void textdomain_init(const std::string& domain, const std::string& 
path);
 
        std::istream& in_;
        int current_;
Index: wesnoth/src/tstring.cpp
diff -u wesnoth/src/tstring.cpp:1.1 wesnoth/src/tstring.cpp:1.2
--- wesnoth/src/tstring.cpp:1.1 Sun Mar 27 23:06:17 2005
+++ wesnoth/src/tstring.cpp     Tue Mar 29 20:19:33 2005
@@ -1,4 +1,4 @@
-/* $Id: tstring.cpp,v 1.1 2005/03/27 23:06:17 gruikya Exp $ */
+/* $Id: tstring.cpp,v 1.2 2005/03/29 20:19:33 gruikya Exp $ */
 /*
    Copyright (C) 2004 by Philippe Plantier <address@hidden>
    Part of the Battle for Wesnoth Project http://www.wesnoth.org
@@ -17,6 +17,7 @@
 #include "tstring.hpp"
 #include "wassert.hpp"
 #include "gettext.hpp"
+#include "filesystem.hpp"
 #include "log.hpp"
 
 #define ERR_CF lg::err(lg::config)
@@ -25,9 +26,10 @@
        const char TRANSLATABLE_PART = 0x01;
        const char UNTRANSLATABLE_PART = 0x02;
        const char TEXTDOMAIN_SEPARATOR = 0x03;
+       const char ID_TRANSLATABLE_PART = 0x04;
 
-       //std::vector<std::string> id_to_textdomain;
-       //std::map<std::string, char> textdomain_to_id;
+       std::vector<std::string> id_to_textdomain;
+       std::map<std::string, unsigned int> textdomain_to_id;
 }
 
 t_string::walker::walker(const t_string& string) :
@@ -88,7 +90,10 @@
 
 void t_string::walker::update()
 {
-       static std::string mark = std::string(TRANSLATABLE_PART, 1) + 
UNTRANSLATABLE_PART;
+       unsigned int id;
+
+       static std::string mark = std::string(TRANSLATABLE_PART, 1) + 
UNTRANSLATABLE_PART +
+               ID_TRANSLATABLE_PART;
 
        if(begin_ == string_.size())
                return;
@@ -114,18 +119,44 @@
 
                break;
        }
+       case ID_TRANSLATABLE_PART:
+               if(begin_ + 3 >= string_.size()) {
+                       ERR_CF << "Error: invalid string\n";
+                       begin_ = string_.size();
+                       return;
+               }
+               end_ = string_.find_first_of(mark, begin_ + 3);
+               if(end_ == std::string::npos)
+                       end_ = string_.size();
+
+               id = string_[begin_ + 1] + string_[begin_ + 2] * 256;
+               if(id >= id_to_textdomain.size()) {
+                       ERR_CF << "Error: invalid string\n";
+                       begin_ = string_.size();
+                       return;
+               }
+               textdomain_ = id_to_textdomain[id];
+               begin_ += 3;
+               translatable_ = true;
+
+               break;
+
        case UNTRANSLATABLE_PART:
                end_ = string_.find_first_of(mark, begin_ + 1);
-               if (end_ == std::string::npos)
+               if(end_ == std::string::npos)
                        end_ = string_.size();
 
-               wassert(int(end_) - int(begin_) - 1 >= 0);
+               if(end_ <= begin_ + 1) {
+                       ERR_CF << "Error: invalid string\n";
+                       begin_ = string_.size();
+                       return;
+               }
 
                translatable_ = false;
                textdomain_ = "";
                begin_ += 1;
-
                break;
+
        default:
                end_ = string_.size();
                translatable_ = false;
@@ -154,10 +185,25 @@
 
 t_string::t_string(const std::string& string, const std::string& textdomain) :
        translatable_(true),
-       value_(TRANSLATABLE_PART, 1)
+       value_(1, ID_TRANSLATABLE_PART)
 {
-       value_ += textdomain;
-       value_ += TEXTDOMAIN_SEPARATOR;
+       std::map<std::string, unsigned int>::const_iterator idi = 
textdomain_to_id.find(textdomain);
+       unsigned int id;
+
+       if(idi == textdomain_to_id.end()) {
+               textdomain_to_id[textdomain] = id_to_textdomain.size();
+               id = id_to_textdomain.size();
+               id_to_textdomain.push_back(textdomain);
+
+               // Register and bind this textdomain
+               bindtextdomain(textdomain.c_str(), get_intl_dir().c_str());
+               bind_textdomain_codeset(textdomain.c_str(), "UTF-8");
+       } else {
+               id = idi->second;
+       }
+
+       value_ += char(id & 0xff);
+       value_ += char(id >> 8);
        value_ += string;
 }
 
@@ -173,17 +219,52 @@
 
 t_string t_string::from_serialized(const std::string& string)
 {
-       t_string res(string);
+       t_string orig(string);
        
        if(!string.empty() && (string[0] == TRANSLATABLE_PART || string[0] == 
UNTRANSLATABLE_PART)) {
-               res.translatable_ = true;
+               orig.translatable_ = true;
        } else {
-               res.translatable_ = false;
+               orig.translatable_ = false;
+       }
+       
+       t_string res;
+
+       for(walker w(orig); !w.eos(); w.next()) {
+               std::string substr(w.begin(), w.end());
+
+               if(w.translatable()) {
+                       res += t_string(substr, w.textdomain());
+               } else {
+                       res += substr;
+               }
        }
 
        return res;
 }
 
+std::string t_string::to_serialized() const
+{
+       t_string res;
+
+       for(walker w(*this); !w.eos(); w.next()) {
+               t_string chunk;
+
+               std::string substr(w.begin(), w.end());
+               if(w.translatable()) {
+                       chunk.translatable_ = true;
+                       chunk.value_ = TRANSLATABLE_PART + w.textdomain() + 
+                               TEXTDOMAIN_SEPARATOR + substr;
+               } else {
+                       chunk.translatable_ = false;
+                       chunk.value_ = substr;
+               }
+
+               res += chunk;
+       }
+
+       return res.value();
+}
+
 t_string& t_string::operator=(const t_string& string)
 {
        value_ = string.value_;
Index: wesnoth/src/tstring.hpp
diff -u wesnoth/src/tstring.hpp:1.2 wesnoth/src/tstring.hpp:1.3
--- wesnoth/src/tstring.hpp:1.2 Mon Mar 28 07:39:48 2005
+++ wesnoth/src/tstring.hpp     Tue Mar 29 20:19:33 2005
@@ -1,4 +1,4 @@
-/* $Id: tstring.hpp,v 1.2 2005/03/28 07:39:48 silene Exp $ */
+/* $Id: tstring.hpp,v 1.3 2005/03/29 20:19:33 gruikya Exp $ */
 /*
    Copyright (C) 2004 by Philippe Plantier <address@hidden>
    Part of the Battle for Wesnoth Project http://www.wesnoth.org
@@ -50,6 +50,7 @@
        ~t_string();
 
        static t_string from_serialized(const std::string& string);
+       std::string to_serialized() const;
 
        t_string& operator=(const t_string&);
        t_string& operator=(const std::string&);