Re: [gnugo-devel] tactical move ordering tuning

[Gunnar Farneback]

I wrote:
> Included below is the script I'm using to get regression results with
> node counts. You need to have Pike installed to run it.

Appended is a new version of this script which also reports all GTP
failures while running the regressions. An example of this is shown
below:

manyfaces                              59.78   8439866   16644    35859
niki                                  102.60  15337870   19128    73797
trevor:         ? cannot open or parse 'games/nngs/theDoor-gnugo-3.3.11-2002-111
30142.sgf'
trevor:         ? cannot open or parse 'games/nngs/theDoor-gnugo-3.3.11-2002-111
30142.sgf'
trevor                                 89.41  13329594   28602    39041
tactics                                40.01   6871058    4124    40433

I've corrected the bad filename in trevor.tst directly in the CVS.

/Gunnar

#!/usr/bin/env pike

static Thread.Queue write_queue;
static Thread.Condition cond;
static Thread.Mutex condmutex;
static Thread.MutexKey condmutexkey;
static int debug = 0;
static object machine;
static mapping(int:string) correct_results = ([]);
static multiset expected_failures = (<>);
static int timebase;
static float last_time;
static float total_time = 0.0;
static int total_pass = 0;
static int total_fail = 0;

class Testsuite
{
  string name;
  int reading_nodes;
  int owl_nodes;
  int connection_nodes;
  float time;
  
  array(int) pass;
  array(int) fail;
  array(int) PASS;
  array(int) FAIL;

  void create(string s)
  {
    name = s;
    reading_nodes = 0;
    owl_nodes = 0;
    connection_nodes = 0;
    time = 0.0;
    pass = ({});
    fail = ({});
    PASS = ({});
    FAIL = ({});
  }
}

array(Testsuite) testsuites = ({});
Testsuite current_testsuite;

void Send(string|void s)
{
  if (!s) {
    if (debug)
      werror("Finishing sending.\n");
    write_queue->write("");
  }
  else {
    if (debug)
      werror("Sent: " + s + "\n");
    write_queue->write(s + "\n");
  }
}

static void Finish()
{
  write("%-37s %6.2f %9d %7d %8d\n", current_testsuite->name,
        current_testsuite->time,
        current_testsuite->reading_nodes, current_testsuite->owl_nodes,
        current_testsuite->connection_nodes);
  cond->signal();
}

static void ProgramReader(object f)
{
  int move;
  string stored = "";
  array(int) nodes = ({0, 0, 0});
  array(int) total_nodes = ({0, 0, 0});
  array(int) unexpected_failures = ({});
  array(int) unexpected_passes = ({});
  int test_number;
  while (1) {
    string s = f->gets();
    float current_time = time(timebase);
    if (!s)
      break;
    if (debug)
      werror("Recv: " + s + "\n");
    
    int number;
    string answer;
    if (sscanf(s, "=%d %s", number, answer)) {
      if (number < 10000) {
        test_number = (int) number;
        string correct = correct_results[test_number];
        int negate = 0;
        if (correct[0] == '!') {
          correct = correct[1..];
          negate = 1;
        }
        correct = "^" + correct + "$";
        object re = Regexp(correct);
        string result = (negate ^ re->match(answer)) ? "pass" : "fail";
        
        if (result == "pass" && expected_failures[test_number]) {
          result = "PASS";
        }
        if (result == "fail" && !expected_failures[test_number]) {
          result = "FAIL";
        }
        current_testsuite[result] += ({test_number});
        current_testsuite->time += (current_time - last_time);
        
        if (result == "PASS" || result == "FAIL")
          write("%-15s %s %s [%s]\n",
                current_testsuite->name + ":" + test_number, result, answer,
                correct_results[test_number]);
        last_time = current_time;
      }
      else if (number == 10000)
        current_testsuite->reading_nodes += (int) answer;
      else if (number == 10001)
        current_testsuite->owl_nodes += (int) answer;
      else if (number == 10002)
        current_testsuite->connection_nodes += (int) answer;
      else if (number == 10003)
        break;
    }
    else if (sscanf(s, "?%s", answer))
      write("%-15s ?%s\n", current_testsuite->name + ":", answer);
  }
  f->close();
  if (debug)
    werror("Reader closed down.\n");
  Finish();
}

static void ProgramWriter(object f)
{
  while (1) {
    string s = write_queue->read();
    if (s == "")
      break;
    f->write(s);
  }
  f->close();
  if (debug)
    werror("Writer closed down\n");
}

int run_testsuite(string suite_name, array(string) program_args)
{
  array(string) program_start_array = ({"../interface/gnugo", "--quiet",
                                        "--mode", "gtp"});
  write_queue = Thread.Queue();
  object f1 = Stdio.File();
  object pipe1 = f1->pipe();
  object f2 = Stdio.FILE();
  object pipe2 = f2->pipe();
  machine = Process.create_process(program_start_array + program_args,
                                   (["stdin":pipe1, "stdout":pipe2]));
  thread_create(ProgramReader, f2);
  thread_create(ProgramWriter, f1);

  string testsuite = Stdio.read_file(suite_name);
  if (!testsuite) {
    werror("Couldn't find " + suite_name + "\n");
    exit(1);
  }
  int number;
  string answer;
  string expected;
  
  timebase = time();
  last_time = time(timebase);
  
  correct_results = ([]);
  expected_failures = (<>);
  current_testsuite = Testsuite(suite_name - ".tst");
  testsuites += ({current_testsuite});
  
  foreach (testsuite/"\n", string s) {
    if (sscanf(s, "%d", number) == 1) {
      if (number >= 10000)
        continue;
      Send("reset_reading_node_counter");
      Send("reset_owl_node_counter");
      Send("reset_connection_node_counter");
      Send(s);
      Send("10000 get_reading_node_counter");
      Send("10001 get_owl_node_counter");
      Send("10002 get_connection_node_counter");
    }
    else if (sscanf(s, "#? [%[^]]]%s", answer, expected)) {
      correct_results[(int)number] = answer;
      if (expected == "*")
        expected_failures[(int)number] = 1;
    }
    else
      Send(s);
  }
  
  Send("10003 quit");
  Send();
  cond->wait(condmutexkey);
}

void final_report()
{
  float total_time = 0.0;
  int reading_nodes = 0;
  int owl_nodes = 0;
  int connection_nodes = 0;

  foreach (testsuites, Testsuite t) {
    total_time       += t->time;
    reading_nodes    += t->reading_nodes;
    owl_nodes        += t->owl_nodes;
    connection_nodes += t->connection_nodes;
  }
  write("Total nodes: %d %d %d\n", reading_nodes, owl_nodes,
        connection_nodes);
  write("Total time: %7.2f\n", total_time);
}

int main(int argc, array(string) argv)
{
  cond = Thread.Condition();
  condmutex = Thread.Mutex();
  condmutexkey = condmutex->lock();

  string makefile = Stdio.read_bytes("Makefile.am");
  foreach (makefile / "\n", string s) {
    string filename;
    if (sscanf(s, "%*sregress.sh $(srcdir) %s ", filename) == 2)
      run_testsuite(filename, argv[1..]);
  }
  final_report();
}