octave-maintainers
[Top][All Lists]
Advanced

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

updated test scripts


From: Paul Kienzle
Subject: updated test scripts
Date: Tue, 2 May 2000 07:09:22 +0100 (BST)

test.m

## Copyright (C) 2000 Paul Kienzle
##
## This program is free software and may be used for any purpose. This
## copyright notice must be maintained.  Paul Kienzle is not responsible
## for the consequences of using this software.

## test('name')
##   Perform all tests in the first file matching 'name', 'name.m' or
##   'name.cc' in your loadpath.  Tests are done interactively, stopping
##   when they encounter the first error.
##
## test('name', ['quiet'|'normal'|'verbose'])
##   'quiet': Don't do tests which require user interaction
##       (e.g., looking at graphs or examining results vectors).
##   'normal': Do tests which require user interaction, but don't
##       display tests as they happen.
##   'verbose': Display tests as they happen.
##   Tests are done interactively, stopping at the first error.
##
## test('name', ['quiet'|'normal'|'verbose'], fid)
##   'quiet': Don't log all the tests as they happen, just the errors.
##   'normal': Log all tests as they happen, but don't do tests
##       which require user interaction.
##   'verbose': Do tests which require user interaction.
##   Batch processing.  Write errors to already open file fid (hopefully
##   then when octave crashes this file will tell you what was happening
##   when it did).  You can use stdout if you want to see the results as
##   they happen.
##
## success = test(...)
##   return true if all the tests succeeded.
##
## Lines which start with %! in file xxxx are test lines written in
## in Octave which are evaluated when you call the function test('xxxx').
## You can even embed them in C or C++ files if you do something like:
##    #if 0
##    %! disp('this is a test')
##    #endif
##
## Since eval() will stop at the first error it encounters, you must
## divide your tests up into blocks, with anything in a separate
## block (indicated by << >>) evaluated separately.  Blocks may not
## nest.
##
## There are a number of special block types:
##    <<shared x,y,z>> 
##        Variables which are shared between all remaining blocks.
##        Variables are all initialized to [], even if they are alreay
##        defined.
##    <<assert code::answer>>
##        Evaluates code and fail if !all(all(ans=answer)).  If ::answer
##        is not specified, then all(all(ans)) must evaluate to true. 
##        The code may use any shared variables.
##    <<usage name(args)>>
##        Verify that usage info is printed when name(args) is evaluated.
##        If name(args) is not given, then the name passed to test is 
##        used without arguments.
##    <<error code>>
##        The code should generate an error message.  Test fails
##        otherwise.  You may use any shared variables in the code.
##    <<noisy code>>
##        The user will be asked to pause after this block of code
##        is executed.  Put all graphs and print statements in this
##        kind of block.
##    << code>>
##        Arbitrary code.  Don't do any user interaction (i.e., graphs
##        or print statements).  Assign values needed to shared 
##        variables but don't perform any tests.  Use <<assert condition>> 
##        to test results. (well, if you do do tests, you can signal
##        failure by a call to error("");
##    <<# whatever>>
##        don't process the block.  Useful if you want to temporarily
##        disable some tests. WARNING: because of my rather simplistic
##        parsing, #<< whatever>> does NOT comment out the block.

## TODO: Consider protecting __xxx variables by eliminating them
## TODO:    from the shared list automatically.
## TODO: Assertion values should be logged in the test file in
## TODO:    batch mode.  How to output an arbitrary variable
## TODO:    pretty-printed to a file?  "disp(v,fid)" would be nice.
## TODO: Verify that the error returned is a usage error.  How??
## TODO: Detect warnings?
## TODO: Determine if error is elegant or if it is something obscure
## TODO:    like index out of range or matrix contains empty
## TODO: Assertions need a tolerance variable for equality.

function __success_r = test (__name, __flag, __fid)
  if (nargin < 2 || isempty(__flag))
    __flag = 'normal'; 
  endif
  if (nargin < 3) 
    __fid = []; 
  endif
  if (nargin < 1 || nargin > 3 || !isstr(__name) || !isstr(__flag))
    usage("success = test('name', ['quiet'|'normal'|'verbose'], fid)");
  endif
  __batch = (!isempty(__fid));

  if (strcmp(__flag, "normal"))
    __alltests = !__batch;
    __verbose = __batch;
  elseif (strcmp(__flag, "quiet"))
    __alltests = 0;
    __verbose = 0;
  elseif (strcmp(__flag, "verbose"))
    __alltests = 1;
    __verbose = 1;
  else
    error(["test unknown flag '" __flag "'"]);
  endif
    
  ## information from test will be introduced by "key" 
  __status = "***** ";
  __fail = "!!!!! ";

  ## decide if error messages should be collected
  if (__batch)
    fputs (__fid, [__status "processing " __name "\n" ]);
  else
    __fid = stdout;
  endif

  ## locate the file to test
  __file = file_in_loadpath (__name);
  if (isempty (__file)) 
    __file = file_in_loadpath ([__name ".m"]);
  endif
  if (isempty (__file))
    __file = file_in_loadpath ([__name ".cc"]);
  endif
  if (isempty (__file))
    fputs(__fid, [__fail __name " does not exist in path\n" ]);
    if (nargout > 0) __success_r = 0; endif
    return;
  endif

  ## grab the test code from the file
  __body = system([ "sed -n 's/^%!//p' " __file]);
  if (isempty (__body))
    fputs(__fid, [ __fail __file " has no tests available\n" ]);
    if (nargout > 0) __success_r = 0; endif
    return;
  endif

  ## chop it up into blocks for evaluation
  __blockstart = findstr ("<<", __body);
  __blockend = findstr (">>", __body);
  if (any (size (__blockstart) != size (__blockend)) 
      || any (__blockstart > __blockend))
    fputs(__fid, [ __fail __file " has invalid << block structure>>\n" ]);
    if (nargout > 0) __success_r = 0; endif
    return;
  endif

  ## ready to start tests ... if in batch mode, tell us what is happening
  if (__verbose)
    disp ([ __status __file ]);
  endif

  ## assume all tests will pass
  __all_success = 1;

  ## if no blocks given, pretend we have one big block
  if isempty(__blockstart)
    __blockstart = 1;
    __blockend = length (__body) + 3;
    __body = ["<< " __body ">>"];
  endif

  ## process each block separately, initially with no shared variables
  __shared = " ";
  __shared_r = " ";
  for __i=1:length(__blockstart)

    ## extract the block
    __block = __body(__blockstart(__i)+2:__blockend(__i)-1);
    
    ## let the user/logfile know what is happening
    if (__verbose)
      fputs (__fid, [__status "<<" __block ">>\n"]);
    endif

    ## pad with blanks for indexing purposes
    __block = [ __block "        " ];
    
    ## assume the block will succeed;
    __success = 1;
    __msg = [];
    __show_assert = 0;
    
### SHARED
    if strcmp (__block (1:6), "shared")
      __code = deblank (__block (7:length(__block)));
      ## variables need default values
      try
        eval([strrep(__code,",","=[];") "=[];"]);
        __success = 1; # in case the user is fiddling them
        __msg = []; 
        if (length (__code) == 0)
          __shared = " ";
          __shared_r = " ";
        else
          __shared = __code;
          __shared_r = ["[ " __code "] = "];
        endif
      catch
        __success = 0;
        __msg = [ __fail "shared variable initialization\n"];
      end_try_catch
      
### ASSERT
    elseif strcmp (__block (1:6), "assert")
      __code = deblank (__block (7:length (__block)));

      ## extract answer experssion
      __answer_idx = index(__code, "::");
      if (__answer_idx != 0)
        __answer_code = __code(__answer_idx+2:length(__code));
        __code = __code(1:__answer_idx-1);
      endif

      ## add __value to the returned variables
      if (!strcmp(__shared, " "))
        __returns = [ "[ __value, " __shared "] = " ];
      else
        __returns = "__value =";
      endif

      ## evaluate test expression
      try
        eval(["function " __returns "__test__(" __shared ")\n" ...
              __code ";\n__value = ans;\nendfunction"]);
        eval([ __returns "__test__(" __shared ");"]);
        __test_value = __value;
      catch
        __success = 0;
        __msg = [ __fail "assert expression caused an error!\n" \
                 __error_text__];
      end_try_catch
      clear __test__;


      ## evaluate answer
      if (__success)
        if(__answer_idx>0)
          try
            eval(["function " __returns "__test__(" __shared ")\n" \
                  __answer_code ";\n__value = ans;\nendfunction"]);
            eval([ __returns "__test__(" __shared ");"]);
            if (!all(size(__test_value)==size(__value)) || 
                !all(all(__test_value == __value)))
              __success = 0;
              __show_assert = 2;
              __msg = [ __fail "assert expression != answer\n"];
            endif
          catch
            __success = 0;
            __msg = [ __fail "assert answer caused an error!\n" \
                     __error_text__];
          end_try_catch
          clear __test__;
        elseif (!all (all (__test_value)))
          __success = 0;
          __show_assert = 1;
          __msg = [ __fail "assert expression not all true\n"];
        endif
      endif
      
### USAGE
    elseif strcmp (__block(1:5), "usage")
      __code = deblank(__block (6:length (__block)));
      if (length (__code) == 0)
        __code = __name;
      endif
      __success = 0;
      __msg = [ __fail "usage test does not cause an error!\n"];
      try
        eval ([__code ";"]);
      catch
        ## if ( length (__error_text__) > 7 ... 
        ##    && strcmp (__error_text__(1:7), "usage: ") );
        __success = 1;
        __msg = [ __status "usage produced:\n" __error_text__];
        ## else
        ##  __success = 0;
        ##  __msg = [ __fail "usage produced an error!\n" __error_text__];
        ## endif
      end_try_catch
      
### ERROR
    elseif strcmp (__block(1:5), "error")
      __code = deblank(__block (6:length (__block)));
      try
        eval(["function " __shared_r "__test__(" __shared ")\n" ...
              __code "\nendfunction"]);
      catch
        __success = 0;
        __msg = [ __fail "error test  syntax error!\n" __error_text__];
      end_try_catch
      
      if (__success)
        __success = 0;
        __msg = [ __fail "error test does not cause an error!\n" ];
        try
          eval([ __shared_r "__test__(" __shared ");"]);
        catch
          __success = 1;
          __msg = [ __status "error test caused the following error:\n" ...
                   __error_text__ ];
        end_try_catch
        clear __test__;
      endif
      
### NOISY
    elseif strcmp (__block(1:5), "noisy")
      if (__alltests)
        __code = deblank(__block (6:length (__block)));
        try
          eval(["function " __shared_r "__test__(" __shared ")\n" ...
                __code "\nendfunction"]);
          eval([__shared_r "__test__(" __shared ");"]);
          input("Press <enter> to continue: ");
        catch
          __success = 0;
          __msg = [ __status "noisy test caused an error!\n" __error_text__];
        end_try_catch
        clear __test__;
      else
        __msg = [ __status "skipping\n" ];
      endif
      
### comment block
    elseif strcmp (__block(1:1), "#")
      __msg = [ __status "skipping\n" ];
      
### default code block
    elseif isspace(__block(1))
      __code = deblank(__block (2:length (__block)));
      try
        eval(["function " __shared_r "__test__(" __shared ")\n" ...
              __code "\nendfunction"]);
        eval([__shared_r "__test__(" __shared ");"]);
      catch
        __success = 0;
        __msg = [ __fail "code error\n" __error_text__];
      end_try_catch
      clear __test__;
      
### unkown block
    else
      __success = 0;
      __msg = [ __fail "unknown test type!\n"];
    endif
    
    ## All done.  Remember if we were successful and print any messages
    if (__success == 0)
      __all_success = 0;
      ## make sure the user knows what caused the error
      if (!__verbose)
        fputs (__fid, [__status "<<" __block ">>\n"]);
      endif
      fputs (__fid, __msg);
      if (!__batch)
        ## display assert values
        if (__show_assert >= 1)
          disp([__fail "assert expression evaluates to:"]);
          disp(__test_value);
        endif
        if (__show_assert == 2)
          disp([__fail "assert answer evaluates to:"]);
          disp(__value);
        endif
        ## stop after one error if not in batch mode
        if (nargout > 0) __success_r = 0; endif
        return;
      endif
    elseif (__verbose && !isempty(__msg))
      fputs (__fid, __msg);
    endif
  endfor
  if (nargout > 0) __success_r = 0; endif
endfunction

%! text outside the blocks is not processed.
%! <<usage  test>>
%! <<usage  test(1,2,3,4)>>
%! <<error  test("test", 'bogus'); >>

%! <<shared a,b,c>>
%! <<assert [isempty(a) isempty(b) isempty(c)]>>
%! <<       a=1; b=2; c=3;>>
%! <<assert [a b c]::[1 2 3]>>
%! <<noisy  disp("here are the values for a b and c:")
%!          a, b, c>>
%! <<       c=6;>>
%! <<assert [a b c]::[1 2 6]>>
%! <<shared __msg,__success>>
%! <<  __msg = "invisible only if 'quiet' since otherwise it is verbose\n";>>
%! <<assert !exist("a")>>

%! <<#this is a comment block. it can contain anything.>>
%! <<#
%! it is the # after the block open < < which determines this.>>

%! ## failure tests.
%! << error("Failure tests.  To see them all, use test('test','verbose',1)");
%! >>
%! <<assert 
%! "assertion is false, so fail";
%! [a b c]::[1 3 6]>>
%! <<shared a,b,c>>
%! <<bogus>>
%! <<usage toeplitz([1,2,3])>>
%! <<  syntax errors in this code>>
%! <<shared garbage in and garbage out>>
%! <<assert 5::5::5>>
%! <<usage garbage>>
%! <<error when it is a syntax error, then failure fails>>
%! <<error "this is an error because it succeeds.";>>
%! <<noisy syntax error)>>
%! <<error  test('/etc/passwd');
%!          test("nonexistent file");
%! ## These don't signal an error, so the test for an error fails. Note 
%! ## that the call doesn't reference the current fid (it is unavailable),
%! ## so of course the informational message is not printed in the log.>>



reply via email to

[Prev in Thread] Current Thread [Next in Thread]