/Users/paul/Documents/phlogo/core/tokenizertest.cpp

#define BOOST_TEST_MAIN
#include <boost/test/unit_test.hpp>

#include "Tokenizer.h"
#include "Verbs.h"
#include "Exceptions.h"

using namespace std;
using namespace phlogo;
using namespace boost;

BOOST_AUTO_TEST_CASE( numbers )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);

        BOOST_CHECK_EQUAL(tk.tokenizeToString("1"), "1");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("20"), "20");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("1000"), "1000");
}

BOOST_AUTO_TEST_CASE( discrimination )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);

        BOOST_CHECK_EQUAL(tk.tokenizeToString("1 1"), "1 1");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("1 + 1"), "$+ 1 1");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("1 1 + 1 1"), "1 $+ 1 1 1");
}

BOOST_AUTO_TEST_CASE( subexpr )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);

        BOOST_CHECK_EQUAL(tk.tokenizeToString("1 (1) 1"), "1 ( 1 ) 1");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("1 + 1 (2 2) 1"), "$+ 1 1 ( 2 2 ) 1");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("1 (1 + 1) 1"), "1 ( $+ 1 1 ) 1");
}

BOOST_AUTO_TEST_CASE( numberexpr )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);

        BOOST_CHECK_EQUAL(tk.tokenizeToString("1+2"), "$+ 1 2");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("1+(3+4)"), "$+ 1 ( $+ 3 4 )");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("2 + 3+100"), "$+ $+ 2 3 100");
}

BOOST_AUTO_TEST_CASE( tokens )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);

        BOOST_CHECK_EQUAL(tk.tokenizeToString("1 2"), "1 2");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("\"abced \"fghi"), "abced fghi");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("[2]"), "[2]");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("[2 3]"), "[2,3]");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("[aaa bbb]"), "[aaa,bbb]");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("[[aaa bbb] [ccc ddd]]"), "[[aaa,bbb],[ccc,ddd]]");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("{2}"), "{2}");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("{2 3}"), "{2,3}");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("{aaa bbb}"), "{aaa,bbb}");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("{{aaa bbb} {ccc ddd}}"), "{{aaa,bbb},{ccc,ddd}}");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("{[aaa bbb] {ccc ddd}}"), "{[aaa,bbb],{ccc,ddd}}");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("[{aaa bbb} {ccc ddd}]"), "[{aaa,bbb},{ccc,ddd}]");
}

BOOST_AUTO_TEST_CASE( smoke )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);
        
        BOOST_CHECK_EQUAL(tk.tokenizeToString("print \"X"), "$1.1 X");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("print \"X print \"Y"), "$1.1 X $1.1 Y");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("print \"X print \"Y print \"Z"), "$1.1 X $1.1 Y $1.1 Z");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("print \"X make \"Y [2]"), "$1.1 X $1.3 Y [2]");
}

BOOST_AUTO_TEST_CASE( error )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);
        
        try {
                tk.tokenizeToString("print hello");
                BOOST_CHECK( false );
        }
        catch ( dont_know_how_exception &x ) {
        }
}

BOOST_AUTO_TEST_CASE( comments )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);
        
        BOOST_CHECK_EQUAL(tk.tokenizeToString("print \"hello ; This is a comment"), "$1.1 hello");
}

BOOST_AUTO_TEST_CASE( multinumericargs )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);
        
        BOOST_CHECK_EQUAL(tk.tokenizeToString("print (array 4 1)"), "$1.1 ( $2.7 4 1 )");
}

BOOST_AUTO_TEST_CASE( compressLists )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);
        
        BOOST_CHECK_EQUAL(tk.tokenizeToString("[hello world]"), "[hello,world]");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("[hello]"), "[hello]");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("[[[hello world] x]]"), "[[[hello,world],x]]");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("{{{hello world} x}}"), "{{{hello,world},x}}");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("{{[hello world] x}}"), "{{[hello,world],x}}");
}

BOOST_AUTO_TEST_CASE( literalList )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);
        
        BOOST_CHECK_EQUAL(tk.tokenizeToString("[\"hello \"world]"), "[hello,world]");
}

BOOST_AUTO_TEST_CASE( tokenizeExpression1 )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);
        
        BOOST_CHECK_EQUAL(tk.tokenizeToString("1+2"), "$+ 1 2");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("1+  2"), "$+ 1 2");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("1+   2"), "$+ 1 2");
        
        // but leave numerics with spaces alone.
        BOOST_CHECK_EQUAL(tk.tokenizeToString("1 2"), "1 2");

        // leave variables with spaces alone
        BOOST_CHECK_EQUAL(tk.tokenizeToString("1 :X"), "1 :X");

        // and literals with spaces
        BOOST_CHECK_EQUAL(tk.tokenizeToString("\"aaa :X"), "aaa :X");
}

BOOST_AUTO_TEST_CASE( tokenizeExpression2 )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);
        
        BOOST_CHECK_EQUAL(tk.tokenizeToString("(gensym)"), "$2.13");
        BOOST_CHECK_EQUAL(tk.tokenizeToString("(gensym gensym word \"X \"Y)"), "$2.13 $2.13 $2.1 X Y");
}

BOOST_AUTO_TEST_CASE( tokenizeExpression3 )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);
        
        BOOST_CHECK_EQUAL(tk.tokenizeToString("(thing :var)+1"), "$+ ( $1.4 :var ) 1");
}

BOOST_AUTO_TEST_CASE( tokenizeExpressionWithLists )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);
        
        BOOST_CHECK_EQUAL(tk.tokenizeToString("(mdarray [1 2])"), "$2.8 [1,2]");
}

BOOST_AUTO_TEST_CASE( varhasdot )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);
        
        BOOST_CHECK_EQUAL(tk.tokenizeToString(":dot.var"), ":dot.var");
}

BOOST_AUTO_TEST_CASE( varhasnum )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);
        
        BOOST_CHECK_EQUAL(tk.tokenizeToString(":word1"), ":word1");
}

BOOST_AUTO_TEST_CASE( literalhasdot )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);
        
        BOOST_CHECK_EQUAL(tk.tokenizeToString("\"dot.var"), "dot.var");
}

BOOST_AUTO_TEST_CASE( listErrors )
{
        Verbs verbs(0, 0);
        Tokenizer tk(&verbs);
        
        try {
                tk.tokenizeToString("[hello world");
                BOOST_CHECK( false );
        }
        catch ( dont_know_how_exception &x ) {
        }
        
        try {
                tk.tokenizeToString("{[hello world}");
        }
        catch ( dont_know_how_exception &x ) {
        }
}