Index: trunk/src/sql-tokenizer.l
===================================================================
--- trunk/src/sql-tokenizer.l	(revision 369)
+++ trunk/src/sql-tokenizer.l	Mon Apr 21 17:45:15 CEST 2008
@@ -76,6 +76,7 @@
 	S(TK_SQL_CHECK),
 	S(TK_SQL_COLLATE),
 	S(TK_SQL_COLUMN),
+	S(TK_SQL_COLUMNS),
 	S(TK_SQL_CONDITION),
 	S(TK_SQL_CONSTRAINT),
 	S(TK_SQL_CONTINUE),
@@ -128,6 +129,7 @@
 	S(TK_SQL_FOREIGN),
 	S(TK_SQL_FROM),
 	S(TK_SQL_FULLTEXT),
+	S(TK_SQL_FULL),
 	S(TK_SQL_GRANT),
 	S(TK_SQL_GROUP),
 	S(TK_SQL_HAVING),
@@ -192,6 +194,7 @@
 	S(TK_SQL_NULL),
 	S(TK_SQL_NUMERIC),
 	S(TK_SQL_ON),
+	S(TK_SQL_OPEN),
 	S(TK_SQL_OPTIMIZE),
 	S(TK_SQL_OPTION),
 	S(TK_SQL_OPTIONALLY),
@@ -241,9 +244,11 @@
 	S(TK_SQL_SQLSTATE),
 	S(TK_SQL_SQLWARNING),
 	S(TK_SQL_SSL),
+	S(TK_SQL_STATUS),
 	S(TK_SQL_STARTING),
 	S(TK_SQL_STRAIGHT_JOIN),
 	S(TK_SQL_TABLE),
+	S(TK_SQL_TEMPORARY),
 	S(TK_SQL_TERMINATED),
 	S(TK_SQL_TEXT),
 	S(TK_SQL_THEN),
@@ -256,6 +261,7 @@
 	S(TK_SQL_TRAILING),
 	S(TK_SQL_TRIGGER),
 	S(TK_SQL_TRUE),
+	S(TK_SQL_TRUNCATE),
 	S(TK_SQL_UNDO),
 	S(TK_SQL_UNION),
 	S(TK_SQL_UNIQUE),
Index: trunk/tests/suite/bugs/t/bug_tokenizer.test
===================================================================
--- trunk/tests/suite/bugs/t/bug_tokenizer.test	Wed Apr 23 10:03:44 CEST 2008
+++ trunk/tests/suite/bugs/t/bug_tokenizer.test	Wed Apr 23 10:03:44 CEST 2008
@@ -0,0 +1,10 @@
+# Test new tokens
+SELECT FULL COLUMNS FROM tbl_name;
+CREATE TEMPORARY TABLE tbl_name (id INT);
+TRUNCATE TABLE tbl_name;
+SHOW TABLE STATUS;
+SHOW OPEN TABLES;
+
+# Misinterpreted tokens
+INSERT INTO a(col_name) VALUES('test');
+CREATE TABLE a(id INT);
Index: trunk/tests/suite/bugs/r/bug_tokenizer.result
===================================================================
--- trunk/tests/suite/bugs/r/bug_tokenizer.result	Wed Apr 23 10:06:46 CEST 2008
+++ trunk/tests/suite/bugs/r/bug_tokenizer.result	Wed Apr 23 10:06:46 CEST 2008
@@ -0,0 +1,53 @@
+SELECT FULL COLUMNS FROM tbl_name;
+token_name	text
+TK_SQL_SELECT	SELECT
+TK_SQL_FULL	FULL
+TK_SQL_COLUMNS	COLUMNS
+TK_SQL_FROM	FROM
+TK_LITERAL	tbl_name
+CREATE TEMPORARY TABLE tbl_name (id INT);
+token_name	text
+TK_SQL_CREATE	CREATE
+TK_SQL_TEMPORARY	TEMPORARY
+TK_SQL_TABLE	TABLE
+TK_LITERAL	tbl_name
+TK_OBRACE	(
+TK_LITERAL	id
+TK_SQL_INT	INT
+TK_CBRACE	)
+TRUNCATE TABLE tbl_name;
+token_name	text
+TK_SQL_TRUNCATE	TRUNCATE
+TK_SQL_TABLE	TABLE
+TK_LITERAL	tbl_name
+SHOW TABLE STATUS;
+token_name	text
+TK_SQL_SHOW	SHOW
+TK_SQL_TABLE	TABLE
+TK_SQL_STATUS	STATUS
+SHOW OPEN TABLES;
+token_name	text
+TK_SQL_SHOW	SHOW
+TK_SQL_OPEN	OPEN
+TK_LITERAL	TABLES
+INSERT INTO a(col_name) VALUES('test');
+token_name	text
+TK_SQL_INSERT	INSERT
+TK_SQL_INTO	INTO
+TK_LITERAL	a
+TK_OBRACE	(
+TK_LITERAL	col_name
+TK_CBRACE	)
+TK_FUNCTION	VALUES
+TK_OBRACE	(
+TK_STRING	test
+TK_CBRACE	)
+CREATE TABLE a(id INT);
+token_name	text
+TK_SQL_CREATE	CREATE
+TK_SQL_TABLE	TABLE
+TK_LITERAL	a
+TK_OBRACE	(
+TK_LITERAL	id
+TK_SQL_INT	INT
+TK_CBRACE	)
Index: trunk/tests/suite/bugs/t/bug_tokenizer.lua
===================================================================
--- trunk/tests/suite/bugs/t/bug_tokenizer.lua	Wed Apr 23 10:06:46 CEST 2008
+++ trunk/tests/suite/bugs/t/bug_tokenizer.lua	Wed Apr 23 10:06:46 CEST 2008
@@ -0,0 +1,23 @@
+--- Test script that simply tokenizes a query and returns the token_name and text values
+-- @autor Peter Romianowski
+local tokenizer = require("proxy.tokenizer")
+
+function read_query( packet )
+	if packet:byte() == proxy.COM_QUERY then
+        local query = packet:sub(2)
+        local tokens = tokenizer.tokenize(query)
+        local resultRows = {}
+        for _, token in pairs(tokens) do
+            table.insert(resultRows, {token.token_name, token.text})
+        end
+        proxy.response.resultset = {
+            fields = {
+                {type = proxy.MYSQL_TYPE_STRING, name = "token_name"},
+                {type = proxy.MYSQL_TYPE_STRING, name = "text"}
+            },
+            rows = resultRows
+        }
+        proxy.response.type = proxy.MYSQLD_PACKET_OK
+        return proxy.PROXY_SEND_RESULT
+    end
+end
Index: trunk/src/sql-tokenizer.h
===================================================================
--- trunk/src/sql-tokenizer.h	(revision 369)
+++ trunk/src/sql-tokenizer.h	Mon Apr 21 17:45:26 CEST 2008
@@ -83,6 +83,7 @@
 	TK_SQL_CHECK,
 	TK_SQL_COLLATE,
 	TK_SQL_COLUMN,
+	TK_SQL_COLUMNS,
 	TK_SQL_CONDITION,
 	TK_SQL_CONSTRAINT,
 	TK_SQL_CONTINUE,
@@ -135,6 +136,7 @@
 	TK_SQL_FOREIGN,
 	TK_SQL_FROM,
 	TK_SQL_FULLTEXT,
+	TK_SQL_FULL,
 	TK_SQL_GRANT,
 	TK_SQL_GROUP,
 	TK_SQL_HAVING,
@@ -199,6 +201,7 @@
 	TK_SQL_NULL,
 	TK_SQL_NUMERIC,
 	TK_SQL_ON,
+	TK_SQL_OPEN,
 	TK_SQL_OPTIMIZE,
 	TK_SQL_OPTION,
 	TK_SQL_OPTIONALLY,
@@ -249,8 +252,10 @@
 	TK_SQL_SQLWARNING,
 	TK_SQL_SSL,
 	TK_SQL_STARTING,
+	TK_SQL_STATUS,
 	TK_SQL_STRAIGHT_JOIN,
 	TK_SQL_TABLE,
+	TK_SQL_TEMPORARY,
 	TK_SQL_TERMINATED,
 	TK_SQL_TEXT,
 	TK_SQL_THEN,
@@ -263,6 +268,7 @@
 	TK_SQL_TRAILING,
 	TK_SQL_TRIGGER,
 	TK_SQL_TRUE,
+	TK_SQL_TRUNCATE,
 	TK_SQL_UNDO,
 	TK_SQL_UNION,
 	TK_SQL_UNIQUE,