TParser

I was playing around over the christmas period with various things and this kind of spawned itself, it's pretty much uncommented, and probably a completely crap way of doing it but what the heck.
Feel free to post changes, suggestions and/or additions to it here.

The code:

Strict

Rem
Author: Kris Kelly
License: Mozilla Public License 1.1
Version: 0.1a
EndRem

Rem
Type TTokenizer
EndRem

Type TTokenizer
'Fields
	Field File : String
	Field FileHandle
	Field Token : String
	Field Position : Long
	Field Whitespace : String
	Field Delimiters : String
	Field Tokens:TList  'stores a list of all tokens minus the whitespace (delimiters are added seperate from the tokens)
	
'Methods
	Method GetNextToken()
		Local tempToken : String
		Local Char : String

		Char = ReadString(FileHandle, 1)
		
		If Delimiters.Find(Char) > -1 Then 
			Token = Char
			Return
		EndIf
		
		While Whitespace.Find(Char) < 0
		
			If Whitespace.Find(Char) < 0 Then 
				If Delimiters.Find(Char) < 0 Then
					tempToken :+ Char
				Else
					Token = tempToken
					SeekStream(FileHandle, StreamPos(FileHandle) -1)
					Return
				EndIf
			EndIf
			
			If KeyDown(KEY_ESCAPE) Then
				Exit
			EndIf
			
			Try
				Char = ReadString(FileHandle, 1)
			Catch err$
				Notify("Error when reading from source file (" + err$ + ")")
			EndTry
	
		Wend
		Token = tempToken
	End Method
	
	
	Method Tokenize()
		While Not Eof(FileHandle)
			GetNextToken()
			If Len(token)>0 Then ListAddLast(Tokens, Token)
		Wend
	End Method
	
	
	Method Init()
		FileHandle = OpenFile(File,1,0) 'read only
	End Method

	
	Method Delete()
		CloseFile(FileHandle)
		Release Tokens
	End Method

'Constructor	
	Function Create:TTokenizer(FileName : String, Delimiters : String = "',:().~q=+-/#$%~^&*!~r", Whitespace : String = " ~t~n~0")
		Local tempTok:TTokenizer = New TTokenizer
			tempTok.File = FileName
			tempTok.Whitespace = Whitespace
			tempTok.Delimiters = Delimiters
			tempTok.Init()
			tempTok.Tokens = CreateList()
		Return tempTok
	End Function
	
End Type


Type TStringVariable
	Field Identifier : String
	Field Value : String
End Type	


Rem
Type TKeyword
EndRem
Type TKeyword
	Field Keyword : String
	Field Action(Sender:TParser, Token:String)
	Field MiscData : Byte Ptr
End Type


Rem
Type TKeywordsList
EndRem
Type TKeywordsList
	Field Keywords:TList
	
	
	Method AddKeyword(Keyword:String, Action(Sender:TParser, Token:String))'MiscData:Byte Ptr)
		Local tempKeyword:TKeyword = New TKeyword
			tempKeyword.Keyword = Keyword
			tempKeyword.Action = Action
			If Not ListContains(Keywords, tempKeyword) Then			
				ListAddLast(Keywords, tempKeyword)
			Else
				Notify("Keyword already in list (" + Keyword + ")")
			EndIf
	End Method
	
	
	Method Save(FileName:String)

	End Method


	Method Load(FileName:String)
		
	End Method
	
'Constructor
	Function Create:TKeywordsList()
		Local tempSI:TKeywordsList= New TKeywordsList
			tempSI.Keywords = CreateList()
		Return tempSI
	End Function
End Type


Rem
Type TParser
EndRem
Type TParser
	Field Tokenizer : TTokenizer
	Field KeywordList : TKeywordsList
	Field UnknownHandler(Sender:TParser, Token:String)
	Field TokenPos : Short


	Method Init(FileName : String, Delimiters : String = "',:().~q=+-/#$%~^&*!~r", Whitespace : String = " ~t~n~0")
		Tokenizer = TTokenizer.Create(FileName, Delimiters, Whitespace)
		Tokenizer.Tokenize()
	End Method


	Method GetNextToken:String()
		Local Result:String
		If TokenPos < CountList(Tokenizer.Tokens)
			Result = String(Tokenizer.Tokens.ValueAtIndex(TokenPos))
			TokenPos :+ 1
		EndIf
		Return Result
	End Method


	Method Parse()
		Local Token : String
		Local kw : TKeyword
		Local kwFound = False
		
		TokenPos = 0
		
		While TokenPos < CountList(Tokenizer.Tokens)
			
			Token = GetNextToken()
					
			If Not CheckKeywords(Token) Then 'keywords take precedence over vars
				If Not CheckVars(Token) Then 'vars take next precedence
					If UnknownHandler <> Null Then UnknownHandler(Self, Token)
				EndIf
			EndIf
			
		Wend
	End Method
	
	
	Method CheckKeywords(Token : String)
		Local kw : TKeyword
		Local kwFound = False
			For kw = EachIn KeywordList.Keywords
				If Lower(kw.Keyword) = Lower(Token) Then
					If kw.Action <> Null Then kw.Action(Self, Token)
					kwFound = True
					Exit'exit since we found the keyword (no point in running through the rest of the keywords)
				EndIf
			Next
		Return kwFound
	End Method
	
	
	Method CheckVars(Token : String)
		Local sv : TStringVariable
		Local VarFound = False
			
			
		Return VarFound
	End Method


'Constructor
	Function Create:TParser()
		Local tempParser : TParser = New TParser
			tempParser.KeywordList = TKeywordsList.Create()
		Return tempParser
	End Function
End Type


'TEST PARSER
'Rem

Type TSimpleBasic
	Field Parser : TParser
	
	Method _Print(Sender:TParser, token:String)
		Local OutString : String
		'For now, just print the next token
		OutString :+ Sender.GetNextToken()
		Print OutString
	End Method


	Method _HandleUnk(Sender:TParser, token:String)
		Print "Unhandled: " + token
	End Method


	Method _MsgBox(Sender:TParser, token:String)
		Notify(Sender.GetNextToken())
	End Method


	Method _SetVar(Sender:TParser, token:String)
		'assume string type for the moment
		Local tempStrVar:TStringVariable= New TStringVariable
			tempStrVar.Identifier = Sender.GetNextToken()
			If Sender.GetNextToken() = "=" Then
				If Sender.GetNextToken() = "~q" Then
					tempStrVar.Value = Sender.GetNextToken()
					If Sender.GetNextToken() <> "~q" Then Notify("expected ending quote")
				Else
					Notify("expected ~q")
				EndIf
			Else
				Notify("expected =")
			EndIf
		ListAddLast(StringVars, tempStrVar)
	End Method


	Method _GetVar:TStringVariable()
	
	End Method


	Function Create:TSimpleBasic(SourceFile : String)
	'	Parser:TParser = TParser.Create()
	'	Parser.Init(SourceFile)
	'	Parser.KeywordList.AddKeyword("print", _Print)
	'	Parser.KeywordList.AddKeyword("msgbox",_MsgBox)
	'	Parser.KeywordList.AddKeyword("var",_SetVar)
	'	Parser.UnknownHandler = _HandleUnk
	'	Parser.Parse()
	
	End Function

End Type


Global StringVars:TList = CreateList()
	
'Rem

Function _Print(Sender:TParser, token:String)
	Local OutString : String
	'For now, just print the next token
	OutString :+ Sender.GetNextToken()
	Print OutString
End Function


Function _HandleUnk(Sender:TParser, token:String)
	Print "Unhandled: " + token
End Function


Function _MsgBox(Sender:TParser, token:String)
	Notify(Sender.GetNextToken())
End Function


Function _SetVar(Sender:TParser, token:String)
	'assume string type for the moment
	Local tempStrVar:TStringVariable= New TStringVariable
		tempStrVar.Identifier = Sender.GetNextToken()
		If Sender.GetNextToken() = "=" Then
			If Sender.GetNextToken() = "~q" Then
				tempStrVar.Value = Sender.GetNextToken()
				If Sender.GetNextToken() <> "~q" Then Notify("expected ending quote")
			Else
				Notify("expected ~q")
			EndIf
		Else
			Notify("expected =")
		EndIf
	ListAddLast(StringVars, tempStrVar)
	'Print "Var added"
	
End Function


Function _GetVar:TStringVariable()
	
End Function


Local myParser:TParser = TParser.Create()
	myParser.Init("test2.txt")
	myParser.KeywordList.AddKeyword("print", _Print)
	myParser.KeywordList.AddKeyword("msgbox",_MsgBox)
	myParser.KeywordList.AddKeyword("var",_SetVar)
	myParser.UnknownHandler = _HandleUnk
	myParser.Parse()

Release myParser
'EndRem
'EndRem

End

The content of test2.txt:

Print test2
var test = "TestVar"
Print test

Shoudln't ~n be a delimiter too... or ~r be whitespace?
I don't think my editors use ~r for linebreaks by default.

It works nice tho
:-)

In Linux and OSX it's just a linefeed character in win32 it's a Carriage Return + Line Feed so a platform check would probably be necessary.
Should be easy to sort with ?Win32 ?MacOS ?Linux ? though.

*EDIT*
I suppose swapping them around would solve it easy enough.

i.e. ~n as a delimiter and ~r as whitespace.