Marks Code parser (Blitz)

Anybody got a copy of the code parser MS posted a long while back? I think the source code was the actual code used when parsing "Blitz Basic" files

Or, has anyone written a parser which can handle source code lines?
The parser needs to be able to handle various user input styles.
Here, for example, each section should parse out to the same instruction:

if (a=b) or (c=(f-1)*16) then FuncA else FuncB
'
'
'
If a=b or c=(f-1)*16 FuncA Else FuncB
'
'
'
If      ((a =  b))        or         ((  c= (f-1) * 16) Then
      FuncA ' i am a random comment ** ignore
Else
      FuncB ' blah %|^%$
EndIF                   ' or, the user can also use "End If"

(I cannot make much sense of the BlitzMax IDE source code though)

;Very simple compiler.
;
;Blitz does NOT generate code like this! But the parsing is similar...
;
;Statements supported: let, print, end, if, then, else
;
;Operators supported: binary +, -, *, /, <, =, >, <=, <>, >= and unary -
;
;*, / have precedence over +, -, which have precendence over <,=,>,<=,<>,>=
;
;Parenthesis can be used to group subexpressions.

;**** The program *****
;
Data "let a=1"
Data "let b=2"
Data "let c=a*b+a/b"
Data "if a<>TOKE_IDENT SynErr()
		id$=toke$
		AddGlob( toke$ )
		If GetToke()<>TOKE_EQ SynErr()
		t=ParseExpr()
		Print "    mov   [_"+id$+"],eax"
	Case TOKE_IF
		If ParseExpr()<>TOKE_THEN SynErr()
		label=label+1:lab=label
		Print "    and   eax,eax"
		Print "    jz    __"+lab
		t=ParseStmt( GetToke() )
		If t=TOKE_ELSE
			label=label+1:lab2=label
			Print "    jmp   __"+lab2
			Print "__"+lab+":"
			t=ParseStmt( GetToke() )
			lab=lab2
		EndIf
		Print "__"+lab+":"
	Case TOKE_PRINT
		t=ParseExpr()
		Print "    call  PRINT"
	Case TOKE_END
		Print "    ret"
		Print
		For gl.Glob=Each Glob
			Print "_"+gl\name+":"
			Print "    dd    0"
		Next
		Print:Print "Done!":WaitKey
		End
	Default
		SynErr()
	End Select
	Return t
End Function

Function ParseExpr()
	Return ParseComp()
End Function

Function ParseComp()
	t=ParseTerm()
	Repeat
		Select t
		Case TOKE_LT op$="lt"	;these ops are wrong! x86 is weird!
		Case TOKE_GT op$="gt"
		Case TOKE_LE op$="le"
		Case TOKE_GE op$="ge"
		Case TOKE_EQ op$="eq"
		Case TOKE_NE op$="ne"
		Default Return t
		End Select
		Print "    push  eax"
		t=ParseTerm()
		Print "    pop   ecx"
		Print "    cmp   ecx,eax"
		Print "    s"+op$+"   eax"
		Print "    and   eax,255"
	Forever
End Function

Function ParseTerm()
	t=ParseFact()
	Repeat
		Select t
		Case TOKE_ADD
			Print "    push  eax"
			t=ParseFact()
			Print "    pop   ecx"
			Print "    add   eax,ecx"
		Case TOKE_SUB
			Print "    push  eax"
			t=ParseFact()
			Print "    mov   ecx,eax"
			Print "    pop   eax"
			Print "    sub   eax,ecx"
		Default
			Return t
		End Select
	Forever
End Function

Function ParseFact()
	t=ParseLeaf()
	Repeat
		Select t
		Case TOKE_MUL
			Print "    push  eax"
			t=ParseLeaf()
			Print "    pop   ecx"
			Print "    imul  eax,ecx"
		Case TOKE_DIV
			Print "    push  eax"
			t=ParseLeaf()
			Print "    mov   ecx,eax"
			Print "    pop   ecx"
			Print "    idiv  eax,ecx"
		Default
			Return t
		End Select
	Forever
End Function

Function ParseLeaf()
	t=GetToke()
	Select t
	Case TOKE_SUB
		t=ParseLeaf()
		Print "    neg   eax"
	Case TOKE_OPENPAR
		If ParseExpr() <> TOKE_CLOSEPAR SynErr()
		t=GetToke()
	Case TOKE_IDENT
		Print "    mov   eax,[_"+toke$+"]"
		AddGlob( toke$ )
		t=GetToke()
	Case TOKE_CONST
		Print "    mov   eax,"+toke$
		t=GetToke()
	Default
		SynErr()
	End Select
	Return t
End Function

Function AddGlob( name$ )
	For gl.Glob=Each Glob
		If gl\name=name Return
	Next
	gl=New Glob
	gl\name=name
End Function

Function SynErr()
	RuntimeError "Syntax Error"
End Function
	
Function CurrCh$()
	If lin$=""
		Read lin$
		lin$=Lower$( lin$ )+" "
	EndIf
	Return Left( lin$,1 )
End Function

Function NextCh()
	lin$=Mid$( lin$,2 )
End Function

Function GetToke()

	Repeat
		toke$=CurrCh$()
		NextCh()
	Until toke$<>" "
	
	If toke$>="0" And toke$<="9"
		Repeat
			t$=CurrCh$()
			If t$<"0" Or t$>"9" Return TOKE_CONST
			toke$=toke$+t$
			nextCh()
		Forever
	EndIf
	
	If toke$>="a" And toke$<="z"
		Repeat
			t$=CurrCh$()
			If t$<"a" Or t$>="z"
				Select toke$
				Case "let" Return TOKE_LET
				Case "print" Return TOKE_PRINT
				Case "end" Return TOKE_END
				Case "if" Return TOKE_IF
				Case "then" Return TOKE_THEN
				Case "else" Return TOKE_ELSE
				End Select
				Return TOKE_IDENT
			EndIf
			toke$=toke$+t$
			NextCh()
		Forever
	EndIf
	
	Select toke$
	Case "+" Return TOKE_ADD
	Case "-" Return TOKE_SUB
	Case "*" Return TOKE_MUL
	Case "/" Return TOKE_DIV
	Case "(" Return TOKE_OPENPAR
	Case ")" Return TOKE_CLOSEPAR
	Case "="
		If CurrCh$()="<" NextCh():Return TOKE_LE
		If CurrCh$()=">" NextCh():Return TOKE_GE
		Return TOKE_EQ
	Case "<"
		If CurrCh$()="=" NextCh():Return TOKE_LE
		If currCh$()=">" NextCh():Return TOKE_NE
		Return TOKE_LT
	Case ">"
		If CurrCh$()="=" NextCh():Return TOKE_GE
		If CurrCh$()="<" NextCh():Return TOKE_NE
		Return TOKE_GT
	End Select
	
	SynErr()
	
End Function

Try this:
http://www.blitzbasic.com/codearcs/codearcs.php?code=1900

or this:
http://www.blitzbasic.com/codearcs/codearcs.php?code=407

This code I wrote some time ago is faster:
http://www.blitzbasic.com/codearcs/codearcs.php?code=2103
It provides a way to execute simple and small scripts. It is a bit buggy on logic-flow control, but the math evaluator works quite well and it is fast (it is based on byte-code generation so it gets faster when the byte code is already compiled generated). It is a bare-bones script system that should get LOTS of changes to be considerated really a pro script system, but it is a starting point, and it does perform a decent parsing and evaluation.

@JimBrown: Reading again your post, I think you need a decent tokenizer.

This is the tokenizer for my blitzmax-like script system:
(This code is not public domain, use it if you wish in your own project, but don't distribute it outside this forum, please. I'll distribute the whole script package once I feel it is stable.)

'This BMX file was edited with BLIde ( <a href="http://www.blide.org" target="_blank">http://www.blide.org</a> )
Type TTokenizer
'There's another single line comment
	Field Tokens:TToken[]
	rem
		bbdoc: Reads a string containing source code and fills the tokens collection.
	end rem
	Method Tokenize:String(Text:String)
		Local Cur:String
		Local Literal:Int = False
		Local Line:Int = 0
		Local SingleComment:Int = False
		For Local i:Int = 1 To Text.Length
			Local a:String = Mid(Text, I, 1)
			If Not literal Then
				If singleComment = True Then
					If a = Chr(13) Or a = Chr(10) Then
						cur = ""
						singlecomment = False
					End If
				ElseIf (a >= "a" And a <= "z") Or (a >= "A" And a <= "Z") Or (a >= "0" And a <= "9") Or a = "_"
					Cur = cur + a
				ElseIf a = "'" Then
					If cur.Length > 0 Then
						If tokens = Null Then
							tokens = New TToken[1]
						Else
							tokens = tokens[..tokens.length + 1]
						End If
						tokens[tokens.Length - 1] = New TToken
						tokens[tokens.Length - 1].Text = cur
						tokens[tokens.Length - 1].Line = Line
					EndIf
					cur = ""
					SingleComment = True
				ElseIf a = Chr(34) Then
					If cur.Length > 0 Then
						If tokens = Null Then
							tokens = New TToken[1]
						Else
							tokens = tokens[..tokens.length + 1]
						End If
						tokens[tokens.Length - 1] = New TToken
						tokens[tokens.Length - 1].Text = cur
						tokens[tokens.Length - 1].Line = Line

						'Print "TOKEN: " + cur
					EndIf
					cur = a
					literal = True
				Else
					If cur.Length > 0 Then
						If tokens = Null Then
							tokens = New TToken[1]
						Else
							tokens = tokens[..tokens.length + 1]
						End If
						tokens[tokens.Length - 1] = New TToken
						tokens[tokens.Length - 1].Text = cur
						tokens[tokens.Length - 1].Line = Line
						'Print "TOKEN: " + cur
					EndIf
					If a <> Chr(10) Then
						If tokens = Null Then
							tokens = New TToken[1]
						Else
							tokens = tokens[..tokens.length + 1]
						End If
						tokens[tokens.Length - 1] = New TToken
						tokens[tokens.Length - 1].Text = A
						tokens[tokens.Length - 1].Line = Line
						If a <> Chr(13) Then
							'Print "TOKEN: " + A
						Else
							'Print "<BR>"
							Line:+1
						EndIf
					EndIf
					CUR = ""
				End If
			Else
				Cur = cur + a
				If a = Chr(34) Then
					literal = False
				ElseIf a = Chr(13) Or a = Chr(10) Then
					Return "Missing " + Chr(34) + " at string literal termination."
				End If
			EndIf
		Next
		'Get last:
		If cur <> "" Then
			If tokens = Null Then
				tokens = New TToken[1]
			Else
				tokens = tokens[..tokens.length + 1]
			End If
			tokens[tokens.Length - 1] = New TToken
			tokens[tokens.Length - 1].Text = cur
			tokens[tokens.Length - 1].Line = Line
			'Print "TOKEN: " + cur
		End If
		CleanTokens(Self)
	End Method
	rem
		bbdoc: Gets a token from the collection by idex
	endrem
	Method Item:Ttoken(I:Int)
		Return Self.Tokens[i]
	End Method
	rem
		bbdoc: Gets the number of tokens in the collection
	endrem
	Method Count:Int()
		Return Self.Tokens.Length
	End Method
	rem
		bbdoc: Gets the number of the latest index in the collection
	endrem
	Method LastIndex:Int()
		Return Self.Tokens.Length - 1
	End Method
	rem
		bbdoc: Removes an index from the collection at a given index
	endrem
	Method RemoveTokenAt(I:Int)
		Self.Tokens = Self.tokens[0..i] + Self.tokens[i + 1..]
	End Method
	rem
		bbdoc: Inserts an item on the collection at a given index
	endrem
	Method AddTokenAt(I:Int, T:Ttoken)
		Local Ar:TToken[] = New TToken[1]
		Ar[0] = T
		Self.Tokens = Self.tokens[0..i] + Ar + Self.tokens[i + 1..]
	End Method
End Type

Private
rem
	bbdoc: This function removes the void and false separator tokens from the collection
endrem
Function CleanTokens(Tokenizer:TTokenizer)
	'DebugStop
	Local DeleteToken:Int[] = New Int[1]
	Local InsideRem:Int = False
	Local LastEnter:Int = False
	For Local i:Int = 0 To tokenizer.Tokens.Length - 1
		If tokenizer.tokens[i].Text = " " Or tokenizer.tokens[i].Text = Chr(9) Then
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i
		ElseIf i > 0 And tokenizer.tokens[i - 1].Text = "<" And tokenizer.tokens[i].Text = ">" Then
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i - 1
			tokenizer.Tokens[i].text = "<>"
		ElseIf i > 0 And tokenizer.tokens[i - 1].Text = ">" And tokenizer.tokens[i].Text = "=" Then
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i - 1
			tokenizer.Tokens[i].text = ">="
		ElseIf i > 0 And tokenizer.tokens[i - 1].Text = "<" And tokenizer.tokens[i].Text = "=" Then
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i - 1
			tokenizer.Tokens[i].text = "<="
		ElseIf i > 0 And tokenizer.tokens[i - 1].Text = "=" And tokenizer.tokens[i].Text = "<" Then
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i - 1
			tokenizer.Tokens[i].text = "<="
		ElseIf i > 0 And tokenizer.tokens[i - 1].Text = "=" And tokenizer.tokens[i].Text = ">" Then
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i - 1
			tokenizer.Tokens[i].text = ">="
		ElseIf i > 0 And tokenizer.tokens[i - 1].Text = ":" And tokenizer.tokens[i].Text = "+" Then
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i - 1
			tokenizer.Tokens[i].text = ":+"
		ElseIf i > 0 And tokenizer.tokens[i - 1].Text = ":" And tokenizer.tokens[i].Text = "-" Then
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i - 1
			tokenizer.Tokens[i].text = ":-"
		ElseIf i > 0 And tokenizer.tokens[i - 1].Text = ":" And tokenizer.tokens[i].Text = "*" Then
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i - 1
			tokenizer.Tokens[i].text = ":*"
		ElseIf i > 0 And tokenizer.tokens[i - 1].Text = ":" And tokenizer.tokens[i].Text = "/" Then
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i - 1
			tokenizer.Tokens[i].text = ":/"
		ElseIf i > 0 And tokenizer.tokens[i - 1].Text = ":" And tokenizer.tokens[i].Text = "^" Then
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i - 1
			tokenizer.Tokens[i].text = ":^"
		ElseIf (i > 0 And i < tokenizer.Tokens.Length - 1) And ..
				tokenizer.tokens[i].text = "." And ..
				IsOperator (tokenizer.tokens[i - 1]) And ..
				IsIntNumber(tokenizer.tokens[i + 1])
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i + 1
			tokenizer.Tokens[i].text = "." + tokenizer.tokens[i + 1].text

		ElseIf (i > 0 And i < tokenizer.Tokens.Length - 1) And ..
				tokenizer.tokens[i].text = "." And ..
				IsIntNumber(tokenizer.tokens[i - 1]) And ..
				IsIntNumber(tokenizer.tokens[i + 1])
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i - 1
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i + 1
			tokenizer.Tokens[i].text = tokenizer.tokens[i - 1].Text + "." + tokenizer.tokens[i + 1].text
						
		ElseIf (tokenizer.tokens[i].Text.ToLower() = "rem") And (insiderem = False) Then
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i
			InsideRem = True
			
		ElseIf tokenizer.tokens[i].Text.ToLower() = "endrem" Then
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i
			InsideRem = False
			
		ElseIf (i > 1) And (tokenizer.tokens[i].Text.ToLower() = "rem") And (insiderem = True) And (tokenizer.tokens[i - 1].Text.ToLower() = " ") And ((tokenizer.tokens[i - 2].Text.ToLower() = "end")) Then
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i - 2
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i - 1
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i
			insiderem = False
			
		ElseIf insiderem = True Then
			DeleteToken = DeleteToken[..deletetoken.length + 1]
			DeleteToken[deleteToken.length - 1] = i
			tokenizer.tokens[i].Text = ";" 	'Deletion on anex pass!!
		End If
		If tokenizer.tokens[i].Text = Chr(13) Or tokenizer.tokens[i].Text = ";" Then
			If lastenter = True Then
				DeleteToken = DeleteToken[..deletetoken.length + 1]
				DeleteToken[deleteToken.length - 1] = i
			End If
			tokenizer.tokens[i].Text = ";"
			lastenter = True
		Else
			lastenter = False
		EndIf
	Next
	deletetoken.sort()
	Local LastIndex:Int = -1	'Ensure not to delete the same index twice!
	Print "Disposable tokens: " + deletetoken.Length
	For Local i:Int = deletetoken.length - 1 To 1 Step - 1
		'tokenizer.Tokens = Tokenizer.tokens[0..deletetoken[i] ] + tokenizer.tokens[deletetoken[i] + 1..]
		If lastindex <> deletetoken[i] Then
			Tokenizer.RemoveTokenAt(deletetoken[i])
			lastindex = deletetoken[i]
		EndIf
	Next
End Function
Public

'This BMX file was edited with BLIde ( <a href="http://www.blide.org" target="_blank">http://www.blide.org</a> )
rem
	bbdoc: Base token class
endrem
Type TToken
	rem
		bbdoc: Text represented by the token
	endrem
	Field Text:String
	rem
		bbdoc: line in the original source code where the token belongs
	endrem
	Field Line:Int = -1	'-1 means no line
End Type

rem
	bbdoc:Returns TRUE if a token is an operator. Otherwise return false.
end rem
Function IsOperator:Int(Token:TToken)
	Select Token.Text.ToLower()
		Case "+", "-", "*", "/", "^", "[", "]", "(", ")", "=", ":+", ":-", ":*", ":/", ":^",  ..
			"and", "or", "xor", "|", "&", "<>", ">=", "<=", ".", "{", "}", ";"
			Return True
		Default
			Return False
	End Select
End Function

Function IsIntNumber:Int(Token:TToken)
	
	For Local i:Int = 1 To token.Text.Length
		Local Char:String = Mid(token.Text, i, 1)
		Select char
			Case "-"
				If i <> 1 Then Return False
			Case "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"
			Default
				Return False
		End Select
	Next
	Return True
End Function

'This BMX file was edited with BLIde ( <a href="http://www.blide.org" target="_blank">http://www.blide.org</a> )
Function LoadStreamText:TTextStream(url:Object)
	Local format:Int, Size:Int = 0, c:Int, d:Int, e:Int
	Local stream:TStream = ReadStream(url) 
	If stream = Null Then Return Null
	If Not stream.Eof() 
		c = stream.ReadByte() 
		Size:+1
		If Not stream.Eof() 
			d = stream.ReadByte() 
			Size:+1
			If c = $fe And d = $ff
				format = TTextStream.UTF16BE
			Else If c = $ff And d = $fe
				format = TTextStream.UTF16LE
			Else If c = $ef And d = $bb
				If Not stream.Eof() Then
					e = stream.ReadByte() 
					Size:+1
					If e = $bf format = TTextStream.UTF8
				EndIf
			EndIf
		EndIf
	EndIf
	If Not format
		Local Stream2:TTextStream = TTextStream.Create(ReadStream(url), TTextStream.LATIN1) 
		Return Stream2
	EndIf
	stream:TStream = ReadStream(url) 
	Local TStream:TTextStream = TTextStream.Create(stream, format) 
	Return TStream
End Function

(This code is not public domain, use it if you wish in your own project, but don't distribute it outside this forum, please. I'll distribute the whole script package once I feel it is stable.)

By the way, forgot to mention that the code is superstrict. It won't compile as not-strict code.

Good help all (Special thanks Ziggy)