Retrogamecoding(.org) > Other Languages

Adventures in B

<< < (5/5)

Aurel:
and tiny tokenizer


--- Code: ---'microB tokenizer
int tkNULL=0,tkPLUS=1,tkMINUS=2,tkMULTI=3,tkDIVIDE=4
int tkCOLON=5,tkCOMMA=6,tkLPAREN=7,tkRPAREN=8,tkLBRACKET=9,tkRBRACKET=10
int tkPRINT=11,tkDOT=12,tkLINE=13,tkCIRCLE=14 ,tkEOL = 20
string tokList[256] : int typList[256]   'token/type arrays
int start , p = 1 ,start = p ,tp ,n      'init
string code,ch,tk ,crlf=chr(13)+chr(10),bf
code = "var1 + 2.5 " ' test or load_src?

sub tokenizer(src as string) as int
'ch = mid(src,p,1) : print "CH:" + ch' get first char
while p <= len(src)       
     ch = mid(src,p,1)                    'get char

 If asc(ch)=32 : p++ : end if             'skip blank space
           
 If (asc(ch)>96 and asc(ch)<123)          ' [a-z]
       print "CH2:" + ch : p--
   while (asc(ch)>96 and asc(ch)<123) or (asc(ch)>47 and asc(ch)<58) ' [a-z0-9]*
       p++:ch = mid(src,p,1) : print "AZ:" + ch
       tk =tk+ch   
   wend
      print "TOK-AZ:" + tk
       tp++ : tokList[tp] = tk : tk="" :p++       
       'return IDENT;
 Elseif (asc(ch)>47 and asc(ch)<58)                    ' [0-9]
p--
    while (asc(ch)>47 and asc(ch)<58) or (asc(ch)=46)  '[0-9[0.0]]*
        p++ : ch = mid(src,p,1):tk = tk + ch
    wend
       tp++ : tokList[tp] = tk : tk="":p++
       'return NUMBER;
 Elseif asc(ch)=43                                     ' [ + ]
       tk = ch : tp++ : tokList[tp] = tk : tk="" :p++  ' set_token
   
 'elseif...
 End if
wend
return tp
end sub

'call tokenizer..tested(ident,numbers)
int tn: tn = tokenizer(code) : print "number of tokens:" + str(tn)
for n = 1 to tn : bf = bf + tokList[n] + crlf : next n
print  bf
--- End code ---

Tomaaz:
"Adventures in C" you say.  Interesting... ???

Navigation

[0] Message Index

[*] Previous page