python实现词法分析器

Python  /  管理员 发布于 7年前   156

简单Python词法分析器实现，供大家参考，具体内容如下

词法分析器状态转换图：

词法分析器总流程图：

预处理程序：

词法分析器：

词法分析器程序详细设计

详细代码实现:

#!/usr/bin/env python3.4# coding=utf-8import sysimport stringkeywards = {}# 关键字部分keywards['False'] = 101keywards['class'] = 102keywards['finally'] = 103keywards['is'] = 104keywards['return'] = 105keywards['None'] = 106keywards['continue'] = 107keywards['for'] = 108keywards['lambda'] = 109keywards['try'] = 110keywards['True'] = 111keywards['def'] = 112keywards['from'] = 113keywards['nonlocal'] = 114keywards['while'] = 115keywards['and'] = 116keywards['del'] = 117keywards['global'] = 118keywards['not'] = 119keywards['with'] = 120keywards['as'] = 121keywards['elif'] = 122keywards['if'] = 123keywards['or'] = 124keywards['yield'] = 125keywards['assert'] = 126keywards['else'] = 127keywards['import'] = 128keywards['pass'] = 129keywards['break'] = 130keywards['except'] = 131keywards['in'] = 132keywards['raise'] = 133# 符号keywards['+'] = 201keywards['-'] = 202keywards['*'] = 203keywards['/'] = 204keywards['='] = 205keywards[':'] = 206keywards['<'] = 207keywards['>'] = 208keywards['%'] = 209keywards['&'] = 210keywards['!'] = 211keywards['('] = 212keywards[')'] = 213keywards['['] = 214keywards[']'] = 215keywards['{'] = 216keywards['}'] = 217keywards['#'] = 218keywards['|'] = 219keywards[','] = 220# 变量# keywards['var'] = 301# 常量# keywards['const'] = 401# Error# keywards['const'] = 501signlist = {}# 预处理函数，将文件中的空格，换行等无关字符处理掉def pretreatment(file_name): try:  fp_read = open(file_name, 'r')  fp_write = open('file.tmp', 'w')  sign = 0  while True:   read = fp_read.readline()   if not read:    break   length = len(read)   i = -1   while i < length - 1:    i += 1    if sign == 0:     if read[i] == ' ':      continue    if read[i] == '#':     break    elif read[i] == ' ':     if sign == 1:      continue     else:      sign = 1      fp_write.write(' ')    elif read[i] == '\t':     if sign == 1:      continue     else:      sign = 1      fp_write.write(' ')    elif read[i] == '\n':     if sign == 1:      continue     else:      fp_write.write(' ')      sign = 1    elif read[i] == '"':     fp_write.write(read[i])     i += 1     while i < length and read[i] != '"':      fp_write.write(read[i])      i += 1     if i >= length:      break     fp_write.write(read[i])    elif read[i] == "'":     fp_write.write(read[i])     i += 1     while i < length and read[i] != "'":      fp_write.write(read[i])      i += 1     if i >= length:      break     fp_write.write(read[i])    else:     sign = 3     fp_write.write(read[i]) except Exception:  print(file_name, ': This FileName Not Found!')def save(string): if string in keywards.keys():  if string not in signlist.keys():   signlist[string] = keywards[string] else:  try:   float(string)   save_const(string)  except ValueError:   save_var(string)def save_var(string): if string not in signlist.keys():  if len(string.strip()) < 1:   pass  else:   if is_signal(string) == 1:    signlist[string] = 301   else:    signlist[string] = 501def save_const(string): if string not in signlist.keys():  signlist[string] = 401def save_error(string): if string not in signlist.keys():  signlist[string] = 501def is_signal(s): if s[0] == '_' or s[0] in string.ascii_letters:  for i in s:   if i in string.ascii_letters or i == '_' or i in string.digits:    pass   else:    return 0  return 1 else:  return 0def recognition(filename): try:  fp_read = open(filename, 'r')  string = ""  sign = 0  while True:   read = fp_read.read(1)   if not read:    break   if read == ' ':    if len(string.strip()) < 1:     sign = 0     pass    else:     if sign == 1 or sign == 2:      string += read     else:      save(string)      string = ""      sign = 0   elif read == '(':    if sign == 1 or sign == 2:     string += read    else:     save(string)     string = ""     save('(')   elif read == ')':    if sign == 1 or sign == 2:     string += read    else:     save(string)     string = ""     save(')')   elif read == '[':    if sign == 1 or sign == 2:     string += read    else:     save(string)     string = ""     save('[')   elif read == ']':    if sign == 1 or sign == 2:     string += read    else:     save(string)     string = ""     save(']')   elif read == '{':    if sign == 1 or sign == 2:     string += read    else:     save(string)     string = ""     save('{')   elif read == '}':    if sign == 1 or sign == 2:     string += read    else:     save(string)     string = ""     save('}')   elif read == '<':    save(string)    string = ""    save('<')   elif read == '>':    save(string)    string = ""    save('>')   elif read == ',':    save(string)    string = ""    save(',')   elif read == "'":    string += read    if sign == 1:     sign = 0     save_const(string)     string = ""    else:     if sign != 2:      sign = 1   elif read == '"':    string += read    if sign == 2:     sign = 0     save_const(string)     string = ""    else:     if sign != 1:      sign = 2   elif read == ':':    if sign == 1 or sign == 2:     string += read    else:     save(string)     string = ""     save(':')   elif read == '+':    save(string)    string = ""    save('+')   elif read == '=':    save(string)    string = ""    save('=')   else:    string += read except Exception as e:  print(e)def main(): if len(sys.argv) < 2:  print("Please Input FileName") else:  pretreatment(sys.argv[1]) recognition('file.tmp') for i in signlist.keys():  print("(", signlist[i], ",", i, ")")if __name__ == '__main__': main()

以上就是本文的全部内容，希望对大家的学习有所帮助，也希望大家多多支持。