text_file.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286
  1. """text_file
  2. provides the TextFile class, which gives an interface to text files
  3. that (optionally) takes care of stripping comments, ignoring blank
  4. lines, and joining lines with backslashes."""
  5. import sys, io
  6. class TextFile:
  7. """Provides a file-like object that takes care of all the things you
  8. commonly want to do when processing a text file that has some
  9. line-by-line syntax: strip comments (as long as "#" is your
  10. comment character), skip blank lines, join adjacent lines by
  11. escaping the newline (ie. backslash at end of line), strip
  12. leading and/or trailing whitespace. All of these are optional
  13. and independently controllable.
  14. Provides a 'warn()' method so you can generate warning messages that
  15. report physical line number, even if the logical line in question
  16. spans multiple physical lines. Also provides 'unreadline()' for
  17. implementing line-at-a-time lookahead.
  18. Constructor is called as:
  19. TextFile (filename=None, file=None, **options)
  20. It bombs (RuntimeError) if both 'filename' and 'file' are None;
  21. 'filename' should be a string, and 'file' a file object (or
  22. something that provides 'readline()' and 'close()' methods). It is
  23. recommended that you supply at least 'filename', so that TextFile
  24. can include it in warning messages. If 'file' is not supplied,
  25. TextFile creates its own using 'io.open()'.
  26. The options are all boolean, and affect the value returned by
  27. 'readline()':
  28. strip_comments [default: true]
  29. strip from "#" to end-of-line, as well as any whitespace
  30. leading up to the "#" -- unless it is escaped by a backslash
  31. lstrip_ws [default: false]
  32. strip leading whitespace from each line before returning it
  33. rstrip_ws [default: true]
  34. strip trailing whitespace (including line terminator!) from
  35. each line before returning it
  36. skip_blanks [default: true}
  37. skip lines that are empty *after* stripping comments and
  38. whitespace. (If both lstrip_ws and rstrip_ws are false,
  39. then some lines may consist of solely whitespace: these will
  40. *not* be skipped, even if 'skip_blanks' is true.)
  41. join_lines [default: false]
  42. if a backslash is the last non-newline character on a line
  43. after stripping comments and whitespace, join the following line
  44. to it to form one "logical line"; if N consecutive lines end
  45. with a backslash, then N+1 physical lines will be joined to
  46. form one logical line.
  47. collapse_join [default: false]
  48. strip leading whitespace from lines that are joined to their
  49. predecessor; only matters if (join_lines and not lstrip_ws)
  50. errors [default: 'strict']
  51. error handler used to decode the file content
  52. Note that since 'rstrip_ws' can strip the trailing newline, the
  53. semantics of 'readline()' must differ from those of the builtin file
  54. object's 'readline()' method! In particular, 'readline()' returns
  55. None for end-of-file: an empty string might just be a blank line (or
  56. an all-whitespace line), if 'rstrip_ws' is true but 'skip_blanks' is
  57. not."""
  58. default_options = { 'strip_comments': 1,
  59. 'skip_blanks': 1,
  60. 'lstrip_ws': 0,
  61. 'rstrip_ws': 1,
  62. 'join_lines': 0,
  63. 'collapse_join': 0,
  64. 'errors': 'strict',
  65. }
  66. def __init__(self, filename=None, file=None, **options):
  67. """Construct a new TextFile object. At least one of 'filename'
  68. (a string) and 'file' (a file-like object) must be supplied.
  69. They keyword argument options are described above and affect
  70. the values returned by 'readline()'."""
  71. if filename is None and file is None:
  72. raise RuntimeError("you must supply either or both of 'filename' and 'file'")
  73. # set values for all options -- either from client option hash
  74. # or fallback to default_options
  75. for opt in self.default_options.keys():
  76. if opt in options:
  77. setattr(self, opt, options[opt])
  78. else:
  79. setattr(self, opt, self.default_options[opt])
  80. # sanity check client option hash
  81. for opt in options.keys():
  82. if opt not in self.default_options:
  83. raise KeyError("invalid TextFile option '%s'" % opt)
  84. if file is None:
  85. self.open(filename)
  86. else:
  87. self.filename = filename
  88. self.file = file
  89. self.current_line = 0 # assuming that file is at BOF!
  90. # 'linebuf' is a stack of lines that will be emptied before we
  91. # actually read from the file; it's only populated by an
  92. # 'unreadline()' operation
  93. self.linebuf = []
  94. def open(self, filename):
  95. """Open a new file named 'filename'. This overrides both the
  96. 'filename' and 'file' arguments to the constructor."""
  97. self.filename = filename
  98. self.file = io.open(self.filename, 'r', errors=self.errors)
  99. self.current_line = 0
  100. def close(self):
  101. """Close the current file and forget everything we know about it
  102. (filename, current line number)."""
  103. file = self.file
  104. self.file = None
  105. self.filename = None
  106. self.current_line = None
  107. file.close()
  108. def gen_error(self, msg, line=None):
  109. outmsg = []
  110. if line is None:
  111. line = self.current_line
  112. outmsg.append(self.filename + ", ")
  113. if isinstance(line, (list, tuple)):
  114. outmsg.append("lines %d-%d: " % tuple(line))
  115. else:
  116. outmsg.append("line %d: " % line)
  117. outmsg.append(str(msg))
  118. return "".join(outmsg)
  119. def error(self, msg, line=None):
  120. raise ValueError("error: " + self.gen_error(msg, line))
  121. def warn(self, msg, line=None):
  122. """Print (to stderr) a warning message tied to the current logical
  123. line in the current file. If the current logical line in the
  124. file spans multiple physical lines, the warning refers to the
  125. whole range, eg. "lines 3-5". If 'line' supplied, it overrides
  126. the current line number; it may be a list or tuple to indicate a
  127. range of physical lines, or an integer for a single physical
  128. line."""
  129. sys.stderr.write("warning: " + self.gen_error(msg, line) + "\n")
  130. def readline(self):
  131. """Read and return a single logical line from the current file (or
  132. from an internal buffer if lines have previously been "unread"
  133. with 'unreadline()'). If the 'join_lines' option is true, this
  134. may involve reading multiple physical lines concatenated into a
  135. single string. Updates the current line number, so calling
  136. 'warn()' after 'readline()' emits a warning about the physical
  137. line(s) just read. Returns None on end-of-file, since the empty
  138. string can occur if 'rstrip_ws' is true but 'strip_blanks' is
  139. not."""
  140. # If any "unread" lines waiting in 'linebuf', return the top
  141. # one. (We don't actually buffer read-ahead data -- lines only
  142. # get put in 'linebuf' if the client explicitly does an
  143. # 'unreadline()'.
  144. if self.linebuf:
  145. line = self.linebuf[-1]
  146. del self.linebuf[-1]
  147. return line
  148. buildup_line = ''
  149. while True:
  150. # read the line, make it None if EOF
  151. line = self.file.readline()
  152. if line == '':
  153. line = None
  154. if self.strip_comments and line:
  155. # Look for the first "#" in the line. If none, never
  156. # mind. If we find one and it's the first character, or
  157. # is not preceded by "\", then it starts a comment --
  158. # strip the comment, strip whitespace before it, and
  159. # carry on. Otherwise, it's just an escaped "#", so
  160. # unescape it (and any other escaped "#"'s that might be
  161. # lurking in there) and otherwise leave the line alone.
  162. pos = line.find("#")
  163. if pos == -1: # no "#" -- no comments
  164. pass
  165. # It's definitely a comment -- either "#" is the first
  166. # character, or it's elsewhere and unescaped.
  167. elif pos == 0 or line[pos-1] != "\\":
  168. # Have to preserve the trailing newline, because it's
  169. # the job of a later step (rstrip_ws) to remove it --
  170. # and if rstrip_ws is false, we'd better preserve it!
  171. # (NB. this means that if the final line is all comment
  172. # and has no trailing newline, we will think that it's
  173. # EOF; I think that's OK.)
  174. eol = (line[-1] == '\n') and '\n' or ''
  175. line = line[0:pos] + eol
  176. # If all that's left is whitespace, then skip line
  177. # *now*, before we try to join it to 'buildup_line' --
  178. # that way constructs like
  179. # hello \\
  180. # # comment that should be ignored
  181. # there
  182. # result in "hello there".
  183. if line.strip() == "":
  184. continue
  185. else: # it's an escaped "#"
  186. line = line.replace("\\#", "#")
  187. # did previous line end with a backslash? then accumulate
  188. if self.join_lines and buildup_line:
  189. # oops: end of file
  190. if line is None:
  191. self.warn("continuation line immediately precedes "
  192. "end-of-file")
  193. return buildup_line
  194. if self.collapse_join:
  195. line = line.lstrip()
  196. line = buildup_line + line
  197. # careful: pay attention to line number when incrementing it
  198. if isinstance(self.current_line, list):
  199. self.current_line[1] = self.current_line[1] + 1
  200. else:
  201. self.current_line = [self.current_line,
  202. self.current_line + 1]
  203. # just an ordinary line, read it as usual
  204. else:
  205. if line is None: # eof
  206. return None
  207. # still have to be careful about incrementing the line number!
  208. if isinstance(self.current_line, list):
  209. self.current_line = self.current_line[1] + 1
  210. else:
  211. self.current_line = self.current_line + 1
  212. # strip whitespace however the client wants (leading and
  213. # trailing, or one or the other, or neither)
  214. if self.lstrip_ws and self.rstrip_ws:
  215. line = line.strip()
  216. elif self.lstrip_ws:
  217. line = line.lstrip()
  218. elif self.rstrip_ws:
  219. line = line.rstrip()
  220. # blank line (whether we rstrip'ed or not)? skip to next line
  221. # if appropriate
  222. if (line == '' or line == '\n') and self.skip_blanks:
  223. continue
  224. if self.join_lines:
  225. if line[-1] == '\\':
  226. buildup_line = line[:-1]
  227. continue
  228. if line[-2:] == '\\\n':
  229. buildup_line = line[0:-2] + '\n'
  230. continue
  231. # well, I guess there's some actual content there: return it
  232. return line
  233. def readlines(self):
  234. """Read and return the list of all logical lines remaining in the
  235. current file."""
  236. lines = []
  237. while True:
  238. line = self.readline()
  239. if line is None:
  240. return lines
  241. lines.append(line)
  242. def unreadline(self, line):
  243. """Push 'line' (a string) onto an internal buffer that will be
  244. checked by future 'readline()' calls. Handy for implementing
  245. a parser with line-at-a-time lookahead."""
  246. self.linebuf.append(line)