root / tags / v1_0_2_Build_915 / extensions / extScripting / scripts / jython / Lib / sgmllib.py @ 12217
History | View | Annotate | Download (17 KB)
1 | 5782 | jmvivo | """A parser for SGML, using the derived class as a static DTD."""
|
---|---|---|---|
2 | |||
3 | # XXX This only supports those SGML features used by HTML.
|
||
4 | |||
5 | # XXX There should be a way to distinguish between PCDATA (parsed
|
||
6 | # character data -- the normal case), RCDATA (replaceable character
|
||
7 | # data -- only char and entity references and end tags are special)
|
||
8 | # and CDATA (character data -- only end tags are special).
|
||
9 | |||
10 | |||
11 | import re |
||
12 | import string |
||
13 | |||
14 | __all__ = ["SGMLParser"]
|
||
15 | |||
16 | # Regular expressions used for parsing
|
||
17 | |||
18 | interesting = re.compile('[&<]')
|
||
19 | incomplete = re.compile('&([a-zA-Z][a-zA-Z0-9]*|#[0-9]*)?|'
|
||
20 | '<([a-zA-Z][^<>]*|'
|
||
21 | '/([a-zA-Z][^<>]*)?|'
|
||
22 | '![^<>]*)?')
|
||
23 | |||
24 | entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]')
|
||
25 | charref = re.compile('&#([0-9]+)[^0-9]')
|
||
26 | |||
27 | starttagopen = re.compile('<[>a-zA-Z]')
|
||
28 | shorttagopen = re.compile('<[a-zA-Z][-.a-zA-Z0-9]*/')
|
||
29 | shorttag = re.compile('<([a-zA-Z][-.a-zA-Z0-9]*)/([^/]*)/')
|
||
30 | piopen = re.compile('<\?')
|
||
31 | piclose = re.compile('>')
|
||
32 | endtagopen = re.compile('</[<>a-zA-Z]')
|
||
33 | endbracket = re.compile('[<>]')
|
||
34 | special = re.compile('<![^<>]*>')
|
||
35 | commentopen = re.compile('<!--')
|
||
36 | commentclose = re.compile(r'--\s*>')
|
||
37 | tagfind = re.compile('[a-zA-Z][-_.a-zA-Z0-9]*')
|
||
38 | attrfind = re.compile( |
||
39 | r'\s*([a-zA-Z_][-.a-zA-Z_0-9]*)(\s*=\s*'
|
||
40 | r'(\'[^\']*\'|"[^"]*"|[-a-zA-Z0-9./:;+*%?!&$\(\)_#=~\'"]*))?')
|
||
41 | |||
42 | declname = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9]*\s*')
|
||
43 | declstringlit = re.compile(r'(\'[^\']*\'|"[^"]*")\s*')
|
||
44 | |||
45 | |||
46 | class SGMLParseError(RuntimeError): |
||
47 | """Exception raised for all parse errors."""
|
||
48 | pass
|
||
49 | |||
50 | |||
51 | # SGML parser base class -- find tags and call handler functions.
|
||
52 | # Usage: p = SGMLParser(); p.feed(data); ...; p.close().
|
||
53 | # The dtd is defined by deriving a class which defines methods
|
||
54 | # with special names to handle tags: start_foo and end_foo to handle
|
||
55 | # <foo> and </foo>, respectively, or do_foo to handle <foo> by itself.
|
||
56 | # (Tags are converted to lower case for this purpose.) The data
|
||
57 | # between tags is passed to the parser by calling self.handle_data()
|
||
58 | # with some data as argument (the data may be split up in arbitrary
|
||
59 | # chunks). Entity references are passed by calling
|
||
60 | # self.handle_entityref() with the entity reference as argument.
|
||
61 | |||
62 | class SGMLParser: |
||
63 | |||
64 | # Interface -- initialize and reset this instance
|
||
65 | def __init__(self, verbose=0): |
||
66 | self.verbose = verbose
|
||
67 | self.reset()
|
||
68 | |||
69 | # Interface -- reset this instance. Loses all unprocessed data
|
||
70 | def reset(self): |
||
71 | self.rawdata = '' |
||
72 | self.stack = []
|
||
73 | self.lasttag = '???' |
||
74 | self.nomoretags = 0 |
||
75 | self.literal = 0 |
||
76 | |||
77 | # For derived classes only -- enter literal mode (CDATA) till EOF
|
||
78 | def setnomoretags(self): |
||
79 | self.nomoretags = self.literal = 1 |
||
80 | |||
81 | # For derived classes only -- enter literal mode (CDATA)
|
||
82 | def setliteral(self, *args): |
||
83 | self.literal = 1 |
||
84 | |||
85 | # Interface -- feed some data to the parser. Call this as
|
||
86 | # often as you want, with as little or as much text as you
|
||
87 | # want (may include '\n'). (This just saves the text, all the
|
||
88 | # processing is done by goahead().)
|
||
89 | def feed(self, data): |
||
90 | self.rawdata = self.rawdata + data |
||
91 | self.goahead(0) |
||
92 | |||
93 | # Interface -- handle the remaining data
|
||
94 | def close(self): |
||
95 | self.goahead(1) |
||
96 | |||
97 | # Internal -- handle data as far as reasonable. May leave state
|
||
98 | # and data to be processed by a subsequent call. If 'end' is
|
||
99 | # true, force handling all data as if followed by EOF marker.
|
||
100 | def goahead(self, end): |
||
101 | rawdata = self.rawdata
|
||
102 | i = 0
|
||
103 | n = len(rawdata)
|
||
104 | while i < n:
|
||
105 | if self.nomoretags: |
||
106 | self.handle_data(rawdata[i:n])
|
||
107 | i = n |
||
108 | break
|
||
109 | match = interesting.search(rawdata, i) |
||
110 | if match: j = match.start(0) |
||
111 | else: j = n
|
||
112 | if i < j: self.handle_data(rawdata[i:j]) |
||
113 | i = j |
||
114 | if i == n: break |
||
115 | if rawdata[i] == '<': |
||
116 | if starttagopen.match(rawdata, i):
|
||
117 | if self.literal: |
||
118 | self.handle_data(rawdata[i])
|
||
119 | i = i+1
|
||
120 | continue
|
||
121 | k = self.parse_starttag(i)
|
||
122 | if k < 0: break |
||
123 | i = k |
||
124 | continue
|
||
125 | if endtagopen.match(rawdata, i):
|
||
126 | k = self.parse_endtag(i)
|
||
127 | if k < 0: break |
||
128 | i = k |
||
129 | self.literal = 0 |
||
130 | continue
|
||
131 | if commentopen.match(rawdata, i):
|
||
132 | if self.literal: |
||
133 | self.handle_data(rawdata[i])
|
||
134 | i = i+1
|
||
135 | continue
|
||
136 | k = self.parse_comment(i)
|
||
137 | if k < 0: break |
||
138 | i = i+k |
||
139 | continue
|
||
140 | if piopen.match(rawdata, i):
|
||
141 | if self.literal: |
||
142 | self.handle_data(rawdata[i])
|
||
143 | i = i+1
|
||
144 | continue
|
||
145 | k = self.parse_pi(i)
|
||
146 | if k < 0: break |
||
147 | i = i+k |
||
148 | continue
|
||
149 | match = special.match(rawdata, i) |
||
150 | if match:
|
||
151 | if self.literal: |
||
152 | self.handle_data(rawdata[i])
|
||
153 | i = i+1
|
||
154 | continue
|
||
155 | # This is some sort of declaration; in "HTML as
|
||
156 | # deployed," this should only be the document type
|
||
157 | # declaration ("<!DOCTYPE html...>").
|
||
158 | k = self.parse_declaration(i)
|
||
159 | if k < 0: break |
||
160 | i = k |
||
161 | continue
|
||
162 | elif rawdata[i] == '&': |
||
163 | match = charref.match(rawdata, i) |
||
164 | if match:
|
||
165 | name = match.group(1)
|
||
166 | self.handle_charref(name)
|
||
167 | i = match.end(0)
|
||
168 | if rawdata[i-1] != ';': i = i-1 |
||
169 | continue
|
||
170 | match = entityref.match(rawdata, i) |
||
171 | if match:
|
||
172 | name = match.group(1)
|
||
173 | self.handle_entityref(name)
|
||
174 | i = match.end(0)
|
||
175 | if rawdata[i-1] != ';': i = i-1 |
||
176 | continue
|
||
177 | else:
|
||
178 | raise SGMLParseError('neither < nor & ??') |
||
179 | # We get here only if incomplete matches but
|
||
180 | # nothing else
|
||
181 | match = incomplete.match(rawdata, i) |
||
182 | if not match: |
||
183 | self.handle_data(rawdata[i])
|
||
184 | i = i+1
|
||
185 | continue
|
||
186 | j = match.end(0)
|
||
187 | if j == n:
|
||
188 | break # Really incomplete |
||
189 | self.handle_data(rawdata[i:j])
|
||
190 | i = j |
||
191 | # end while
|
||
192 | if end and i < n: |
||
193 | self.handle_data(rawdata[i:n])
|
||
194 | i = n |
||
195 | self.rawdata = rawdata[i:]
|
||
196 | # XXX if end: check for empty stack
|
||
197 | |||
198 | # Internal -- parse comment, return length or -1 if not terminated
|
||
199 | def parse_comment(self, i): |
||
200 | rawdata = self.rawdata
|
||
201 | if rawdata[i:i+4] != '<!--': |
||
202 | raise SGMLParseError('unexpected call to parse_comment()') |
||
203 | match = commentclose.search(rawdata, i+4)
|
||
204 | if not match: |
||
205 | return -1 |
||
206 | j = match.start(0)
|
||
207 | self.handle_comment(rawdata[i+4: j]) |
||
208 | j = match.end(0)
|
||
209 | return j-i
|
||
210 | |||
211 | # Internal -- parse declaration.
|
||
212 | def parse_declaration(self, i): |
||
213 | rawdata = self.rawdata
|
||
214 | j = i + 2
|
||
215 | # in practice, this should look like: ((name|stringlit) S*)+ '>'
|
||
216 | while 1: |
||
217 | c = rawdata[j:j+1]
|
||
218 | if c == ">": |
||
219 | # end of declaration syntax
|
||
220 | self.handle_decl(rawdata[i+2:j]) |
||
221 | return j + 1 |
||
222 | if c in "\"'": |
||
223 | m = declstringlit.match(rawdata, j) |
||
224 | if not m: |
||
225 | # incomplete or an error?
|
||
226 | return -1 |
||
227 | j = m.end() |
||
228 | elif c in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ": |
||
229 | m = declname.match(rawdata, j) |
||
230 | if not m: |
||
231 | # incomplete or an error?
|
||
232 | return -1 |
||
233 | j = m.end() |
||
234 | elif i == len(rawdata): |
||
235 | # end of buffer between tokens
|
||
236 | return -1 |
||
237 | else:
|
||
238 | raise SGMLParseError(
|
||
239 | "unexpected char in declaration: %s" % `rawdata[i]`)
|
||
240 | assert 0, "can't get here!" |
||
241 | |||
242 | # Internal -- parse processing instr, return length or -1 if not terminated
|
||
243 | def parse_pi(self, i): |
||
244 | rawdata = self.rawdata
|
||
245 | if rawdata[i:i+2] != '<?': |
||
246 | raise SGMLParseError('unexpected call to parse_pi()') |
||
247 | match = piclose.search(rawdata, i+2)
|
||
248 | if not match: |
||
249 | return -1 |
||
250 | j = match.start(0)
|
||
251 | self.handle_pi(rawdata[i+2: j]) |
||
252 | j = match.end(0)
|
||
253 | return j-i
|
||
254 | |||
255 | __starttag_text = None
|
||
256 | def get_starttag_text(self): |
||
257 | return self.__starttag_text |
||
258 | |||
259 | # Internal -- handle starttag, return length or -1 if not terminated
|
||
260 | def parse_starttag(self, i): |
||
261 | self.__starttag_text = None |
||
262 | start_pos = i |
||
263 | rawdata = self.rawdata
|
||
264 | if shorttagopen.match(rawdata, i):
|
||
265 | # SGML shorthand: <tag/data/ == <tag>data</tag>
|
||
266 | # XXX Can data contain &... (entity or char refs)?
|
||
267 | # XXX Can data contain < or > (tag characters)?
|
||
268 | # XXX Can there be whitespace before the first /?
|
||
269 | match = shorttag.match(rawdata, i) |
||
270 | if not match: |
||
271 | return -1 |
||
272 | tag, data = match.group(1, 2) |
||
273 | self.__starttag_text = '<%s/' % tag |
||
274 | tag = tag.lower() |
||
275 | k = match.end(0)
|
||
276 | self.finish_shorttag(tag, data)
|
||
277 | self.__starttag_text = rawdata[start_pos:match.end(1) + 1] |
||
278 | return k
|
||
279 | # XXX The following should skip matching quotes (' or ")
|
||
280 | match = endbracket.search(rawdata, i+1)
|
||
281 | if not match: |
||
282 | return -1 |
||
283 | j = match.start(0)
|
||
284 | # Now parse the data between i+1 and j into a tag and attrs
|
||
285 | attrs = [] |
||
286 | if rawdata[i:i+2] == '<>': |
||
287 | # SGML shorthand: <> == <last open tag seen>
|
||
288 | k = j |
||
289 | tag = self.lasttag
|
||
290 | else:
|
||
291 | match = tagfind.match(rawdata, i+1)
|
||
292 | if not match: |
||
293 | raise SGMLParseError('unexpected call to parse_starttag') |
||
294 | k = match.end(0)
|
||
295 | tag = rawdata[i+1:k].lower()
|
||
296 | self.lasttag = tag
|
||
297 | while k < j:
|
||
298 | match = attrfind.match(rawdata, k) |
||
299 | if not match: break |
||
300 | attrname, rest, attrvalue = match.group(1, 2, 3) |
||
301 | if not rest: |
||
302 | attrvalue = attrname |
||
303 | elif attrvalue[:1] == '\'' == attrvalue[-1:] or \ |
||
304 | attrvalue[:1] == '"' == attrvalue[-1:]: |
||
305 | attrvalue = attrvalue[1:-1] |
||
306 | attrs.append((attrname.lower(), attrvalue)) |
||
307 | k = match.end(0)
|
||
308 | if rawdata[j] == '>': |
||
309 | j = j+1
|
||
310 | self.__starttag_text = rawdata[start_pos:j]
|
||
311 | self.finish_starttag(tag, attrs)
|
||
312 | return j
|
||
313 | |||
314 | # Internal -- parse endtag
|
||
315 | def parse_endtag(self, i): |
||
316 | rawdata = self.rawdata
|
||
317 | match = endbracket.search(rawdata, i+1)
|
||
318 | if not match: |
||
319 | return -1 |
||
320 | j = match.start(0)
|
||
321 | tag = rawdata[i+2:j].strip().lower()
|
||
322 | if rawdata[j] == '>': |
||
323 | j = j+1
|
||
324 | self.finish_endtag(tag)
|
||
325 | return j
|
||
326 | |||
327 | # Internal -- finish parsing of <tag/data/ (same as <tag>data</tag>)
|
||
328 | def finish_shorttag(self, tag, data): |
||
329 | self.finish_starttag(tag, [])
|
||
330 | self.handle_data(data)
|
||
331 | self.finish_endtag(tag)
|
||
332 | |||
333 | # Internal -- finish processing of start tag
|
||
334 | # Return -1 for unknown tag, 0 for open-only tag, 1 for balanced tag
|
||
335 | def finish_starttag(self, tag, attrs): |
||
336 | try:
|
||
337 | method = getattr(self, 'start_' + tag) |
||
338 | except AttributeError: |
||
339 | try:
|
||
340 | method = getattr(self, 'do_' + tag) |
||
341 | except AttributeError: |
||
342 | self.unknown_starttag(tag, attrs)
|
||
343 | return -1 |
||
344 | else:
|
||
345 | self.handle_starttag(tag, method, attrs)
|
||
346 | return 0 |
||
347 | else:
|
||
348 | self.stack.append(tag)
|
||
349 | self.handle_starttag(tag, method, attrs)
|
||
350 | return 1 |
||
351 | |||
352 | # Internal -- finish processing of end tag
|
||
353 | def finish_endtag(self, tag): |
||
354 | if not tag: |
||
355 | found = len(self.stack) - 1 |
||
356 | if found < 0: |
||
357 | self.unknown_endtag(tag)
|
||
358 | return
|
||
359 | else:
|
||
360 | if tag not in self.stack: |
||
361 | try:
|
||
362 | method = getattr(self, 'end_' + tag) |
||
363 | except AttributeError: |
||
364 | self.unknown_endtag(tag)
|
||
365 | else:
|
||
366 | self.report_unbalanced(tag)
|
||
367 | return
|
||
368 | found = len(self.stack) |
||
369 | for i in range(found): |
||
370 | if self.stack[i] == tag: found = i |
||
371 | while len(self.stack) > found: |
||
372 | tag = self.stack[-1] |
||
373 | try:
|
||
374 | method = getattr(self, 'end_' + tag) |
||
375 | except AttributeError: |
||
376 | method = None
|
||
377 | if method:
|
||
378 | self.handle_endtag(tag, method)
|
||
379 | else:
|
||
380 | self.unknown_endtag(tag)
|
||
381 | del self.stack[-1] |
||
382 | |||
383 | # Overridable -- handle start tag
|
||
384 | def handle_starttag(self, tag, method, attrs): |
||
385 | method(attrs) |
||
386 | |||
387 | # Overridable -- handle end tag
|
||
388 | def handle_endtag(self, tag, method): |
||
389 | method() |
||
390 | |||
391 | # Example -- report an unbalanced </...> tag.
|
||
392 | def report_unbalanced(self, tag): |
||
393 | if self.verbose: |
||
394 | print '*** Unbalanced </' + tag + '>' |
||
395 | print '*** Stack:', self.stack |
||
396 | |||
397 | # Example -- handle character reference, no need to override
|
||
398 | def handle_charref(self, name): |
||
399 | try:
|
||
400 | n = int(name)
|
||
401 | except ValueError: |
||
402 | self.unknown_charref(name)
|
||
403 | return
|
||
404 | if not 0 <= n <= 255: |
||
405 | self.unknown_charref(name)
|
||
406 | return
|
||
407 | self.handle_data(chr(n)) |
||
408 | |||
409 | # Definition of entities -- derived classes may override
|
||
410 | entitydefs = \ |
||
411 | {'lt': '<', 'gt': '>', 'amp': '&', 'quot': '"', 'apos': '\''} |
||
412 | |||
413 | # Example -- handle entity reference, no need to override
|
||
414 | def handle_entityref(self, name): |
||
415 | table = self.entitydefs
|
||
416 | if table.has_key(name):
|
||
417 | self.handle_data(table[name])
|
||
418 | else:
|
||
419 | self.unknown_entityref(name)
|
||
420 | return
|
||
421 | |||
422 | # Example -- handle data, should be overridden
|
||
423 | def handle_data(self, data): |
||
424 | pass
|
||
425 | |||
426 | # Example -- handle comment, could be overridden
|
||
427 | def handle_comment(self, data): |
||
428 | pass
|
||
429 | |||
430 | # Example -- handle declaration, could be overridden
|
||
431 | def handle_decl(self, decl): |
||
432 | pass
|
||
433 | |||
434 | # Example -- handle processing instruction, could be overridden
|
||
435 | def handle_pi(self, data): |
||
436 | pass
|
||
437 | |||
438 | # To be overridden -- handlers for unknown objects
|
||
439 | def unknown_starttag(self, tag, attrs): pass |
||
440 | def unknown_endtag(self, tag): pass |
||
441 | def unknown_charref(self, ref): pass |
||
442 | def unknown_entityref(self, ref): pass |
||
443 | |||
444 | |||
445 | class TestSGMLParser(SGMLParser): |
||
446 | |||
447 | def __init__(self, verbose=0): |
||
448 | self.testdata = "" |
||
449 | SGMLParser.__init__(self, verbose)
|
||
450 | |||
451 | def handle_data(self, data): |
||
452 | self.testdata = self.testdata + data |
||
453 | if len(`self.testdata`) >= 70: |
||
454 | self.flush()
|
||
455 | |||
456 | def flush(self): |
||
457 | data = self.testdata
|
||
458 | if data:
|
||
459 | self.testdata = "" |
||
460 | print 'data:', `data` |
||
461 | |||
462 | def handle_comment(self, data): |
||
463 | self.flush()
|
||
464 | r = `data` |
||
465 | if len(r) > 68: |
||
466 | r = r[:32] + '...' + r[-32:] |
||
467 | print 'comment:', r |
||
468 | |||
469 | def unknown_starttag(self, tag, attrs): |
||
470 | self.flush()
|
||
471 | if not attrs: |
||
472 | print 'start tag: <' + tag + '>' |
||
473 | else:
|
||
474 | print 'start tag: <' + tag, |
||
475 | for name, value in attrs: |
||
476 | print name + '=' + '"' + value + '"', |
||
477 | print '>' |
||
478 | |||
479 | def unknown_endtag(self, tag): |
||
480 | self.flush()
|
||
481 | print 'end tag: </' + tag + '>' |
||
482 | |||
483 | def unknown_entityref(self, ref): |
||
484 | self.flush()
|
||
485 | print '*** unknown entity ref: &' + ref + ';' |
||
486 | |||
487 | def unknown_charref(self, ref): |
||
488 | self.flush()
|
||
489 | print '*** unknown char ref: &#' + ref + ';' |
||
490 | |||
491 | def close(self): |
||
492 | SGMLParser.close(self)
|
||
493 | self.flush()
|
||
494 | |||
495 | |||
496 | def test(args = None): |
||
497 | import sys |
||
498 | |||
499 | if not args: |
||
500 | args = sys.argv[1:]
|
||
501 | |||
502 | if args and args[0] == '-s': |
||
503 | args = args[1:]
|
||
504 | klass = SGMLParser |
||
505 | else:
|
||
506 | klass = TestSGMLParser |
||
507 | |||
508 | if args:
|
||
509 | file = args[0]
|
||
510 | else:
|
||
511 | file = 'test.html'
|
||
512 | |||
513 | if file == '-': |
||
514 | f = sys.stdin |
||
515 | else:
|
||
516 | try:
|
||
517 | f = open(file, 'r') |
||
518 | except IOError, msg: |
||
519 | print file, ":", msg |
||
520 | sys.exit(1)
|
||
521 | |||
522 | data = f.read() |
||
523 | if f is not sys.stdin: |
||
524 | f.close() |
||
525 | |||
526 | x = klass() |
||
527 | for c in data: |
||
528 | x.feed(c) |
||
529 | x.close() |
||
530 | |||
531 | |||
532 | if __name__ == '__main__': |
||
533 | test() |