root / tags / v1_0_2_Build_900 / libraries / libIverUtiles / src / com / iver / utiles / console / jedit / TokenMarker.java @ 10562
History | View | Annotate | Download (8.95 KB)
1 |
package com.iver.utiles.console.jedit; |
---|---|
2 |
/*
|
3 |
* TokenMarker.java - Generic token marker
|
4 |
* Copyright (C) 1998, 1999 Slava Pestov
|
5 |
*
|
6 |
* You may use and modify this package for any purpose. Redistribution is
|
7 |
* permitted, in both source and binary form, provided that this notice
|
8 |
* remains intact in all source distributions of this package.
|
9 |
*/
|
10 |
|
11 |
import javax.swing.text.Segment; |
12 |
|
13 |
/**
|
14 |
* A token marker that splits lines of text into tokens. Each token carries
|
15 |
* a length field and an indentification tag that can be mapped to a color
|
16 |
* for painting that token.<p>
|
17 |
*
|
18 |
* For performance reasons, the linked list of tokens is reused after each
|
19 |
* line is tokenized. Therefore, the return value of <code>markTokens</code>
|
20 |
* should only be used for immediate painting. Notably, it cannot be
|
21 |
* cached.
|
22 |
*
|
23 |
* @author Slava Pestov
|
24 |
* @version $Id$
|
25 |
*
|
26 |
* @see org.gjt.sp.jedit.syntax.Token
|
27 |
*/
|
28 |
public abstract class TokenMarker |
29 |
{ |
30 |
/**
|
31 |
* A wrapper for the lower-level <code>markTokensImpl</code> method
|
32 |
* that is called to split a line up into tokens.
|
33 |
* @param line The line
|
34 |
* @param lineIndex The line number
|
35 |
*/
|
36 |
public Token markTokens(Segment line, int lineIndex) |
37 |
{ |
38 |
if(lineIndex >= length)
|
39 |
{ |
40 |
throw new IllegalArgumentException("Tokenizing invalid line: " |
41 |
+ lineIndex); |
42 |
} |
43 |
|
44 |
lastToken = null;
|
45 |
|
46 |
LineInfo info = lineInfo[lineIndex]; |
47 |
LineInfo prev; |
48 |
if(lineIndex == 0) |
49 |
prev = null;
|
50 |
else
|
51 |
prev = lineInfo[lineIndex - 1];
|
52 |
|
53 |
byte oldToken = info.token;
|
54 |
byte token = markTokensImpl(prev == null ? |
55 |
Token.NULL : prev.token,line,lineIndex); |
56 |
|
57 |
info.token = token; |
58 |
|
59 |
/*
|
60 |
* This is a foul hack. It stops nextLineRequested
|
61 |
* from being cleared if the same line is marked twice.
|
62 |
*
|
63 |
* Why is this necessary? It's all JEditTextArea's fault.
|
64 |
* When something is inserted into the text, firing a
|
65 |
* document event, the insertUpdate() method shifts the
|
66 |
* caret (if necessary) by the amount inserted.
|
67 |
*
|
68 |
* All caret movement is handled by the select() method,
|
69 |
* which eventually pipes the new position to scrollTo()
|
70 |
* and calls repaint().
|
71 |
*
|
72 |
* Note that at this point in time, the new line hasn't
|
73 |
* yet been painted; the caret is moved first.
|
74 |
*
|
75 |
* scrollTo() calls offsetToX(), which tokenizes the line
|
76 |
* unless it is being called on the last line painted
|
77 |
* (in which case it uses the text area's painter cached
|
78 |
* token list). What scrollTo() does next is irrelevant.
|
79 |
*
|
80 |
* After scrollTo() has done it's job, repaint() is
|
81 |
* called, and eventually we end up in paintLine(), whose
|
82 |
* job is to paint the changed line. It, too, calls
|
83 |
* markTokens().
|
84 |
*
|
85 |
* The problem was that if the line started a multiline
|
86 |
* token, the first markTokens() (done in offsetToX())
|
87 |
* would set nextLineRequested (because the line end
|
88 |
* token had changed) but the second would clear it
|
89 |
* (because the line was the same that time) and therefore
|
90 |
* paintLine() would never know that it needed to repaint
|
91 |
* subsequent lines.
|
92 |
*
|
93 |
* This bug took me ages to track down, that's why I wrote
|
94 |
* all the relevant info down so that others wouldn't
|
95 |
* duplicate it.
|
96 |
*/
|
97 |
if(!(lastLine == lineIndex && nextLineRequested))
|
98 |
nextLineRequested = (oldToken != token); |
99 |
|
100 |
lastLine = lineIndex; |
101 |
|
102 |
addToken(0,Token.END);
|
103 |
|
104 |
return firstToken;
|
105 |
} |
106 |
|
107 |
/**
|
108 |
* An abstract method that splits a line up into tokens. It
|
109 |
* should parse the line, and call <code>addToken()</code> to
|
110 |
* add syntax tokens to the token list. Then, it should return
|
111 |
* the initial token type for the next line.<p>
|
112 |
*
|
113 |
* For example if the current line contains the start of a
|
114 |
* multiline comment that doesn't end on that line, this method
|
115 |
* should return the comment token type so that it continues on
|
116 |
* the next line.
|
117 |
*
|
118 |
* @param token The initial token type for this line
|
119 |
* @param line The line to be tokenized
|
120 |
* @param lineIndex The index of the line in the document,
|
121 |
* starting at 0
|
122 |
* @return The initial token type for the next line
|
123 |
*/
|
124 |
protected abstract byte markTokensImpl(byte token, Segment line, |
125 |
int lineIndex);
|
126 |
|
127 |
/**
|
128 |
* Returns if the token marker supports tokens that span multiple
|
129 |
* lines. If this is true, the object using this token marker is
|
130 |
* required to pass all lines in the document to the
|
131 |
* <code>markTokens()</code> method (in turn).<p>
|
132 |
*
|
133 |
* The default implementation returns true; it should be overridden
|
134 |
* to return false on simpler token markers for increased speed.
|
135 |
*/
|
136 |
public boolean supportsMultilineTokens() |
137 |
{ |
138 |
return true; |
139 |
} |
140 |
|
141 |
/**
|
142 |
* Informs the token marker that lines have been inserted into
|
143 |
* the document. This inserts a gap in the <code>lineInfo</code>
|
144 |
* array.
|
145 |
* @param index The first line number
|
146 |
* @param lines The number of lines
|
147 |
*/
|
148 |
public void insertLines(int index, int lines) |
149 |
{ |
150 |
if(lines <= 0) |
151 |
return;
|
152 |
length += lines; |
153 |
ensureCapacity(length); |
154 |
int len = index + lines;
|
155 |
System.arraycopy(lineInfo,index,lineInfo,len,
|
156 |
lineInfo.length - len); |
157 |
|
158 |
for(int i = index + lines - 1; i >= index; i--) |
159 |
{ |
160 |
lineInfo[i] = new LineInfo();
|
161 |
} |
162 |
} |
163 |
|
164 |
/**
|
165 |
* Informs the token marker that line have been deleted from
|
166 |
* the document. This removes the lines in question from the
|
167 |
* <code>lineInfo</code> array.
|
168 |
* @param index The first line number
|
169 |
* @param lines The number of lines
|
170 |
*/
|
171 |
public void deleteLines(int index, int lines) |
172 |
{ |
173 |
if (lines <= 0) |
174 |
return;
|
175 |
int len = index + lines;
|
176 |
length -= lines; |
177 |
System.arraycopy(lineInfo,len,lineInfo,
|
178 |
index,lineInfo.length - len); |
179 |
} |
180 |
|
181 |
/**
|
182 |
* Returns the number of lines in this token marker.
|
183 |
*/
|
184 |
public int getLineCount() |
185 |
{ |
186 |
return length;
|
187 |
} |
188 |
|
189 |
/**
|
190 |
* Returns true if the next line should be repainted. This
|
191 |
* will return true after a line has been tokenized that starts
|
192 |
* a multiline token that continues onto the next line.
|
193 |
*/
|
194 |
public boolean isNextLineRequested() |
195 |
{ |
196 |
return nextLineRequested;
|
197 |
} |
198 |
|
199 |
// protected members
|
200 |
|
201 |
/**
|
202 |
* The first token in the list. This should be used as the return
|
203 |
* value from <code>markTokens()</code>.
|
204 |
*/
|
205 |
protected Token firstToken;
|
206 |
|
207 |
/**
|
208 |
* The last token in the list. New tokens are added here.
|
209 |
* This should be set to null before a new line is to be tokenized.
|
210 |
*/
|
211 |
protected Token lastToken;
|
212 |
|
213 |
/**
|
214 |
* An array for storing information about lines. It is enlarged and
|
215 |
* shrunk automatically by the <code>insertLines()</code> and
|
216 |
* <code>deleteLines()</code> methods.
|
217 |
*/
|
218 |
protected LineInfo[] lineInfo; |
219 |
|
220 |
/**
|
221 |
* The number of lines in the model being tokenized. This can be
|
222 |
* less than the length of the <code>lineInfo</code> array.
|
223 |
*/
|
224 |
protected int length; |
225 |
|
226 |
/**
|
227 |
* The last tokenized line.
|
228 |
*/
|
229 |
protected int lastLine; |
230 |
|
231 |
/**
|
232 |
* True if the next line should be painted.
|
233 |
*/
|
234 |
protected boolean nextLineRequested; |
235 |
|
236 |
/**
|
237 |
* Creates a new <code>TokenMarker</code>. This DOES NOT create
|
238 |
* a lineInfo array; an initial call to <code>insertLines()</code>
|
239 |
* does that.
|
240 |
*/
|
241 |
protected TokenMarker()
|
242 |
{ |
243 |
lastLine = -1;
|
244 |
} |
245 |
|
246 |
/**
|
247 |
* Ensures that the <code>lineInfo</code> array can contain the
|
248 |
* specified index. This enlarges it if necessary. No action is
|
249 |
* taken if the array is large enough already.<p>
|
250 |
*
|
251 |
* It should be unnecessary to call this under normal
|
252 |
* circumstances; <code>insertLine()</code> should take care of
|
253 |
* enlarging the line info array automatically.
|
254 |
*
|
255 |
* @param index The array index
|
256 |
*/
|
257 |
protected void ensureCapacity(int index) |
258 |
{ |
259 |
if(lineInfo == null) |
260 |
lineInfo = new LineInfo[index + 1]; |
261 |
else if(lineInfo.length <= index) |
262 |
{ |
263 |
LineInfo[] lineInfoN = new LineInfo[(index + 1) * 2]; |
264 |
System.arraycopy(lineInfo,0,lineInfoN,0, |
265 |
lineInfo.length); |
266 |
lineInfo = lineInfoN; |
267 |
} |
268 |
} |
269 |
|
270 |
/**
|
271 |
* Adds a token to the token list.
|
272 |
* @param length The length of the token
|
273 |
* @param id The id of the token
|
274 |
*/
|
275 |
protected void addToken(int length, byte id) |
276 |
{ |
277 |
if(id >= Token.INTERNAL_FIRST && id <= Token.INTERNAL_LAST)
|
278 |
throw new InternalError("Invalid id: " + id); |
279 |
|
280 |
if(length == 0 && id != Token.END) |
281 |
return;
|
282 |
|
283 |
if(firstToken == null) |
284 |
{ |
285 |
firstToken = new Token(length,id);
|
286 |
lastToken = firstToken; |
287 |
} |
288 |
else if(lastToken == null) |
289 |
{ |
290 |
lastToken = firstToken; |
291 |
firstToken.length = length; |
292 |
firstToken.id = id; |
293 |
} |
294 |
else if(lastToken.next == null) |
295 |
{ |
296 |
lastToken.next = new Token(length,id);
|
297 |
lastToken = lastToken.next; |
298 |
} |
299 |
else
|
300 |
{ |
301 |
lastToken = lastToken.next; |
302 |
lastToken.length = length; |
303 |
lastToken.id = id; |
304 |
} |
305 |
} |
306 |
|
307 |
/**
|
308 |
* Inner class for storing information about tokenized lines.
|
309 |
*/
|
310 |
public class LineInfo |
311 |
{ |
312 |
/**
|
313 |
* Creates a new LineInfo object with token = Token.NULL
|
314 |
* and obj = null.
|
315 |
*/
|
316 |
public LineInfo()
|
317 |
{ |
318 |
} |
319 |
|
320 |
/**
|
321 |
* Creates a new LineInfo object with the specified
|
322 |
* parameters.
|
323 |
*/
|
324 |
public LineInfo(byte token, Object obj) |
325 |
{ |
326 |
this.token = token;
|
327 |
this.obj = obj;
|
328 |
} |
329 |
|
330 |
/**
|
331 |
* The id of the last token of the line.
|
332 |
*/
|
333 |
public byte token; |
334 |
|
335 |
/**
|
336 |
* This is for use by the token marker implementations
|
337 |
* themselves. It can be used to store anything that
|
338 |
* is an object and that needs to exist on a per-line
|
339 |
* basis.
|
340 |
*/
|
341 |
public Object obj; |
342 |
} |
343 |
} |