休息了大半个月，没写自己的代码了。国庆过了，再不为自己写写代码就有负罪感了。这篇文章所提到的所有工具的代码都可以在
Vczh Library++ 3.0的页面找到。
上一篇文章提到了一个状态机绘图工具，直到最近我终于把他的第一个部分给做好了。现在可以画图之后产生一个可以供
这里所描述的高亮控件所使用的着色器了。第一步我们要使用TokenizerBuilder绘制一个状态机：
然后点击“Generate to Clipboard”按钮，就会要求你输入一个类名然后产生着色器插件的代码了。上面这个是一个简化过的C#代码着色状态机，其中Id有个星号代表不是所有走到这个状态的东西都可以叫Id（其实是关键字，懒得改了）。中括号里面的是颜色的名称。这些名称最终是要拿来生成代码的，所以必须是C#接受的可以拿来做变量名的东西，不过我也没检查，只管拼接字符串生成。
我把生成后的代码贴在了CodeForm工程的CSharpTokenizer.Generated.cs里：
第三步就是要自己建立一个CSharpColorizer.Configuration.cs了。C#的partial class真是伟大，简直就是为了代码生成而设计出来的。在这里我们看看生成后的CSharpColorizer.Generated.cs的代码：
1 using System;
2 using System.Collections.Generic;
3 using System.Linq;
4 using System.Text;
5 using CodeBoxControl;
6 using System.Drawing;
7
8 namespace CodeForm
9 {
10 partial class CSharpColorizer : ITextEditorColorizer
11 {
12 //public const int NormalColorId = 0;
13 public const int IdColorId = NormalColorId + 1;
14 public const int StringColorId = NormalColorId + 2;
15 public const int CommentColorId = NormalColorId + 3;
16
17 //private readonly Color HighlightColor = Color.FromArgb(173, 214, 255);
18 //private readonly Color NormalColor = Color.FromArgb(0, 0, 0);
19 //private readonly Color IdColor = Color.FromArgb(0, 0, 0);
20 //private readonly Color StringColor = Color.FromArgb(0, 0, 0);
21 //private readonly Color CommentColor = Color.FromArgb(0, 0, 0);
22
23 private const int StartState = 0;
24 private const int NormalState = 1;
25 private const int IdStateId = 2;
26 private const int InStringStateId = 3;
27 private const int InStringEscapingStateId = 4;
28 private const int InCharStateId = 5;
29 private const int InCharEscapingStateId = 6;
30 private const int StringStateId = 7;
31 private const int CommentStartStateId = 8;
32 private const int SingleLineCommentStateId = 9;
33 private const int InMultipleLineCommentStateId = 10;
34 private const int InMultipleLineCommentWaitingToFinishStateId = 11;
35 private const int MultipleLineCommentStateId = 12;
36
37 private TextEditorColorItem[] colorItems = new TextEditorColorItem[NormalColorId + 4];
38 private int[] charset = new int[65536];
39 private int[,] transitions = new int[13, 11];
40 private bool[] finalStates = new bool[13];
41 private int[] stateColors = new int[13];
42
43 public TextEditorColorItem[] ColorItems
44 {
45 get
46 {
47 return this.colorItems;
48 }
49 }
50
51 public CSharpColorizer()
52 {
53 this.colorItems[NormalColorId] = new TextEditorColorItem()
54 {
55 Text = NormalColor,
56 HighlightText = NormalColor,
57 Highlight = HighlightColor
58 };
59 this.colorItems[IdColorId] = new TextEditorColorItem()
60 {
61 Text = IdColor,
62 HighlightText = IdColor,
63 Highlight = HighlightColor
64 };
65 this.colorItems[StringColorId] = new TextEditorColorItem()
66 {
67 Text = StringColor,
68 HighlightText = StringColor,
69 Highlight = HighlightColor
70 };
71 this.colorItems[CommentColorId] = new TextEditorColorItem()
72 {
73 Text = CommentColor,
74 HighlightText = CommentColor,
75 Highlight = HighlightColor
76 };
77 // You should write your own CreateAdditionalColors() implementation to add additional colors.
78 // You can modify the NormalColorId and put all additional colors ids before NormalColorId.
79 // It is recommended to use another partial class to store all customized code.
80 CreateAdditionalColors();
81 CreateStateMachine();
82 }
83 public int ColorizeLine(char[] items, int length, int initialState, int[] colors)
84 {
85 int state = initialState;
86 int itemStart = 0;
87 int lastFinalState = StartState;
88
89 for (int i = 0; i <= length; i++)
90 {
91 if (i != length)
92 {
93 state = transitions[state, charset[items[i]]];
94 if (state == StartState)
95 {
96 state = transitions[state, charset[items[i]]];
97 }
98 }
99 else
100 {
101 lastFinalState = state;
102 }
103
104 if (i == length || lastFinalState != state && lastFinalState != StartState)
105 {
106 int color = stateColors[lastFinalState];
107 switch (lastFinalState)
108 {
109 case IdStateId:
110 // You should write your own IsValidId implementation.
111 color = IsValidId(new string(items, itemStart, Math.Min(i, length) - itemStart)) ? stateColors[lastFinalState] : NormalColorId;
112 break;
113 }
114 for (int j = itemStart; j < i; j++)
115 {
116 colors[j] = color;
117 }
118 itemStart = i;
119 }
120 lastFinalState = finalStates[state] ? state : StartState;
121 }
122
123 return transitions[state, charset['\n']];
124 }
125
126 private void CreateStateMachine()
127 {
128 for (int i = 0; i < 10; i++)
129 charset[i] = 0;
130 for (int i = 10; i < 11; i++)
131 charset[i] = 1;
132 for (int i = 11; i < 34; i++)
133 charset[i] = 0;
134 for (int i = 34; i < 35; i++)
135 charset[i] = 2;
136 for (int i = 35; i < 36; i++)
137 charset[i] = 3;
138 for (int i = 36; i < 39; i++)
139 charset[i] = 0;
140 for (int i = 39; i < 40; i++)
141 charset[i] = 4;
142 for (int i = 40; i < 42; i++)
143 charset[i] = 0;
144 for (int i = 42; i < 43; i++)
145 charset[i] = 5;
146 for (int i = 43; i < 47; i++)
147 charset[i] = 0;
148 for (int i = 47; i < 48; i++)
149 charset[i] = 6;
150 for (int i = 48; i < 58; i++)
151 charset[i] = 7;
152 for (int i = 58; i < 65; i++)
153 charset[i] = 0;
154 for (int i = 65; i < 91; i++)
155 charset[i] = 8;
156 for (int i = 91; i < 92; i++)
157 charset[i] = 0;
158 for (int i = 92; i < 93; i++)
159 charset[i] = 9;
160 for (int i = 93; i < 95; i++)
161 charset[i] = 0;
162 for (int i = 95; i < 96; i++)
163 charset[i] = 8;
164 for (int i = 96; i < 97; i++)
165 charset[i] = 0;
166 for (int i = 97; i < 123; i++)
167 charset[i] = 8;
168 for (int i = 123; i < 65536; i++)
169 charset[i] = 0;
170
171 finalStates[0] = false;
172 finalStates[1] = true;
173 finalStates[2] = true;
174 finalStates[3] = false;
175 finalStates[4] = false;
176 finalStates[5] = false;
177 finalStates[6] = false;
178 finalStates[7] = true;
179 finalStates[8] = false;
180 finalStates[9] = true;
181 finalStates[10] = false;
182 finalStates[11] = false;
183 finalStates[12] = true;
184
185 stateColors[0] = NormalColorId + 0;
186 stateColors[1] = NormalColorId + 0;
187 stateColors[2] = NormalColorId + 1;
188 stateColors[3] = NormalColorId + 2;
189 stateColors[4] = NormalColorId + 2;
190 stateColors[5] = NormalColorId + 2;
191 stateColors[6] = NormalColorId + 2;
192 stateColors[7] = NormalColorId + 2;
193 stateColors[8] = NormalColorId + 0;
194 stateColors[9] = NormalColorId + 3;
195 stateColors[10] = NormalColorId + 3;
196 stateColors[11] = NormalColorId + 3;
197 stateColors[12] = NormalColorId + 3;
198
199 transitions[0, 0] = 1;
200 transitions[0, 1] = 1;
201 transitions[0, 2] = 3;
202 transitions[0, 3] = 2;
203 transitions[0, 4] = 5;
204 transitions[0, 5] = 1;
205 transitions[0, 6] = 8;
206 transitions[0, 7] = 1;
207 transitions[0, 8] = 2;
208 transitions[0, 9] = 1;
209 transitions[0, 10] = 1;
210 transitions[1, 0] = 0;
211 transitions[1, 1] = 0;
212 transitions[1, 2] = 0;
213 transitions[1, 3] = 0;
214 transitions[1, 4] = 0;
215 transitions[1, 5] = 0;
216 transitions[1, 6] = 0;
217 transitions[1, 7] = 0;
218 transitions[1, 8] = 0;
219 transitions[1, 9] = 0;
220 transitions[1, 10] = 0;
221 transitions[2, 0] = 0;
222 transitions[2, 1] = 0;
223 transitions[2, 2] = 0;
224 transitions[2, 3] = 0;
225 transitions[2, 4] = 0;
226 transitions[2, 5] = 0;
227 transitions[2, 6] = 0;
228 transitions[2, 7] = 2;
229 transitions[2, 8] = 2;
230 transitions[2, 9] = 0;
231 transitions[2, 10] = 0;
232 transitions[3, 0] = 3;
233 transitions[3, 1] = 0;
234 transitions[3, 2] = 7;
235 transitions[3, 3] = 3;
236 transitions[3, 4] = 3;
237 transitions[3, 5] = 3;
238 transitions[3, 6] = 3;
239 transitions[3, 7] = 3;
240 transitions[3, 8] = 3;
241 transitions[3, 9] = 4;
242 transitions[3, 10] = 0;
243 transitions[4, 0] = 3;
244 transitions[4, 1] = 0;
245 transitions[4, 2] = 3;
246 transitions[4, 3] = 3;
247 transitions[4, 4] = 3;
248 transitions[4, 5] = 3;
249 transitions[4, 6] = 3;
250 transitions[4, 7] = 3;
251 transitions[4, 8] = 3;
252 transitions[4, 9] = 3;
253 transitions[4, 10] = 0;
254 transitions[5, 0] = 5;
255 transitions[5, 1] = 0;
256 transitions[5, 2] = 5;
257 transitions[5, 3] = 5;
258 transitions[5, 4] = 7;
259 transitions[5, 5] = 5;
260 transitions[5, 6] = 5;
261 transitions[5, 7] = 5;
262 transitions[5, 8] = 5;
263 transitions[5, 9] = 6;
264 transitions[5, 10] = 0;
265 transitions[6, 0] = 5;
266 transitions[6, 1] = 0;
267 transitions[6, 2] = 5;
268 transitions[6, 3] = 5;
269 transitions[6, 4] = 5;
270 transitions[6, 5] = 5;
271 transitions[6, 6] = 5;
272 transitions[6, 7] = 5;
273 transitions[6, 8] = 5;
274 transitions[6, 9] = 5;
275 transitions[6, 10] = 0;
276 transitions[7, 0] = 0;
277 transitions[7, 1] = 0;
278 transitions[7, 2] = 0;
279 transitions[7, 3] = 0;
280 transitions[7, 4] = 0;
281 transitions[7, 5] = 0;
282 transitions[7, 6] = 0;
283 transitions[7, 7] = 0;
284 transitions[7, 8] = 0;
285 transitions[7, 9] = 0;
286 transitions[7, 10] = 0;
287 transitions[8, 0] = 1;
288 transitions[8, 1] = 1;
289 transitions[8, 2] = 1;
290 transitions[8, 3] = 1;
291 transitions[8, 4] = 1;
292 transitions[8, 5] = 10;
293 transitions[8, 6] = 9;
294 transitions[8, 7] = 1;
295 transitions[8, 8] = 1;
296 transitions[8, 9] = 1;
297 transitions[8, 10] = 1;
298 transitions[9, 0] = 9;
299 transitions[9, 1] = 0;
300 transitions[9, 2] = 9;
301 transitions[9, 3] = 9;
302 transitions[9, 4] = 9;
303 transitions[9, 5] = 9;
304 transitions[9, 6] = 9;
305 transitions[9, 7] = 9;
306 transitions[9, 8] = 9;
307 transitions[9, 9] = 9;
308 transitions[9, 10] = 0;
309 transitions[10, 0] = 10;
310 transitions[10, 1] = 10;
311 transitions[10, 2] = 10;
312 transitions[10, 3] = 10;
313 transitions[10, 4] = 10;
314 transitions[10, 5] = 11;
315 transitions[10, 6] = 0;
316 transitions[10, 7] = 10;
317 transitions[10, 8] = 10;
318 transitions[10, 9] = 10;
319 transitions[10, 10] = 0;
320 transitions[11, 0] = 10;
321 transitions[11, 1] = 10;
322 transitions[11, 2] = 10;
323 transitions[11, 3] = 10;
324 transitions[11, 4] = 10;
325 transitions[11, 5] = 10;
326 transitions[11, 6] = 12;
327 transitions[11, 7] = 10;
328 transitions[11, 8] = 10;
329 transitions[11, 9] = 10;
330 transitions[11, 10] = 0;
331 transitions[12, 0] = 0;
332 transitions[12, 1] = 0;
333 transitions[12, 2] = 0;
334 transitions[12, 3] = 0;
335 transitions[12, 4] = 0;
336 transitions[12, 5] = 0;
337 transitions[12, 6] = 0;
338 transitions[12, 7] = 0;
339 transitions[12, 8] = 0;
340 transitions[12, 9] = 0;
341 transitions[12, 10] = 0;
342 }
343 }
344 }
345
这是一个典型的状态机，里面定义了所有状态、颜色和对外可见的颜色标号。这里使用partial class是因为颜色是要你自己填写的，因此你可以看见我在代码的一开始注释掉了几行，所以他们就会出现在CSharpColorizer.Configuration.cs里：
1 using System;
2 using System.Collections.Generic;
3 using System.Linq;
4 using System.Text;
5 using CodeBoxControl;
6 using System.Drawing;
7
8 namespace CodeForm
9 {
10 partial class CSharpColorizer
11 {
12 public const int BreakPointColorId = 0;
13 public const int BlockPointColorId = 1;
14 public const int NormalColorId = 2;
15
16 private readonly Color HighlightColor = Color.FromArgb(173, 214, 255);
17 private readonly Color NormalColor = Color.FromArgb(0, 0, 0);
18 private readonly Color IdColor = Color.FromArgb(0, 0, 255);
19 private readonly Color StringColor = Color.FromArgb(163, 21, 21);
20 private readonly Color CommentColor = Color.FromArgb(0, 128, 0);
21
22 private readonly Color BreakPointColor = Color.FromArgb(255, 255, 255);
23 private readonly Color BreakPointHighlightColor = Color.FromArgb(123, 119, 166);
24 private readonly Color BlockPointColor = Color.Gray;
25
26 private void CreateAdditionalColors()
27 {
28 this.colorItems[BreakPointColorId] = new TextEditorColorItem()
29 {
30 Text = BreakPointColor,
31 HighlightText = BreakPointColor,
32 Highlight = BreakPointHighlightColor
33 };
34 this.colorItems[BlockPointColorId] = new TextEditorColorItem()
35 {
36 Text = BlockPointColor,
37 HighlightText = BlockPointColor,
38 Highlight = HighlightColor
39 };
40 }
41
42 private bool IsValidId(string token)
43 {
44 return token[0] == '#' || Array.BinarySearch(keywords, token) >= 0;
45 }
46
47 private static string[] keywords ={
48 "abstract",
49 "as",
50 "base",
51 "bool",
52 "break",
53 "byte",
54 "case",
55 "catch",
56 "char",
57 "checked",
58 "class",
59 "const",
60 "continue",
61 "decimal",
62 "default",
63 "delegate",
64 "do",
65 "double",
66 "else",
67 "enum",
68 "event",
69 "explicit",
70 "extern",
71 "false",
72 "finally",
73 "fixed",
74 "float",
75 "for",
76 "foreach",
77 "goto",
78 "if",
79 "implicit",
80 "in",
81 "int",
82 "interface",
83 "internal",
84 "is",
85 "lock",
86 "long",
87 "namespace",
88 "new",
89 "null",
90 "object",
91 "operator",
92 "out",
93 "override",
94 "params",
95 "private",
96 "protected",
97 "public",
98 "readonly",
99 "ref",
100 "return",
101 "sbyte",
102 "sealed",
103 "short",
104 "sizeof",
105 "stackalloc",
106 "static",
107 "string",
108 "struct",
109 "switch",
110 "this",
111 "throw",
112 "true",
113 "try",
114 "typeof",
115 "unit",
116 "ulong",
117 "unchecked",
118 "unsafe",
119 "ushort",
120 "using",
121 "virtual",
122 "void",
123 "volatile",
124 "while"
125 };
126 }
127 }
128
这个Configuration做了三件事情。第一、定义了额外的颜色。在这里我们修改了NormalColorId，这是系统颜色的第一个颜色序号。我们只需要修改了它，然后把自己的颜色加在它前面就行了。第二件事情是定义了什么是Id。我们在Id状态里打了个星号，那么生成的代码就会要求我们实现一个IsValidId函数。第三件事情就是我们指定了所有记号的真正的颜色。
为什么要做成partial class呢？我们可以很容易看出来，每一次生成代码的时候，我们只需要修改namespace和类名以及注释掉NormalColorId和所有颜色的声明，就可以让我们的自定义部分得到保留。这无疑很方便我们对着色器进行修改。只需要修改一下状态机，生成一份代码，再做一点很小的改动就行了。
最重要的是，这个着色器的性能跟
手写的基本一样优秀，因此从现在开始，我们开发一个上下文无关的着色器控件就基本不需要写代码了，只要是用Vczh Library++ 3.0实验库里面的TextEditorControl，再用这个TokenizerBuilder画个状态机生成个着色器的代码就搞定了，哇哈哈。下面要做的就是增强TokenizerBuilder，从状态机生成词法分析器了，然后就可以继续智能提示的实验了。
posted on 2010-10-08 06:05
陈梓瀚(vczh) 阅读(6729)
评论(8) 编辑 收藏 引用 所属分类:
开发自己的IDE