@@ -194,17 +194,31 @@ var DefaultHTMLConfig = HTMLConfig{
194194}
195195
196196func Print (s * scanner.Scanner , w io.Writer , p Printer ) error {
197+ accum := ""
198+ lastKind := Whitespace
197199 tok := s .Scan ()
198200 for tok != scanner .EOF {
199- tokText := s .TokenText ()
200- err := p .Print (w , tokenKind (tok , tokText ), tokText )
201- if err != nil {
202- return err
201+ text := s .TokenText ()
202+ kind := tokenKind (tok , text )
203+ if kind != lastKind {
204+ if accum != "" {
205+ if err := p .Print (w , lastKind , accum ); err != nil {
206+ return err
207+ }
208+ accum = ""
209+ }
210+ lastKind = kind
203211 }
204-
212+ accum += text
205213 tok = s .Scan ()
206214 }
207215
216+ if accum != "" {
217+ if err := p .Print (w , lastKind , accum ); err != nil {
218+ return err
219+ }
220+ }
221+
208222 return nil
209223}
210224
@@ -214,20 +228,38 @@ func Annotate(src []byte, a Annotator) (annotate.Annotations, error) {
214228 var anns annotate.Annotations
215229 read := 0
216230
231+ accum := ""
232+ lastKind := Whitespace
217233 tok := s .Scan ()
218234 for tok != scanner .EOF {
219- tokText := s .TokenText ()
235+ text := s .TokenText ()
236+ kind := tokenKind (tok , text )
237+ if kind != lastKind {
238+ if accum != "" {
239+ ann , err := a .Annotate (read , lastKind , accum )
240+ if err != nil {
241+ return nil , err
242+ }
243+ read += len (accum )
244+ if ann != nil {
245+ anns = append (anns , ann )
246+ }
247+ accum = ""
248+ }
249+ lastKind = kind
250+ }
251+ accum += text
252+ tok = s .Scan ()
253+ }
220254
221- ann , err := a .Annotate (read , tokenKind (tok , tokText ), tokText )
255+ if accum != "" {
256+ ann , err := a .Annotate (read , lastKind , accum )
222257 if err != nil {
223258 return nil , err
224259 }
225- read += len (tokText )
226260 if ann != nil {
227261 anns = append (anns , ann )
228262 }
229-
230- tok = s .Scan ()
231263 }
232264
233265 return anns , nil
0 commit comments