added more specific classification to tokens

diff --git a/colormap.ml b/colormap.ml
index 2ea2220fbb2ec94f01194270da64eff9c9664a07..68b398035cd3b6b920fd9ab365351e2e5d4bf029 100644 (file)
--- a/colormap.ml
+++ b/colormap.ml
@@ -2,15 +2,10 @@ open Lexing
 
 exception Eof
 
-type style = Normal | Comment | Constant | Keyword | Type | PreProcessor
-
-module Span = struct
-  type t = { start : int; stop : int; style : int }
-  let compare a b =
-    if a.stop < b.start then -1
-    else if a.start > b.stop then 1
-    else 0
-end
+type style =
+  | Normal | Comment | Constant | String | Char | Number | Boolean | Float
+  | Variable | Function | Keyword | Operator | PreProcessor | Type | Statement
+  | Special
 
 type ctx = {
   lbuf : lexbuf;
@@ -23,20 +18,30 @@ type lexer = {
 }
 
 let get_color = function
-  | Normal       -> 0
-  | Comment      -> 1
-  | Constant     -> 2
-  | Keyword      -> 3
-  | Type         -> 4
-  | PreProcessor -> 5
+  | Normal ->       1 * 16
+  | Comment ->      2 * 16
+  | Constant ->     3 * 16
+  | String ->       4 * 16
+  | Char ->         5 * 16
+  | Number ->       6 * 16
+  | Boolean ->      7 * 16
+  | Float ->        8 * 16
+  | Variable ->     9 * 16
+  | Function ->     10 * 16
+  | Keyword ->      11 * 16
+  | Operator ->     12 * 16
+  | PreProcessor -> 13 * 16
+  | Type ->         14 * 16
+  | Statement ->    15 * 16
+  | Special ->      16 * 16
 
 let set_color ctx clr =
   Printf.printf "%d,%d,%d\n"
-    (lexeme_start ctx.lbuf) ((lexeme_end ctx.lbuf) - 1) (get_color clr)
+    (lexeme_start ctx.lbuf) (lexeme_end ctx.lbuf) (get_color clr)
 
 let range_start ctx =
   ctx.pos <- (lexeme_start ctx.lbuf)
 
 let range_stop ctx clr =
   Printf.printf "%d,%d,%d\n"
-    ctx.pos ((lexeme_end ctx.lbuf) - 1) (get_color clr)
+    ctx.pos (lexeme_end ctx.lbuf) (get_color clr)

diff --git a/lex_cpp.mll b/lex_cpp.mll
index 48fea7cedb4ea47515940fb5428e0937be468ee7..658b0da76d49b45f0bf09e74c898380cbfe47265 100644 (file)
--- a/lex_cpp.mll
+++ b/lex_cpp.mll
@@ -40,9 +40,9 @@ let typedef = "bool" | "short" | "int" | "long" | "unsigned" | "signed" | "char"
 rule scan ctx = parse
   | "/*"       { range_start ctx; comment ctx lexbuf }
   | ln_cmt     { set_color ctx Comment }
-  | number     { set_color ctx Constant }
-  | character  { set_color ctx Constant }
-  | string     { set_color ctx Constant }
+  | number     { set_color ctx Number }
+  | character  { set_color ctx Char }
+  | string     { set_color ctx String }
   | const      { set_color ctx Constant }
   | keyword    { set_color ctx Keyword }
   | typedef    { set_color ctx Type }

diff --git a/lex_ocaml.mll b/lex_ocaml.mll
index 6e23b6c99c34f065907c2e505ce2e6b05b03a0c2..5e350738081430a51e785f2b7e52cb55454c0960 100644 (file)
--- a/lex_ocaml.mll
+++ b/lex_ocaml.mll
@@ -24,9 +24,9 @@ let keyword = "and" | "as" | "assert" | "begin" | "class" | "constraint" | "do"
 
 rule scan ctx = parse
   | "(*"       { range_start ctx; comment ctx lexbuf }
-  | number     { set_color ctx Constant }
-  | character  { set_color ctx Constant }
-  | string     { set_color ctx Constant }
+  | number     { set_color ctx Number }
+  | character  { set_color ctx Char }
+  | string     { set_color ctx String }
   | const      { set_color ctx Constant }
   | keyword    { set_color ctx Keyword }
   | typedef    { set_color ctx Type }

diff --git a/lex_ruby.mll b/lex_ruby.mll
index e5c479c7dbb42a47f73c292a1f919356899e8cc8..785b0206bd9440ffa977e581798b204acc008810 100644 (file)
--- a/lex_ruby.mll
+++ b/lex_ruby.mll
@@ -31,8 +31,8 @@ let string = (
 
 rule scan ctx = parse
   | ln_cmt     { set_color ctx Comment }
-  | number     { set_color ctx Constant }
-  | string     { set_color ctx Constant }
+  | number     { set_color ctx Number }
+  | string     { set_color ctx String }
   | keyword    { set_color ctx Keyword }
   | typedef    { set_color ctx Type }
   | identifier { (* skip *) }

diff --git a/main.ml b/main.ml
index 86d808e365cc61050a8cf19b533c71364a27bb16..ae9631443b5744480838d08fa2cd1990f0069ee0 100644 (file)
--- a/main.ml
+++ b/main.ml
@@ -1,4 +1,3 @@
-
 type filetype = {
   syntax : Colormap.ctx -> Lexing.lexbuf -> unit;
   names : string list;
@@ -32,16 +31,19 @@ let pick_syntax path =
   in
   (List.find_opt match_ftype filetypes)
 
-let scan_string lexfn string =
+let rec scan_string lexfn string =
   let lbuf = Lexing.from_string string in
   let ctx = Colormap.({ lbuf = lbuf; pos = 0; }) in
   try while true do lexfn ctx lbuf done
   with Colormap.Eof ->
     Printf.printf "0,0,0\n";
-    flush stdout
+    flush stdout;
+    scan_input lexfn
 
-let rec scan_input lexfn =
-  try scan_string lexfn (really_input_string stdin (read_int ())) with
+and scan_input lexfn =
+  try
+    scan_string lexfn (really_input_string stdin (read_int ()));
+  with
   | Failure _ -> scan_input lexfn
   | End_of_file -> ()