From: Michael D. Lowis <mike.lowis@gentex.com>
Date: Fri, 9 Mar 2018 15:35:43 +0000 (-0500)
Subject: added more specific classification to tokens
X-Git-Url: https://git.mdlowis.com/?a=commitdiff_plain;h=797447f67ecad210c8dd9809e17a7eb0a733363e;p=archive%2Ftide-hl.git

added more specific classification to tokens
---

diff --git a/colormap.ml b/colormap.ml
index 2ea2220..68b3980 100644
--- a/colormap.ml
+++ b/colormap.ml
@@ -2,15 +2,10 @@ open Lexing
 
 exception Eof
 
-type style = Normal | Comment | Constant | Keyword | Type | PreProcessor
-
-module Span = struct
-  type t = { start : int; stop : int; style : int }
-  let compare a b =
-    if a.stop < b.start then -1
-    else if a.start > b.stop then 1
-    else 0
-end
+type style =
+  | Normal | Comment | Constant | String | Char | Number | Boolean | Float
+  | Variable | Function | Keyword | Operator | PreProcessor | Type | Statement
+  | Special
 
 type ctx = {
   lbuf : lexbuf;
@@ -23,20 +18,30 @@ type lexer = {
 }
 
 let get_color = function
-  | Normal       -> 0
-  | Comment      -> 1
-  | Constant     -> 2
-  | Keyword      -> 3
-  | Type         -> 4
-  | PreProcessor -> 5
+  | Normal ->       1 * 16
+  | Comment ->      2 * 16
+  | Constant ->     3 * 16
+  | String ->       4 * 16
+  | Char ->         5 * 16
+  | Number ->       6 * 16
+  | Boolean ->      7 * 16
+  | Float ->        8 * 16
+  | Variable ->     9 * 16
+  | Function ->     10 * 16
+  | Keyword ->      11 * 16
+  | Operator ->     12 * 16
+  | PreProcessor -> 13 * 16
+  | Type ->         14 * 16
+  | Statement ->    15 * 16
+  | Special ->      16 * 16
 
 let set_color ctx clr =
   Printf.printf "%d,%d,%d\n"
-    (lexeme_start ctx.lbuf) ((lexeme_end ctx.lbuf) - 1) (get_color clr)
+    (lexeme_start ctx.lbuf) (lexeme_end ctx.lbuf) (get_color clr)
 
 let range_start ctx =
   ctx.pos <- (lexeme_start ctx.lbuf)
 
 let range_stop ctx clr =
   Printf.printf "%d,%d,%d\n"
-    ctx.pos ((lexeme_end ctx.lbuf) - 1) (get_color clr)
+    ctx.pos (lexeme_end ctx.lbuf) (get_color clr)
diff --git a/lex_cpp.mll b/lex_cpp.mll
index 48fea7c..658b0da 100644
--- a/lex_cpp.mll
+++ b/lex_cpp.mll
@@ -40,9 +40,9 @@ let typedef = "bool" | "short" | "int" | "long" | "unsigned" | "signed" | "char"
 rule scan ctx = parse
   | "/*"       { range_start ctx; comment ctx lexbuf }
   | ln_cmt     { set_color ctx Comment }
-  | number     { set_color ctx Constant }
-  | character  { set_color ctx Constant }
-  | string     { set_color ctx Constant }
+  | number     { set_color ctx Number }
+  | character  { set_color ctx Char }
+  | string     { set_color ctx String }
   | const      { set_color ctx Constant }
   | keyword    { set_color ctx Keyword }
   | typedef    { set_color ctx Type }
diff --git a/lex_ocaml.mll b/lex_ocaml.mll
index 6e23b6c..5e35073 100644
--- a/lex_ocaml.mll
+++ b/lex_ocaml.mll
@@ -24,9 +24,9 @@ let keyword = "and" | "as" | "assert" | "begin" | "class" | "constraint" | "do"
 
 rule scan ctx = parse
   | "(*"       { range_start ctx; comment ctx lexbuf }
-  | number     { set_color ctx Constant }
-  | character  { set_color ctx Constant }
-  | string     { set_color ctx Constant }
+  | number     { set_color ctx Number }
+  | character  { set_color ctx Char }
+  | string     { set_color ctx String }
   | const      { set_color ctx Constant }
   | keyword    { set_color ctx Keyword }
   | typedef    { set_color ctx Type }
diff --git a/lex_ruby.mll b/lex_ruby.mll
index e5c479c..785b020 100644
--- a/lex_ruby.mll
+++ b/lex_ruby.mll
@@ -31,8 +31,8 @@ let string = (
 
 rule scan ctx = parse
   | ln_cmt     { set_color ctx Comment }
-  | number     { set_color ctx Constant }
-  | string     { set_color ctx Constant }
+  | number     { set_color ctx Number }
+  | string     { set_color ctx String }
   | keyword    { set_color ctx Keyword }
   | typedef    { set_color ctx Type }
   | identifier { (* skip *) }
diff --git a/main.ml b/main.ml
index 86d808e..ae96314 100644
--- a/main.ml
+++ b/main.ml
@@ -1,4 +1,3 @@
-
 type filetype = {
   syntax : Colormap.ctx -> Lexing.lexbuf -> unit;
   names : string list;
@@ -32,16 +31,19 @@ let pick_syntax path =
   in
   (List.find_opt match_ftype filetypes)
 
-let scan_string lexfn string =
+let rec scan_string lexfn string =
   let lbuf = Lexing.from_string string in
   let ctx = Colormap.({ lbuf = lbuf; pos = 0; }) in
   try while true do lexfn ctx lbuf done
   with Colormap.Eof ->
     Printf.printf "0,0,0\n";
-    flush stdout
+    flush stdout;
+    scan_input lexfn
 
-let rec scan_input lexfn =
-  try scan_string lexfn (really_input_string stdin (read_int ())) with
+and scan_input lexfn =
+  try
+    scan_string lexfn (really_input_string stdin (read_int ()));
+  with
   | Failure _ -> scan_input lexfn
   | End_of_file -> ()