From: Michael D. Lowis <mike@mdlowis.com>
Date: Tue, 12 Dec 2017 02:23:59 +0000 (-0500)
Subject: tweaked lexers for text and ruby
X-Git-Url: https://git.mdlowis.com/?a=commitdiff_plain;h=f5bd8875cfaac70361f75c6b4fc60928a366c244;p=archive%2Ftide-ocaml.git

tweaked lexers for text and ruby
---

diff --git a/Makefile b/Makefile
index 0d98c92..60af2a7 100644
--- a/Makefile
+++ b/Makefile
@@ -38,7 +38,10 @@ BINSRCS = \
 	unittests.ml
 
 LEXERS = \
-	lib/lexers/lex_cpp.ml
+	lib/lexers/lex_text.ml \
+	lib/lexers/lex_cpp.ml \
+	lib/lexers/lex_ruby.ml \
+	lib/lexers/lex_ocaml.ml
 
 LIBSRCS = \
 	lib/misc.ml \
@@ -76,7 +79,7 @@ all: $(BINS) lib/lexers/lex_cpp.ml
 clean:
 	$(RM) *.byte *.bin *.cm* *.o *.a
 	$(RM) lib/*.cm* lib/*.o tests/*.cm* tests/*.o
-	$(RM) lib/lexers/*.cm* lib/lexers/*.ml
+	$(RM) lib/lexers/*.cm* lib/lexers/*.ml lib/lexers/*.o
 
 # Executable targets
 edit.$(BINEXT): tide.$(LIBEXT) edit.$(OBJEXT)
diff --git a/lib/lexers/lex_ruby.mll b/lib/lexers/lex_ruby.mll
index 48fea7c..e5c479c 100644
--- a/lib/lexers/lex_ruby.mll
+++ b/lib/lexers/lex_ruby.mll
@@ -3,60 +3,38 @@
 let oct = ['0'-'9']
 let dec = ['0'-'9']
 let hex = ['0'-'9' 'a'-'f' 'A'-'F']
-let exp = ['e''E'] ['+''-']? dec+
 
 let alpha_ = ['a'-'z' 'A'-'Z' '_']
 let alnum_ = (alpha_ | dec)
 
-let fstyle = ['f' 'F' 'l' 'L']
-let istyle = ['u' 'U' 'l' 'L']
+let ln_cmt = '#' [^'\n']*
+let typedef = ['A'-'Z'] alnum_*
+let identifier = ['@' '$']? ['a'-'z'] alnum_* ['!' '=' '?']?
 
-let ln_cmt = "//" [^ '\n']*
-let character = "'" ([^'\\' '\''] | '\\' _) "'"
-let string = '"' ([^'\\' '"'] | '\\' _)* ['"' '\n']
-let identifier = alpha_ alnum_*
-let preprocess = "#" [' ' '\t']* alpha_+
-let sys_incl = (' '|'\t')* '<' [^ '\n' '>']* '>'
+let keyword = "if" | "not" | "then" | "else" | "elsif" | "end" | "def" | "do"
+    | "exit" | "nil" | "begin" | "rescue" | "raise" | "pass" | "class" | "goto"
+    | "break" | "return" | "continue" | "case" | "default" | "switch" | "while"
+    | "for"
 
 let number = (
-    dec+ istyle*
-  | '0' ['x''X'] hex+ istyle*
-  | dec+ exp? fstyle?
-  | dec* '.' dec+ exp? fstyle?
-  | dec+ '.' dec* exp? fstyle?
+    dec+
+  | '0' ['o''O'] oct+
+  | '0' ['d''D'] dec+
+  | '0' ['x''X'] hex+
 )
 
-let const = "true" | "false" | "NULL"
-
-let keyword = "goto" | "break" | "return" | "continue" | "asm" | "case"
-    | "default" | "if" | "else" | "switch" | "while" | "for" | "do" | "sizeof"
-
-let typedef = "bool" | "short" | "int" | "long" | "unsigned" | "signed" | "char"
-    | "size_t" | "void" | "extern" | "static" | "inline" | "struct" | "enum"
-    | "typedef" | "union" | "volatile" | "auto" | "const" | "int8_t" | "int16_t"
-    | "int32_t" | "int64_t" | "uint8_t" | "uint16_t" | "uint32_t" | "uint64_t"
-    | "float" | "double"
+let string = (
+    '"' ([^'\\' '"'] | '\\' _)* '"'
+  | '/' ([^'\\' '/'] | '\\' _)* '/'
+  | '\'' ([^'\\' '\''] | '\\' _)* '\''
+)
 
 rule scan ctx = parse
-  | "/*"       { range_start ctx; comment ctx lexbuf }
   | ln_cmt     { set_color ctx Comment }
   | number     { set_color ctx Constant }
-  | character  { set_color ctx Constant }
   | string     { set_color ctx Constant }
-  | const      { set_color ctx Constant }
   | keyword    { set_color ctx Keyword }
   | typedef    { set_color ctx Type }
-  | preprocess { set_color ctx PreProcessor; preproc ctx lexbuf }
   | identifier { (* skip *) }
   | _          { scan ctx lexbuf }
   | eof        { raise Eof }
-
-and comment ctx = parse
-  | "*/" { range_stop ctx Comment }
-  | _    { comment ctx lexbuf }
-  | eof  { raise Eof }
-
-and preproc ctx = parse
-  | sys_incl { set_color ctx Constant }
-  | _        { (* skip *) }
-  | eof      { raise Eof }
diff --git a/lib/lexers/lex_text.mll b/lib/lexers/lex_text.mll
index b58b158..6800d33 100644
--- a/lib/lexers/lex_text.mll
+++ b/lib/lexers/lex_text.mll
@@ -1,8 +1,5 @@
 { open Colormap }
 
-let ident = ['a'-'z' 'A'-'Z']+
-
 rule scan ctx = parse
-  | ident { scan ctx lexbuf }
-  | _     { scan ctx lexbuf }
-  | eof   { raise Eof }
+  | _   { raise Eof }
+  | eof { raise Eof }