added clasue parsing and ruleset extension logic

diff --git a/nlshell.rb b/nlshell.rb
index 16fa1fe3227237e6f95b8fec1a81f68b45fb527b..dfd73c30334185e5cff66b45fd1ea54d085969e3 100755 (executable)
--- a/nlshell.rb
+++ b/nlshell.rb
@@ -32,17 +32,15 @@ module Lexer
   end
 
   # Tried in order at each token start; first match wins. Because we are always
-  # at a token boundary, no lookbehinds are needed.
-  LITERALS = {
-    string:  /"[^"]*"|`[^`]*`|'[^']*'/,
-    url:     %r{[a-z][a-z0-9+.-]*://\S+}i,
-    env:     /\$\{?[a-z_]\w*\}?/i,
-    glob:    %r@[\w.~/-]*(?:\*|\[[^\]\s]+\]|\{[^}\s]+\})[\w.~/*?\[\]{},-]*@,
-    path:    %r@(?:(?:~|\.{1,2})?(?:/[^\s/]+)+/?|[\w.-]+(?:/[\w.-]+)+/?|~(?=\s|\z))@,
-    flag:    /--?[a-z][\w-]*(?:=\S+)?/i,
-    number:  /\d+(?:\.\d+)?(?:kb|mb|gb|tb|ms|min|[kmgshd%])?(?![\w]|\.\w)/i,
-    file:    %r{(?:\.[a-z][\w.-]*|[\w-]+(?:\.[\w-]+)*\.[a-z0-9]{1,5})(?![\w/])}i,
-  }.freeze
+  # at a token boundary, no lookbehinds are needed. Set via Lexer.configure.
+  class << self
+    attr_accessor :literals, :connectives
+  end
+
+  def self.configure(ruleset)
+    self.literals    = ruleset.literal_patterns
+    self.connectives = ruleset.connective_patterns
+  end
 
   # Sentence punctuation that greedy patterns may swallow ("rm ~/old.log.")
   TRAILING = /[.,;:!?]+\z/
@@ -62,7 +60,7 @@ module Lexer
   end
 
   def self.read_literal(sc)
-    LITERALS.each do |kind, re|
+    literals.each do |kind, re|
       text = sc.scan(re) or next # Match or skip to the next
 
       if kind == :string
@@ -132,18 +130,11 @@ module Lexer
 
   KNOWN_ACTION_WORDS = Set.new(%w[run]).freeze
 
-  ConnectivePatterns = [
-    ["after", "that"],
-    ["and", "then"],
-    ["then"],
-    ["next"],
-  ]
-
   # Length in tokens of a connective starting at i, or 0 if none matches.
   def self.connective_length(tokens, i)
     length = 0
     t, n = tokens[i], tokens[i + 1]
-    ConnectivePatterns.each do |pattern|
+    connectives.each do |pattern|
       slice = tokens[i..(i+pattern.length-1)].map{|e| (e ? e.text : "") }
       if slice == pattern
         return pattern.length
@@ -171,10 +162,15 @@ end
 module Ruleset
   class RulesModule
     def types(hash)
+      @types = hash
     end
 
     def connectives(list)
+      @connectives = list
     end
+
+    def literal_patterns    = @types
+    def connective_patterns = @connectives
   end
 
   def self.create(name, &block)
@@ -185,8 +181,8 @@ module Ruleset
 end
 
 # Create default ruleset
-Ruleset.create("builtins") do
-  types {
+builtins = Ruleset.create("builtins") do
+  types(
     string:  /"[^"]*"|`[^`]*`|'[^']*'/,
     url:     %r{[a-z][a-z0-9+.-]*://\S+}i,
     env:     /\$\{?[a-z_]\w*\}?/i,
@@ -195,7 +191,7 @@ Ruleset.create("builtins") do
     flag:    /--?[a-z][\w-]*(?:=\S+)?/i,
     number:  /\d+(?:\.\d+)?(?:kb|mb|gb|tb|ms|min|[kmgshd%])?(?![\w]|\.\w)/i,
     file:    %r{(?:\.[a-z][\w.-]*|[\w-]+(?:\.[\w-]+)*\.[a-z0-9]{1,5})(?![\w/])}i,
-  }
+  )
 
   connectives [
     ["after", "that"],
@@ -203,9 +199,10 @@ Ruleset.create("builtins") do
     ["then"],
     ["next"],
   ]
-
 end
 
+Lexer.configure(builtins)
+
 # Start the REPL now
 loop do
   print ":> "