From cfdd97f726fa1faef7909c77e8c83f9ced17677f Mon Sep 17 00:00:00 2001 From: Mike Lowis Date: Wed, 30 Sep 2026 21:25:35 -0400 Subject: [PATCH] added tier 1 matching --- nlshell.rb | 151 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 142 insertions(+), 9 deletions(-) diff --git a/nlshell.rb b/nlshell.rb index dfd73c3..d6c720c 100755 --- a/nlshell.rb +++ b/nlshell.rb @@ -20,7 +20,9 @@ module Lexer # text: the word/punct as typed, or the literal's exact value # kind: literal kind (:path, :string, ...) or nil Token = Struct.new(:type, :text, :kind) do - def word? = type == :word + # :command matches as a word (it's just a word that shadows a PATH entry); + # it still isn't a :literal, so it can't fill a rule's :slot. + def word? = type == :word || type == :command def literal? = type == :literal def punct? = type == :punct @@ -34,12 +36,13 @@ module Lexer # Tried in order at each token start; first match wins. Because we are always # at a token boundary, no lookbehinds are needed. Set via Lexer.configure. class << self - attr_accessor :literals, :connectives + attr_accessor :literals, :connectives, :verbs end def self.configure(ruleset) self.literals = ruleset.literal_patterns self.connectives = ruleset.connective_patterns + self.verbs = ruleset.verb_patterns end # Sentence punctuation that greedy patterns may swallow ("rm ~/old.log.") @@ -100,8 +103,10 @@ module Lexer end # Lowercase words only. Literals keep their exact case by construction. + # Keeps :command tagged as :command (word? is true for both) so that info + # isn't lost once case-normalized. def self.downcase_words(tokens) - tokens.map { |t| t.word? ? Token.new(:word, t.text.downcase) : t } + tokens.map { |t| t.word? ? Token.new(t.type, t.text.downcase) : t } end # Split into sentences at . ? ! ; (the terminator stays with its sentence). @@ -128,8 +133,6 @@ module Lexer parts.reject(&:empty?) end - KNOWN_ACTION_WORDS = Set.new(%w[run]).freeze - # Length in tokens of a connective starting at i, or 0 if none matches. def self.connective_length(tokens, i) length = 0 @@ -142,9 +145,9 @@ module Lexer end # Check for "and " or ", and " patterns - if t.text == "and" && n&.word? && KNOWN_ACTION_WORDS.include?(n.text) + if t.text == "and" && n&.word? && verbs.include?(n.text) length = 1 - elsif t.text == "," && n&.text == "and" && tokens[i + 2]&.word? && KNOWN_ACTION_WORDS.include?(tokens[i + 2].text) + elsif t.text == "," && n&.text == "and" && tokens[i + 2]&.word? && verbs.include?(tokens[i + 2].text) length = 2 end @@ -158,6 +161,76 @@ module Lexer end end +# Stage: CLS. Tags a clause as teach/conditional/loop/query/action by +# matching its leading words against configured phrases, checked in the +# configured order (first match wins). +module Classifier + class << self + attr_accessor :patterns + end + + def self.configure(ruleset) + self.patterns = ruleset.classifier_patterns + end + + # Returns the matched kind, or nil if no configured phrase matches. + # TODO: fall through to Tier 3 (LLM) instead of nil once that tier exists. + def self.classify(clause) + patterns.each do |kind, phrases| + return kind if phrases.any? { |phrase| leading_match?(clause, phrase) } + end + nil + end + + def self.leading_match?(clause, phrase) + return false if clause.length < phrase.length + + clause.first(phrase.length).map { |t| t.word? ? t.text : nil } == phrase + end +end + +# Stage: Tier 1. Exact/template match against configured rule patterns. A +# pattern is a string of literal words and :slot placeholders, matched +# word-for-word against a clause (same length, no fuzzing - that's Tier 2). +module Tier1 + class << self + attr_accessor :rules + end + + def self.configure(ruleset) + self.rules = ruleset.rule_patterns + end + + # Returns an action node (Hash) on an exact match, or nil on a miss. + def self.lookup(clause) + rules.each do |pattern, action_name| + args = match(pattern.split, clause) + return { type: "action", name: action_name, args: args } if args + end + nil + end + + def self.match(words, clause) + return nil unless words.length == clause.length + + args = {} + words.each_with_index do |word, i| + token = clause[i] + if word.start_with?(":") + # Not restricted to :literal - a bare word (e.g. an extensionless + # filename like "Makefile") is a legitimate slot value too. Keeping + # it un-normalized is Tier 2's job when canonicalizing, not ours. + return nil if token.punct? + + args[word.delete_prefix(":").to_sym] = token + else + return nil unless token.word? && token.text == word + end + end + args + end +end + module Ruleset class RulesModule @@ -169,8 +242,26 @@ module Ruleset @connectives = list end + def classifiers(hash) + @classifiers = hash + end + + def rules(hash) + @rules = hash + end + + def verbs(list) + @verbs = Set.new(list) + end + def literal_patterns = @types def connective_patterns = @connectives + def rule_patterns = @rules + def verb_patterns = @verbs + + # :action phrases come from verbs, not the classifiers() call, so the + # two lists can't drift apart. + def classifier_patterns = @classifiers.merge(action: @verbs.to_a.map { |v| [v] }) end def self.create(name, &block) @@ -199,9 +290,39 @@ builtins = Ruleset.create("builtins") do ["then"], ["next"], ] + + # Checked in this order; first matching kind wins. + classifiers( + teach: [ + ["teach"], ["learn"], ["remember"] + ], + conditional: [ + ["if"], ["unless"], ["when"] + ], + loop: [ + ["for", "each"], ["repeat"], ["while"], ["every"] + ], + query: [ + ["what"], ["how"], ["why"], ["who"], ["where"], ["is"], ["are"], ["does"], ["list"], ["show"] + ], + # action: derived from verbs, below. + ) + + rules( + "delete :target" => "delete", + "remove :target" => "delete", + "archive :target" => "archive", + "copy :target to :destination" => "copy", + "move :target" => "move", + ) + + # Verbs recognized by connective_length's bare "and " check. + verbs %w[run delete remove copy move create rename archive search find locate pick transfer] end Lexer.configure(builtins) +Classifier.configure(builtins) +Tier1.configure(builtins) # Start the REPL now loop do @@ -220,8 +341,20 @@ loop do # Debug only: print the sentences found Lexer.sentences(tokens).each_with_index do |sentence, i| puts "#{i}: #{sentence.map(&:to_s).join(' ')}" - Lexer.clauses(sentence).each_with_index do |clause, j| - puts " clause #{j}: #{clause.map(&:to_s).join(' ')}" + + clauses = Lexer.clauses(sentence) + kinds = clauses.map { |clause| Classifier.classify(clause) } + if kinds.any?(&:nil?) + puts " couldn't tell what kind of request this is, please rephrase" + next + end + + clauses.each_with_index do |clause, j| + puts " clause #{j} [#{kinds[j]}]: #{clause.map(&:to_s).join(' ')}" + if kinds[j] == :action + node = Tier1.lookup(clause) + puts " #{node ? "tier1 hit: #{node}" : 'tier1 miss'}" + end end puts " match: #{Lexer.match_text(sentence)}" end -- 2.55.0