# text: the word/punct as typed, or the literal's exact value
# kind: literal kind (:path, :string, ...) or nil
Token = Struct.new(:type, :text, :kind) do
- def word? = type == :word
+ # :command matches as a word (it's just a word that shadows a PATH entry);
+ # it still isn't a :literal, so it can't fill a rule's :slot.
+ def word? = type == :word || type == :command
def literal? = type == :literal
def punct? = type == :punct
# Tried in order at each token start; first match wins. Because we are always
# at a token boundary, no lookbehinds are needed. Set via Lexer.configure.
class << self
- attr_accessor :literals, :connectives
+ attr_accessor :literals, :connectives, :verbs
end
def self.configure(ruleset)
self.literals = ruleset.literal_patterns
self.connectives = ruleset.connective_patterns
+ self.verbs = ruleset.verb_patterns
end
# Sentence punctuation that greedy patterns may swallow ("rm ~/old.log.")
end
# Lowercase words only. Literals keep their exact case by construction.
+ # Keeps :command tagged as :command (word? is true for both) so that info
+ # isn't lost once case-normalized.
def self.downcase_words(tokens)
- tokens.map { |t| t.word? ? Token.new(:word, t.text.downcase) : t }
+ tokens.map { |t| t.word? ? Token.new(t.type, t.text.downcase) : t }
end
# Split into sentences at . ? ! ; (the terminator stays with its sentence).
parts.reject(&:empty?)
end
- KNOWN_ACTION_WORDS = Set.new(%w[run]).freeze
-
# Length in tokens of a connective starting at i, or 0 if none matches.
def self.connective_length(tokens, i)
length = 0
end
# Check for "and <verb>" or ", and <verb>" patterns
- if t.text == "and" && n&.word? && KNOWN_ACTION_WORDS.include?(n.text)
+ if t.text == "and" && n&.word? && verbs.include?(n.text)
length = 1
- elsif t.text == "," && n&.text == "and" && tokens[i + 2]&.word? && KNOWN_ACTION_WORDS.include?(tokens[i + 2].text)
+ elsif t.text == "," && n&.text == "and" && tokens[i + 2]&.word? && verbs.include?(tokens[i + 2].text)
length = 2
end
end
end
+# Stage: CLS. Tags a clause as teach/conditional/loop/query/action by
+# matching its leading words against configured phrases, checked in the
+# configured order (first match wins).
+module Classifier
+ class << self
+ attr_accessor :patterns
+ end
+
+ def self.configure(ruleset)
+ self.patterns = ruleset.classifier_patterns
+ end
+
+ # Returns the matched kind, or nil if no configured phrase matches.
+ # TODO: fall through to Tier 3 (LLM) instead of nil once that tier exists.
+ def self.classify(clause)
+ patterns.each do |kind, phrases|
+ return kind if phrases.any? { |phrase| leading_match?(clause, phrase) }
+ end
+ nil
+ end
+
+ def self.leading_match?(clause, phrase)
+ return false if clause.length < phrase.length
+
+ clause.first(phrase.length).map { |t| t.word? ? t.text : nil } == phrase
+ end
+end
+
+# Stage: Tier 1. Exact/template match against configured rule patterns. A
+# pattern is a string of literal words and :slot placeholders, matched
+# word-for-word against a clause (same length, no fuzzing - that's Tier 2).
+module Tier1
+ class << self
+ attr_accessor :rules
+ end
+
+ def self.configure(ruleset)
+ self.rules = ruleset.rule_patterns
+ end
+
+ # Returns an action node (Hash) on an exact match, or nil on a miss.
+ def self.lookup(clause)
+ rules.each do |pattern, action_name|
+ args = match(pattern.split, clause)
+ return { type: "action", name: action_name, args: args } if args
+ end
+ nil
+ end
+
+ def self.match(words, clause)
+ return nil unless words.length == clause.length
+
+ args = {}
+ words.each_with_index do |word, i|
+ token = clause[i]
+ if word.start_with?(":")
+ # Not restricted to :literal - a bare word (e.g. an extensionless
+ # filename like "Makefile") is a legitimate slot value too. Keeping
+ # it un-normalized is Tier 2's job when canonicalizing, not ours.
+ return nil if token.punct?
+
+ args[word.delete_prefix(":").to_sym] = token
+ else
+ return nil unless token.word? && token.text == word
+ end
+ end
+ args
+ end
+end
+
module Ruleset
class RulesModule
@connectives = list
end
+ def classifiers(hash)
+ @classifiers = hash
+ end
+
+ def rules(hash)
+ @rules = hash
+ end
+
+ def verbs(list)
+ @verbs = Set.new(list)
+ end
+
def literal_patterns = @types
def connective_patterns = @connectives
+ def rule_patterns = @rules
+ def verb_patterns = @verbs
+
+ # :action phrases come from verbs, not the classifiers() call, so the
+ # two lists can't drift apart.
+ def classifier_patterns = @classifiers.merge(action: @verbs.to_a.map { |v| [v] })
end
def self.create(name, &block)
["then"],
["next"],
]
+
+ # Checked in this order; first matching kind wins.
+ classifiers(
+ teach: [
+ ["teach"], ["learn"], ["remember"]
+ ],
+ conditional: [
+ ["if"], ["unless"], ["when"]
+ ],
+ loop: [
+ ["for", "each"], ["repeat"], ["while"], ["every"]
+ ],
+ query: [
+ ["what"], ["how"], ["why"], ["who"], ["where"], ["is"], ["are"], ["does"], ["list"], ["show"]
+ ],
+ # action: derived from verbs, below.
+ )
+
+ rules(
+ "delete :target" => "delete",
+ "remove :target" => "delete",
+ "archive :target" => "archive",
+ "copy :target to :destination" => "copy",
+ "move :target" => "move",
+ )
+
+ # Verbs recognized by connective_length's bare "and <verb>" check.
+ verbs %w[run delete remove copy move create rename archive search find locate pick transfer]
end
Lexer.configure(builtins)
+Classifier.configure(builtins)
+Tier1.configure(builtins)
# Start the REPL now
loop do
# Debug only: print the sentences found
Lexer.sentences(tokens).each_with_index do |sentence, i|
puts "#{i}: #{sentence.map(&:to_s).join(' ')}"
- Lexer.clauses(sentence).each_with_index do |clause, j|
- puts " clause #{j}: #{clause.map(&:to_s).join(' ')}"
+
+ clauses = Lexer.clauses(sentence)
+ kinds = clauses.map { |clause| Classifier.classify(clause) }
+ if kinds.any?(&:nil?)
+ puts " couldn't tell what kind of request this is, please rephrase"
+ next
+ end
+
+ clauses.each_with_index do |clause, j|
+ puts " clause #{j} [#{kinds[j]}]: #{clause.map(&:to_s).join(' ')}"
+ if kinds[j] == :action
+ node = Tier1.lookup(clause)
+ puts " #{node ? "tier1 hit: #{node}" : 'tier1 miss'}"
+ end
end
puts " match: #{Lexer.match_text(sentence)}"
end
projects / proto / nlshell.git / commitdiff