Add tracer for eval frameworks + debugging

 ## About

This change adds lightweight, built‑in tracing primitives so eval frameworks
(and debugging tools) can capture spans and events around model interactions.

The motivation is to provide a structured, zero‑dependency way to observe
chat/respond lifecycles without external instrumentation, while keeping
the API simple and opt‑in.

Author: 0x1eef

README.md

@@ -116,6 +116,7 @@ bot.messages.select(&:assistant?).each { |m| puts "[#{m.role}] #{m.content}" }
 - 📦  Zero runtime deps (stdlib-only)
 - 🧩  Pluggable JSON adapters (JSON, Oj, Yajl, etc)
 - ♻️  Optional persistent HTTP pool (net-http-persistent)
+- 🧭  Built-in tracing primitives for eval frameworks
 
 #### Chat, Agents
 - 🧠  Stateless + stateful chat (completions + responses)
@@ -300,6 +301,21 @@ end
 bot.chat(prompt)
 ```
 
+#### Tracer
+
+The following example demonstrates how to use the built-in tracing primitives
+to capture spans and events for a chat interaction. This can be useful for
+debugging, logging, or feeding into an eval framework:
+
+```ruby
+require "llm"
+
+llm = LLM.openai(key: ENV["OPENAI_SECRET"], trace: true)
+bot = LLM::Bot.new(llm)
+bot.chat "Hello world"
+pp bot.tracer.to_h
+```
+
 ### Schema
 
 All LLM providers except Anthropic and DeepSeek allow a client to describe

lib/llm.rb

@@ -2,6 +2,7 @@
 
 module LLM
   require "stringio"
+  require_relative "llm/tracer"
   require_relative "llm/json_adapter"
   require_relative "llm/error"
   require_relative "llm/contract"

lib/llm/bot.rb

@@ -41,6 +41,7 @@ class Bot
     # @option params [Array<LLM::Function>, nil] :tools Defaults to nil
     def initialize(provider, params = {})
       @provider = provider
+      @tracer = provider.tracer
       @params = {model: provider.default_model, schema: nil}.compact.merge!(params)
       @messages = LLM::Buffer.new(provider)
     end
@@ -58,13 +59,20 @@ def initialize(provider, params = {})
     #   response = bot.chat("Hello, what is your name?")
     #   puts response.choices[0].content
     def chat(prompt, params = {})
+      span = start_span("llm.chat", provider: @provider.class.name, model: @params[:model])
       prompt, params, messages = fetch(prompt, params)
       params = params.merge(messages: [*@messages.to_a, *messages])
       params = @params.merge(params)
+      event("llm.chat.input", {
+        prompt_size: prompt.to_s.bytesize,
+        message_count: params[:messages]&.length,
+        stream: !!params[:stream]
+      })
       res = @provider.complete(prompt, params)
       @messages.concat [LLM::Message.new(params[:role] || :user, prompt)]
       @messages.concat messages
       @messages.concat [res.choices[-1]]
+      end_span(span)
       res
     end
 
@@ -82,14 +90,24 @@ def chat(prompt, params = {})
     #   res = bot.respond("What is the capital of France?")
     #   puts res.output_text
     def respond(prompt, params = {})
+      span = start_span("llm.respond", {
+        provider: @provider.class.name,
+        model: @params[:model]
+      })
       prompt, params, messages = fetch(prompt, params)
       res_id = @messages.find(&:assistant?)&.response&.response_id
       params = params.merge(previous_response_id: res_id, input: messages).compact
       params = @params.merge(params)
+      event("llm.respond.input", {
+        prompt_size: prompt.to_s.bytesize,
+        message_count: messages.length,
+        stream: !!params[:stream]
+      })
       res = @provider.responses.create(prompt, params)
       @messages.concat [LLM::Message.new(params[:role] || :user, prompt)]
       @messages.concat messages
       @messages.concat [res.choices[-1]]
+      end_span(span)
       res
     end
 
@@ -101,6 +119,13 @@ def inspect
       "@messages=#{@messages.inspect}>"
     end
 
+    ##
+    # Returns the tracer for this bot
+    # @return [LLM::Tracer::Tracer, LLM::Tracer::Null]
+    def tracer
+      @tracer
+    end
+
     ##
     # Returns an array of functions that can be called
     # @return [Array<LLM::Function>]
@@ -173,5 +198,17 @@ def fetch(prompt, params)
       params.merge!(role: prompt.role)
       [prompt.content, params, messages]
     end
+
+    def start_span(...)
+      @tracer.start_span(...)
+    end
+
+    def end_span(...)
+      @tracer.end_span(...)
+    end
+
+    def event(...)
+      @tracer.event(...)
+    end
   end
 end

lib/llm/provider.rb

@@ -30,16 +30,23 @@ def self.clients = @@clients
   # @param [Boolean] persistent
   #  Whether to use a persistent connection.
   #  Requires the net-http-persistent gem.
-  def initialize(key:, host:, port: 443, timeout: 60, ssl: true, persistent: false)
+  def initialize(key:, host:, port: 443, timeout: 60, ssl: true, persistent: false, trace: false)
     @key = key
     @host = host
     @port = port
     @timeout = timeout
     @ssl = ssl
     @client = persistent ? persistent_client : transient_client
+    @tracer = trace ? LLM::Tracer::Tracer.new : LLM::Tracer::Null.new
     @base_uri = URI("#{ssl ? "https" : "http"}://#{host}:#{port}/")
   end
 
+  ##
+  # @return [LLM::Tracer::Tracer, LLM::Tracer::Null]
+  def tracer
+    @tracer
+  end
+
   ##
   # Returns an inspection of the provider object
   # @return [String]

lib/llm/tracer.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+##
+# The {LLM::Tracer LLM::Tracer} provides a tracer for debugging,
+# logging, or feeding into an eval framework.
+module LLM::Tracer
+  require_relative "tracer/span"
+  require_relative "tracer/event"
+  require_relative "tracer/tracer"
+  require_relative "tracer/null"
+end

lib/llm/tracer/event.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "time"
+
+module LLM::Tracer
+  class Event
+    ##
+    # A point-in-time event.
+    # @see LLM::Tracer::Tracer
+    ##
+    # @return [String]
+    attr_reader :name, :time, :attrs, :span_id
+
+    ##
+    # @param [String, Symbol] name
+    # @param [Hash] attrs
+    # @param [String, nil] span_id
+    def initialize(name, attrs = {}, span_id: nil)
+      @name = name.to_s
+      @time = Time.now.utc
+      @attrs = attrs.dup
+      @span_id = span_id
+    end
+
+    ##
+    # @return [Hash]
+    def to_h
+      {
+        name: @name,
+        time: @time.iso8601(6),
+        span_id: @span_id,
+        attrs: @attrs.dup
+      }
+    end
+  end
+end

lib/llm/tracer/null.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module LLM::Tracer
+  class Null
+    ##
+    # A no-op tracer that discards spans and events.
+    # @see LLM::Tracer::Tracer
+    ##
+    # @return [nil]
+    def start_span(*) = nil
+    ##
+    # @return [nil]
+    def end_span(*) = nil
+    ##
+    # @return [nil]
+    def event(*) = nil
+    ##
+    # @return [nil]
+    def error(*) = nil
+
+    ##
+    # @yield [span] Yields nil
+    # @return [void]
+    def with_span(*)
+      yield(nil)
+    end
+  end
+end

lib/llm/tracer/span.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "time"
+
+module LLM::Tracer
+  ##
+  # Minimal tracing primitives for eval frameworks.
+  #
+  # @example Basic usage
+  #   tracer = LLM::Tracer::Tracer.new
+  #   # ... perform requests
+  #   pp tracer.to_h
+  ##
+  # A timed span with attributes.
+  # @see LLM::Tracer::Tracer
+  class Span
+    ##
+    # @return [String]
+    attr_reader :id, :name, :parent_id, :started_at, :ended_at, :attrs
+
+    ##
+    # @param [String, Symbol] name
+    # @param [Hash] attrs
+    # @param [String, nil] parent_id
+    def initialize(name, attrs = {}, parent_id: nil)
+      @id = SecureRandom.hex(8)
+      @name = name.to_s
+      @parent_id = parent_id
+      @attrs = attrs.dup
+      @started_at = Time.now.utc
+      @start_mono = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      @ended_at = nil
+    end
+
+    ##
+    # @param [Hash] attrs
+    # @return [LLM::Tracer::Span]
+    def finish(attrs = {})
+      return self if @ended_at
+      @ended_at = Time.now.utc
+      duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - @start_mono) * 1000.0).round(3)
+      @attrs.merge!(attrs) if attrs && !attrs.empty?
+      @attrs[:duration_ms] ||= duration_ms
+      self
+    end
+
+    ##
+    # @return [Hash]
+    def to_h
+      {
+        id: @id,
+        name: @name,
+        parent_id: @parent_id,
+        started_at: @started_at.iso8601(6),
+        ended_at: @ended_at&.iso8601(6),
+        attrs: @attrs.dup
+      }
+    end
+  end
+end

lib/llm/tracer/tracer.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module LLM::Tracer
+  ##
+  # A simple in-memory tracer for prototyping.
+  #
+  # @example
+  #   tracer = LLM::Tracer::Tracer.new
+  #   # ... perform requests
+  #   puts tracer.to_h
+  class Tracer
+    ##
+    # @return [Array<LLM::Tracer::Span>]
+    attr_reader :spans, :events
+
+    ##
+    # @return [LLM::Tracer::Tracer]
+    def initialize
+      @spans = []
+      @events = []
+    end
+
+    ##
+    # Start a span and set it as current in this thread.
+    # @param [String, Symbol] name
+    # @param [Hash] attrs
+    # @return [LLM::Tracer::Span]
+    def start_span(name, attrs = {})
+      span = Span.new(name, attrs, parent_id: current_span&.id)
+      @spans << span
+      stack << span
+      span
+    end
+
+    ##
+    # End a span.
+    # @param [LLM::Tracer::Span, nil] span
+    # @param [Hash] attrs
+    # @return [LLM::Tracer::Span, nil]
+    def end_span(span = nil, attrs = {})
+      span ||= current_span
+      return unless span
+      span.finish(attrs)
+      stack.pop if stack.last == span
+      span
+    end
+
+    ##
+    # Record a point-in-time event.
+    # @param [String, Symbol] name
+    # @param [Hash] attrs
+    # @return [LLM::Tracer::Event]
+    def event(name, attrs = {})
+      @events << Event.new(name, attrs, span_id: current_span&.id)
+    end
+
+    ##
+    # Record an error event.
+    # @param [Exception] err
+    # @param [Hash] attrs
+    # @return [LLM::Tracer::Event]
+    def error(err, attrs = {})
+      event("error", {
+        error: {class: err.class.name, message: err.message}
+      }.merge(attrs))
+    end
+
+    ##
+    # Start a span, yield it, then end the span.
+    # @param [String, Symbol] name
+    # @param [Hash] attrs
+    # @yield [span] The new span
+    # @return [void]
+    def with_span(name, attrs = {})
+      span = start_span(name, attrs)
+      yield(span)
+    ensure
+      end_span(span)
+    end
+
+    ##
+    # Serialize spans and events.
+    # @return [Hash]
+    def to_h
+      {spans: @spans.map(&:to_h), events: @events.map(&:to_h)}
+    end
+
+    private
+
+    def stack
+      Thread.current[stack_key] ||= []
+    end
+
+    def current_span
+      stack.last
+    end
+
+    def stack_key
+      @stack_key ||= :"llm.tracer.stack.#{object_id}"
+    end
+  end
+end