Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions cmd/lk/simulate.go
Original file line number Diff line number Diff line change
Expand Up @@ -655,6 +655,16 @@ func simulationJobDashboardURL(projectID, runID, jobID string) string {
return fmt.Sprintf("%s?job=%s", base, jobID)
}

// simulationItemDashboardURL points at a single chat item within a job, the
// target of a <ref> citation in the run summary.
func simulationItemDashboardURL(projectID, runID, jobID, itemID string) string {
base := simulationJobDashboardURL(projectID, runID, jobID)
if base == "" || itemID == "" {
return base
}
return fmt.Sprintf("%s&item=%s", base, itemID)
}

func cancelSimulationRun(client *lksdk.AgentSimulationClient, runID string) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
Expand Down
127 changes: 127 additions & 0 deletions cmd/lk/simulate_refs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
// Copyright 2026 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"fmt"
"regexp"
"strings"
"unicode"
"unicode/utf8"

"github.com/charmbracelet/lipgloss"

"github.com/livekit/livekit-cli/v2/pkg/util"
)

// The summarization model cites the conversation turns behind a finding with
// <ref job="..." item="...">quoted text</ref>. Attribute order is not
// guaranteed, so the tag is matched loosely and the attributes are extracted
// separately.
var (
summaryRefPattern = regexp.MustCompile(`(?s)<ref\s([^>]*)>(.*?)</ref>`)
summaryRefAttrPattern = regexp.MustCompile(`([a-zA-Z]+)\s*=\s*"([^"]*)"`)
)

// summaryRefStyle marks cited text as a link, for terminals that render OSC 8
// hyperlinks no differently from surrounding text.
func summaryRefStyle() lipgloss.Style {
return lipgloss.NewStyle().Foreground(util.Brand()).Underline(true)
}

// A citation's number is an invitation to press that digit, so only as many
// citations as there are digits to press carry one.
const maxNumberedSummaryRefs = 9

// summaryRefTarget is the chat item a numbered citation points at.
type summaryRefTarget struct {
job string
item string
}

// summaryRefIndex numbers citations as they are rendered. The number a reader
// sees has to select the same citation when pressed, so one index is threaded
// through every block of a summary and numbering follows render order.
type summaryRefIndex struct {
targets []summaryRefTarget
}

// add records a citation and returns its 1-based number, or false once every
// digit is spoken for.
func (x *summaryRefIndex) add(attrs map[string]string) (int, bool) {
if len(x.targets) >= maxNumberedSummaryRefs {
return 0, false
}
x.targets = append(x.targets, summaryRefTarget{job: attrs["job"], item: attrs["item"]})
return len(x.targets), true
}

// linkSummaryRefs replaces each <ref> in summary prose with its quoted text,
// numbered so the digit keys can open the cited turn, and hyperlinked to the
// cited item when the run has a dashboard URL. A ref naming no job cites
// nothing openable and degrades to the quoted text alone.
func linkSummaryRefs(text, projectID, runID string, refs *summaryRefIndex) string {
return replaceSummaryRefs(text, func(attrs map[string]string, label string) string {
if attrs["job"] == "" {
return label
}
n, ok := refs.add(attrs)
if !ok {
return label
}
rendered := summaryRefStyle().Render(label)
if url := simulationItemDashboardURL(projectID, runID, attrs["job"], attrs["item"]); url != "" {
rendered = util.Hyperlink(url, rendered)
}
return rendered + dimStyle.Render(fmt.Sprintf(" [%d]", n))
})
}

// stripSummaryRefs reduces each <ref> in summary prose to its quoted text, for
// output that cannot carry a link (files, CI logs, redirected stdout).
func stripSummaryRefs(text string) string {
return replaceSummaryRefs(text, func(_ map[string]string, label string) string {
return label
})
}

// replaceSummaryRefs rewrites every <ref> in text through render. Citations are
// often appended to a sentence with no separator, either directly after the
// full stop or back-to-back with each other, so a ref that abuts the text
// before it gains a leading space; without one the quotes run together into a
// single unreadable phrase.
func replaceSummaryRefs(text string, render func(attrs map[string]string, label string) string) string {
var b strings.Builder
end := 0
for _, m := range summaryRefPattern.FindAllStringSubmatchIndex(text, -1) {
b.WriteString(text[end:m[0]])
if m[0] > 0 && !endsWithSpace(text[:m[0]]) {
b.WriteString(" ")
}
attrs := make(map[string]string)
for _, attr := range summaryRefAttrPattern.FindAllStringSubmatch(text[m[2]:m[3]], -1) {
attrs[strings.ToLower(attr[1])] = attr[2]
}
b.WriteString(render(attrs, text[m[4]:m[5]]))
end = m[1]
}
b.WriteString(text[end:])
return b.String()
}

func endsWithSpace(s string) bool {
r, _ := utf8.DecodeLastRuneInString(s)
return unicode.IsSpace(r)
}
91 changes: 91 additions & 0 deletions cmd/lk/simulate_refs_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
// Copyright 2026 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"strings"
"testing"

"github.com/stretchr/testify/require"
)

const refProse = `it kept asking for details in <ref job="SRJ_Bzb9ZaoJFJyp" item="item_dd0ee81187bd">"I've had a few, sure"</ref> and <ref item="item_13b90227fe38" job="SRJ_Bzb9ZaoJFJyp">"I'm totally fine to drive"</ref>.`

func TestStripSummaryRefs(t *testing.T) {
require.Equal(t,
`it kept asking for details in "I've had a few, sure" and "I'm totally fine to drive".`,
stripSummaryRefs(refProse),
)

// footnote-style citations: appended to a sentence and to each other
require.Equal(t,
"left out the passport requirement. accepted cards exchange rate posting",
stripSummaryRefs(`left out the passport requirement.<ref job="J" item="i1">accepted cards</ref><ref job="J" item="i2">exchange rate posting</ref>`),
)

// prose without refs, and a ref spanning a newline
require.Equal(t, "nothing to strip", stripSummaryRefs("nothing to strip"))
require.Equal(t, "a\nquote", stripSummaryRefs("<ref job=\"J\" item=\"I\">a\nquote</ref>"))
}

func TestLinkSummaryRefs(t *testing.T) {
var refs summaryRefIndex
linked := linkSummaryRefs(refProse, "proj", "run", &refs)

require.NotContains(t, linked, "<ref")
require.NotContains(t, linked, "</ref>")
// both refs link to their own item, whatever the attribute order
require.Contains(t, linked, "runs/run?job=SRJ_Bzb9ZaoJFJyp&item=item_dd0ee81187bd")
require.Contains(t, linked, "runs/run?job=SRJ_Bzb9ZaoJFJyp&item=item_13b90227fe38")
require.Contains(t, linked, `"I've had a few, sure"`)
require.Equal(t, 2, strings.Count(linked, "\x1b]8;;"+dashboardBaseURL()))

// the number a label carries selects the citation recorded under it
require.Contains(t, linked, "[1]")
require.Contains(t, linked, "[2]")
require.Equal(t, []summaryRefTarget{
{job: "SRJ_Bzb9ZaoJFJyp", item: "item_dd0ee81187bd"},
{job: "SRJ_Bzb9ZaoJFJyp", item: "item_13b90227fe38"},
}, refs.targets)
}

func TestLinkSummaryRefsWithoutTarget(t *testing.T) {
// a ref naming no job cites nothing that can be opened: quoted text alone
var unopenable summaryRefIndex
require.Equal(t, "quoted", linkSummaryRefs(`<ref item="item_x">quoted</ref>`, "proj", "run", &unopenable))
require.Empty(t, unopenable.targets)

// with no dashboard URL the job is still openable from the TUI, so the
// citation keeps its number and loses only the hyperlink
var local summaryRefIndex
linked := linkSummaryRefs(refProse, "", "", &local)
require.NotContains(t, linked, "\x1b]8;;")
require.Contains(t, linked, "[1]")
require.Len(t, local.targets, 2)
}

func TestSummaryRefIndexStopsAtTheLastDigit(t *testing.T) {
var b strings.Builder
for range maxNumberedSummaryRefs + 2 {
b.WriteString(`<ref job="J" item="i">q</ref>`)
}

var refs summaryRefIndex
linked := linkSummaryRefs(b.String(), "proj", "run", &refs)

require.Len(t, refs.targets, maxNumberedSummaryRefs)
require.Contains(t, linked, "[9]")
require.NotContains(t, linked, "[10]")
}
8 changes: 4 additions & 4 deletions cmd/lk/simulate_report.go
Original file line number Diff line number Diff line change
Expand Up @@ -255,15 +255,15 @@ func writeRunSummary(w io.Writer, run *livekit.SimulationRun, summary *livekit.S
if summary.GoingWell != "" {
fmt.Fprintln(w)
fmt.Fprintln(w, "Going well:")
for line := range strings.SplitSeq(summary.GoingWell, "\n") {
for line := range strings.SplitSeq(stripSummaryRefs(summary.GoingWell), "\n") {
fmt.Fprintf(w, " %s\n", line)
}
}

if summary.ToImprove != "" {
fmt.Fprintln(w)
fmt.Fprintln(w, "To improve:")
for line := range strings.SplitSeq(summary.ToImprove, "\n") {
for line := range strings.SplitSeq(stripSummaryRefs(summary.ToImprove), "\n") {
fmt.Fprintf(w, " %s\n", line)
}
}
Expand All @@ -272,9 +272,9 @@ func writeRunSummary(w io.Writer, run *livekit.SimulationRun, summary *livekit.S
fmt.Fprintln(w)
fmt.Fprintln(w, "Issues:")
for i, issue := range summary.Issues {
fmt.Fprintf(w, " %d. %s\n", i+1, issue.Description)
fmt.Fprintf(w, " %d. %s\n", i+1, stripSummaryRefs(issue.Description))
if issue.Suggestion != "" {
fmt.Fprintf(w, " Suggestion: %s\n", issue.Suggestion)
fmt.Fprintf(w, " Suggestion: %s\n", stripSummaryRefs(issue.Suggestion))
}
}
}
Expand Down
Loading
Loading