feat(dialtesting): extend Lightpanda browser checks

dialtesting/README.md

@@ -28,6 +28,18 @@ Browser tasks require these binaries on the dial node:
 Chrome can be configured by `Task.SetOption()["chrome_path"]`. Lightpanda can be
 configured by `Task.SetOption()["lightpanda_path"]`.
 
+Lightpanda also reads these node-level task options:
+
+- `browser_ca_cert_file`: absolute path to a custom CA certificate file.
+- `browser_ca_cert_dir`: absolute path to a custom CA certificate directory.
+- `browser_proxy_url`: default HTTP proxy URL.
+- `browser_block_private_network`: set to `true` to block private-network requests.
+- `browser_block_cidrs`: comma-separated CIDRs to block instead of all private ranges.
+
+When a custom CA is configured, the runner passes a detected system CA directory
+and the custom CA path as separate Lightpanda arguments so both trust sources are
+loaded without moving certificate files.
+
 ## Task Fields
 
 Browser tasks use the normal common task fields, including:
@@ -189,13 +201,18 @@ Supported step actions:
 | --- | --- | --- |
 | `goto` | `url` or top-level `target` | Navigate to a page. |
 | `wait_for_selector` | `selector` | Wait until the selector appears. |
+| `wait_for_url` | `contains`, `equals`, or `text` | Wait until the current URL matches. |
 | `click` | `selector` | Click the selector. |
 | `fill` | `selector`, plus `value` or `value_from` | Fill an input. |
 | `assert_title` | `contains`, `equals`, or `text` | Assert the page title. |
 | `assert_url` | `contains`, `equals`, or `text` | Assert the current URL. |
 | `assert_text` | `selector`, plus `contains`, `equals`, or `text` | Assert element text. |
 | `eval` | `value` or `text` | Evaluate JavaScript in the page. |
 
+`wait_for_url`, `assert_title`, `assert_url`, and `assert_text` poll until they
+match or time out. A step `timeout_ms` takes precedence; otherwise the script
+timeout applies.
+
 Recorder tools should generate this schema directly. The Chrome extension code
 does not need to live in this repository; only the generated `browser_config`
 must match this contract.

dialtesting/browser.go

@@ -40,6 +40,11 @@ const (
 
 	optionLightpandaPath      = "lightpanda_path"
 	optionLightpandaPathCamel = "lightpandaPath"
+	optionBrowserCACertFile   = "browser_ca_cert_file"
+	optionBrowserCACertDir    = "browser_ca_cert_dir"
+	optionBrowserProxyURL     = "browser_proxy_url"
+	optionBlockPrivateNetwork = "browser_block_private_network"
+	optionBrowserBlockCIDRs   = "browser_block_cidrs"
 	optionChromePath          = "chrome_path"
 	optionChromePathCamel     = "chromePath"
 
@@ -346,6 +351,11 @@ func (t *BrowserTask) runBrowserDialEmbedded(path string, viewport BrowserViewpo
 		Tags:                t.Tags,
 		EngineName:          engineName,
 		LightpandaPath:      t.lightpandaPath(),
+		CACertFile:          t.browserCACertFile(),
+		CACertDir:           t.browserCACertDir(),
+		DefaultProxyURL:     t.browserDefaultProxyURL(),
+		BlockPrivateNetwork: t.browserBlockPrivateNetwork(),
+		PrivateNetworkCIDRs: t.browserPrivateNetworkCIDRs(),
 		ChromePath:          t.chromePath(),
 		StartupTimeout:      5 * time.Second,
 		ScreenshotOnFailure: t.AdvanceOptions != nil && t.AdvanceOptions.ScreenshotOnFailure,
@@ -505,6 +515,37 @@ func (t *BrowserTask) lightpandaPath() string {
 	return ""
 }
 
+func (t *BrowserTask) browserCACertFile() string {
+	return strings.TrimSpace(t.GetOption()[optionBrowserCACertFile])
+}
+
+func (t *BrowserTask) browserCACertDir() string {
+	return strings.TrimSpace(t.GetOption()[optionBrowserCACertDir])
+}
+
+func (t *BrowserTask) browserDefaultProxyURL() string {
+	return strings.TrimSpace(t.GetOption()[optionBrowserProxyURL])
+}
+
+func (t *BrowserTask) browserBlockPrivateNetwork() bool {
+	return strings.EqualFold(strings.TrimSpace(t.GetOption()[optionBlockPrivateNetwork]), "true")
+}
+
+func (t *BrowserTask) browserPrivateNetworkCIDRs() []string {
+	value := strings.TrimSpace(t.GetOption()[optionBrowserBlockCIDRs])
+	if value == "" {
+		return nil
+	}
+	parts := strings.Split(value, ",")
+	cidrs := make([]string, 0, len(parts))
+	for _, part := range parts {
+		if cidr := strings.TrimSpace(part); cidr != "" {
+			cidrs = append(cidrs, cidr)
+		}
+	}
+	return cidrs
+}
+
 func (t *BrowserTask) chromePath() string {
 	if value := t.GetOption()[optionChromePath]; value != "" {
 		return value

dialtesting/browser_test.go

@@ -58,7 +58,12 @@ func TestBrowserTaskRunSetsLightpandaPath(t *testing.T) {
 	task, err := NewTask("", browserTask)
 	require.NoError(t, err)
 	task.SetOption(map[string]string{
-		optionLightpandaPath: "/opt/datakit/lightpanda",
+		optionLightpandaPath:      "/opt/datakit/lightpanda",
+		optionBrowserCACertFile:   "/etc/datakit/certs/internal-ca.pem",
+		optionBrowserCACertDir:    "/etc/datakit/certs/roots",
+		optionBrowserProxyURL:     "http://proxy.example.com:8080",
+		optionBlockPrivateNetwork: "true",
+		optionBrowserBlockCIDRs:   "10.0.0.0/8, 192.168.0.0/16",
 	})
 
 	var gotOptions browserrunner.EngineOptions
@@ -70,6 +75,11 @@ func TestBrowserTaskRunSetsLightpandaPath(t *testing.T) {
 	assert.Equal(t, "OK", tags["status"])
 	assert.Equal(t, int64(1), fields["success"])
 	assert.Equal(t, "/opt/datakit/lightpanda", gotOptions.LightpandaPath)
+	assert.Equal(t, "/etc/datakit/certs/internal-ca.pem", gotOptions.CACertFile)
+	assert.Equal(t, "/etc/datakit/certs/roots", gotOptions.CACertDir)
+	assert.Equal(t, "http://proxy.example.com:8080", gotOptions.ProxyURL)
+	assert.True(t, gotOptions.BlockPrivateNetwork)
+	assert.Equal(t, []string{"10.0.0.0/8", "192.168.0.0/16"}, gotOptions.PrivateNetworkCIDRs)
 }
 
 func TestBrowserTaskRunSetsChromePath(t *testing.T) {
@@ -691,6 +701,22 @@ func TestBrowserTaskLightpandaPathOption(t *testing.T) {
 	assert.Empty(t, task.lightpandaPath())
 }
 
+func TestBrowserTaskLightpandaRuntimeOptions(t *testing.T) {
+	task := &BrowserTask{Task: &Task{}}
+	task.SetOption(map[string]string{
+		optionBrowserCACertFile:   " /etc/datakit/certs/internal-ca.pem ",
+		optionBrowserCACertDir:    " /etc/datakit/certs/roots ",
+		optionBrowserProxyURL:     " http://proxy.example.com:8080 ",
+		optionBlockPrivateNetwork: "TRUE",
+		optionBrowserBlockCIDRs:   "10.0.0.0/8, 192.168.0.0/16",
+	})
+	assert.Equal(t, "/etc/datakit/certs/internal-ca.pem", task.browserCACertFile())
+	assert.Equal(t, "/etc/datakit/certs/roots", task.browserCACertDir())
+	assert.Equal(t, "http://proxy.example.com:8080", task.browserDefaultProxyURL())
+	assert.True(t, task.browserBlockPrivateNetwork())
+	assert.Equal(t, []string{"10.0.0.0/8", "192.168.0.0/16"}, task.browserPrivateNetworkCIDRs())
+}
+
 func TestBrowserTaskChromePathOption(t *testing.T) {
 	task := &BrowserTask{Task: &Task{}}
 	task.SetOption(map[string]string{optionChromePath: "/opt/chrome"})

dialtesting/browserdial/lightpanda/lightpanda.go

@@ -28,6 +28,11 @@ import (
 
 const defaultHost = "127.0.0.1"
 
+var systemCACertificateDirectories = []string{
+	"/etc/ssl/certs",
+	"/etc/pki/tls/certs",
+}
+
 type Engine struct {
 	ctx       context.Context
 	cancel    context.CancelFunc
@@ -48,11 +53,15 @@ type requestInfo struct {
 }
 
 func NewEngine(ctx context.Context, options runner.EngineOptions) (runner.Engine, error) {
+	arguments, err := lightpandaArguments(options)
+	if err != nil {
+		return nil, err
+	}
 	executable, err := resolveExecutable(options.LightpandaPath)
 	if err != nil {
 		return nil, err
 	}
-	activeSession, err := start(ctx, executable, options.StartupTimeout)
+	activeSession, err := start(ctx, executable, options.StartupTimeout, arguments)
 	if err != nil {
 		return nil, err
 	}
@@ -89,6 +98,65 @@ func NewEngine(ctx context.Context, options runner.EngineOptions) (runner.Engine
 	return engine, nil
 }
 
+func lightpandaArguments(options runner.EngineOptions) ([]string, error) {
+	return lightpandaArgumentsWithSystemCADirectories(options, systemCACertificateDirectories)
+}
+
+func lightpandaArgumentsWithSystemCADirectories(options runner.EngineOptions, systemDirectories []string) ([]string, error) {
+	arguments := []string{}
+	caCertFile := strings.TrimSpace(options.CACertFile)
+	caCertDir := strings.TrimSpace(options.CACertDir)
+	if caCertFile != "" || caCertDir != "" {
+		for _, directory := range systemDirectories {
+			directory = strings.TrimSpace(directory)
+			if directory == "" || directory == caCertDir {
+				continue
+			}
+			info, err := os.Stat(directory)
+			if err == nil && info.IsDir() {
+				arguments = append(arguments, "--ca-path", directory)
+				break
+			}
+		}
+	}
+	if caCertFile != "" {
+		if !filepath.IsAbs(caCertFile) {
+			return nil, fmt.Errorf("browser CA certificate file path must be absolute: %s", caCertFile)
+		}
+		arguments = append(arguments, "--ca-cert", caCertFile)
+	}
+	if caCertDir != "" {
+		if !filepath.IsAbs(caCertDir) {
+			return nil, fmt.Errorf("browser CA certificate directory path must be absolute: %s", caCertDir)
+		}
+		arguments = append(arguments, "--ca-path", caCertDir)
+	}
+	if proxyURL := strings.TrimSpace(options.ProxyURL); proxyURL != "" {
+		parsed, err := url.Parse(proxyURL)
+		if err != nil || parsed.Host == "" || !strings.EqualFold(parsed.Scheme, "http") {
+			return nil, fmt.Errorf("lightpanda proxy URL must use the http scheme and include a host")
+		}
+		arguments = append(arguments, "--http-proxy", proxyURL)
+	}
+	cidrs := make([]string, 0, len(options.PrivateNetworkCIDRs))
+	for _, value := range options.PrivateNetworkCIDRs {
+		cidr := strings.TrimSpace(value)
+		if cidr == "" {
+			continue
+		}
+		if _, _, err := net.ParseCIDR(cidr); err != nil {
+			return nil, fmt.Errorf("invalid private network CIDR %q: %w", cidr, err)
+		}
+		cidrs = append(cidrs, cidr)
+	}
+	if len(cidrs) > 0 {
+		arguments = append(arguments, "--block-cidrs", strings.Join(cidrs, ","))
+	} else if options.BlockPrivateNetwork {
+		arguments = append(arguments, "--block-private-networks")
+	}
+	return arguments, nil
+}
+
 func (e *Engine) ConfigureBrowser(ctx context.Context, config runner.BrowserConfig) error {
 	actionCtx, cancel := e.actionContext(ctx)
 	defer cancel()
@@ -362,7 +430,7 @@ type session struct {
 	logs     *limitedBuffer
 }
 
-func start(parent context.Context, executable string, startupTimeout time.Duration) (*session, error) {
+func start(parent context.Context, executable string, startupTimeout time.Duration, extraArguments []string) (*session, error) {
 	if startupTimeout <= 0 {
 		startupTimeout = 5 * time.Second
 	}
@@ -373,7 +441,9 @@ func start(parent context.Context, executable string, startupTimeout time.Durati
 	endpoint := fmt.Sprintf("http://%s:%d", defaultHost, port)
 	procCtx, cancel := context.WithCancel(parent)
 	logs := &limitedBuffer{limit: 8_000}
-	cmd := exec.CommandContext(procCtx, executable, "serve", "--host", defaultHost, "--port", strconv.Itoa(port))
+	arguments := []string{"serve", "--host", defaultHost, "--port", strconv.Itoa(port)}
+	arguments = append(arguments, extraArguments...)
+	cmd := exec.CommandContext(procCtx, executable, arguments...)
 	cmd.Stdout = logs
 	cmd.Stderr = logs
 	if err := cmd.Start(); err != nil {

dialtesting/browserdial/lightpanda/lightpanda_args_test.go

@@ -0,0 +1,87 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the MIT License.
+// This product includes software developed at Guance Cloud (https://www.guance.com/).
+// Copyright 2021-present Guance, Inc.
+
+package lightpanda
+
+import (
+	"path/filepath"
+	"reflect"
+	"testing"
+
+	"github.com/GuanceCloud/cliutils/dialtesting/browserdial/runner"
+)
+
+func TestLightpandaArguments(t *testing.T) {
+	root := t.TempDir()
+	caFile := filepath.Join(root, "internal-ca.pem")
+	caDir := filepath.Join(root, "roots")
+	arguments, err := lightpandaArgumentsWithSystemCADirectories(runner.EngineOptions{
+		CACertFile: caFile,
+		CACertDir:  caDir,
+		ProxyURL:   "http://user:password@proxy.example.com:8080",
+	}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []string{
+		"--ca-cert", caFile,
+		"--ca-path", caDir,
+		"--http-proxy", "http://user:password@proxy.example.com:8080",
+	}
+	if !reflect.DeepEqual(arguments, want) {
+		t.Fatalf("arguments = %#v, want %#v", arguments, want)
+	}
+
+	arguments, err = lightpandaArguments(runner.EngineOptions{BlockPrivateNetwork: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(arguments, []string{"--block-private-networks"}) {
+		t.Fatalf("private network block argument is missing: %#v", arguments)
+	}
+	arguments, err = lightpandaArguments(runner.EngineOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(arguments) != 0 {
+		t.Fatalf("zero-value options should preserve existing behavior: %#v", arguments)
+	}
+}
+
+func TestLightpandaArgumentsPreserveSystemCAAndCustomCIDRs(t *testing.T) {
+	systemDirectory := t.TempDir()
+	customDirectory := t.TempDir()
+	arguments, err := lightpandaArgumentsWithSystemCADirectories(runner.EngineOptions{
+		CACertDir:           customDirectory,
+		PrivateNetworkCIDRs: []string{"10.0.0.0/8", "192.168.0.0/16"},
+	}, []string{systemDirectory})
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []string{
+		"--ca-path", systemDirectory,
+		"--ca-path", customDirectory,
+		"--block-cidrs", "10.0.0.0/8,192.168.0.0/16",
+	}
+	if !reflect.DeepEqual(arguments, want) {
+		t.Fatalf("arguments = %#v, want %#v", arguments, want)
+	}
+}
+
+func TestLightpandaArgumentsRejectInvalidValues(t *testing.T) {
+	tests := []runner.EngineOptions{
+		{CACertFile: "internal-ca.pem"},
+		{CACertDir: "certs"},
+		{ProxyURL: "https://proxy.example.com:8443"},
+		{ProxyURL: "socks5://proxy.example.com:1080"},
+		{ProxyURL: "http:///missing-host"},
+		{PrivateNetworkCIDRs: []string{"not-a-cidr"}},
+	}
+	for _, options := range tests {
+		if _, err := lightpandaArgumentsWithSystemCADirectories(options, nil); err == nil {
+			t.Fatalf("expected invalid options to fail: %#v", options)
+		}
+	}
+}