Add collection support

docs/collections.md

@@ -0,0 +1,41 @@
+# Collections
+
+Requires the engine changes in [dagger/dagger#14221](https://github.com/dagger/dagger/pull/14221).
+
+A collection has stored keys and a function that returns one item for a key.
+The engine supplies `keys`, `get`, `list`, and `subset`. Other exposed
+functions appear under `batch`.
+
+```typescript
+import { collection, delta, func, get, keys, object, CollectionDelta } from "@dagger.io/dagger"
+
+@object()
+class Item {
+  @func() name: string = ""
+}
+
+@object()
+@collection()
+class Items {
+  @func()
+  @keys()
+  paths: string[] = []
+
+  @func()
+  @delta()
+  selection?: CollectionDelta
+
+  @func()
+  @get()
+  item(key: string): Item {
+    return Object.assign(new Item(), { name: key })
+  }
+}
+```
+
+The generated TypeScript and Dang entrypoints both carry collection metadata.
+Self-call clients use the projected collection schema.
+
+The engine fills the optional delta field before a module call. It compares the
+current keys with the original keys. Copies preserve the internal base state.
+A new object starts a new base. The internal state is not an exposed field.

helpers/codegen/generator/typescript/templates/collections_test.go

@@ -0,0 +1,24 @@
+package templates
+
+import (
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestCollectionDangMetadata(t *testing.T) {
+	obj := &TypedefObject{Name: "Items", IsCollection: true,
+		Properties: map[string]*TypedefProperty{
+			"names":     {Name: "names", Alias: "paths", IsExposed: true, IsCollectionKeys: true, Type: &TypedefType{Kind: KindList, TypeDef: &TypedefType{Kind: KindString}}},
+			"selection": {Name: "selection", IsExposed: true, IsCollectionDelta: true, Type: &TypedefType{Kind: KindObject, Name: "CollectionDelta"}},
+		},
+		Methods: map[string]*TypedefFunction{
+			"item": {Name: "item", Alias: "lookup", IsCollectionGet: true, ReturnType: &TypedefType{Kind: KindObject, Name: "Item"}},
+		},
+	}
+	c := &dangFuncCtx{module: &TypedefModule{Name: "Main"}}
+	got := c.dangObjectEntry(obj)
+	require.Contains(t, got, ".withCollection")
+	require.Contains(t, got, `.withCollectionKeys("paths")`)
+	require.Contains(t, got, `.withCollectionDelta("selection")`)
+	require.Contains(t, got, `.withCollectionGet("lookup")`)
+}

helpers/codegen/generator/typescript/templates/entrypoint_dang.go

@@ -219,13 +219,30 @@ func (c *dangFuncCtx) dangTypeEntries() []string {
 
 func (c *dangFuncCtx) dangObjectEntry(obj *TypedefObject) string {
 	calls := []string{c.dangObjectDef(obj)}
+	if obj.IsCollection {
+		calls = append(calls, "withCollection")
+	}
 
 	for _, name := range sortedFunctionKeys(obj.Methods) {
-		calls = append(calls, c.dangWrappedCall("withFunction", c.dangFunctionExpr(obj.Methods[name])))
+		method := obj.Methods[name]
+		calls = append(calls, c.dangWrappedCall("withFunction", c.dangFunctionExpr(method)))
+		if method.IsCollectionGet {
+			member := method.Name
+			if method.Alias != "" {
+				member = method.Alias
+			}
+			calls = append(calls, fmt.Sprintf("withCollectionGet(%s)", dangString(member)))
+		}
 	}
 	for _, name := range sortedPropertyKeys(obj.Properties) {
 		if prop := obj.Properties[name]; prop.IsExposed {
 			calls = append(calls, c.dangFieldCall(prop))
+			if prop.IsCollectionKeys {
+				calls = append(calls, fmt.Sprintf("withCollectionKeys(%s)", dangString(propFieldName(prop))))
+			}
+			if prop.IsCollectionDelta {
+				calls = append(calls, fmt.Sprintf("withCollectionDelta(%s)", dangString(propFieldName(prop))))
+			}
 		}
 	}
 	// The engine identifies an entrypoint module's main object by its

helpers/codegen/generator/typescript/templates/entrypoint_functions.go

@@ -3,7 +3,9 @@ package templates
 import (
 	"encoding/json"
 	"fmt"
+	"maps"
 	"path/filepath"
+	"slices"
 	"sort"
 	"strings"
 	"text/template"
@@ -292,7 +294,21 @@ func (c *entrypointFuncCtx) renderObjectDef(obj *TypedefObject) string {
 	if sm := sourceMapExpr(obj.Location); sm != "" {
 		opts["sourceMap"] = sm
 	}
-	return fmt.Sprintf("dag.typeDef().withObject(%s%s)", jsString(obj.Name), optsLit(opts))
+	result := fmt.Sprintf("dag.typeDef().withObject(%s%s)", jsString(obj.Name), optsLit(opts))
+	if obj.IsCollection {
+		result += ".withCollection()"
+	}
+	for _, name := range slices.Sorted(maps.Keys(obj.Methods)) {
+		method := obj.Methods[name]
+		if method.IsCollectionGet {
+			member := method.Name
+			if method.Alias != "" {
+				member = method.Alias
+			}
+			result += fmt.Sprintf(".withCollectionGet(%s)", jsString(member))
+		}
+	}
+	return result
 }
 
 func (c *entrypointFuncCtx) renderFieldCall(prop *TypedefProperty) string {
@@ -306,7 +322,14 @@ func (c *entrypointFuncCtx) renderFieldCall(prop *TypedefProperty) string {
 	if sm := sourceMapExpr(prop.Location); sm != "" {
 		opts["sourceMap"] = sm
 	}
-	return fmt.Sprintf(".withField(%s, %s%s)", jsString(propFieldName(prop)), c.renderTypeDef(prop.Type), optsLit(opts))
+	result := fmt.Sprintf(".withField(%s, %s%s)", jsString(propFieldName(prop)), c.renderTypeDef(prop.Type), optsLit(opts))
+	if prop.IsCollectionKeys {
+		result += fmt.Sprintf(".withCollectionKeys(%s)", jsString(propFieldName(prop)))
+	}
+	if prop.IsCollectionDelta {
+		result += fmt.Sprintf(".withCollectionDelta(%s)", jsString(propFieldName(prop)))
+	}
+	return result
 }
 
 func (c *entrypointFuncCtx) renderEnumDef(e *TypedefEnum) string {

helpers/codegen/generator/typescript/templates/entrypoint_typedef.go

@@ -17,6 +17,7 @@ type TypedefModule struct {
 }
 
 type TypedefObject struct {
+	IsCollection    bool                        `json:"isCollection"`
 	Name            string                      `json:"name"`
 	Kind            string                      `json:"kind"` // "class" | "object"
 	IsExported      bool                        `json:"isExported"`
@@ -35,18 +36,19 @@ type TypedefConstructor struct {
 }
 
 type TypedefFunction struct {
-	Name        string             `json:"name"`
-	Alias       string             `json:"alias,omitempty"`
-	Cache       string             `json:"cache,omitempty"`
-	Description string             `json:"description"`
-	Deprecated  string             `json:"deprecated,omitempty"`
-	IsCheck     bool               `json:"isCheck"`
-	IsGenerator bool               `json:"isGenerator"`
-	IsUp        bool               `json:"isUp"`
-	IsAgent     bool               `json:"isAgent"`
-	Location    *TypedefLocation   `json:"location,omitempty"`
-	ReturnType  *TypedefType       `json:"returnType,omitempty"`
-	Arguments   []*TypedefArgument `json:"arguments"`
+	IsCollectionGet bool               `json:"isCollectionGet"`
+	Name            string             `json:"name"`
+	Alias           string             `json:"alias,omitempty"`
+	Cache           string             `json:"cache,omitempty"`
+	Description     string             `json:"description"`
+	Deprecated      string             `json:"deprecated,omitempty"`
+	IsCheck         bool               `json:"isCheck"`
+	IsGenerator     bool               `json:"isGenerator"`
+	IsUp            bool               `json:"isUp"`
+	IsAgent         bool               `json:"isAgent"`
+	Location        *TypedefLocation   `json:"location,omitempty"`
+	ReturnType      *TypedefType       `json:"returnType,omitempty"`
+	Arguments       []*TypedefArgument `json:"arguments"`
 }
 
 type TypedefArgument struct {
@@ -65,13 +67,15 @@ type TypedefArgument struct {
 }
 
 type TypedefProperty struct {
-	Name        string           `json:"name"`
-	Alias       string           `json:"alias,omitempty"`
-	Description string           `json:"description,omitempty"`
-	Deprecated  string           `json:"deprecated,omitempty"`
-	IsExposed   bool             `json:"isExposed"`
-	Type        *TypedefType     `json:"type,omitempty"`
-	Location    *TypedefLocation `json:"location,omitempty"`
+	IsCollectionKeys  bool             `json:"isCollectionKeys"`
+	IsCollectionDelta bool             `json:"isCollectionDelta"`
+	Name              string           `json:"name"`
+	Alias             string           `json:"alias,omitempty"`
+	Description       string           `json:"description,omitempty"`
+	Deprecated        string           `json:"deprecated,omitempty"`
+	IsExposed         bool             `json:"isExposed"`
+	Type              *TypedefType     `json:"type,omitempty"`
+	Location          *TypedefLocation `json:"location,omitempty"`
 }
 
 type TypedefEnum struct {

library/src/module/decorators.ts

@@ -10,6 +10,15 @@ import { registry } from "./registry.js"
  */
 export const object = registry.object
 
+/** Declare a collection object. */
+export const collection = registry.collection
+/** Select the stored keys field. */
+export const keys = registry.keys
+/** Select the item lookup method. */
+export const get = registry.get
+/** Select the field that receives the collection delta. */
+export const delta = registry.delta
+
 /**
  * The definition of @func decorator that should be on top of any
  * class' method that must be exposed to the Dagger API.

library/src/module/entrypoint/load.ts

@@ -20,6 +20,9 @@ import {
 import { TypeDef } from "../introspector/typedef.js"
 import { InvokeCtx } from "./context.js"
 
+// Opaque engine state, excluded from the module schema.
+const collectionBaseField = "__daggerCollectionBase"
+
 /**
  * Import all given typescript files so that trigger their decorators
  * and register their class and functions inside the Registry.
@@ -125,6 +128,10 @@ export async function loadParentState(
   const parentState: Args = {}
 
   for (const [key, value] of Object.entries(ctx.parentArgs)) {
+    if (key === collectionBaseField) {
+      parentState[key] = value
+      continue
+    }
     const property = object.properties[key]
     if (!property) {
       throw new Error(`could not find parent property ${key}`)
@@ -287,6 +294,10 @@ export async function loadResult(
     const state: any = {}
 
     for (const [key, value] of Object.entries(result)) {
+      if (key === collectionBaseField) {
+        state[key] = value
+        continue
+      }
       const property = Object.values(object.properties).find(
         (p) => p.name === key,
       )

library/src/module/entrypoint/register.ts

@@ -55,10 +55,16 @@ export class Register {
 
       // Register the class Typedef object in Dagger
       let typeDef = dag.typeDef().withObject(object.name, objectOpts)
+      if (object.isCollection) {
+        typeDef = typeDef.withCollection()
+      }
 
       // Register all functions (methods) to this object
       Object.values(object.methods).forEach((method) => {
         typeDef = typeDef.withFunction(this.addFunction(method))
+        if (method.isCollectionGet) {
+          typeDef = typeDef.withCollectionGet(method.alias ?? method.name)
+        }
       })
 
       // Register all fields that belong to this object
@@ -75,6 +81,12 @@ export class Register {
             addTypeDef(field.type!),
             fieldOpts,
           )
+          if (field.isCollectionKeys) {
+            typeDef = typeDef.withCollectionKeys(field.alias ?? field.name)
+          }
+          if (field.isCollectionDelta) {
+            typeDef = typeDef.withCollectionDelta(field.alias ?? field.name)
+          }
         }
       })