diff --git a/.dagger/modules/e2e/dagger.json b/.dagger/modules/e2e/dagger.json index 3267257..33a7dcf 100644 --- a/.dagger/modules/e2e/dagger.json +++ b/.dagger/modules/e2e/dagger.json @@ -12,11 +12,6 @@ { "name": "python-sdk-runtime", "source": "../../../runtime" - }, - { - "name": "sdk-sdk", - "source": "github.com/dagger/sdk-sdk", - "pin": "334448911a8292fba0d677e5f31926c79ad80ad3" } ] } diff --git a/.dagger/modules/e2e/fixtures/handover/reach.py b/.dagger/modules/e2e/fixtures/handover/reach.py new file mode 100644 index 0000000..50652a3 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/handover/reach.py @@ -0,0 +1,282 @@ +"""What module code can read of its caller's files: nothing it was not handed. + +Copied into a module that runs on an entrypoint. `reach` takes every ID the +SDK's loader holds, however it holds it, and drives each down every route on a +`ModuleSource` (and a `Module`, and a `Workspace`) that could rebuild a +directory from somewhere other than the source's own loaded files: the caller's +host, its `currentWorkspace`, or a workspace the source secretly retained. That +last shape is the one that defeated an earlier fix, where a +`Workspace.moduleSource` result reloaded its context from the workspace on +`withIncludes`. The handed-over source is a DIR-kind source with no workspace, +so the same routes should read nothing; this proves it against the engine +rather than by reading the engine's code. + +Each route reads a file that exists only at the caller's workspace root and is +never handed to the module. A route that returns it is a finding. A route that +fails is only reassuring if it failed because the capability was refused, not +because the field does not exist on this engine: the two are told apart, and +the summary reports the split so a run that proved nothing cannot read as a +pass. +""" + +from __future__ import annotations + +import dataclasses + +from dagger._exceptions import QueryError +from dagger.client import _load +from dagger.client._core import Arg, Context + +# A leak by climbing is only reachable when the number of "../" lands exactly +# on the caller's workspace root: too few stays inside the module, too many +# escapes the context and the engine refuses the whole pattern. The module's +# depth is not known here, so every route that climbs is tried at each depth. +DEPTHS = range(1, 13) + + +def _ups() -> list[str]: + return ["../" * n for n in DEPTHS] + + +def _held_ids() -> list[str]: + """Every ID-shaped string the loader holds, however it holds it.""" + found: list[str] = [] + + def walk(value, depth=0): + if depth > 4: + return + if isinstance(value, str) and len(value) > 20: + found.append(value) + elif isinstance(value, dict): + for v in value.values(): + walk(v, depth + 1) + elif isinstance(value, (list, tuple, set, frozenset)): + for v in value: + walk(v, depth + 1) + elif hasattr(value, "__dict__") and not isinstance(value, type): + walk(vars(value), depth + 1) + + for name, value in vars(_load).items(): + if name.startswith("_") and not name.startswith("__"): + walk(value) + return found + + +def _read_dir(dctx: Context, secret: str) -> dict[str, Context]: + """Reach the secret from a Directory: a plain read in case a reload already + pulled it in at the root, and a climb at each depth, by file path and by + stepping up with `directory("../")` first. + """ + routes = {"file": _file(dctx, secret)} + for up in _ups(): + n = up.count("../") + routes[f"file(../x{n})"] = _file(dctx, up + secret) + routes[f"directory(../x{n}).file"] = _file( + dctx.select("Directory", "directory", [Arg("path", up)]), secret + ) + return routes + + +def _file(dctx: Context, path: str) -> Context: + return dctx.select("Directory", "file", [Arg("path", path)]).select( + "File", "contents", [] + ) + + +def _source_dirs(source: Context, secret: str) -> dict[str, Context]: + """Directory-producing routes on a ModuleSource: every field that rebuilds + or exposes a context, each a chance to reach past the loaded files. + + Rejected, with reason: + - asString/pin/digest/version/commit/cloneRef/cloneURL/htmlURL/ + repoRootPath/sourceSubpath/originalSubpath: scalars, no directory. + - localContextDirectoryPath: a host path string, and only for a local + source; the handed source is DIR. No contents. + - introspectionSchemaJSON/clientSchemaIntrospectionJSON: schema JSON of + the module, not a directory of files. + - generate(workspace:): needs a Workspace argument, which is the very + capability the module does not have. + - withDependencies/withBlueprint/withToolchains/withUpdate*: attach other + modules by ref; they do not rebuild this source's context from the + caller's workspace, and any ref resolves in the module's own nested + context. Their result is still walked through `dependencies` below. + - withClient/withUpdatedClients: mutate the client list, not the context + root; the result's contextDirectory is still the source's own, which + the `withName`/`withSDK`/`withEngineVersion` mutations already cover. + """ + routes: dict[str, Context] = {} + + def add(label: str, dctx: Context): + for how, ctx in _read_dir(dctx, secret).items(): + routes[f"{label} {how}"] = ctx + + def src_ctx(s: Context) -> Context: + return s.select("ModuleSource", "contextDirectory", []) + + add("contextDirectory", src_ctx(source)) + # withIncludes an escaping pattern reloads the context from wherever the + # source came from; at the exact depth this reaches the workspace root. + # withSourceSubpath and directory climb the same way. Each depth is its own + # route: the engine refuses a whole pattern list if any pattern escapes. + for up in _ups(): + n = up.count("../") + add( + f"withIncludes(../x{n}).contextDirectory", + src_ctx( + source.select( + "ModuleSource", "withIncludes", [Arg("patterns", [up + secret])] + ) + ), + ) + add( + f"withSourceSubpath(../x{n}).contextDirectory", + src_ctx( + source.select("ModuleSource", "withSourceSubpath", [Arg("path", up)]) + ), + ) + add( + f"directory(../x{n})", + source.select("ModuleSource", "directory", [Arg("path", up)]), + ) + add( + "generatedContextDirectory", + source.select("ModuleSource", "generatedContextDirectory", []), + ) + add( + "updatedConfigDirectory", + source.select("ModuleSource", "updatedConfigDirectory", []), + ) + add( + "generatedContextChangeset.layer", + source.select("ModuleSource", "generatedContextChangeset", []).select( + "Changeset", "layer", [] + ), + ) + # Mutations that clone and may reload the context. + for label, field, arg in ( + ("withName", "withName", Arg("name", "reach")), + ("withSDK", "withSDK", Arg("source", "")), + ("withEngineVersion", "withEngineVersion", Arg("version", "v1.0.0-0")), + ): + add( + f"{label}.contextDirectory", + src_ctx(source.select("ModuleSource", field, [arg])), + ) + return routes + + +async def _sub_source_ids(source: Context) -> list[str]: + """One level down: the source's dependencies, and its own asModule.source, + each a ModuleSource whose routes are worth the same walk. + """ + ids: list[str] = [] + try: + deps = await source.select("ModuleSource", "dependencies", []).execute( + list[_Ref] + ) + ids += [d.id for d in deps] + except Exception: # noqa: BLE001 - a source may have no dependencies + pass + try: + ids.append( + await source.select("ModuleSource", "asModule", []) + .select("Module", "source", []) + .select("ModuleSource", "id", []) + .execute(str) + ) + except Exception: # noqa: BLE001 - not every ID is a loadable module + pass + return ids + + +@dataclasses.dataclass +class _Ref: + id: str + + +def _all_routes(held: str, secret: str) -> dict[str, Context]: + """Every route from one held ID: as a workspace, as a module source, and as + a Module whose source is walked the same way. + """ + routes: dict[str, Context] = { + "workspace.file": Context() + .select_id("Workspace", held) + .select("Workspace", "file", [Arg("path", "/" + secret)]) + .select("File", "contents", []), + "workspace.directory": _file( + Context() + .select_id("Workspace", held) + .select("Workspace", "directory", [Arg("path", "/")]), + secret, + ), + } + as_source = Context().select_id("ModuleSource", held) + for label, ctx in _source_dirs(as_source, secret).items(): + routes[f"source {label}"] = ctx + mod_source = Context().select_id("Module", held).select("Module", "source", []) + for label, ctx in _source_dirs(mod_source, secret).items(): + routes[f"module.source {label}"] = ctx + return routes + + +def _is_absent(error: QueryError) -> bool: + """The field does not exist on this engine: a validation error, before any + resolver runs. Anything else means a resolver ran and refused. + """ + for e in error.errors: + if e.path is None and e.extensions.get("code") == "GRAPHQL_VALIDATION_FAILED": + return True + return False + + +async def _probe(ctx: Context) -> tuple[str, str]: + try: + value = await ctx.execute(str) + except QueryError as e: + return ("absent" if _is_absent(e) else "refused", str(e)[:120]) + except Exception as e: # noqa: BLE001 + return ("errored", str(e)[:120]) + return ("read", value.strip() if value else "") + + +async def reach(secret: str) -> str: + held = _held_ids() + # The held IDs, plus one level of sub-sources reached from each. + sources = list(held) + for i in held: + sources += await _sub_source_ids(Context().select_id("ModuleSource", i)) + + reads: list[str] = [] + counts = {"read": 0, "refused": 0, "absent": 0, "errored": 0} + absent_routes: list[str] = [] + total = 0 + for i in sources: + for label, ctx in _all_routes(i, secret).items(): + status, detail = await _probe(ctx) + total += 1 + counts[status] += 1 + # The file exists only at the caller's workspace root, never in the + # module's loaded files, so any successful read of it is a leak, + # whatever its contents. + if status == "read": + reads.append(f"{label}: {detail}") + elif status == "absent": + absent_routes.append(label) + + own = ( + Context() + .root_select("currentWorkspace", []) + .select("Workspace", "directory", [Arg("path", "/")]) + ) + status, detail = await _probe(_file(own, secret)) + total += 1 + counts[status] += 1 + if status == "read": + reads.append(f"currentWorkspace.directory: {detail}") + + absent = ",".join(sorted(set(absent_routes))) + return ( + f"held={len(held)} sources={len(sources)} routes={total} " + f"reads={reads} refused={counts['refused']} absent={counts['absent']} " + f"errored={counts['errored']} absentRoutes=[{absent}]" + ) diff --git a/.dagger/modules/e2e/fixtures/lookup/app/clients/linter/pyproject.toml b/.dagger/modules/e2e/fixtures/lookup/app/clients/linter/pyproject.toml new file mode 100644 index 0000000..58dfbcc --- /dev/null +++ b/.dagger/modules/e2e/fixtures/lookup/app/clients/linter/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "dagger-clients-linter" + +[tool.dagger] +generated = "client" diff --git a/.dagger/modules/e2e/fixtures/lookup/app/sdk/pyproject.toml b/.dagger/modules/e2e/fixtures/lookup/app/sdk/pyproject.toml new file mode 100644 index 0000000..223654d --- /dev/null +++ b/.dagger/modules/e2e/fixtures/lookup/app/sdk/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "dagger-io" + +[tool.dagger] +generated = "runtime" diff --git a/.dagger/modules/e2e/fixtures/plain/clients/core/pyproject.toml b/.dagger/modules/e2e/fixtures/plain/clients/core/pyproject.toml new file mode 100644 index 0000000..9d9d6a5 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/plain/clients/core/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "dagger-clients-core" + +[tool.dagger] +generated = "core" diff --git a/.dagger/modules/e2e/fixtures/plain/pyproject.toml b/.dagger/modules/e2e/fixtures/plain/pyproject.toml new file mode 100644 index 0000000..b1698d3 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/plain/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "plain-scope" +version = "0.1.0" +requires-python = ">=3.14" +dependencies = ["dagger-io"] diff --git a/.dagger/modules/e2e/fixtures/plain/src/plain_scope/__init__.py b/.dagger/modules/e2e/fixtures/plain/src/plain_scope/__init__.py new file mode 100644 index 0000000..bda9196 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/plain/src/plain_scope/__init__.py @@ -0,0 +1,5 @@ +from dagger_clients.core import core + + +async def hostname() -> str: + return await core().container().from_("alpine:3.22").with_exec(["hostname"]).stdout() diff --git a/.dagger/modules/e2e/main.dang b/.dagger/modules/e2e/main.dang index f21cc8d..f7ede2d 100644 --- a/.dagger/modules/e2e/main.dang +++ b/.dagger/modules/e2e/main.dang @@ -15,13 +15,47 @@ type E2e { let runtimeModulePath: String! = fixtureRoot + "/runtime/app" let tomlGenerateModulePath: String! = fixtureRoot + "/toml-generate/app" let clientDepPath: String! = fixtureRoot + "/clients/dep" + let plainScopePath: String! = fixtureRoot + "/plain" let staticTypesModulePath: String! = fixtureRoot + "/static-types" let runtimeGreeting: String! = "served by the python-sdk runtime" let mixedDiscoveryModulePath: String! = fixtureRoot + "/mixed-discovery/ancestor/work/app" let mixedDiscoveryNestedPath: String! = mixedDiscoveryModulePath + "/nested/deeper" - let generatedMarkerPath: String! = "sdk/src/dagger/client/gen.py" + let legacyBindingsPath: String! = "sdk/src/dagger/client/gen.py" let generatedMarkerContents: String! = "Code generated by dagger." + let sdkMemberFile: String! = "sdk/pyproject.toml" + let coreMemberFile: String! = "clients/core/pyproject.toml" + let coreInitPath: String! = "clients/core/src/dagger_clients/core/__init__.py" + let clientDepMember: String! = "clients/client-dep" + let clientDepTargetPath: String! = clientDepMember + "/src/dagger_clients/client_dep/_target.py" + let globalPackage: String! = "dagger_global" + + """ + The tree every generated scope has, module or not: the SDK files with no + generated code in them, and core, each a member marked as the SDK's. + """ + let assertScopeTree(generated: Workspace!, scope: String!): Void { + let root = generated.directory("/" + scope) + assertContains(root.file(sdkMemberFile).contents, "generated = \"runtime\"", scope + ": sdk/ is not marked as the SDK's member") + assert(root.exists("sdk/src/dagger/__init__.py"), scope + ": sdk/ lacks the SDK files") + assert(root.exists(legacyBindingsPath) == false, scope + ": sdk/ still holds generated bindings") + assertContains(root.file(coreMemberFile).contents, "generated = \"core\"", scope + ": clients/core is not marked as core") + assertContains(root.file(coreInitPath).contents, generatedMarkerContents, scope + ": core was not generated") + let scopeFile = root.file("pyproject.toml").contents + assertContainsAll(scopeFile, [ + "[tool.uv.workspace]", + "\"sdk\"", + "\"clients/core\"", + "dagger-io = { workspace = true }", + "dagger-clients-core = { workspace = true }", + ]) + null + } + + let coreDigestOf(generated: Workspace!, scope: String!): String! { + let m = generated.file("/" + scope + "/" + coreInitPath).contents.match("(?m)^CORE_DIGEST = \"([^\"]*)\"") + if (m == null) { raise scope + ": core carries no CORE_DIGEST" } else { m.captures[0] ?? "" } + } let assert(condition: Boolean!, message: String!): Void { if (condition == false) { @@ -110,8 +144,9 @@ type E2e { """ findClientRoot answers with the directory of the nearest pyproject.toml as a - workspace-root-relative path, with the module rather than its vendored client - library, and with null where there is none. + workspace-root-relative path, with the scope rather than one of its + generated members, with the module rather than its vendored client library + from before the marker, and with null where there is none. """ pub findClientRootCheck(ws: Workspace!): Void @check { assert(pythonSdk.findClientRoot(ws.withWorkdir(lookupNestedPath)) == lookupModulePath, "findClientRoot should find the module owning a nested path") @@ -123,15 +158,23 @@ type E2e { assert(pythonSdk.findClientRoot(nestedSdkWs.withWorkdir(runtimeModulePath + "/sdk/src/dagger/client")) == runtimeModulePath + "/sdk", "findClientRoot should answer with a nested module named sdk, not lift it into its parent") assert(pythonSdk.findClientRoot(ws) == null, "findClientRoot should return null when it finds no client root") + # A generated member lifts to its scope: the marker tells, not the name. + # The members are fixtures on disk, because findUp does not see files + # added to the workspace. + assert(pythonSdk.findClientRoot(ws.withWorkdir(lookupModulePath + "/clients/linter")) == lookupModulePath, "findClientRoot should lift a generated client to its scope") + assert(pythonSdk.findClientRoot(ws.withWorkdir(lookupModulePath + "/sdk")) == lookupModulePath, "findClientRoot should lift a marked sdk/ to its scope") + assert(pythonSdk.findClientRoot(ws.withWorkdir(plainScopePath + "/clients/core")) == plainScopePath, "findClientRoot should lift core to a scope without a module") + null } + """ generateScope initializes a module that has no config: the template, a dagger-module.toml from the manifest builder and no dagger.json, and the - generated client library, all under the scope, with the workspace cwd and - existing files untouched. SDK settings select the template and its - pyproject.toml values. + scope tree, all under the scope, with the workspace cwd and existing files + untouched. A new module imports core from dagger_clients and gets no global + client. SDK settings select the template and its pyproject.toml values. """ pub generateScopeInitCheck(ws: Workspace!): Void @check { let scope = outputRoot + "/scope-init" @@ -147,26 +190,34 @@ type E2e { assert(contains(changes.addedPaths, scope + "/dagger.json") == false, "initializing a scope must not write a dagger.json") assertAdded(changes, scope + "/pyproject.toml") assertAdded(changes, scope + "/src/scope_init/__init__.py") - assertAdded(changes, scope + "/" + generatedMarkerPath) + assertAdded(changes, scope + "/" + sdkMemberFile) + assertAdded(changes, scope + "/" + coreInitPath) + assertScopeTree(generated.withWorkdir("."), scope) assert(changes.modifiedPaths.length == 0, "initializing a scope modified existing files: " + changes.modifiedPaths.join(", ")) assert(changes.removedPaths.length == 0, "initializing a scope removed existing files: " + changes.removedPaths.join(", ")) - assertContainsAll(changes.layer.file(scope + "/dagger-module.toml").contents, [ + let manifest = changes.layer.file(scope + "/dagger-module.toml").contents + assertContainsAll(manifest, [ "name = \"scope-init\"", - "engineVersion = \"", - "[runtime]", - "source = \"python\"", + "[entrypoint]", + "kind = \"dang\"", + "source = \"" + sharedEntrypointSource + "\"", ]) + # The entrypoint runs the module, on the engine's own version. + assertContainsNone(manifest, ["[runtime]", "engineVersion", "[[dependencies]]"]) let defaultSource = changes.layer.file(scope + "/src/scope_init/__init__.py").contents assertContainsAll(defaultSource, [ "class ScopeInit:", - "ws: dagger.Workspace", - "def container(self) -> dagger.Container:", + "from dagger_clients.core import", + "ws: Workspace", + "def container(self) -> Container:", + "core()", ]) - assertContainsNone(defaultSource, ["def __init__", "{{"]) + assertContainsNone(defaultSource, ["def __init__", "{{", "dag."]) let defaultPyproject = changes.layer.file(scope + "/pyproject.toml").contents assertContains(defaultPyproject, ">=3.14", "default settings should keep the template python version") - assertNotContains(defaultPyproject, "[tool.dagger]", "default settings should not write a [tool.dagger] table") - assertContains(changes.layer.file(scope + "/" + generatedMarkerPath).contents, generatedMarkerContents, "generateScope did not generate the client library") + assertNotContains(defaultPyproject, "[tool.dagger]", "a new module should get no [tool.dagger] table, so no global client") + assertContains(defaultPyproject, "\"dagger-clients-core\"", "a new module should depend on core") + assert(generated.directory("/" + scope).exists("sdk/src/" + globalPackage) == false, "a new module got the global client") let configuredScope = outputRoot + "/scope-init-configured" let configuredScoped = ws.withNewDirectory("/" + configuredScope, directory).withWorkdir(configuredScope) @@ -180,93 +231,439 @@ type E2e { "use-uv = false", "python:3.13-slim", "dagger-io", + "dagger-clients-core", + "workspace = true", ]) + assertNotContains(configuredChanges.layer.file(configuredScope + "/pyproject.toml").contents, "global-client", "a new module got the global client") null } """ generateScope preserves a current manifest. It migrates a pre-1.0 - dagger.json to dagger-module.toml. Both cases generate the client library. + dagger.json to dagger-module.toml. Both cases get the scope tree, and the + vendored source of the layout before it becomes a workspace source. """ pub generateScopeExistingCheck(ws: Workspace!): Void @check { - let current = pythonSdk + let currentWs = pythonSdk .generateScope(ws.withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: []) .withWorkdir(".") - .changes(ws) - assertAdded(current, tomlGenerateModulePath + "/" + generatedMarkerPath) - assertContains(current.after.file(tomlGenerateModulePath + "/dagger-module.toml").contents, "engineVersion = \"v1.0.0-0\"", "regenerating did not preserve engineVersion") - - let legacy = pythonSdk + let current = currentWs.changes(ws) + assertAdded(current, tomlGenerateModulePath + "/" + sdkMemberFile) + assertScopeTree(currentWs, tomlGenerateModulePath) + let currentManifest = current.after.file(tomlGenerateModulePath + "/dagger-module.toml").contents + assertContains(currentManifest, "source = \"" + sharedEntrypointSource + "\"", "regenerating did not move the module onto the entrypoint") + assertContainsNone(currentManifest, ["[runtime]", "engineVersion"]) + assertNotContains(current.after.file(tomlGenerateModulePath + "/pyproject.toml").contents, "path = \"sdk\"", "the vendored source survived") + + # The runtime reads the members as TOML defines them, so a quoted header, + # valid TOML, generates, and the runtime reads the members it lists. The + # input is a scope generation wrote, with only its header rewritten. + let pyprojectPath = "/" + tomlGenerateModulePath + "/pyproject.toml" + let written = currentWs.file(pyprojectPath).contents + assert(written.contains("\n[tool.uv.workspace]\n"), "generation wrote no [tool.uv.workspace] header to rewrite") + let quotedToml = written.replace("[tool.uv.workspace]", "[\"tool\".\"uv\".\"workspace\"]") + let quoted = pythonSdk + .generateScope(currentWs.withNewFile(pyprojectPath, quotedToml).withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: []) + .withWorkdir(".") + let quotedRead = pythonSdkRuntime.scopeMembers(quoted.file(pyprojectPath)) + assert(quotedRead.filter { m => m == "sdk" or m == "clients/core" }.length == 2, "the runtime did not read the members under a quoted header: " + quotedRead.join(", ")) + + # Nor a member the runtime would read and uv would not: a nested array is + # no member, so the runtime refuses to read it rather than flatten it, + # and generation refuses the file. + let nestedToml = written.replace("\"clients/core\"]", "\"clients/core\", [\"clients/unowned\"]]") + assert(nestedToml != written, "generation wrote no members array to nest into") + let runtimeRead = pythonSdkRuntime.scopeMembers(directory.withNewFile("pyproject.toml", nestedToml).file("pyproject.toml")) rescue { + err: Error => ["refused"] + } + assert(runtimeRead == ["refused"], "the runtime read a nested members array as " + runtimeRead.join(", ")) + let nested = pythonSdk + .generateScope(currentWs.withNewFile(pyprojectPath, nestedToml).withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: []) + .cwd rescue { + err: Error => err.message + } + assertContainsAll(nested, ["the module runtime cannot read: members is not an array of strings", "Write members as an array of strings, not a nested array", "then run dagger generate again"]) + # The members it lists are fine; naming them would point away from the cause. + assertNotContains(nested, "misses", "the refusal listed the members the runtime did not get to read") + + let legacyWs = pythonSdk .generateScope(ws.withWorkdir(generateModulePath), isModule: true, name: "generate-app", clients: []) .withWorkdir(".") - .changes(ws) - assertAdded(legacy, generateModulePath + "/" + generatedMarkerPath) + let legacy = legacyWs.changes(ws) + assertScopeTree(legacyWs, generateModulePath) assertAdded(legacy, generateModulePath + "/dagger-module.toml") assert(contains(legacy.removedPaths, generateModulePath + "/dagger.json"), "migration did not remove dagger.json") - assertContainsAll(legacy.after.file(generateModulePath + "/dagger-module.toml").contents, [ + let legacyManifest = legacy.after.file(generateModulePath + "/dagger-module.toml").contents + assertContainsAll(legacyManifest, [ "name = \"generate-app\"", - "source = \"python\"", + "source = \"" + sharedEntrypointSource + "\"", ]) + assertContainsNone(legacyManifest, ["[runtime]", "engineVersion"]) null } """ - In a module scope the client set becomes the module's dependency set: the - manifest lists each client, the generated bindings include its types, and a - client that is no longer requested is dropped again. A scope without a - module is left alone, and standalone clients are refused rather than - silently skipped. + A client is a member of the scope that declares it: adding one writes + exactly its member, its source, its `members` entry and its dependency, and + removing it undoes all four. The member loads its module by the path from + the workspace root and was generated against the scope's core. No client + reaches the manifest, and the [[dependencies]] of the layout before go. """ pub generateScopeClientsCheck(ws: Workspace!): Void @check { - let scoped = ws.withWorkdir(tomlGenerateModulePath) - let manifestPath = tomlGenerateModulePath + "/dagger-module.toml" - let bindingsPath = tomlGenerateModulePath + "/" + generatedMarkerPath + let manifestPath = "/" + tomlGenerateModulePath + "/dagger-module.toml" + let scopeFilePath = "/" + tomlGenerateModulePath + "/pyproject.toml" + let member = "/" + tomlGenerateModulePath + "/" + clientDepMember let client = ws.moduleSource("/" + clientDepPath) + let seeded = ws.withNewFile(manifestPath, ws.file(manifestPath).contents + + "\n[[dependencies]]\nname = \"client-dep\"\nsource = \"../../clients/dep\"\n") + let without = pythonSdk + .generateScope(seeded.withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: []) + .withWorkdir(".") + assertNotContains(without.file(manifestPath).contents, "[[dependencies]]", "generation kept the [[dependencies]] of the layout before") - let untouched = pythonSdk.generateScope(scoped, isModule: false, name: "toml-generate-app", clients: []).withWorkdir(".").changes(ws) - assert(untouched.isEmpty, "a scope without a module should not be generated") - - let refused = pythonSdk.generateScope(scoped, isModule: false, name: "toml-generate-app", clients: [client]).cwd rescue "raised" - assert(refused == "raised", "generateScope should refuse standalone clients") + let with = pythonSdk + .generateScope(without.withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: [client]) + .withWorkdir(".") + let added = with.changes(without) + assertScopeTree(with, tomlGenerateModulePath) + assert(added.modifiedPaths.length == 1 and contains(added.modifiedPaths, tomlGenerateModulePath + "/pyproject.toml"), "adding a client modified more than the scope file: " + added.modifiedPaths.join(", ")) + assert(added.removedPaths.length == 0, "adding a client removed files: " + added.removedPaths.join(", ")) + assert(added.addedPaths.filter { path => path.hasPrefix(tomlGenerateModulePath + "/" + clientDepMember + "/") == false }.length == 0, "adding a client added files outside its member") + assertNotContains(with.file(manifestPath).contents, "client-dep", "a client reached the manifest") + + let memberFile = with.file(member + "/pyproject.toml").contents + assertContainsAll(memberFile, ["name = \"dagger-clients-client-dep\"", "generated = \"client\"", "module-name = \"dagger_clients.client_dep\""]) + let target = with.file("/" + tomlGenerateModulePath + "/" + clientDepTargetPath).contents + assertContainsAll(target, [ + "NAME = \"client-dep\"", + "REF = \"/" + clientDepPath + "\"", + "PIN = None", + "CORE_DIGEST = \"" + coreDigestOf(with, tomlGenerateModulePath) + "\"", + ]) + assertContains(with.file(member + "/src/dagger_clients/client_dep/__init__.py").contents, "def client_dep(", "the client has no entry function") - let withClient = pythonSdk.generateScope(scoped, isModule: true, name: "toml-generate-app", clients: [client]) - let added = withClient.withWorkdir(".").changes(ws) - assert(contains(added.modifiedPaths, manifestPath), "adding a client should record it in the manifest") - assertContainsAll(added.after.file(manifestPath).contents, ["[[dependencies]]", "name = \"client-dep\"", "source = \"../../clients/dep\""]) - assert(contains(added.addedPaths, tomlGenerateModulePath + "/dagger.json") == false, "adding a client should not add a manifest the module did not have") - assertContains(added.after.file(bindingsPath).contents, "class ClientDep", "the generated bindings should include the client's types") + let scopeDiff = with.file(scopeFilePath).contents + assertContainsAll(scopeDiff, [ + "\"clients/client-dep\"", + "dagger-clients-client-dep = { workspace = true }", + "\"dagger-clients-client-dep\"", + ]) - let withoutClient = pythonSdk.generateScope(withClient, isModule: true, name: "toml-generate-app", clients: []) - let removed = withoutClient.withWorkdir(".").changes(ws) - assertNotContains(removed.after.file(manifestPath).contents, "client-dep", "removing the last client should drop the dependency") - assertNotContains(removed.after.file(bindingsPath).contents, "class ClientDep", "removing the client should drop its types from the bindings") + let removed = pythonSdk + .generateScope(with.withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: []) + .withWorkdir(".") + assert(removed.changes(without).isEmpty, "removing the client did not undo adding it") null } - """Git pins follow lock, and JSON is removed in both modes.""" + """ + Git pins follow lock into the client's descriptor, never into the manifest, + and JSON is removed in both modes. + """ pub generateScopePinsCheck(ws: Workspace!): Void @check { let commit = "64645f1967d3dba6fce951dd61ae4acd8d9b0861" let target = moduleSource("github.com/dagger/sdk-helpers@main", refPin: commit) + let descriptor = "clients/sdk-helpers/src/dagger_clients/sdk_helpers/_target.py" let input = ws.withWorkdir(tomlGenerateModulePath).withNewFile("keep.txt", "keep") let unlocked = pythonSdk.generateScope(input, isModule: true, name: "toml-generate-app", clients: [target]) - assertNotContains(unlocked.file("dagger-module.toml").contents, "pin =", "default output contains a pin") - assertContains(unlocked.file("dagger-module.toml").contents, target.asString, "Git source changed") + assertContainsAll(unlocked.file(descriptor).contents, ["REF = \"" + target.asString + "\"", "PIN = None"]) + assertNotContains(unlocked.file("dagger-module.toml").contents, "sdk-helpers", "a client reached the manifest") let locked = pythonSdk(lock: true).generateScope(unlocked, isModule: true, name: "toml-generate-app", clients: [target]) - assertContains(locked.file("dagger-module.toml").contents, "pin = \"" + commit + "\"", "selected commit was lost") + assertContains(locked.file(descriptor).contents, "PIN = \"" + commit + "\"", "selected commit was lost") let unlockedAgain = pythonSdk.generateScope(locked.withNewFile("dagger.json", "stale"), isModule: true, name: "toml-generate-app", clients: [target]) - assertNotContains(unlockedAgain.file("dagger-module.toml").contents, "pin =", "disabling lock retained a pin") + assertContains(unlockedAgain.file(descriptor).contents, "PIN = None", "disabling lock retained a pin") assert(!unlockedAgain.directory(".").exists("dagger.json"), "generation retained JSON") assert(unlockedAgain.cwd == input.cwd, "generation changed cwd") assert(unlockedAgain.file("keep.txt").contents == "keep", "generation changed another file") - assertContains(unlockedAgain.file("dagger-module.toml").contents, "engineVersion = \"v1.0.0-0\"", "generation changed the engine version") + assertNotContains(unlockedAgain.file("dagger-module.toml").contents, "engineVersion", "an entrypoint manifest kept the engine version only a runtime reads") let repeated = pythonSdk.generateScope(unlockedAgain, isModule: true, name: "toml-generate-app", clients: [target]) assert(repeated.changes(unlockedAgain).isEmpty, "repeated generation changed the workspace") null } + """ + Core is read in the view the module's session runs in, and every client is + read in the same view. On an entrypoint that is the engine's own version, + whatever engineVersion the runtime manifest before it declared. A module + that keeps a runtime of its own runs in the view it declares, and so does + its core. + + Not covered yet: a client in a module older than v1.0.0. That view names + each module type's ID scalar and loader with no @sourceMap, so the + generator counts them as core and refuses the client as skew. + """ + pub generateScopeViewCheck(ws: Workspace!): Void @check { + let manifestPath = "/" + tomlGenerateModulePath + "/dagger-module.toml" + let older = "name = \"toml-generate-app\"\nengineVersion = \"v0.21.9\"\n\n[runtime]\nsource = " + let migrated = pythonSdk + .generateScope(ws.withNewFile(manifestPath, older + "\"python\"\n").withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: []) + .withWorkdir(".") + let own = pythonSdk + .generateScope(ws.withNewFile(manifestPath, older + "\"../../../../../../runtime\"\n").withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: []) + .withWorkdir(".") + # Workspace.cwd is newer than v0.21.9. + assertContains(migrated.file("/" + tomlGenerateModulePath + "/" + coreInitPath).contents, "def cwd(", "a module moved onto the entrypoint kept the view of its runtime") + assertNotContains(own.file("/" + tomlGenerateModulePath + "/" + coreInitPath).contents, "def cwd(", "core was not generated in the view of the module's own runtime") + + null + } + + """ + A module upgraded from the bindings of the layout before gets the temporary + global client once: the flag is written, dagger_global joins the SDK files, + which then depend on core and each client, and the old bindings go. + Turning the flag off with `mod config set` removes the global client at the + next generation. + """ + pub generateScopeGlobalClientCheck(ws: Workspace!): Void @check { + let root = "/" + runtimeModulePath + let client = ws.moduleSource("/" + clientDepPath) + let upgraded = pythonSdk + .generateScope(ws.withWorkdir(runtimeModulePath), isModule: true, name: "runtime-app", clients: [client]) + .withWorkdir(".") + assertContains(upgraded.file(root + "/pyproject.toml").contents, "global-client = true", "an upgraded module did not get the flag") + assert(upgraded.directory(root).exists(legacyBindingsPath) == false, "the bindings of the layout before survived the upgrade") + assertContains(upgraded.file(root + "/sdk/src/" + globalPackage + "/__init__.py").contents, "class Client(", "the global client was not generated") + assertContainsAll(upgraded.file(root + "/" + sdkMemberFile).contents, [ + "module-name = [\"dagger\", \"dagger_global\"]", + "\"dagger-clients-core\",", + "\"dagger-clients-client-dep\",", + ]) + + let again = pythonSdk + .generateScope(upgraded.withWorkdir(runtimeModulePath), isModule: true, name: "runtime-app", clients: [client]) + .withWorkdir(".") + assert(again.changes(upgraded).isEmpty, "a second generation changed an upgraded module") + + let off = upgraded.withChanges(pythonSdk.mod(upgraded, path: runtimeModulePath, findUp: false).config.set(globalClient: false)) + assertNotContains(off.file(root + "/pyproject.toml").contents, "global-client", "mod config set left the flag") + let without = pythonSdk + .generateScope(off.withWorkdir(runtimeModulePath), isModule: true, name: "runtime-app", clients: [client]) + .withWorkdir(".") + assert(without.directory(root).exists("sdk/src/" + globalPackage) == false, "turning the flag off kept the global client") + let sdkFile = without.file(root + "/" + sdkMemberFile).contents + assertContains(sdkFile, "module-name = \"dagger\"", "turning the flag off kept dagger_global in the SDK files") + assertNotContains(sdkFile, "dagger-clients-client-dep", "turning the flag off kept the SDK files' dependency on the client") + assertNotContains(without.file(root + "/pyproject.toml").contents, "global-client", "generation wrote the flag again") + + # A module on a published dagger-io kept its generated bindings next to + # its code. They go with the upgrade, since the SDK files no longer read + # them; a hand-written file of that name is the user's and stays. + let published = "/" + tomlGenerateModulePath + let userBindings = ws.withNewFile(published + "/src/dagger_gen.py", "# Code generated by dagger. DO NOT EDIT.\n") + let onPublished = pythonSdk + .generateScope(userBindings.withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: []) + .withWorkdir(".") + assertContains(onPublished.file(published + "/pyproject.toml").contents, "global-client = true", "a module with src/dagger_gen.py was not upgraded") + assert(onPublished.directory(published).exists("sdk/src/" + globalPackage), "a module with src/dagger_gen.py got no global client") + assert(onPublished.directory(published).exists("src/dagger_gen.py") == false, "the generated src/dagger_gen.py survived the upgrade") + + let handWritten = ws.withNewFile(published + "/src/dagger_gen.py", "from dagger import * # mine\n") + let onHandWritten = pythonSdk + .generateScope(handWritten.withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: []) + .withWorkdir(".") + assertContains(onHandWritten.file(published + "/pyproject.toml").contents, "global-client = true", "a module with a hand-written src/dagger_gen.py was not upgraded") + assert(onHandWritten.file(published + "/src/dagger_gen.py").contents == "from dagger import * # mine\n", "generation changed a hand-written src/dagger_gen.py") + let offPublished = onHandWritten.withChanges(pythonSdk.mod(onHandWritten, path: tomlGenerateModulePath, findUp: false).config.set(globalClient: false)) + let stays = pythonSdk + .generateScope(offPublished.withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: []) + .withWorkdir(".") + assertNotContains(stays.file(published + "/pyproject.toml").contents, "global-client", "src/dagger_gen.py turned the global client on again") + assert(stays.directory(published).exists("sdk/src/" + globalPackage) == false, "src/dagger_gen.py brought the global client back") + + null + } + + """ + The flag alone decides: set on a new module, generation emits the global + client and the SDK files' dependency on each client; cleared, generation + removes both. + """ + pub generateScopeGlobalClientFlagCheck(ws: Workspace!): Void @check { + let root = "/" + tomlGenerateModulePath + let client = ws.moduleSource("/" + clientDepPath) + let fresh = pythonSdk + .generateScope(ws.withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: [client]) + .withWorkdir(".") + assert(fresh.directory(root).exists("sdk/src/" + globalPackage) == false, "a module without the flag got the global client") + + let flagged = fresh.withChanges(pythonSdk.mod(fresh, path: tomlGenerateModulePath, findUp: false).config.set(globalClient: true)) + let on = pythonSdk + .generateScope(flagged.withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: [client]) + .withWorkdir(".") + assertContains(on.file(root + "/sdk/src/" + globalPackage + "/__init__.py").contents, "def client_dep(", "the global client lacks the client") + assertContains(on.file(root + "/" + sdkMemberFile).contents, "\"dagger-clients-client-dep\",", "the SDK files do not depend on the client the global client imports") + + let cleared = on.withChanges(pythonSdk.mod(on, path: tomlGenerateModulePath, findUp: false).config.set(globalClient: false)) + let off = pythonSdk + .generateScope(cleared.withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: [client]) + .withWorkdir(".") + assert(off.directory(root).exists("sdk/src/" + globalPackage) == false, "clearing the flag kept the global client") + assertNotContains(off.file(root + "/" + sdkMemberFile).contents, "dagger-clients-client-dep", "clearing the flag kept the SDK files' dependency on the client") + assert( + off.directory(root + "/sdk").digest == fresh.directory(root + "/sdk").digest, + "clearing the flag did not give back the SDK files of a module that never had it", + ) + + null + } + + """ + A scope without a module gets the same tree as a module, and no module + config. Its scope file is created when it has none. The client member is + byte for byte the one a module scope gets, because a member's files do not + depend on the scope that holds them. + """ + pub generateScopePlainCheck(ws: Workspace!): Void @check { + let scope = outputRoot + "/plain-scope" + let client = ws.moduleSource("/" + clientDepPath) + let bare = ws.withNewDirectory("/" + scope, directory).withWorkdir(scope) + + let empty = pythonSdk.generateScope(bare, isModule: false, name: "plain-scope", clients: []).withWorkdir(".") + assertScopeTree(empty, scope) + assert(empty.directory("/" + scope).exists("dagger-module.toml") == false, "a scope without a module got a module config") + + let plain = pythonSdk.generateScope(bare, isModule: false, name: "plain-scope", clients: [client]).withWorkdir(".") + assertScopeTree(plain, scope) + assertContainsAll(plain.file("/" + scope + "/pyproject.toml").contents, ["\"clients/client-dep\"", "dagger-clients-client-dep = { workspace = true }"]) + assertNotContains(plain.file("/" + scope + "/pyproject.toml").contents, "[project]", "the SDK invented a [project] table") + + # The committed fixture has a [project] table, so the client becomes a dependency. + let fixture = pythonSdk.generateScope(ws.withWorkdir(plainScopePath), isModule: false, name: "plain", clients: [client]).withWorkdir(".") + assertContains(fixture.file("/" + plainScopePath + "/pyproject.toml").contents, "\"dagger-clients-client-dep\"", "the client did not become a dependency of the scope") + + let module = pythonSdk.generateScope(ws.withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: [client]).withWorkdir(".") + assert( + plain.directory("/" + scope + "/" + clientDepMember).digest == module.directory("/" + tomlGenerateModulePath + "/" + clientDepMember).digest, + "one client differs between a scope without a module and a module", + ) + assert( + plain.directory("/" + scope + "/clients/core").digest == module.directory("/" + tomlGenerateModulePath + "/clients/core").digest, + "core differs between a scope without a module and a module", + ) + + null + } + + """ + The SDK owns its entries in the scope file and its members under clients/, + nothing else: user tables, comments and members survive byte for byte, a + user member under clients/ stays in the workspace, a directory under + clients/ without the marker is never deleted, and one is never written over. + """ + pub generateScopeUserContentCheck(ws: Workspace!): Void @check { + let scope = outputRoot + "/user-content" + let client = ws.moduleSource("/" + clientDepPath) + let userFile = + "# the user's own comment\n" + + "[project]\nname = \"user-content\"\nversion = \"0.1.0\"\n" + + "dependencies = [\n \"dagger-io\",\n \"httpx>=0.27\", # a pin of the user's\n]\n\n" + + "[tool.uv.workspace]\nmembers = [\"tools/mine\", \"clients/tools\"]\n\n" + + "[tool.uv.sources]\nmine = { workspace = true }\n\n" + + "[tool.mine]\nanswer = 42\n" + let seeded = ws + .withNewFile("/" + scope + "/pyproject.toml", userFile) + .withNewFile("/" + scope + "/tools/mine/pyproject.toml", "[project]\nname = \"mine\"\n") + .withNewFile("/" + scope + "/clients/tools/pyproject.toml", "[project]\nname = \"tools\"\n") + .withNewFile("/" + scope + "/clients/notes/README.md", "not a member\n") + .withNewFile("/" + scope + "/clients/handmade/pyproject.toml", "[project]\nname = \"handmade\"\ndescription = \"\"\"\n[tool.dagger]\ngenerated = \"client\"\n\"\"\"\n") + .withNewFile("/" + scope + "/clients/handmade/secret.txt", "mine\n") + # A marker that cannot be read never licenses a delete. + .withNewFile("/" + scope + "/clients/brokentoml/pyproject.toml", "this is not = valid toml [[[\n") + .withNewFile("/" + scope + "/clients/brokentoml/secret.txt", "mine\n") + .withNewFile("/" + scope + "/clients/numbermark/pyproject.toml", "[tool.dagger]\ngenerated = 42\n") + .withNewFile("/" + scope + "/clients/numbermark/secret.txt", "mine\n") + .withNewFile("/" + scope + "/clients/emptyfile/pyproject.toml", "") + .withNewFile("/" + scope + "/clients/emptyfile/secret.txt", "mine\n") + .withNewFile("/" + scope + "/clients/nofile/secret.txt", "mine\n") + .withWorkdir(scope) + let generated = pythonSdk.generateScope(seeded, isModule: false, name: "user-content", clients: [client]).withWorkdir(".") + let after = generated.file("/" + scope + "/pyproject.toml").contents + assertContainsAll(after, [ + "# the user's own comment\n[project]\nname = \"user-content\"\n", + " \"httpx>=0.27\", # a pin of the user's\n", + "\"tools/mine\"", + "\"clients/tools\"", + "mine = { workspace = true }\n", + "[tool.mine]\nanswer = 42\n", + ]) + assert(generated.directory("/" + scope).exists("clients/notes/README.md"), "a directory under clients/ without the marker was deleted") + + let removed = pythonSdk.generateScope(generated.withWorkdir(scope), isModule: false, name: "user-content", clients: []).withWorkdir(".") + assert(removed.directory("/" + scope).exists("clients/notes/README.md"), "removing a client deleted a directory without the marker") + assert(removed.directory("/" + scope).exists("clients/handmade/secret.txt"), "text that only looks like the marker, in a string, got a directory deleted") + ["brokentoml", "numbermark", "emptyfile", "nofile"].each { dir => + assert(generated.directory("/" + scope).exists("clients/" + dir + "/secret.txt"), "generation deleted clients/" + dir + ", whose marker cannot be read") + assert(removed.directory("/" + scope).exists("clients/" + dir + "/secret.txt"), "removing a client deleted clients/" + dir + ", whose marker cannot be read") + null + } + assert(removed.directory("/" + scope).exists(clientDepMember) == false, "a removed client's member was left behind") + let removedFile = removed.file("/" + scope + "/pyproject.toml").contents + assertNotContains(removedFile, "\"" + clientDepMember + "\"", "a removed client stayed a workspace member") + assertContains(removedFile, "\"clients/tools\"", "removing a client dropped the user's member under clients/") + assert(removed.directory("/" + scope).exists("clients/tools/pyproject.toml"), "removing a client deleted the user's member under clients/") + + let squatted = ws.withNewFile("/" + scope + "/" + clientDepMember + "/mine.txt", "mine\n").withWorkdir(scope) + let over = pythonSdk.generateScope(squatted, isModule: false, name: "user-content", clients: [client]).cwd rescue "raised" + assert(over == "raised", "a client was written over a directory without the marker") + + let unknown = ws + .withNewFile("/" + scope + "/clients/core/pyproject.toml", "[project]\nname = \"core\"\n\n[tool.dagger]\ngenerated = \"hand-written\"\n") + .withNewFile("/" + scope + "/clients/core/mine.txt", "mine\n") + .withWorkdir(scope) + let overUnknown = pythonSdk.generateScope(unknown, isModule: false, name: "user-content", clients: []).cwd rescue "raised" + assert(overUnknown == "raised", "core was written over a member marked with a kind the SDK does not write") + + # Nor an overwrite, even at a path generation certainly owns. + let corrupted = ws + .withNewFile("/" + scope + "/clients/core/pyproject.toml", "this is not = valid toml [[[\n") + .withNewFile("/" + scope + "/clients/core/mine.txt", "mine\n") + .withWorkdir(scope) + let overCorrupted = pythonSdk.generateScope(corrupted, isModule: false, name: "user-content", clients: []).cwd rescue { + err: Error => err.message + } + assertContainsAll(overCorrupted, ["clients/core/ exists in", "is not a generated core; move it away"]) + + # sdk/ from an earlier SDK carries no marker either, so only the header of + # its generated bindings tells it from one the user wrote. + let vendored = ws + .withNewFile("/" + scope + "/sdk/src/dagger/__init__.py", "# my own dagger\n") + .withNewFile("/" + scope + "/sdk/src/dagger/client/gen.py", "# written by hand\n") + .withWorkdir(scope) + let overSdk = pythonSdk.generateScope(vendored, isModule: false, name: "user-content", clients: []).cwd rescue { + err: Error => err.message + } + assertContainsAll(overSdk, ["sdk/ exists", "was not written by dagger generate", "move or rename it"]) + + null + } + + """ + `mod generate` writes the same tree as the scope generation, with the + clients the workspace declares for the module's scope. + """ + pub modGenerateClientsCheck(ws: Workspace!): Void @check { + # A client added from the scope names its module from there. + let scoped = ws.withWorkdir(tomlGenerateModulePath) + .withClient(module: "../../clients/dep", sdk: "python") + .withWorkdir(".") + let generated = pythonSdk.mod(scoped, path: tomlGenerateModulePath).generated + assertScopeTree(generated, tomlGenerateModulePath) + assertContains(generated.file("/" + tomlGenerateModulePath + "/" + clientDepTargetPath).contents, "REF = \"/" + clientDepPath + "\"", "mod generate missed the scope's client") + let viaScope = pythonSdk + .generateScope(ws.withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: [ws.moduleSource("/" + clientDepPath)]) + .withWorkdir(".") + assert( + generated.directory("/" + tomlGenerateModulePath).withoutFile("dagger-module.toml").digest == + viaScope.directory("/" + tomlGenerateModulePath).withoutFile("dagger-module.toml").digest, + "mod generate and the scope generation wrote different trees", + ) + null + } + """ Generation reports only what it produced: dagger.json and the committed .gitattributes are already on disk, so a changeset adding either describes @@ -284,14 +681,15 @@ type E2e { } """ - Generating an existing module should produce generated files rooted at that - module without touching unrelated paths. + Generating a pre-1.0 module is still left to the engine's builtin Python + SDK, which vendors its bindings; the module is not upgraded behind the + user's back. """ pub generateCheck(ws: Workspace!): Void @check { let changes = pythonSdk.mod(ws, path: generateModulePath).generate - assertAdded(changes, generateModulePath + "/" + generatedMarkerPath) - assertContains(changes.layer.file(generateModulePath + "/" + generatedMarkerPath).contents, generatedMarkerContents, "generate did not include the SDK code-generated marker") + assertAdded(changes, generateModulePath + "/" + legacyBindingsPath) + assertContains(changes.layer.file(generateModulePath + "/" + legacyBindingsPath).contents, generatedMarkerContents, "generate did not include the SDK code-generated marker") null } @@ -303,23 +701,22 @@ type E2e { """ pub tomlGenerateCheck(ws: Workspace!): Void @check { let changes = pythonSdk.mod(ws, path: tomlGenerateModulePath).generate - let genPath = tomlGenerateModulePath + "/" + generatedMarkerPath - assertAdded(changes, genPath) - assertContains(changes.layer.file(genPath).contents, generatedMarkerContents, "generate did not produce code-generated bindings") + assertAdded(changes, tomlGenerateModulePath + "/" + sdkMemberFile) + assertAdded(changes, tomlGenerateModulePath + "/" + coreInitPath) assert( contains(changes.addedPaths, tomlGenerateModulePath + "/sdk/runtime/dagger.json") == false, "the module was generated by the engine's builtin Python SDK, not this one", ) - # Vendored at some point and must not come back: a module needs only the - # importable library. + # Vendored at some point and must not come back: a scope needs only the + # importable library, and its bindings are members of their own. assertNoneAdded( changes, [ tomlGenerateModulePath + "/sdk/codegen/pyproject.toml", tomlGenerateModulePath + "/sdk/uv.lock", - tomlGenerateModulePath + "/sdk/src/dagger/provisioning/__init__.py", + tomlGenerateModulePath + "/" + legacyBindingsPath, ], "generate", ) @@ -328,16 +725,13 @@ type E2e { } """ - A module on this repository's runtime builds and runs, driven through a - released CLI. The fixture points `[runtime] source` at `runtime/` by relative + A module on this repository's runtime builds and runs, driven through the + floor release's CLI. The fixture points `[runtime] source` at `runtime/` by relative path, so the runtime in this working tree answers. """ pub runtimeCallCheck(ws: Workspace!): Void @check { - let run = sdkSdk - .target(ws.directory("/"), ".") - .runInstalled(["call", "-m", runtimeFixturePath, "greeting"]) - run.assertSuccess - assertContains(run.stdout, runtimeGreeting, "the module did not run on this repository's runtime") + let called = cli(playground(ws), ["-m", runtimeModulePath, "call", "greeting"]).stdout + assertContains(called, runtimeGreeting, "the module did not run on this repository's runtime") null } @@ -348,77 +742,571 @@ type E2e { builtin honours too. """ pub runtimeRequiresGeneratedFilesCheck(ws: Workspace!): Void @check { - let stripped = ws - .directory("/") - .withoutFile(runtimeModulePath + "/sdk/pyproject.toml") - let run = sdkSdk - .target(stripped, ".") - .runInstalled(["call", "-m", runtimeFixturePath, "greeting"]) - run.assertFailure - assertContains(run.stderr, "run `dagger generate` and commit", "the runtime did not report the missing generated file") + let stripped = playgroundTree(ws).withoutFile(runtimeModulePath + "/sdk/pyproject.toml") + let refused = playgroundOf(stripped) + .withExec(["dagger", "-m", runtimeModulePath, "call", "greeting"], experimentalPrivilegedNesting: true, expect: ReturnType.FAILURE) + .stderr + assertContains(refused, "run `dagger generate` and commit", "the runtime did not report the missing generated file") + + null + } + + """ + A module scope builds from its workspace: the SDK files, core and the + client are installed from their members, not editable, whether the module + builds with uv, with pip, or from the uv.lock that generation locked again. + """ + pub runtimeScopeInstallCheck(ws: Workspace!): Void @check { + let pyprojectPath = "/" + tomlGenerateModulePath + "/pyproject.toml" + let lockPath = "/" + tomlGenerateModulePath + "/uv.lock" + let pip = ws.withNewFile(pyprojectPath, ws.file(pyprojectPath).contents + "\n[tool.dagger]\nuse-uv = false\n") + # A lock of the layout before, which names none of the members. + let stale = ws.withNewFile(lockPath, "version = 1\nrequires-python = \">=3.14\"\n") + + let built = assertInstalled(ws, "uv") + assertInstalled(pip, "pip") + let locked = assertInstalled(stale, "uv.lock") + assertContains(locked.file(lockPath).contents, "name = \"dagger-clients-client-dep\"", "generation did not lock the client") + + # A member the scope lists and the tree lacks is refused before the build. + let withoutCore = built.withoutFile("/" + tomlGenerateModulePath + "/" + coreMemberFile) + let refused = pythonSdkRuntime + .moduleRuntime(modSource: runtimeSource(withoutCore, tomlGenerateModulePath), introspectionJson: null) + .withExec(["true"]) + .stdout rescue "raised" + assert(refused == "raised", "a scope without its core member was built") + + # Only the members the project depends on are installed, as `uv sync` + # installs them: a user member nothing depends on may not even install + # here, and pip or an unlocked uv must not try. + # The dependencies are the ones a real project writes: extras, an escaped + # quote in a marker, a literal string with a backslash, and a multi-line + # string in [project] holding text that looks like keys (a readme, since + # uv_build wants a one-line description). The markers are false, so + # nothing is fetched. + let members = ws + .withNewFile(pyprojectPath, ws.file(pyprojectPath).contents + .replace("dependencies = [\"dagger-io\"]", + "dependencies = [\n" + + " \"dagger-io\",\n" + + " \"used[extra]>=0.1\",\n" + + " \"tomli; python_version < \\\"3.0\\\"\", # an escaped quote\n" + + " 'tomli; python_version < \"3.0\" and os_name == \"C:\\dir\"', # a literal string\n" + + "]\n" + + "readme = { text = \"\"\"\ndependencies = 42\nmembers = [[\"nope\"]]\n\"\"\", content-type = \"text/plain\" }") + .replace("[tool.uv.sources]\n", "[tool.uv.sources]\nused = { workspace = true }\n") + + "\n[tool.uv.workspace]\nmembers = [\"tools/used\", \"tools/unused\"]\n") + .withNewFile("/" + tomlGenerateModulePath + "/tools/used/pyproject.toml", userMember("used", ">=3.10")) + .withNewFile("/" + tomlGenerateModulePath + "/tools/used/src/used/__init__.py", "") + .withNewFile("/" + tomlGenerateModulePath + "/tools/unused/pyproject.toml", userMember("unused", ">=99")) + .withNewFile("/" + tomlGenerateModulePath + "/tools/unused/src/unused/__init__.py", "") + assertMembersInstalled(members, "uv") + assertMembersInstalled(members.withNewFile(pyprojectPath, members.file(pyprojectPath).contents + "\n[tool.dagger]\nuse-uv = false\n"), "pip") + + # A member whose pyproject.toml does not read is named, not shown as a + # traceback from the reader. + let broken = pythonSdk + .generateScope(members.withNewFile("/" + tomlGenerateModulePath + "/tools/unused/pyproject.toml", "[project\n").withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: []) + .withWorkdir(".") + let brokenBuild = pythonSdkRuntime + .moduleRuntime(modSource: runtimeSource(broken, tomlGenerateModulePath), introspectionJson: null) + .withExec(["true"]) + .stdout rescue { + err: Error => err.message + } + assertContains(brokenBuild, "tools/unused/pyproject.toml is not valid TOML", "a broken member was not named: " + brokenBuild) + assertNotContains(brokenBuild, "Traceback", "a broken member surfaced as a traceback") + + null + } + + let userMember(name: String!, requiresPython: String!): String! { + "[project]\nname = \"" + name + "\"\nversion = \"0.1.0\"\nrequires-python = \"" + requiresPython + "\"\n" + + "[project.optional-dependencies]\nextra = []\n\n" + + "[build-system]\nrequires = [\"uv_build>=0.8.4,<0.12.0\"]\nbuild-backend = \"uv_build\"\n" + } + let assertMembersInstalled(seeded: Workspace!, what: String!): Void { + let generated = assertInstalled(seeded, what) + let out = pythonSdkRuntime + .moduleRuntime(modSource: runtimeSource(generated, tomlGenerateModulePath), introspectionJson: null) + .withExec(["python", "-c", + "import importlib.util as u, sys\n" + + "sys.exit('used is missing' if u.find_spec('used') is None else 'unused was installed' if u.find_spec('unused') else print('members') or 0)\n", + ]) + .stdout + assertContains(out, "members", what + ": " + out) null } + """The generated module, after its build found each member installed.""" + let assertInstalled(seeded: Workspace!, what: String!): Workspace! { + let generated = pythonSdk + .generateScope(seeded.withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: [seeded.moduleSource("/" + clientDepPath)]) + .withWorkdir(".") + let out = pythonSdkRuntime + .moduleRuntime(modSource: runtimeSource(generated, tomlGenerateModulePath), introspectionJson: null) + .withExec(["python", "-c", installedProbe]) + .stdout + assertContains(out, "installed", what + ": " + out) + generated + } + + """ + A generated module as the runtime module takes it, which is how both + entrypoints build it too: with its entrypoint table swapped for the builtin + runtime, since a module source handed to a runtime names one. + """ + let runtimeSource(generated: Workspace!, path: String!): ModuleSource! { + let manifestPath = "/" + path + "/dagger-module.toml" + generated + .withNewFile(manifestPath, generated.file(manifestPath).contents.replaceMatches("(?ms)^\\[entrypoint\\].*?(^\\[|\\z)", "[runtime]\n source = \"python\"\n\n$1")) + .moduleSource("/" + path) + } + """ - The fixture as seen from sdk-sdk's scratch workspace, where this repository - is vendored. + Finds each package without importing it, so the check holds whatever the + SDK files import. The module's source is mounted under /src, so an origin + there is an editable install. """ - let runtimeFixturePath: String! = "vendor/sdk-workspace/" + fixtureRoot + "/runtime/app" + let installedProbe: String! = + "import importlib.util as u, sys\n" + + "for name in ('dagger', 'dagger_clients.core', 'dagger_clients.client_dep'):\n" + + " spec = u.find_spec(name)\n" + + " if spec is None or spec.origin.startswith('/src/'):\n" + + " sys.exit(name + ' is not installed from its member: ' + str(spec and spec.origin))\n" + + "print('installed')\n" + + """ + A module calls the client it declared: its code imports the client and + calls the client's module through the engine, on this repository's + runtime, driven through the floor release's CLI. + """ + pub runtimeClientCallCheck(ws: Workspace!): Void @check { + let sourcePath = "/" + tomlGenerateModulePath + "/src/toml_generate_app/__init__.py" + let calling = ws.withNewFile(sourcePath, + "from dagger import function, object_type\n" + + "from dagger_clients.client_dep import client_dep\n\n\n" + + "@object_type\n" + + "class TomlGenerateApp:\n" + + " @function\n" + + " async def greeting(self) -> str:\n" + + " return await client_dep().greeting()\n") + let generated = pythonSdk + .generateScope(calling.withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: [calling.moduleSource("/" + clientDepPath)]) + .withWorkdir(".") + # The runtime of this working tree, as the runtime fixture names it. + .withNewFile("/" + tomlGenerateModulePath + "/dagger-module.toml", runtimeOnlyManifest("toml-generate-app")) + let called = cli(playground(generated), ["-m", tomlGenerateModulePath, "call", "greeting"]).stdout + assertContains(called, "hello from the client dependency", "the module did not call its client: " + called) + + null + } + + """ + An upgraded module's code runs unchanged through the global client: `dag` + and `dagger.Directory` are core's, on this repository's runtime, driven + through the floor release's CLI. No client is called, so no module is loaded. + """ + pub runtimeGlobalClientCallCheck(ws: Workspace!): Void @check { + let sourcePath = "/" + runtimeModulePath + "/src/runtime_app/__init__.py" + let legacy = ws.withNewFile(sourcePath, + "import dagger\n" + + "from dagger import dag, function, object_type\n\n\n" + + "@object_type\n" + + "class RuntimeApp:\n" + + " @function\n" + + " async def greeting(self) -> str:\n" + + " d = dag.directory().with_new_file(\"a\", \"served through the global client\")\n" + + " assert isinstance(d, dagger.Directory), type(d)\n" + + " return await d.file(\"a\").contents() + \" by \" + type(d).__module__\n") + let upgraded = pythonSdk + .generateScope(legacy.withWorkdir(runtimeModulePath), isModule: true, name: "runtime-app", clients: []) + .withWorkdir(".") + .withNewFile("/" + runtimeModulePath + "/dagger-module.toml", runtimeOnlyManifest("runtime-app")) + let called = cli(playground(upgraded), ["-m", runtimeModulePath, "call", "greeting"]).stdout + assertContains(called, "served through the global client by dagger_clients.core", "the upgraded module did not run through the global client: " + called) + + null + } + + """ + A manifest on this working tree's runtime and nothing else, so a released + CLI runs the module on this repository's build, not a published entrypoint. + """ + let runtimeOnlyManifest(name: String!): String! { + "name = \"" + name + "\"\nengineVersion = \"v1.0.0-0\"\n\n[runtime]\nsource = \"../../../../../../runtime\"\n" + } let sharedEntrypointSource: String! = "dagger.io/sdk/python/entrypoint@v1" """The manifest generating the toml-generate fixture leaves, from a seeded workspace.""" - let fatManifestOf(seeded: Workspace!): String! { + let manifestOf(seeded: Workspace!): String! { + manifestWithClients(seeded, []) + } + + let manifestWithClients(seeded: Workspace!, clients: [ModuleSource!]!): String! { pythonSdk - .generateScope(seeded.withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: []) + .generateScope(seeded.withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: clients) .withWorkdir(".") .file("/" + tomlGenerateModulePath + "/dagger-module.toml").contents } """ - A module on the runtime path gets a fat manifest: the builtin runtime for an - engine that predates entrypoints, and the shared Dang entrypoint for one that - loads manifest version 2. + A module runs on a Dang entrypoint and on nothing else: generation names + the shared entrypoint and no runtime. The builtin runtime this SDK wrote + before goes, with the engine version only a runtime reads. - A stale module-kind entrypoint is replaced, because that engine rejects the + A stale module-kind entrypoint is replaced, because the engine rejects the kind. A Dang entrypoint the user wrote is kept: a pinned version, or a fork. - The manifests carry their keys in either order and a "[" inside a comment, - which must not be read as the start of the next table. + A runtime of the user's own is kept as it is, with no entrypoint put in + front of it for the engine to follow instead, and refused beside one. What + only a runtime manifest + can say, which files the module is and where its source sits, is refused + rather than dropped. The manifests carry their keys in either order and a + "[" inside a comment, which must not be read as the start of the next table. """ - pub fatManifestCheck(ws: Workspace!): Void @check { + pub entrypointManifestCheck(ws: Workspace!): Void @check { let manifestPath = "/" + tomlGenerateModulePath + "/dagger-module.toml" let name = "toml-generate-app" + let head = "name = \"" + name + "\"\nengineVersion = \"v1.0.0-0\"\n" let runtimeTable = "[runtime]\nsource = \"python\"\n" + let noRuntime = ["[runtime]", "engineVersion"] - let plain = fatManifestOf(ws) - assertContainsAll(plain, ["[runtime]", "source = \"python\"", "[entrypoint]", "kind = \"dang\"", "source = \"" + sharedEntrypointSource + "\""]) + let plain = manifestOf(ws) + assertContainsAll(plain, ["[entrypoint]", "kind = \"dang\"", "source = \"" + sharedEntrypointSource + "\""]) + assertContainsNone(plain, noRuntime) - let stale = fatManifestOf(ws.withNewFile(manifestPath, - "name = \"" + name + "\"\nengineVersion = \"v1.0.0-0\"\n\n" + + let stale = manifestOf(ws.withNewFile(manifestPath, head + "\n" + "[entrypoint]\nkind = \"module\"\nsource = \"github.com/dagger/python-sdk/runtime\" # [was this]\n\n" + runtimeTable)) - assertContainsAll(stale, ["[runtime]", "kind = \"dang\"", "source = \"" + sharedEntrypointSource + "\""]) - assertContainsNone(stale, ["kind = \"module\"", "python-sdk/runtime"]) + assertContainsAll(stale, ["kind = \"dang\"", "source = \"" + sharedEntrypointSource + "\""]) + assertContainsNone(stale, noRuntime + ["kind = \"module\"", "python-sdk/runtime"]) let fork = "github.com/someone/python-sdk/entrypoint@entrypoint/v1" - let pinned = fatManifestOf(ws.withNewFile(manifestPath, - "name = \"" + name + "\"\nengineVersion = \"v1.0.0-0\"\n\n" + runtimeTable + "\n" + + let pinned = manifestOf(ws.withNewFile(manifestPath, head + "\n" + runtimeTable + "\n" + "[entrypoint]\n# source first, and a bracket [here] in a comment\n" + "source = \"" + fork + "\"\nkind = \"dang\"\n")) - assertContainsAll(pinned, ["[runtime]", "kind = \"dang\"", "source = \"" + fork + "\""]) + assertContainsAll(pinned, ["kind = \"dang\"", "source = \"" + fork + "\""]) + assertContainsNone(pinned, noRuntime) assertNotContains(pinned, sharedEntrypointSource + "\"", "a Dang entrypoint the user wrote was overwritten") - # A static entrypoint is told from the shared one by its source, not its kind. + let ownRuntime = head + "\n[runtime]\nsource = \"../../../../../../runtime\"\n" + let own = manifestOf(ws.withNewFile(manifestPath, ownRuntime)) + assertContainsAll(own, ["[runtime]", "source = \"../../../../../../runtime\"", "engineVersion = \"v1.0.0-0\""]) + assertNotContains(own, "[entrypoint]", "an entrypoint was put in front of the user's own runtime") + + # A runtime of the user's own beside an entrypoint is refused, whichever + # entrypoint: the engine would run the entrypoint on the session's + # version, and generation would take core from the runtime's. + let bothShared = manifestOf(ws.withNewFile(manifestPath, ownRuntime + "\n[entrypoint]\nkind = \"dang\"\nsource = \"" + sharedEntrypointSource + "\"\n")) rescue { + err: Error => err.message + } + assertContainsAll(bothShared, ["both a runtime", "../../../../../../runtime", sharedEntrypointSource, "Remove [entrypoint]"]) + let bothStatic = manifestOf(ws.withNewFile(manifestPath, ownRuntime + "\n[entrypoint]\nkind = \"dang\"\nsource = \"./sdk/entrypoint\"\n")) rescue { + err: Error => err.message + } + assertContainsAll(bothStatic, ["both a runtime", "./sdk/entrypoint"]) + + let included = manifestOf(ws.withNewFile(manifestPath, head + "include = [\"src\"]\n\n" + runtimeTable)) rescue { + err: Error => err.message + } + assertContainsAll(included, ["cannot carry", "include"]) + let elsewhere = manifestOf(ws.withNewFile(manifestPath, head + "source = \"sub\"\n\n" + runtimeTable)) rescue { + err: Error => err.message + } + assertContainsAll(elsewhere, ["cannot carry", "source"]) + let here = manifestOf(ws.withNewFile(manifestPath, head + "source = \".\"\n\n" + runtimeTable)) + assertContains(here, "[entrypoint]", "source = \".\" was refused") + let jsonIncluded = pythonSdk + .generateScope( + ws.withNewFile("/" + generateModulePath + "/dagger.json", "{\"name\": \"generate-app\", \"engineVersion\": \"v0.20.8\", \"sdk\": {\"source\": \"python\"}, \"exclude\": [\"tmp\"]}").withWorkdir(generateModulePath), + isModule: true, name: "generate-app", clients: [], + ) + .cwd rescue { + err: Error => err.message + } + assertContainsAll(jsonIncluded, ["dagger.json", "cannot carry", "exclude"]) + + # This SDK's static entrypoint is told from any other by its source, the + # path generation writes, not by its kind, its quoting or its key order. let static = ws.withNewFile(manifestPath, - "name = \"" + name + "\"\n\n[entrypoint]\nsource = \"./sdk/entrypoint\"\nkind = \"dang\"\n") + "name = \"" + name + "\"\n\n[ entrypoint ]\nsource = './sdk/entrypoint'\nkind = \"dang\"\n") assert(pythonSdk.mod(static, path: tomlGenerateModulePath, findUp: false).dangEntrypoint, "a static entrypoint written source-first was not recognised") let shared = ws.withNewFile(manifestPath, plain) assert(pythonSdk.mod(shared, path: tomlGenerateModulePath, findUp: false).dangEntrypoint == false, "the shared entrypoint was taken for a static one") + # An entrypoint this SDK did not write is the user's, and is kept as + # written: a path of their own in the module, and a fork quoted their way. + let ownPath = head + "\n[entrypoint]\nkind = \"dang\"\nsource = \"./my-entrypoint\"\n" + let ownManifest = manifestOf(ws.withNewFile(manifestPath, ownPath)) + assertContains(ownManifest, "source = \"./my-entrypoint\"", "a Dang entrypoint of the user's own was replaced: " + ownManifest) + assertNotContains(ownManifest, sharedEntrypointSource, "a Dang entrypoint of the user's own was replaced: " + ownManifest) + assert(pythonSdk.mod(ws.withNewFile(manifestPath, ownPath), path: tomlGenerateModulePath, findUp: false).dangEntrypoint == false, "an entrypoint of the user's own was taken for the static one") + let quoted = manifestOf(ws.withNewFile(manifestPath, head + runtimeTable + "\n[entrypoint]\nkind = 'dang'\nsource = '" + fork + "'\n")) + assertContains(quoted, fork, "a single-quoted Dang entrypoint was replaced: " + quoted) + assertNotContains(quoted, sharedEntrypointSource, "a single-quoted Dang entrypoint was replaced: " + quoted) + assertContainsNone(quoted, noRuntime) + + # Never a manifest that cannot run the module: a local client loads only + # through what the SDK's entrypoint hands over, so an entrypoint it does + # not write, which may not, is refused in a scope with one. + let local = [ws.moduleSource("/" + clientDepPath)] + let ownWithClient = manifestWithClients(ws.withNewFile(manifestPath, ownPath), local) rescue { + err: Error => err.message + } + assertContainsAll(ownWithClient, [ + "./my-entrypoint", "does not write", "local clients (client-dep)", + "update the entrypoint to hand them over", "entrypoint/handover.dang", "ClientHandover(workspace: workspace).clients", + "remove the [entrypoint] table", "remove the local clients", + ]) + # A fork that does what the refusal asks is accepted, and kept as written. + let forkPath = "/" + tomlGenerateModulePath + "/my-entrypoint" + let handingFork = manifestWithClients( + ws.withNewFile(manifestPath, ownPath).withDirectory(forkPath, ws.directory("/entrypoint")), + local, + ) + assertContains(handingFork, "source = \"./my-entrypoint\"", "a fork that hands clients over was refused or replaced: " + handingFork) + # Carrying the file is not enough: the fork must call it. + let idleFork = manifestWithClients( + ws.withNewFile(manifestPath, ownPath) + .withNewFile(forkPath + "/handover.dang", ws.file("/entrypoint/handover.dang").contents) + .withNewFile(forkPath + "/main.dang", "type Entrypoint {\n}\n"), + local, + ) rescue { + err: Error => err.message + } + assertContains(idleFork, "update the entrypoint to hand them over", "a fork that never calls the handover was accepted: " + idleFork) + let pinnedShared = "dagger.io/sdk/python/entrypoint@v1.0.0" + let pinnedWithClient = manifestWithClients(ws.withNewFile(manifestPath, head + "\n[entrypoint]\nkind = \"dang\"\nsource = \"" + pinnedShared + "\"\n"), local) rescue { + err: Error => err.message + } + assertContainsAll(pinnedWithClient, [pinnedShared, "does not write", "local clients (client-dep)"]) + let sharedWithClient = manifestWithClients(ws.withNewFile(manifestPath, plain), local) + assertContains(sharedWithClient, "source = \"" + sharedEntrypointSource + "\"", "the SDK's own entrypoint was refused with a local client: " + sharedWithClient) + null } + """ + The released CLI lists this checkout as the + python SDK, initializes a module from it, and runs it. The module's manifest names the shared + entrypoint and no runtime; it runs on this checkout's copy of that + entrypoint, since the published one predates this layout. + """ + pub cliSdkCheck(ws: Workspace!): Void @check { + let path = ".dagger/modules/sdk-smoke" + let listed = cli(playground(ws), ["sdk", "list"]) + assert(listed.stdout.contains("python"), "the development Python SDK was not listed") + + let initialized = cli(listed, ["module", "init", "python", "--auto-apply", "--name", "sdk-smoke", "--path", path]) + .withExec(["test", "!", "-e", path + "/dagger.json"]) + .withExec(["test", "-f", path + "/pyproject.toml"]) + .withExec(["test", "-f", path + "/src/sdk_smoke/__init__.py"]) + .withExec(["test", "-f", path + "/sdk/pyproject.toml"]) + .withExec(["test", "-f", path + "/clients/core/src/dagger_clients/core/__init__.py"]) + .withExec(["test", "!", "-e", path + "/" + legacyBindingsPath]) + let manifest = initialized.file(playgroundRoot + "/" + path + "/dagger-module.toml").contents + assertContainsAll(manifest, ["[entrypoint]", "kind = \"dang\"", "source = \"" + sharedEntrypointSource + "\""]) + assertContainsNone(manifest, ["[runtime]", "engineVersion"]) + + let release = cli(onCheckoutEntrypoint(initialized, ws, path, "sdk-smoke"), ["-m", path, "call", "container", "file", "--path", "/etc/alpine-release", "contents"]) + .stdout + assert(release.trimSpace != "", "the generated module did not run") + + cli(initialized, [ + "module", "init", "python", "--auto-apply", + "--name", "sdk-settings", + "--path", ".dagger/modules/sdk-settings", + "--template", "empty", + "--python-version", "3.13", + "--use-uv=false", + "--base-image", "python:3.13-slim", + ]) + .withExec(["grep", "-q", "class SdkSettings:", ".dagger/modules/sdk-settings/src/sdk_settings/__init__.py"]) + .withExec(["grep", "-q", ">=3.13", ".dagger/modules/sdk-settings/pyproject.toml"]) + .withExec(["grep", "-q", "use-uv = false", ".dagger/modules/sdk-settings/pyproject.toml"]) + .withExec(["grep", "-q", "python:3.13-slim", ".dagger/modules/sdk-settings/pyproject.toml"]) + .sync + + null + } + + """ + A module on an entrypoint calls a local client whose module is on an + entrypoint too, in both forms: this checkout's shared entrypoint, and the + one `--dang-entrypoint` generates into the module. The released CLI drives + it from `dagger module init` to `dagger call`. + + Under an entrypoint a module's code is not the module to the engine, and + its own workspace is its container's: the client's target resolves only + through what the entrypoint hands over, and the target's entrypoint, + called from that process, finds its module only in its own source. + + What it hands over is the declared client and never the caller's + workspace: the module's code tries every ID it holds against a file that + is only in the caller's workspace, and reads nothing. A function that + declares a Workspace parameter still gets the caller's, because the caller + passes it. + """ + pub entrypointClientCallCheck(ws: Workspace!): Void @check { + entrypointClientCall(ws, "shared", "Shared") + entrypointClientCall(ws, "static", "Static") + null + } + + let entrypointClientCall(ws: Workspace!, form: String!, title: String!): Void { + let target = "ep-target-" + form + let caller = "ep-caller-" + form + let targetPath = ".dagger/modules/" + target + let callerPath = ".dagger/modules/" + caller + let static = if (form == "static") { ["--dang-entrypoint"] } else { [] :: [String!]! } + let greeting = "hello from the " + form + " target" + let targetSource = + "from dagger import function, object_type\n\n\n" + + "@object_type\n" + + "class EpTarget" + title + ":\n" + + " @function\n" + + " def greeting(self) -> str:\n" + + " return \"" + greeting + "\"\n" + let callerSource = + "from dagger import function, object_type\n" + + "from dagger_clients.core import Workspace\n" + + "from dagger_clients.ep_target_" + form + " import ep_target_" + form + "\n\n" + + "from .reach import reach\n\n\n" + + "@object_type\n" + + "class EpCaller" + title + ":\n" + + " @function\n" + + " async def greeting(self) -> str:\n" + + " return await ep_target_" + form + "().greeting()\n\n" + + " @function\n" + + " async def reach(self) -> str:\n" + + " await ep_target_" + form + "().greeting()\n" + + " return await reach(\"" + callerSecretPath + "\")\n\n" + + " @function\n" + + " async def declared(self, ws: Workspace) -> str:\n" + + " return await ws.file(\"/" + callerSecretPath + "\").contents()\n" + + let withTarget = cli(playground(ws), ["module", "init", "python", "--auto-apply", "--name", target, "--path", targetPath] + static) + .withNewFile(targetPath + "/src/ep_target_" + form + "/__init__.py", targetSource) + # A static entrypoint carries the digests of the source it was generated from. + let targetReady = if (form == "static") { + cli(withTarget, ["-y", "call", "python-sdk", "mod", "--path", targetPath, "generate"]) + } else { + onCheckoutEntrypoint(withTarget, ws, targetPath, target) + } + # The caller's code is in place before its client is added, so the static + # entrypoint generated with the client describes it. + let withClient = cli(targetReady, ["module", "init", "python", "--auto-apply", "--name", caller, "--path", callerPath] + static) + .withNewFile(callerPath + "/src/ep_caller_" + form + "/__init__.py", callerSource) + .withFile(callerPath + "/src/ep_caller_" + form + "/reach.py", ws.file("/" + fixtureRoot + "/handover/reach.py")) + .withWorkdir(callerPath) + let added = cli(withClient, ["module", "client", "add", "../" + target, "--sdk", "python", "--auto-apply"]) + .withWorkdir(playgroundRoot) + let callerReady = if (form == "static") { added } else { onCheckoutEntrypoint(added, ws, callerPath, caller) } + let manifest = callerReady.file(playgroundRoot + "/" + callerPath + "/dagger-module.toml").contents + assertContains(manifest, "[entrypoint]", form + ": the caller is not on an entrypoint: " + manifest) + assertNotContains(manifest, "[runtime]", form + ": the caller names a runtime: " + manifest) + + let called = cli(callerReady, ["-m", callerPath, "call", "greeting"]).stdout + assertContains(called, greeting, form + ": the module did not call its client: " + called) + + let secret = "only in the caller's workspace, " + form + let withSecret = callerReady.withNewFile(callerSecretPath, secret) + let reached = cli(withSecret, ["-m", callerPath, "call", "reach"]).stdout + assertContains(reached, "reads=[]", form + ": module code read a caller file: " + reached) + assertNotContains(reached, secret, form + ": module code read a caller file: " + reached) + # The probe proves something only if it held an ID to try, exercised many + # routes, and hit no unknown field: an absent route errored for the wrong + # reason and is no evidence of safety (fixtures/handover/reach.py). + assertNotContains(reached, "held=0 ", form + ": the module held nothing to try, so this proves nothing: " + reached) + assertContains(reached, "absent=0 ", form + ": a probe route hit an unknown field, so it proves nothing: " + reached) + assertNotContains(reached, "routes=0 ", form + ": the probe ran no routes: " + reached) + let declared = cli(withSecret, ["-m", callerPath, "call", "declared"]).stdout + assertContains(declared, secret, form + ": a declared Workspace parameter did not reach the caller's workspace: " + declared) + null + } + + """ + A file at the root of the caller's workspace, which no module is handed. + """ + let callerSecretPath: String! = "review-secret.txt" + + """ + The first release that runs Dang entrypoints and serves serveModule, which + is this SDK's floor. Its engine image carries its CLI, and git. + """ + let floorVersion: String! = "1.0.0-beta.14" + let floorImage: String! { "registry.dagger.io/engine:v" + floorVersion } + + let playgroundRoot: String! = "/work/python-sdk" + + """ + This checkout, with the workspace config that registers it as the python + SDK, in a container with the floor release's CLI. The CLI is a nested + client of the engine running the checks, and its workspace is this tree. + + So the engine is whichever runs the checks. A pass proves the floor only + on the floor's engine, which the floor job (hack/e2e-floor.sh) runs them on + and asserts with assertFloorEngine; an ordinary run takes the engine it + finds. + """ + let playground(ws: Workspace!): Container! { + playgroundOf(playgroundTree(ws)) + } + + """ + The files of a workspace that the playground mounts. + """ + let playgroundTree(ws: Workspace!): Directory! { + ws.directory("/", exclude: [".git", "**/.venv", "**/__pycache__", outputRoot]) + } + + """ + The playground over a tree a check has changed. + """ + let playgroundOf(tree: Directory!): Container! { + container + .from(floorImage) + .withoutEntrypoint + .withMountedDirectory(playgroundRoot, tree) + .withWorkdir(playgroundRoot) + .withExec(["git", "init", "--quiet"]) + } + + """ + The floor job's precondition, not a check of its own: the engine the + playground's CLI talks to is the floor release, by the version it reports + itself. A release reports its build after a "+", which is not part of the + version; a dev or later build is refused. hack/e2e-floor.sh calls it before + the checks, so that job cannot pass on another engine. + """ + pub assertFloorEngine(ws: Workspace!): Void { + let answered = playground(ws).withExec(["dagger", "query"], stdin: "{ version }", experimentalPrivilegedNesting: true).stdout + let reported = json.withContents((answered :: Dagger.JSON!)).field(["version"]).asString + if ((reported.split("+")[0] ?? "") != "v" + floorVersion) { + raise "the checks drive the floor release's CLI, v" + floorVersion + ", against the engine running them, which is " + + reported + ": a pass would not prove the floor. Run the floor job, which provisions its engine: hack/e2e-floor.sh" + } + null + } + + let cli(ctr: Container!, args: [String!]!): Container! { + ctr.withExec(["dagger"] + args, experimentalPrivilegedNesting: true) + } + + """ + A module in the playground on this checkout's shared entrypoint, not the + published one its manifest names, which predates this layout. The copy sits + inside the module: the engine takes an entrypoint only from a git ref or a + path in the module. + """ + let onCheckoutEntrypoint(ctr: Container!, ws: Workspace!, path: String!, name: String!): Container! { + ctr + .withDirectory(path + "/checkout-entrypoint", ws.directory("/entrypoint")) + .withNewFile(path + "/dagger-module.toml", "name = \"" + name + "\"\n\n[entrypoint]\nkind = \"dang\"\nsource = \"./checkout-entrypoint\"\n") + } + """ runtime/build.dang as the shared entrypoint carries it. @@ -562,6 +1450,18 @@ type E2e { assertContains(partial.after.file(configuredPyproj).contents, "use-uv = false", "omitting useUv should leave it untouched") assertContains(partial.after.file(configuredPyproj).contents, "python:3.12-slim", "omitting baseImage should leave it untouched") + # The global client flag reads and writes like use-uv: unset is null, and + # false, the default, writes nothing. + assert(appValues.globalClient == null, "globalClient should be reported as unset") + let on = app.set(globalClient: true) + assert(on.modifiedPaths.length == 1, "setting globalClient modified more than pyproject.toml") + assertContains(on.after.file(pyproj).contents, "global-client = true", "set did not write the global client flag") + let onWs = ws.withChanges(on) + assert(pythonSdk.mod(onWs, path: configModulePath).config.get.globalClient == true, "globalClient should read true once set") + let off = pythonSdk.mod(onWs, path: configModulePath).config.set(globalClient: false) + assertNotContains(off.after.file(pyproj).contents, "global-client", "setting globalClient to false should remove the flag") + assert(off.after.file(pyproj).contents == on.before.file(pyproj).contents, "setting globalClient true then false changed more than the flag") + null } @@ -584,15 +1484,25 @@ type E2e { "unexpected static manifest: " + manifest, ) assertAdded(changes, scope + "/src/static_init/__init__.py") - # Fields the oldest schema view lacks: the bindings came from the current one. - assertContainsAll(changes.layer.file(scope + "/" + generatedMarkerPath).contents, ["def cwd(", "def with_new_file("]) + # Fields the oldest schema view lacks: core came from the current one. + assertContainsAll(changes.layer.file(scope + "/" + coreInitPath).contents, ["def cwd(", "def with_new_file("]) let main = changes.layer.file(scope + "/sdk/entrypoint/main.dang").contents assertContainsAll(main, [ "implements ModuleEntrypoint", "let moduleName: String! = \"static-init\"", - "let modulePath: String! = \"" + scope + "\"", + "clients: ClientHandover(workspace: workspace).clients", + "let module = currentModule.source", ]) + # The declared clients go with the call, never the workspace, and the + # static entrypoint hands them over with the shared one's own code. + assertNotContains(main, "workspace.id", "the static entrypoint hands the module the caller's workspace") + assert( + changes.layer.file(scope + "/sdk/entrypoint/handover.dang").contents == ws.file("/entrypoint/handover.dang").contents, + "the static entrypoint does not carry the shared entrypoint's handover", + ) + # Found through its own source, so a moved module needs no regeneration. + assertNotContains(main, "modulePath", "the entrypoint still finds the module through the workspace") ["pyproject.toml", "src/static_init/__init__.py"].each { path => let digest = changes.after.directory(scope).file(path).digest(excludeMetadata: true) assertContains(main, "SourceFile(path: \"" + path + "\", digest: \"" + digest + "\")", "the baked digest of " + path + " is not the engine's") @@ -611,7 +1521,7 @@ type E2e { """ A module moves between the two paths by flipping the setting: the static Dang entrypoint appears, then disappears again, then comes back the same. Switching - back leaves the fat manifest: the runtime, and the shared Dang entrypoint. + back leaves the shared Dang entrypoint, and no runtime. """ pub staticScopeSwitchCheck(ws: Workspace!): Void @check { let manifestPath = tomlGenerateModulePath + "/dagger-module.toml" @@ -628,7 +1538,8 @@ type E2e { let dynamic = pythonSdk.generateScope(static.withWorkdir(tomlGenerateModulePath), isModule: true, name: name, clients: []).withWorkdir(".") let manifest = dynamic.file("/" + manifestPath).contents - assertContainsAll(manifest, ["name = \"toml-generate-app\"", "[runtime]", "source = \"python\""]) + assertContains(manifest, "name = \"toml-generate-app\"", "switching back lost the module name") + assertContainsNone(manifest, ["[runtime]", "engineVersion"]) assertNotContains(manifest, "./sdk/entrypoint", "switching back kept the static entrypoint in the manifest") assertContainsAll(manifest, ["kind = \"dang\"", "source = \"" + sharedEntrypointSource + "\""]) assert(dynamic.directory("/" + tomlGenerateModulePath).exists("sdk/entrypoint") == false, "switching back left the entrypoint behind") @@ -650,8 +1561,11 @@ type E2e { let manifestPath = "/" + tomlGenerateModulePath + "/dagger-module.toml" let runtimeManifest = "name = \"toml-generate-app\"\nengineVersion = \"v1.0.0-0\"\n\n[runtime]\nsource = \"python\"\n" - let withClient = static.generateScope(scoped, isModule: true, name: name, clients: [ws.moduleSource("/" + clientDepPath)]).cwd rescue "raised" - assert(withClient == "raised", "a static entrypoint accepted a dependency") + # A client is a member of the scope, not a dependency the manifest must + # carry, so the static path takes one like the runtime path. + let withClient = static.generateScope(scoped, isModule: true, name: name, clients: [ws.moduleSource("/" + clientDepPath)]) + assertContains(withClient.file(clientDepTargetPath).contents, "REF = \"/" + clientDepPath + "\"", "a static entrypoint refused a client") + assertContains(withClient.file("dagger-module.toml").contents, "./sdk/entrypoint", "a client took the module off the static path") let legacyScope = outputRoot + "/static-legacy" let legacy = pythonSdk(dangEntrypoint: true, template: "legacy") @@ -665,6 +1579,14 @@ type E2e { let included = static.generateScope(ws.withNewFile(manifestPath, "include = [\"src\"]\n" + runtimeManifest).withWorkdir(tomlGenerateModulePath), isModule: true, name: name, clients: []).cwd rescue "raised" assert(included == "raised", "an include list was dropped silently") + # Read as TOML: the builtin runtime quoted another way is still the one + # this SDK wrote, not a runtime of the user's own. + let quotedRuntime = static.generateScope(ws.withNewFile(manifestPath, "name = 'toml-generate-app'\n\n[runtime]\nsource = 'python'\n").withWorkdir(tomlGenerateModulePath), isModule: true, name: name, clients: []) + .file("/" + tomlGenerateModulePath + "/dagger-module.toml").contents rescue { + err: Error => err.message + } + assertContains(quotedRuntime, "./sdk/entrypoint", "a single-quoted builtin runtime was refused on the static path: " + quotedRuntime) + # A module rooted at its own directory is what an entrypoint manifest # assumes, so this one generates instead of being refused. let selfSourced = static.generateScope(ws.withNewFile(manifestPath, "source = \".\"\n" + runtimeManifest).withWorkdir(tomlGenerateModulePath), isModule: true, name: name, clients: []) @@ -691,11 +1613,9 @@ type E2e { .generateScope(ws.withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: []) .withWorkdir(".") let types = generated.file("/" + tomlGenerateModulePath + "/sdk/entrypoint/types.dang") - let run = sdkSdk - .target(ws.directory("/").withFile(staticTypesModulePath + "/types.dang", types), ".") - .runInstalled(["call", "-m", "vendor/sdk-workspace/" + staticTypesModulePath, "count"]) - run.assertSuccess - assert(run.stdout.trimSpace == "1", "types.dang did not evaluate to the fixture's one type: " + run.stdout) + let tree = playgroundTree(ws).withFile(staticTypesModulePath + "/types.dang", types) + let counted = cli(playgroundOf(tree), ["-m", staticTypesModulePath, "call", "count"]).stdout + assert(counted.trimSpace == "1", "types.dang did not evaluate to the fixture's one type: " + counted) null } diff --git a/.dagger/modules/engine-e2e/dagger-module.toml b/.dagger/modules/engine-e2e/dagger-module.toml deleted file mode 100644 index d370e49..0000000 --- a/.dagger/modules/engine-e2e/dagger-module.toml +++ /dev/null @@ -1,9 +0,0 @@ -name = "engine-e2e" -engineVersion = "v1.0.0-0" - -[runtime] - source = "dang" - -[[dependencies]] - name = "engine-dev" - source = "github.com/dagger/dagger/.dagger/modules/engine-dev@6bf59d50654ce9244ebeee1cc090b7dce3fe3083" diff --git a/.dagger/modules/engine-e2e/main.dang b/.dagger/modules/engine-e2e/main.dang deleted file mode 100644 index 88a0b82..0000000 --- a/.dagger/modules/engine-e2e/main.dang +++ /dev/null @@ -1,67 +0,0 @@ -""" -Checks the Python SDK against the v1.0.0-beta.13 engine, pinned by commit. -Keep engineCommit and the engine-dev dependency in dagger-module.toml aligned. -""" -type EngineE2e { - let engineCommit: String! = "6bf59d50654ce9244ebeee1cc090b7dce3fe3083" - let modulePath: String! = ".dagger/modules/sdk-smoke" - - let assert(condition: Boolean!, message: String!): Void { - if (condition == false) { - raise message - } - null - } - - """ - The pinned main engine should list this checkout as the python SDK, - initialize a Python module from it, and run the module. - """ - pub devSdkCheck(ws: Workspace!): Void @check { - let sdkSource = ws.directory("/", exclude: [".git", ".remember", "**/.venv", "**/__pycache__", ".dagger/modules/e2e/out"]) - .withFile("dagger.toml", ws.file("/.dagger/modules/engine-e2e/workspace.toml")) - let engineSource = git("https://github.com/dagger/dagger") - .ref(engineCommit) - .asWorkspace - let client = engineDev(ws: engineSource) - .playground - .withMountedDirectory("./python-sdk", sdkSource) - .withWorkdir("./python-sdk") - .withExec(["git", "init", "--quiet"]) - - let listed = client.withExec(["dagger", "sdk", "list"]) - assert(listed.stdout.contains("python"), "the development Python SDK was not listed") - - let checked = listed.withExec(["dagger", "check"]) - - let initialized = checked - .withExec(["dagger", "module", "init", "python", "--auto-apply", "--name", "sdk-smoke", "--path", modulePath]) - .withExec(["test", "-f", modulePath + "/dagger-module.toml"]) - .withExec(["test", "!", "-e", modulePath + "/dagger.json"]) - .withExec(["test", "-f", modulePath + "/pyproject.toml"]) - .withExec(["test", "-f", modulePath + "/src/sdk_smoke/__init__.py"]) - .withExec(["test", "-f", modulePath + "/sdk/src/dagger/client/gen.py"]) - - let release = initialized - .withExec(["dagger", "-m", modulePath, "call", "container", "file", "--path", "/etc/alpine-release", "contents"]) - .stdout - assert(release.trimSpace != "", "the generated module did not run") - - initialized - .withExec([ - "dagger", "module", "init", "python", "--auto-apply", - "--name", "sdk-settings", - "--path", ".dagger/modules/sdk-settings", - "--template", "empty", - "--python-version", "3.13", - "--use-uv=false", - "--base-image", "python:3.13-slim", - ]) - .withExec(["grep", "-q", "class SdkSettings:", ".dagger/modules/sdk-settings/src/sdk_settings/__init__.py"]) - .withExec(["grep", "-q", ">=3.13", ".dagger/modules/sdk-settings/pyproject.toml"]) - .withExec(["grep", "-q", "use-uv = false", ".dagger/modules/sdk-settings/pyproject.toml"]) - .withExec(["grep", "-q", "python:3.13-slim", ".dagger/modules/sdk-settings/pyproject.toml"]) - - null - } -} diff --git a/.dagger/modules/engine-e2e/workspace.toml b/.dagger/modules/engine-e2e/workspace.toml deleted file mode 100644 index 063324d..0000000 --- a/.dagger/modules/engine-e2e/workspace.toml +++ /dev/null @@ -1,42 +0,0 @@ -# Dagger workspace configuration -# Install modules with: dagger module install -# Example: -# dagger module install github.com/dagger/dagger/modules/wolfi - -[modules.e2e] -source = ".dagger/modules/e2e" - -[modules.python-sdk] -source = "." -check.skip = ["*"] - -[sdks.python] -module = "python-sdk" - -[sdks.python.scopes.".dagger/modules/e2e/fixtures/generate/app"] -is-module = true -name = "generate-app" - -[sdks.python.scopes.".dagger/modules/e2e/fixtures/lookup/app"] -is-module = true -name = "lookup-app" - -[sdks.python.scopes.".dagger/modules/e2e/fixtures/toml/app"] -is-module = true -name = "toml-app" - -[sdks.python.scopes.".dagger/modules/e2e/fixtures/mixed-discovery/ancestor/work/app"] -is-module = true -name = "mixed-discovery-app" - -[sdks.python.scopes.".dagger/modules/e2e/fixtures/config/app"] -is-module = true -name = "config-app" - -[sdks.python.scopes.".dagger/modules/e2e/fixtures/config/configured"] -is-module = true -name = "config-configured" - -[sdks.python.scopes.".dagger/modules/e2e/fixtures/toml-generate/app"] -is-module = true -name = "toml-generate-app" diff --git a/README.md b/README.md index 32f8f06..95732b6 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ It uses the engine's native `Workspace` and `ModuleSource` APIs. It uses | --- | --- | | `python-sdk.dang`, `mod.dang`, `templates/` | authoring: `findClientRoot`, `generateScope`, `mod` (generate, config), templates | | `sdk/` | the `dagger-io` client library and code generator | -| `runtime/` | the module runtime the engine calls to run a module, and the container build the static entrypoint shares | +| `runtime/` | the container build both entrypoints share, and a module runtime for a manifest that names it | | `entrypoint/` | the shared Dang `ModuleEntrypoint`, served from this repository to any module that names it | Code generation happens at `dagger generate`, which calls `generateScope` for @@ -32,90 +32,86 @@ When a managed pre-1.0 `dagger.json` scope is generated, the SDK writes `dagger-module.toml` and removes `dagger.json`. An unmanaged legacy module keeps using the Python SDK that is built into the engine. -## Two runtimes, one name +## How a module runs -Python modules reach one of two implementations, and which one is decided by -the module's config format: +A module this SDK generates runs on a Dang entrypoint and on nothing else: +the shared one below, or with `--dang-entrypoint` one generated into the +module. Its manifest names no `[runtime]`. That needs an engine that runs +Dang entrypoints and serves `serveModule`: `v1.0.0-beta.14` or later. -- **Legacy** — an unmanaged `dagger.json` with `"sdk": {"source": "python"}` - resolves to the runtime built into the engine (`dagger/dagger`'s - `sdk/python`). It still generates bindings at module load. -- **Modern** — a `dagger-module.toml` can point `[runtime] source` at this - repository's `runtime/`, which is the no-codegen path above. Either a module - ref or a path relative to the module works, for both `dagger generate` and - `dagger call`. +Generating a module that has a manifest already: -The engine resolves the short name `python` to exactly one target, the -engine-baked runtime, so the modern path is reached by module ref rather than -by name. The manifest `generateScope` writes for a new module therefore still -names `python`; it moves to `github.com/dagger/python-sdk/runtime` in a -follow-up, once `runtime/` exists on the default branch for that ref to -resolve to. See -[`future/done/self-contained-python-sdk.md`](./future/done/self-contained-python-sdk.md) -for the full reasoning and for the engine change that would let one name serve -both. +- `[runtime] source = "python"`, which this SDK wrote before, is replaced by + the entrypoint, and `engineVersion` and `[[dependencies]]` go with it. An + entrypoint runs a module on the engine's own version. +- Any other `[runtime]` is the user's choice, and the manifest keeps it as + written, with no entrypoint added: the engine follows an entrypoint over a + runtime. `runtime/` is such a runtime, named by module ref or by a path + relative to the module. +- `include`, `exclude`, or a `source` other than `.` next to the builtin + runtime are refused, because an entrypoint manifest cannot carry them and + dropping them would change which files the module is. -### Trying this repository's runtime - -A module created today names the `python` runtime, so it runs on the -engine's runtime. To move one onto this repository's runtime, point it there by -hand: - -```toml -# /dagger-module.toml -[runtime] -source = "github.com/dagger/python-sdk/runtime" -``` - -Then `dagger generate` the module and `dagger call` it as usual. The generated -files are identical either way — generation is this SDK's regardless of which -runtime runs the module — so switching back is just editing the line again. - -Within this repository, a path relative to the module works too, which is how -the end-to-end fixture exercises the runtime before the ref exists. +An unmanaged legacy `dagger.json` with `"sdk": {"source": "python"}` still +resolves to the runtime built into the engine (`dagger/dagger`'s +`sdk/python`), which generates bindings at module load. ## Shared entrypoint `entrypoint/` is one `ModuleEntrypoint`, written in Dang, that backs every Python module at once, with nothing generated into the module. `dagger generate` names it in the manifest of every module that does not use -`--dang-entrypoint`, next to the builtin runtime: +`--dang-entrypoint`: ```toml # /dagger-module.toml name = "my-module" -engineVersion = "v1.0.0" - -[runtime] -source = "python" [entrypoint] kind = "dang" source = "dagger.io/sdk/python/entrypoint@v1" ``` -One manifest then loads on both kinds of engine. An engine that predates -entrypoints ignores the table and runs the module on the runtime. An engine -that loads manifest version 2 drives the module through the entrypoint and -ignores `[runtime]`; when the module has `[[dependencies]]` it reads the -manifest the old way instead, because manifest version 2 has no dependency -list, and the runtime runs the module. - A Dang entrypoint already in the manifest is kept as written, so a module can -pin a version of the shared entrypoint or point at a fork. A static entrypoint -is told from the shared one by its source, a path inside the module. - -The entrypoint finds the module it serves through the workspace it is handed, -whose working directory is that module's directory. It reads the module's name -from that directory's manifest, builds the module's container with the same -build the runtime uses, and asks the module to describe itself -(`python -m dagger.mod describe`) or to run one call +pin a version of the shared entrypoint, point at a fork, or name one of its +own. Generation replaces only the static entrypoint it writes itself, told by +its source, `./sdk/entrypoint`. The manifest is read with a TOML parser, so +quoting and key order are the user's. One exception is refused rather than kept: an +entrypoint this SDK does not write, in a scope with a local client, that does +not hand clients over. A local client loads only through what an entrypoint +hands the module (below), so generation stops and says so instead of writing +a module that fails at its first call. A fork hands them over by carrying +`handover.dang` unchanged and sending +`clients: ClientHandover(workspace: workspace).clients…` with each call, as +`main.dang` does; generation reads the entrypoint where the engine does and +accepts it then. + +Inside an entrypoint `currentModule` is the module it serves, so the +entrypoint builds that module's container from `currentModule.source`, with +the same build the runtime uses, however the module was loaded. It asks the +module to describe itself (`python -m dagger.mod describe`) or to run one call (`python -m dagger.mod call`). The types it returns are rebuilt from that description in the engine's own session. +The module's code runs in an exec the entrypoint starts, which the engine does +not make the module: its own current workspace is the one found in its +container, so a client to a local module cannot resolve its path there. The +entrypoint resolves the clients the caller's `dagger.toml` declares on the +module's scope, and each call carries them by name, each as a module source +over only the files the engine loaded for that client. A client to a local +module loads through its entry (`node(id:)` → `asModule` → `serve`); a git +client goes through `serveModule`, as in a plain program. + +The caller's workspace never reaches the module's code: an ID is a +capability, and a module is third-party code. Not the workspace, and not the +source `Workspace.moduleSource` returns either, because that one reloads its +context from the workspace when asked for more files. A function that +declares a `Workspace` parameter still gets one, because its caller passes it. + | File | What it is | | --- | --- | | `main.dang` | the `ModuleEntrypoint`: `types` and `call` | +| `handover.dang` | the declared clients each call carries; a static entrypoint carries a copy | | `build.dang` | the container build, generated from `runtime/build.dang` | `build.dang` is generated, not hand-edited: the engine copies only the `.dang` @@ -144,8 +140,8 @@ the engine loads without running Python: dagger module init python --name my-module --dang-entrypoint ``` -Generating the module then writes an entrypoint manifest instead of a runtime -manifest, and `sdk/entrypoint/` next to the vendored library: +Generating the module then names that entrypoint in the manifest instead of +the shared one, and writes `sdk/entrypoint/` next to the vendored library: | File | What it is | | --- | --- | @@ -161,7 +157,7 @@ change them; a call after an edit is refused with a message to run only file contents count, not permissions. What the static path cannot do yet, and refuses at `dagger generate`: -module clients (an entrypoint manifest has no dependencies), any `cache=` value +any `cache=` value on a function (the entrypoint's exec is content-cached and receives no per-call signal), the `legacy` template, and a manifest with `include`, `disableDefaultFunctionCaching`, a runtime other than `python`, a `source` @@ -177,9 +173,7 @@ settings in `dagger.toml`, then `dagger generate`. Generating one module directly, with `dagger call python-sdk mod --path generate`, keeps the mode that module is in; switching modes goes through `dagger module init python --path ` as above. Switching back removes -`sdk/entrypoint/` and rewrites a runtime manifest with the generating -engine's version. Loading a static module needs an engine that reads an -entrypoint manifest (dagger/dagger#14038); see +`sdk/entrypoint/` and names the shared entrypoint again. See [`future/done/static-module-entrypoint.md`](./future/done/static-module-entrypoint.md) for the design and the plan to make it the default. diff --git a/dagger-module.toml b/dagger-module.toml index f6bc375..1ee6429 100644 --- a/dagger-module.toml +++ b/dagger-module.toml @@ -1,5 +1,5 @@ name = "python-sdk" -engineVersion = "v1.0.0-beta.11" +engineVersion = "v1.0.0-beta.14" include = ["!.dagger", "!future", "!docs"] [runtime] diff --git a/dagger.json b/dagger.json index 8da7783..0863906 100644 --- a/dagger.json +++ b/dagger.json @@ -1,6 +1,6 @@ { "name": "python-sdk", - "engineVersion": "v1.0.0-beta.11", + "engineVersion": "v1.0.0-beta.14", "sdk": { "source": "dang" }, diff --git a/dagger.toml b/dagger.toml index 2eec7e7..24bb764 100644 --- a/dagger.toml +++ b/dagger.toml @@ -1,4 +1,40 @@ -# Run SDK checks inside the development engine. +# This checkout as the workspace's python SDK, and the e2e checks of it, which +# run on the local engine: `dagger check`, or hack/e2e-local.sh for a copy. -[modules.engine-e2e] -source = ".dagger/modules/engine-e2e" +[modules.e2e] +source = ".dagger/modules/e2e" + +[modules.python-sdk] +source = "." +check.skip = ["*"] + +[sdks.python] +module = "python-sdk" + +[sdks.python.scopes.".dagger/modules/e2e/fixtures/generate/app"] +is-module = true +name = "generate-app" + +[sdks.python.scopes.".dagger/modules/e2e/fixtures/lookup/app"] +is-module = true +name = "lookup-app" + +[sdks.python.scopes.".dagger/modules/e2e/fixtures/toml/app"] +is-module = true +name = "toml-app" + +[sdks.python.scopes.".dagger/modules/e2e/fixtures/mixed-discovery/ancestor/work/app"] +is-module = true +name = "mixed-discovery-app" + +[sdks.python.scopes.".dagger/modules/e2e/fixtures/config/app"] +is-module = true +name = "config-app" + +[sdks.python.scopes.".dagger/modules/e2e/fixtures/config/configured"] +is-module = true +name = "config-configured" + +[sdks.python.scopes.".dagger/modules/e2e/fixtures/toml-generate/app"] +is-module = true +name = "toml-generate-app" diff --git a/entrypoint/build.dang b/entrypoint/build.dang index 55dd310..a3732d5 100644 --- a/entrypoint/build.dang +++ b/entrypoint/build.dang @@ -33,8 +33,8 @@ type PythonModuleBuild { let modSourceDirPath: String! = "/src" let venvPath: String! = "/opt/venv" let genDir: String! = "sdk" + let clientsDir: String! = "clients" let sdkGenPath: String! = "src/dagger/client/gen.py" - let userGenPath: String! = "src/dagger_gen.py" let projectCfg: String! = "pyproject.toml" let pipCompileLock: String! = "requirements.lock" let uvLock: String! = "uv.lock" @@ -62,8 +62,7 @@ type PythonModuleBuild { raise "no python files found in module source" } else { let cfg = pyConfig(source) - let vendorPath = vendorPathFor(source, cfg) - checkGeneratedFiles(source, moduleName, vendorPath) + checkGeneratedFiles(source, moduleName, requiredGeneratedFiles(source, cfg)) let baseImage = baseImageFor(source, cfg) let uvImage = uvImageFor(cfg) @@ -83,6 +82,13 @@ type PythonModuleBuild { } } + """ + The workspace members of a scope's pyproject.toml, read as the build does. + """ + pub scopeMembers(pyproject: File!): [String!]! { + pyConfig(directory.withFile(projectCfg, pyproject)).members + } + """ The class name of a module's main object, converted like the engine's builtin Python runtime (strcase.ToCamel) so a module loads the same on both. @@ -155,15 +161,18 @@ type PythonModuleBuild { } """ - The vendored library is installed non-editable so uv compiles its bytecode - once, not on every call into a throwaway mount; the module's own package - stays editable. + The SDK files and the generated clients are installed non-editable so uv + compiles their bytecode once, not on every call into a throwaway mount; the + module's own package stays editable. """ let install(ctr: Container!, source: Directory!, cfg: PyConfig!): Container! { let compiled = ctr.withEnvVariable("UV_COMPILE_BYTECODE", "1") + let packages = localPackages(source, cfg) if (cfg.useUv and source.exists(uvLock)) { # --locked: fail loudly on a stale lockfile instead of re-resolving. + # In a scope, sync installs the members the project depends on, which + # --no-install-project leaves in. compiled .withExec(["uv", "sync", "--no-dev", "--locked", "--no-editable", "--no-install-project"]) .withEnvVariable("VIRTUAL_ENV", "$UV_PROJECT_ENVIRONMENT", expand: true) @@ -178,22 +187,139 @@ type PythonModuleBuild { ["-r", projectCfg] } compiled - .withExec(["uv", "pip", "install", "--no-editable", "./" + genDir] + deps) + .withExec(["uv", "pip", "install", "--no-editable"] + packages + deps) .withExec(["uv", "pip", "install", "--no-deps", "-e", "."]) } else { - compiled.withExec(["pip", "install", "./" + genDir, "-e", "."]) + # pip reads no uv sources, so every local package goes in by path, in + # the same command that resolves the project's dependencies on them. + compiled.withExec(["pip", "install"] + packages + ["-e", "."]) } } """ - Fail early when the committed generated files are missing. + What the module installs from its own tree: the members of a scope's + workspace that the project depends on, directly or through another member, + the way `uv sync` picks them; or the vendored library of the layout before. + A member nothing depends on is not installed: it may not even install here. + """ + let localPackages(source: Directory!, cfg: PyConfig!): [String!]! { + if (cfg.isScope) { + workspaceRead(source) + .filter { line => line.hasPrefix("local\t") } + .map { line => line.trimPrefix("local\t") } + } else { + ["./" + genDir] + } + } + + """ + The workspace as TOML defines it, read by tomllib in the pinned default + image: `member\t` for each member, then `local\t./` for each + member directory to install. [project] dependencies and a member's are the + user's own content, with escapes, literal and multi-line strings, so the + regular expressions of pyConfig must not read them. Only the + pyproject.toml files that can be a member's go in, so the read is cached + until one of them changes. A file that does not read is named. """ - let checkGeneratedFiles(source: Directory!, modName: String!, vendorPath: String!): Void { - let required = if (vendorPath == "") { - [userGenPath] + let workspaceRead(source: Directory!): [String!]! { + let lines = container + .from(defaultBaseImage) + .withMountedDirectory("/scope", source.filter( + include: ["**/" + projectCfg], + # No member lives in these, and a virtualenv alone holds hundreds of + # pyproject.toml files that would reach the read and its cache key. + exclude: ["**/.venv", "**/__pycache__", "**/node_modules", "**/.git"], + )) + .withExec(["python", "-c", workspaceReader, "/scope"]) + .stdout + .split("\n") + .filter { line => line != "" } + let errors = lines.filter { line => line.hasPrefix("error\t") } + if (errors.length > 0) { + raise errors.map { line => line.trimPrefix("error\t") }.join("; ") } else { - [vendorPath + "/" + projectCfg, vendorPath + "/" + sdkGenPath] + lines } + } + + let workspaceReader: String! = + "import glob, os, re, sys, tomllib\n" + + "root = sys.argv[1]\n" + + "def fail(message):\n" + + " print('error\\t' + message)\n" + + " sys.exit(0)\n" + + "def load(rel):\n" + + " path = os.path.normpath(os.path.join(rel, 'pyproject.toml'))\n" + + " try:\n" + + " with open(os.path.join(root, path), 'rb') as f:\n" + + " return tomllib.load(f)\n" + + " except (tomllib.TOMLDecodeError, UnicodeDecodeError, OSError) as e:\n" + + " fail(f'{path} is not valid TOML: {e}')\n" + + "def table(doc, *keys):\n" + + " for key in keys:\n" + + " doc = doc.get(key) if isinstance(doc, dict) else None\n" + + " return doc if isinstance(doc, dict) else {}\n" + + "def strings(value, key, where):\n" + + " if value is None:\n" + + " return []\n" + + " if not isinstance(value, list) or not all(isinstance(v, str) for v in value):\n" + + " fail(f'{where}: {key} is not an array of strings; write it as one, such as {key} = [\"a\", \"b\"]')\n" + + " return value\n" + + "def name(requirement):\n" + + " m = re.match(r'\\s*([A-Za-z0-9][A-Za-z0-9._-]*)', requirement if isinstance(requirement, str) else '')\n" + + " return re.sub(r'[-_.]+', '-', m.group(1).lower()) if m else ''\n" + + "doc = load('.')\n" + + "members = strings(table(doc, 'tool', 'uv', 'workspace').get('members'), 'members', 'pyproject.toml')\n" + + "wanted = {name(r) for r in strings(table(doc, 'project').get('dependencies'), 'dependencies', 'pyproject.toml')}\n" + + "dirs = []\n" + + "for member in members:\n" + + " print('member\\t' + member)\n" + + " for found in sorted(glob.glob(os.path.join(root, member, 'pyproject.toml'))):\n" + + " rel = os.path.relpath(os.path.dirname(found), root)\n" + + " if rel not in dirs:\n" + + " dirs.append(rel)\n" + + "projects = {}\n" + + "for rel in dirs:\n" + + " project = table(load(rel), 'project')\n" + + " projects[rel] = (name(project.get('name')), {name(r) for r in strings(project.get('dependencies'), 'dependencies', rel + '/pyproject.toml')})\n" + + "needed = []\n" + + "while True:\n" + + " found = [rel for rel in dirs if rel not in needed and projects[rel][0] in wanted]\n" + + " if not found:\n" + + " break\n" + + " needed += found\n" + + " wanted = set().union(*(projects[rel][1] for rel in found))\n" + + "for rel in dirs:\n" + + " if rel in needed:\n" + + " print('local\\t./' + rel)\n" + + """ + The generated files a module needs before it builds. A scope needs each of + the SDK's members it lists; a user's member is the user's to provide. + """ + let requiredGeneratedFiles(source: Directory!, cfg: PyConfig!): [String!]! { + if (cfg.isScope) { + cfg.members + .filter { member => + (member == genDir or member.hasPrefix(clientsDir + "/")) and member.contains("*") == false + } + .map { member => member + "/" + projectCfg } + } else { + # A published dagger-io brings its own files, and the SDK files no longer + # read bindings from src/dagger_gen.py, so nothing generated is required. + let vendorPath = vendorPathFor(source, cfg) + if (vendorPath == "") { + [] :: [String!]! + } else { + [vendorPath + "/" + projectCfg, vendorPath + "/" + sdkGenPath] + } + } + } + + """ + Fail early when the committed generated files are missing. + """ + let checkGeneratedFiles(source: Directory!, modName: String!, required: [String!]!): Void { required.each { rel => if (source.exists(rel, expectedType: ExistsType.REGULAR_TYPE) == false) { raise "module \"" + modName + "\": generated file \"" + rel + "\" is missing; run `dagger generate` and commit the generated files" @@ -291,7 +417,9 @@ type PythonModuleBuild { """ Not a TOML parser: regular expressions scoped to a table, for the keys the templates and `mod config set` write. Single-line booleans and quoted strings - only; arrays, multi-line strings and dotted keys are not seen. + only; arrays, multi-line strings and dotted keys are not seen. The workspace + members, and the dependencies that pick what to install, are the user's + arrays, so tomllib reads them (workspaceRead). """ let pyConfig(source: Directory!): PyConfig! { let toml = source.file(projectCfg).contents @@ -308,6 +436,9 @@ type PythonModuleBuild { useUv: tomlBool(dagger, "use-uv", true), uvVersion: tomlString(dagger, "uv-version"), vendorPath: inlineTableString(sources, "dagger-io", "path"), + members: workspaceRead(source) + .filter { line => line.hasPrefix("member\t") } + .map { line => line.trimPrefix("member\t") }, indexURL: indexes .filter { table => tomlBool(table, "default", false) } .map { table => tomlString(table, "url") } @@ -371,6 +502,11 @@ type PyConfig { pub useUv: Boolean! pub uvVersion: String! pub vendorPath: String! + + """ + The `[tool.uv.workspace] members` of a scope; empty in the layout before. + """ + pub members: [String!]! pub indexURL: String! pub extraIndexURL: String! @@ -381,6 +517,7 @@ type PyConfig { useUv: Boolean! = true, uvVersion: String! = "", vendorPath: String! = "", + members: [String!]! = [], indexURL: String! = "", extraIndexURL: String! = "", ) { @@ -390,10 +527,19 @@ type PyConfig { self.useUv = useUv self.uvVersion = uvVersion self.vendorPath = vendorPath + self.members = members self.indexURL = indexURL self.extraIndexURL = extraIndexURL self } + + """ + Whether the module is a scope of the unified layout, whose SDK files are + the workspace member sdk/ rather than a library vendored by path. + """ + pub isScope: Boolean! { + members.filter { member => member == "sdk" }.length > 0 + } } """ diff --git a/entrypoint/handover.dang b/entrypoint/handover.dang new file mode 100644 index 0000000..3871910 --- /dev/null +++ b/entrypoint/handover.dang @@ -0,0 +1,124 @@ +""" +What a module entrypoint hands the module's code with each call: the local +clients the caller declared on the module's scope, and nothing else. + +The module's code runs in an exec that the engine does not make the module, so +its own current workspace is its container, and a local client cannot resolve +there. The entrypoint is given the caller's workspace, and resolves each +declared client in it. The code gets each client as a module source over the +files the engine loaded for that client: enough to serve it, and no way back +to the workspace. Not the workspace, which is every caller file; and not the +source Workspace.moduleSource returns, which keeps the workspace it came from +and reloads its context from it when asked for more files. + +A git client needs no workspace: the code serves it through serveModule. + +The shared entrypoint and every static one carry this file as it is, so the +two forms hand over the same thing. + +The list is read from the caller's workspace config at every call, never +baked into a static entrypoint at generation: a baked list lives in the +module's own files, which the module's author controls, while the caller's +config is what the caller agreed to. Do not simplify it into one. +""" +type ClientHandover { + let workspace: Workspace! + + new(workspace: Workspace!) { + self.workspace = workspace + self + } + + """ + One entry per declared local client: the module name its descriptor holds, + and the ID of a module source over its loaded files. + """ + pub clients: [HandedClient!]! { + declared.map { src => + HandedClient( + name: src.moduleName, + source: (loadedFiles(src).asModuleSource(sourceRootPath: src.sourceRootSubpath).id :: String!), + ) + } + } + + """ + The local clients the caller's workspace declares on this module's scope. + The scope is the one registered under this module's name, and only when it + holds this module's own files: a module loaded from elsewhere under the same + name gets nothing. None when no single scope qualifies. + """ + let declared: [ModuleSource!]! { + workspace.sdks.{{name}}.reduce([] :: [ModuleSource!]!) { found, listed => + let sdk = workspace.sdk(name: listed.name) + let scopes = sdk.modules.{{name, source}} + .filter { module => module.name == currentModule.name } + .map { module => normalizePath(module.source) } + .filter { scope => isThisModule(scope) } + if (scopes.length == 1) { + let scope = scopes[0] ?? "" + found + sdk.clients.{{name, source}} + .filter { client => normalizePath(client.name) == scope and isLocal(client.source) } + .map { client => workspace.moduleSource("/" + normalizePath(client.source)) } + } else { + found + } + } + } + + """ + Whether a scope holds this module's files, compared by content. The engine + re-encodes the manifest it serves a module with, so that one is compared + as TOML. + """ + let isThisModule(scope: String!): Boolean! { + let theirs = workspace.moduleSource("/" + scope).directory(".") + let changes = currentModule.source.changes(from: theirs) + let others = changes.modifiedPaths.filter { path => path != manifestName } + changes.addedPaths.length == 0 and changes.removedPaths.length == 0 and others.length == 0 and + sameToml(currentModule.source.file(manifestName), theirs.file(manifestName)) + } + + let manifestName: String! = "dagger-module.toml" + + let sameToml(a: File!, b: File!): Boolean! { + JSON.encode(TOML.decode(a.contents)) == JSON.encode(TOML.decode(b.contents)) + } + + """ + The files the engine loaded for a source, and for its local dependencies, + which a directory source resolves inside the same tree. A local + dependency's root is a path in the same workspace. + """ + let loadedFiles(src: ModuleSource!): Directory! { + src.dependencies.{{kind, sourceRootSubpath}} + .filter { dep => dep.kind == ModuleSourceKind.LOCAL_SOURCE } + .reduce(src.contextDirectory) { dir, dep => + dir.withDirectory(".", loadedFiles(workspace.moduleSource("/" + normalizePath(dep.sourceRootSubpath)))) + } + } + + """ + A workspace path, as the workspace config writes a local client; anything + else is a git ref. + """ + let isLocal(ref: String!): Boolean! { + ref.hasPrefix(".") or ref.hasPrefix("/") + } + + let normalizePath(path: String!): String! { + let normalized = path.trimPrefix("./").trimPrefix("/").trimSuffix("/") + if (normalized == "") { "." } else { normalized } + } +} + +type HandedClient { + pub name: String! + pub source: String! + + new(name: String!, source: String!) { + self.name = name + self.source = source + self + } +} diff --git a/entrypoint/main.dang b/entrypoint/main.dang index 86aa515..d21a4e5 100644 --- a/entrypoint/main.dang +++ b/entrypoint/main.dang @@ -3,8 +3,8 @@ The Python SDK's shared module entrypoint. One copy of this directory, served from this repository, backs every Python module that names it in `[entrypoint] kind = "dang"`. Nothing is generated into -the module: the entrypoint finds the module it serves through the workspace it -is handed, whose working directory is that module's directory. +the module: inside an entrypoint `currentModule` is the module it serves, so +its source holds that module's own files, however the module was loaded. The module still describes its own types the way it always has. This rebuilds them from that description in this session, rather than loading the module the @@ -13,10 +13,10 @@ reads them from. """ type Entrypoint implements ModuleEntrypoint { """ - Every type the module at the workspace cwd defines. + Every type the module defines. """ pub types(workspace: Workspace!): [TypeDef!]! { - let desc = json.withContents(describeJSON(workspace)) + let desc = json.withContents(describeJSON) let objects = desc.field(["objects"]).asArray.{{contents}}.map { raw => objectType(json.withContents(raw.contents)) } @@ -27,10 +27,11 @@ type Entrypoint implements ModuleEntrypoint { } """ - Call one constructor or function of the module at the workspace cwd. + Call one constructor or function of the module. The request and the result are the module's own JSON, so the entrypoint - passes them through without reading them. + passes them through without reading them. The request also carries the + module's declared local clients, never the workspace (see handover.dang). """ pub call( workspace: Workspace!, @@ -44,8 +45,11 @@ type Entrypoint implements ModuleEntrypoint { receiverValue: receiverValue, fnName: fnName, fnArgs: fnArgs, + clients: ClientHandover(workspace: workspace).clients.map { client => + {{name: client.name, source: client.source}} + }, }}) - let result = build(workspace).installed + let result = build.installed .withExec( ["python", "-m", "dagger.mod", "call", "--output", callResultPath], stdin: request, @@ -60,8 +64,8 @@ type Entrypoint implements ModuleEntrypoint { let callResultPath: String! = "/dagger/result.json" """The module's type description, as the JSON its own tools already emit.""" - let describeJSON(workspace: Workspace!): Dagger.JSON! { - (build(workspace).installed + let describeJSON: Dagger.JSON! { + (build.installed .withExec( ["python", "-m", "dagger.mod", "describe", "--output", describePath], experimentalPrivilegedNesting: true, @@ -70,30 +74,25 @@ type Entrypoint implements ModuleEntrypoint { .contents :: Dagger.JSON!) } - """The module this entrypoint was handed: its directory is the workspace cwd.""" - let build(workspace: Workspace!): PythonModuleBuild! { - let path = modulePath(workspace) + """ + The module this entrypoint serves, built from its own source. + + Not from the workspace the engine hands over: that is the caller's, which + holds the module only when the module sits in it. A module loaded from git, + or served into another module's process, is not in it. + """ + let build: PythonModuleBuild! { PythonModuleBuild( - contextDir: workspace.directory( - "/", - include: [if (path == ".") { "**" } else { path + "/**" }], - exclude: ["**/.venv", "**/__pycache__"], - ), - subPath: path, - moduleName: moduleName(workspace, path), + contextDir: directory.withDirectory(".", currentModule.source, exclude: ["**/.venv", "**/__pycache__"]), + subPath: ".", + moduleName: moduleName, ) } - let modulePath(workspace: Workspace!): String! { - let cwd = workspace.cwd.trimPrefix("/").trimSuffix("/") - if (cwd == "") { "." } else { cwd } - } - - let moduleName(workspace: Workspace!, path: String!): String! { - let manifest = if (path == ".") { "dagger-module.toml" } else { path + "/dagger-module.toml" } - let matched = workspace.file("/" + manifest).contents.match("(?m)^\\s*name\\s*=\\s*\"([^\"]*)\"") + let moduleName: String! { + let matched = currentModule.source.file("dagger-module.toml").contents.match("(?m)^\\s*name\\s*=\\s*\"([^\"]*)\"") if (matched == null) { - raise "no module name in " + manifest + raise "no module name in dagger-module.toml" } else { matched.captures[0] ?? "" } diff --git a/hack/designs/2026-09-15-unified-clients.md b/hack/designs/2026-09-15-unified-clients.md new file mode 100644 index 0000000..2f520cc --- /dev/null +++ b/hack/designs/2026-09-15-unified-clients.md @@ -0,0 +1,1092 @@ +# Unified clients for the Python SDK + +Status: design agreed; spikes open (revision 7) +Date: 2026-09-17 +Repo base: `7f4b427` +Spec: "Unified clients" (language-neutral). Prior art: `dagger/java-sdk#23`. + +This is the markdown copy of the HTML design page. Badges: +**[confident]** checked against code or by experiment. +**[decided]** Yves decided it. +**[provisional]** a detail that follows from a decision; change it freely during +implementation. +**[speculative]** not verified; a phase 1 spike must confirm it. +**[open]** a question for Yves, with a recommendation. + +Revision 7, after more discussion: + +- One structure, inside a module and outside one: `src/`, `sdk/`, + `clients/core`, `clients/` (section 4). +- A client is generated **inside the scope that uses it**. No shared location. + Sharing between scopes may come later (5.1). +- No suggested path for generated clients any more. The scope is the address. +- `dagger.toml` joins a scope to its clients, so generation has an order and a + repeatable result (5.4). +- After generation, the user's code needs to know nothing about a client but its + import. The one thing a client needs is the power to load the module it targets. +- Q9 is decided: the SDK writes the client into `[project] dependencies`, + because the scope is now the consumer. +- The load becomes one engine field, `serveModule`, read in Python as + `core().serve_module(…)` (section 13). + +Earlier revisions. 6: all questions decided. 5: self client; signature rule. +4: no `[[dependencies]]`; shared default session. 3: a scope is one +`pyproject.toml`, a uv workspace root. 2: a temporary global client. + +## 1. Summary + +- A scope is one `pyproject.toml`. It is a uv workspace root. A module is a + scope. A test project is a scope. +- Every scope has the same structure: `src/` for the user's code, `sdk/` for the + SDK files, `clients/core` for the core bindings, and `clients/` for each + client. +- A client is generated inside the scope that uses it. A module's client sits in + the module. A test project's client sits in the test project. +- `dagger.toml` records the scope and its clients. `dagger generate` reads that + to know what to generate, and in which order. +- After generation, the user's code only imports a client. It holds no path, no + configuration and no client bookkeeping. +- All generated code lives in the namespace package `dagger_clients`. The SDK + files keep the name `dagger`. +- The way into a client is a function in its own package: `linter()`, `core()`. + The session argument is optional. +- There is one default session per process. The SDK starts it on the first + query. All clients share it. +- Each client package carries a descriptor and loads the module it targets on + first use, with one engine call. No `[[dependencies]]`. +- A module calls itself through a client to itself. The user declares that + client; the SDK adds none. +- An exported signature names core types and the module's own types only. A + client type is for calls inside a function body. +- A temporary global client keeps existing module code working, behind a flag in + `pyproject.toml`. A new module has no flag. +- The SDK files import generated core in many places today. Phase 1 removes + these imports. +- Every scope has its own `clients/core`. The copies are identical, because a + member's files do not depend on its scope. + +## 2. Terms + +The spec terms apply without change: client, scope, consumer, core, runtime, +serve. Added terms: + +| Term | Meaning here | +| --- | --- | +| Scope `pyproject.toml` | The file that defines a scope for this SDK. It is a uv workspace root. It lists the members and their sources. | +| Member | A uv workspace member: a directory with its own `pyproject.toml` inside the scope. `sdk/` and each client are members. | +| Self client | A client to the module that holds it. The module uses it to call its own functions through the engine. | +| Distribution | What a package manager installs, for example `dagger-clients-linter`. | +| Import package | What Python code imports, for example `dagger_clients.linter`. | +| Namespace package | An import package without `__init__.py` (PEP 420). Many distributions can each add one sub-package to it. | +| Client package | The import package inside a client member: `dagger_clients.`. | +| Descriptor | The data that tells a client package where its module is: a workspace path, or a git ref and pin. | +| Default session | The one session per process that a client uses when the caller passes none: `dagger.dag`. | +| Global client | A temporary generated object, `dag`, with one method per core field and per client. It exists only for migration. | + +## 3. What exists today [confident] + +Checked against `7f4b427`. The SDK generates per scope, and each module gets its +own copy of everything. + +- `mod.dang:154` (`vendoredDir`) copies the whole `dagger-io` library into + `/sdk/`. `mod.dang:280` names that directory. +- `mod.dang:208` (`bindings`) generates one file, + `sdk/src/dagger/client/gen.py` (`mod.dang:283`). The file holds core and every + client of the module. The input is the module-facing schema, + `introspectionSchemaJSON` (`mod.dang:155`). +- The generator emits `class Client(Query)` and `dag = Client()` + (`sdk/codegen/src/codegen/generator.py:245-257`). +- Module code reaches a client through core: `dag.client_dep()`. +- `python-sdk.dang:164-165` writes each client as `[[dependencies]]`. The engine + serves the client. Generated code loads nothing. +- `python-sdk.dang:109-114` refuses clients in a scope without a module. +- `findClientRoot` is at `python-sdk.dang:42`, with `pyproject.toml` as the marker. +- A module's `pyproject.toml` names the vendored runtime: + `dagger-io = { path = "sdk", editable = true }` + (`templates/default/pyproject.toml.tmpl`). The module build installs it + (`runtime/build.dang:168`). +- Python SDK settings live in `[tool.dagger]` of the module's `pyproject.toml`: + `use-uv`, `base-image`. `helpers/pyproject/pyproject.go:60-87` reads and writes + them. `runtime/build.dang:305` reads them. +- **New since revision 6.** Every generated module names the shared Dang + entrypoint, `dagger.io/sdk/python/entrypoint@v1` (`python-sdk.dang:78`, + written at `python-sdk.dang:141-166`). The entrypoint asks the module for its + types with `python -m dagger.mod describe` (`entrypoint/main.dang:63`), reads + the JSON (`sdk/src/dagger/mod/_describe.py`, `describe_json`), and replays the + builder calls in its own session (`entrypoint/main.dang:18-19`). So a module's + types already travel as data, not as a generated object. +- **New since revision 6.** The static types path is the setting + `dangEntrypoint` (`python-sdk.dang:32`). It still refuses clients + (`python-sdk.dang:191`). +- `sdk/src/dagger/__init__.py:12-13` already has a hook for extra generated + bindings: `from dagger_gen import *`. +- `SharedConnection` (`sdk/src/dagger/client/_session.py:208`) is a process + singleton. It connects on the first query from `DAGGER_SESSION_PORT` and + `DAGGER_SESSION_TOKEN`. A module and `dagger run` set these. Without them, a + plain program must use `async with dagger.connection()` + (`provisioning/_connection.py:73`). +- The SDK resolves a signature's types in `describe_type` + (`sdk/src/dagger/mod/_describe.py:137`). +- The engine has an experimental `SELF_CALLS` module feature + (`ModuleSourceExperimentalFeature`, `sdk/src/dagger/client/gen.py:193`). +- The engine primitives the spec names exist in the committed core bindings: + `ModuleSource.clientSchemaIntrospectionJSON`, `ModuleSource.withName`, + `Query.moduleSource(refString, refPin)`, `Module.serve`, `SourceMap.module`. +- The code generator already parses schema directives + (`sdk/codegen/src/codegen/ast.py:93`). It can read `@sourceMap`. + +## 4. One structure [decided] + +A module and a plain project get the same tree. The only difference is the +module config file. + +``` +/ + pyproject.toml the scope: a uv workspace root + src/ the user's code, the standard Python layout + sdk/ the SDK files: session, transport, telemetry, module support + sdk/src/dagger_global/ the temporary global client, only with the flag (section 9) + clients/core/ the core bindings + clients// one directory per client +``` + +``` +# a module scope +.../my-module/ + pyproject.toml + dagger-module.toml the module config; no [[dependencies]] + src/my_module/ + sdk/ + clients/core/ + clients/linter/ + clients/my-module/ a self client, only if the user declares one (7.3) + +# a test project scope, in the style of Testcontainers +.../test-project/ + pyproject.toml + src/test_project/ + sdk/ + clients/core/ + clients/linter/ +``` + +- `src/` holds the user's code. Nothing generated goes there. +- `sdk/` holds what is generic: the session, the query transport, telemetry, the + module support. Nothing in it is tied to the types a module exposes. +- `clients/` holds what is generated from a schema: core, and one client per + declared client. +- The names `src`, `sdk` and `clients` are the same in every scope, so a reader + learns the layout once. + +The artifact graph [provisional]. An arrow means "imports". The SDK files import +no generated code. Section 9 adds one optional, temporary exception. + +```mermaid +graph BT + RT["dagger, from sdk/: session, query builder, telemetry, module support"] + CORE["dagger_clients.core, from clients/core"] + L["dagger_clients.linter + descriptor"] + G["dagger_clients.glow + descriptor"] + CORE --> RT + L --> CORE + G --> CORE + L --> RT + G --> RT +``` + +The worked example. A module and a test project each use the linter module. Each +scope holds its own client to it. Both clients load the same module. + +```mermaid +graph LR + subgraph MOD["scope: my-project-dev (a module)"] + MSRC["src/"] + MSDK["sdk/"] + MCORE["clients/core"] + ML["clients/linter"] + MG["clients/glow"] + end + subgraph TP["scope: test-project (a plain project)"] + TSRC["src/"] + TSDK["sdk/"] + TCORE["clients/core"] + TL["clients/linter"] + end + LM["module: linter, in the workspace"] + GLOW["github.com/eunomie/glow, a git module"] + MSRC --> ML + MSRC --> MG + TSRC --> TL + ML -. loads .-> LM + TL -. loads .-> LM + MG -. loads at its pin .-> GLOW +``` + +The two clients to `linter` hold the same bytes, because a client's files do not +depend on the scope that holds it. The copies cost disk, not behaviour. A shared +location would remove the copies; that comes later, if it comes. + +## 5. Scopes and generation + +### 5.1 A client is generated inside the scope that uses it [decided] + +- A client declared on a scope is generated in that scope, in `clients/`. +- A module's clients sit in the module. They travel with it, also in a git + repository. The module's build sees them, because they are inside its own + directory. +- A test project's clients sit in the test project. +- A generated client needs no path from the user. The import is the whole of the + integration. +- A shared location is possible in principle, so that two scopes use one copy. + It is out of scope for this design. + +### 5.2 What a member is [confident] + +A directory with `pyproject.toml` and `src/`, built with `uv_build`. It builds +into a wheel without an engine. + +```toml +# clients/linter/pyproject.toml (generated) +[project] +name = "dagger-clients-linter" +version = "0.0.0" +dependencies = ["dagger-io", "dagger-clients-core"] + +[build-system] +requires = ["uv_build>=0.8.4,<0.12.0"] +build-backend = "uv_build" + +[tool.uv.build-backend] +module-name = "dagger_clients.linter" + +[tool.dagger] +generated = "client" +``` + +``` +clients/linter/src/dagger_clients/ no __init__.py: a namespace package + linter/__init__.py generated types, linter(), as_linter() + linter/_target.py descriptor and the core digest it was generated against + linter/py.typed +``` + +Every generated member has `[tool.dagger] generated`: `"client"`, `"core"` or +`"runtime"`. The SDK uses this marker to find its own members. A member holds no +path and no source entry, so its files do not depend on the scope. One member +generated in two scopes had equal digests. + +**The marker is a deletion boundary, so how it is read matters** [confident]. +Generation deletes and overwrites only what the marker claims, which makes the +reading of that one key as load-bearing as the rule itself. + +- It is read as **TOML**, never as text. A `[` inside a string is not a table + header. A regex over raw text deleted a user's directory whose description + happened to quote a marker. +- Only the three kinds above count. Any other value, `"hand-written"` say, + means a file the SDK does not understand, which is exactly when it must not + delete or overwrite. +- Each place requires its own kind: `clients/core` must say `core`, a client + directory must say `client`, `sdk/` must say `runtime`. +- A member the user wrote under `clients/` therefore survives generation, in the + tree and in the scope file alike. + +### 5.3 The scope `pyproject.toml` [decided] + +It has three parts, and each part has one owner. + +| Part | What it says | Owner | +| --- | --- | --- | +| `[tool.uv.workspace] members` | Which directories are members. | The SDK | +| `[tool.uv.sources]` | Where each client of this scope is. It installs nothing. | The SDK | +| `[project] dependencies` | Which clients this project uses. Only these are installed and importable. | The SDK | + +```toml +# /pyproject.toml +[project] +name = "my-module" +# Core is always a dependency: the module's own code imports dagger_clients.core. +dependencies = ["dagger-io", "dagger-clients-core", "dagger-clients-linter"] + +[tool.uv.workspace] +members = ["sdk", "clients/core", "clients/linter", "clients/my-module"] + +[tool.uv.sources] +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } +dagger-clients-linter = { workspace = true } +dagger-clients-my-module = { workspace = true } +``` + +Tested with uv 0.12.13 [confident]: + +- A scope with a `[project]` table can use its own members. With sources for + three members and a dependency on one, uv installs only that member and core. +- A scope without a `[project]` table also works. +- `uv sync --locked --no-dev` works on that layout. The module build uses that form. +- Removing a member, its source and its directory, then `uv lock`, removes the + client cleanly. +- Two editable members share the `dagger_clients` namespace. mypy and pyright + see types across it when each member has `py.typed`. +- A member builds into a wheel with no engine in reach. + +### 5.4 What `dagger.toml` gives the SDK [decided] + +`dagger.toml` records each scope and the clients of that scope. Because a client +now lives in the scope that declares it, the configuration and the tree say the +same thing. + +- What to generate: the client list of a scope is the generation input. Nothing + is guessed from the tree. +- Order: a client to a local module needs that module's schema. So + `dagger generate` can order the work: a target module first, then a scope whose + client points at it. [provisional] The engine drives the scopes, so the order + is engine-side. +- Repeatable: the same configuration gives the same tree, in the same places. +- Removal: a client that is no longer declared is deleted, with its source and + its member entry. +- No run-time role: generated code never reads `dagger.toml`. A committed client + keeps working when the configuration is gone. + +## 6. Namespacing [decided] + +Generated code goes into the top-level namespace package `dagger_clients`. The +SDK files stay in `dagger`. A client `telemetry` becomes +`dagger_clients.telemetry` and cannot collide with `dagger.telemetry`. A self +client cannot collide with the module's own code: the module is `my_module`, its +self client is `dagger_clients.my_module`. Core is `dagger_clients.core`, so a +client named `core` is refused. + +`dagger.clients.` is not possible: `dagger` is a regular package, and type +checkers do not follow `pkgutil.extend_path` across installs. + +| Input | Rule | Example | +| --- | --- | --- | +| Client name | Lowercase. Replace `-` and `.` with `_`. | `my-project-dev` → `my_project_dev` | +| Refused | Not an identifier, a keyword, starts with `_`, equal to `core`, two clients with the same result. | `class`, `core` | +| Distribution | `dagger-clients-` + name with `-` | `dagger-clients-my-project-dev` | +| Member directory | `clients/` + name with `-` | `clients/my-project-dev` | +| Root class | Today's codegen rule | `MyProjectDev` | +| Entry function | Snake-case name | `my_project_dev()` | + +The SDK writes only into `sdk/` and `clients/`, so a client cannot land on a +user directory. It still refuses to write a member over a directory that exists +without a `[tool.dagger] generated` marker. + +## 7. Entry point + +### 7.1 The entry function [provisional] + +```python +# in a module: src/my_project_dev/__init__.py +from dagger import function, object_type +from dagger_clients.core import Directory +from dagger_clients.linter import linter + +@object_type +class MyProjectDev: + @function + async def lint(self, src: Directory) -> str: + return await linter().lint(src) +``` + +```python +# in a test project: src/test_project/test_lint.py +from dagger_clients.core import core +from dagger_clients.linter import linter + +async def test_lint(): + src = core().host().directory(".") + assert "0 errors" in await linter().lint(src) +``` + +The two files import the same way. Neither one names a path or a session. + +Constructor arguments follow today's rules: required are positional, optional +are keyword-only. The session is an optional keyword-only argument [decided]; +without it the function uses the default session (8.1). A GraphQL argument named +`session` becomes `session_`. + +```python +def linter(source: Directory, *, config: str | None = None, + session: Session | None = None) -> Linter: ... +``` + +### 7.2 Fields a client contributes to core types [decided] + +A field a client contributes to a core type becomes a module-level function with +the core receiver first. The receiver holds its session. + +```python +from dagger_clients.linter import as_linter + +lint = as_linter(binding) # was: binding.as_linter() +``` + +Two core types that contribute a field with one name get one function with +`@typing.overload` per receiver type. + +### 7.3 A module calling itself [decided] + +A module calls one of its own functions through a client to itself. A self +client is a client like any other, and the SDK creates none on its own. + +1. The user declares a client to the module on the module scope. +2. `dagger generate` writes `clients/my-module` and its source. +3. The same generation adds `dagger-clients-my-module` to `[project] dependencies`. +4. The module code imports the self client and calls it. + +```python +from dagger_clients.my_module import my_module + +@object_type +class MyModule: + @function + async def build(self) -> str: ... + + @function + async def release(self) -> str: + return await my_module().build() # a call through the engine +``` + +- Load: the descriptor is the module's own path. In the module's session, the + client asks the engine for the module at that path and loads it. + [speculative] The spike checks whether the engine needs the experimental + `SELF_CALLS` feature for this. +- Order: the self client comes from the module's schema, and the engine reads + that schema by running the module. So the module runs with the previous self + client while the SDK generates the next one. Two rules keep this from + blocking: the dependency comes after the first generation, and a core digest + mismatch is a warning while the SDK registers types (section 10). + [speculative] A spike must confirm it. + +### 7.4 What an exported signature can name [decided] + +This is an engine rule. A module's exported functions, fields and constructor +arguments name core types and the module's own types only. So module A cannot +expose a function that returns a type of module B. A client type is for calls +inside a function body. + +- The SDK refuses a client type in a signature when it registers types. The + check is in `describe_type` (`sdk/src/dagger/mod/_describe.py:137`): a class + from `dagger_clients.`, other than `dagger_clients.core`, is refused. + The message names the function and the type. +- [provisional] The classes of a self client are refused the same way. A + signature uses the module's own classes. + +## 8. Session and load + +### 8.1 The default session [decided] + +- The session is not required. A client called without `session=` uses the + default session, `dagger.dag`. +- There is one default session per process. All clients share it, so objects + pass between clients. +- The SDK starts the default session on the first query. In a module and under + `dagger run`, `SharedConnection` already does this today. +- In a plain program, the SDK also provisions the engine on the first query, and + closes it at exit [confident]. The engine is a `dagger session` subprocess + that ends when its stdin closes, which is sync, so an `atexit` handler closes + it after the program's event loop is gone; `dagger.close()` closes it sooner. + The environment comes first, so a module never provisions. Verified by hand: + a program that only calls a client exits 0 and leaves no session process. +- **A module says it is one** [confident]. The `sdk/` member carries + `dagger.provisioning`, because a plain program run inside a module's own scope + needs it. So both module entrypoints call `mark_module_runtime()` first, and + after that the default session raises "No active engine session to connect to" + rather than provisioning. Without it, the only thing keeping a module from + downloading a CLI into its own container would be the engine always setting + the session in the environment — an accident, not a rule. The signal is the + entrypoint, which knows it serves a module, and not an environment variable + the engine may rename. +- A caller passes `session=` only to use a specific session. + +One session per client does not work: an object belongs to one session, and a +module has exactly one session for a function call. + +### 8.2 What `dagger.dag` is [confident on shape] + +- Without the global client, `dagger.dag` is an instance of a hand-written + `dagger.Session`. It owns the connection, the query transport and the load + memo. It has no API fields. +- The new `Session` wraps today's `SharedConnection`. `dagger.connection()` and + `dagger.Connection` yield a `Session`. +- Without the global client, `dag.container()` does not exist. + `Session.__getattr__` raises `AttributeError` with a migration message that + names `core().container()`. The SDK does not import core for the message. +- A module-level `__getattr__` (PEP 562) on `dagger` does the same for + `dagger.Container` and other core names. + +### 8.3 How a client loads its module [provisional] + +The load is async, and `linter()` is sync and lazy. So the load runs when a +query executes. + +1. The `Context` in the SDK gets the set of descriptors the query needs. +2. `linter()` creates a `Context` that needs the linter descriptor. +3. Chained selections keep the set. +4. `Context.execute` asks the session to load each descriptor first. +5. The session loads each descriptor at most once, with one `anyio.Lock` per entry. +6. An object passed as an argument becomes an ID through its own `execute`, + which loads its own descriptor. +7. The SDK loads an argument from an ID in `dagger/mod/_converter.py:62`. The + generated class carries its descriptor for this. + +```python +# clients/linter/src/dagger_clients/linter/_target.py (generated) +# Plain data, no import: the descriptor is what generation knew. +NAME = "linter" +REF = "/path/to/the/linter/module" # from the workspace root, or "github.com/eunomie/glow" +PIN = None # or the commit the client was generated against +CORE_DIGEST = "sha256:…" +``` + +The package's `__init__.py` builds `_TARGET` from those constants. The name is +private: the package's public names are the generated types, and a user of the +client never handles its descriptor. + +The SDK owns the `Target` class and the load query. The query uses the raw query +builder, not generated core. One field carries both kinds of reference, so the +descriptor has one shape; see section 13. + +### 8.4 The API the generated code targets [decided] + +Generated code calls four hand-written names, and nothing else of the SDK: + +```python +# dagger.client +@dataclass(frozen=True, slots=True) +class Target: + name: str + ref: str + pin: str | None = None + +def client_root(cls: type[T], target: Target | None, field: str | None, + args: list[Arg], *, session: Session | None = None) -> T +def client_select(receiver: Type, target: Target, field: str, + args: list[Arg]) -> Context +def check_core(client: str, expected: str, installed: str) -> None +``` + +- `client_root` starts a query at the root. `target=None` means there is nothing + to load, which is how `core()` is emitted, so core and a client share one + shape. +- `client_select` continues from a core receiver, for a field a client + contributes to a core type (7.2). It keeps the receiver's session and context. + It returns a `Context`, so the generated code wraps it in the return type, or + executes it when the field returns a scalar. +- Both attach the target to the query context. The SDK loads a target at most + once per session (8.3). +- **The generator passes the exact GraphQL field name.** The SDK never derives + it from a class name or from `Target.name`: that would copy the engine's + naming rule into the SDK, where it can drift. +- `check_core` is the import-time staleness check of section 10. Each client + package calls it with its own name, the digest it was generated against, and + the digest of the installed core. Core's own package calls nothing. +- The generated packages import `Session`, `Target`, `client_root`, + `client_select` and `check_core` from `dagger.client`. +- The generator writes no `pyproject.toml`. `generateScope` writes it (12). + +## 9. Temporary global client [decided] + +Existing module code uses `dag.linter().lint()`, `dag.container()` and +`dagger.Container`. A global client keeps that code working during migration, +without an edit by the user. A new module does not get one. + +```toml +# /pyproject.toml +[tool.dagger] +global-client = true +``` + +- The flag is in the same table as `use-uv` and `base-image`. +- No flag means no global client. The templates carry no flag. +- `generateScope` writes the flag once, when it upgrades an existing module: the + module had a config before generation, and has legacy bindings at + `sdk/src/dagger/client/gen.py`. +- `mod config set --global-client=false` removes the flag. The next + `dagger generate` removes the global client. +- Only generation reads the flag. The SDK does not read `pyproject.toml` at run time. + +Where it goes [confident]. The global client is generic in shape but tied to +one scope's clients, so it belongs with the SDK files: the `sdk/` member gains a +second import package, `sdk/src/dagger_global/`, when the flag is on. Then no +new member appears. The cost: that copy of `dagger-io` depends on the scope's +clients while the flag is on. Clearing the flag must give back exactly the +`sdk/` of a module that never had one. + +```python +# sdk/src/dagger_global/__init__.py (generated, temporary) +from dagger import Session +from dagger_clients.core import * # dagger.Container keeps working +from dagger_clients.core import core +from dagger_clients.linter import Linter, linter + +class Client(Session): + def container(self, *, platform=None) -> Container: + return core(session=self).container(platform=platform) + + def linter(self, source: Directory) -> Linter: + return linter(source, session=self) + +dag = Client() +``` + +- Same classes: `dag.container()` returns `dagger_clients.core.Container`. Old + and new calls mix in one module. +- Contributed fields: the global client adds `binding.as_linter()` to the core + class at import, at run time only. Type checkers do not see it, which points + to the migration. +- Dependencies: the global client is a second import package of the `sdk/` + member, not a member of its own, so nothing is added to the scope's + `[project] dependencies`. That member's own `pyproject.toml` gains a + dependency on core and on each client while the flag is on [confident]. +- End of life: a `DeprecationWarning` on import in phase 2. Removal in a later + release. + +The SDK finds it with one optional import, which replaces today's `dagger_gen` +hook (`sdk/src/dagger/__init__.py:12-13`). The import is lazy [confident]: + +```python +# dagger/__init__.py (hand-written) +def __getattr__(name): # PEP 562: on first use, never on import + if name == "dag": + return _sessions.default_session() + ... # names of dagger_global, if it is installed +``` + +An eager `from dagger_global import *` cannot work. The global client imports +core and every client, each of those imports `dagger`, so `dagger` would import +itself whenever a generated package is the first import of the process. A +module-level `__getattr__` runs after `dagger` is built, which breaks the cycle. +`__dir__` and `__all__` follow the same route, so completion and star imports +still see the names. + +`dagger.dag` is the default session (8.1). With the flag, the default session +must be the global client's `dag`, so that a client called without `session=` +shares its loads. The SDK gives the global client one seam for this: +`dagger.client._session.set_default_finder(find)`. `dagger/__init__.py` passes a +function that imports `dagger_global` on first use and returns its `dag`, or +`None`. Only the global client uses the seam. + +Generation: `codegen generate-global -i …` writes the package. It takes +the core schema and each client schema, and it fails if their engine versions +differ. + +This is the only place the SDK files name generated code. It is off for a new +module, and it goes away with the global client. + +An SDK older than this design star-imported its bindings from `dagger_gen`. +Those bindings are no longer loaded. `dagger/__init__.py` finds the module +without importing it (`importlib.util.find_spec`) and warns, so a user who +skipped `dagger generate` is told why `dag` lost its API. + +## 10. Type checking and staleness [decided] + +| When | Signal | Needs the engine | +| --- | --- | --- | +| Type check | `py.typed` and full annotations. A removed or changed function is a type error after `dagger generate`. | No | +| Import | The client passes its `CORE_DIGEST` and the installed core's digest to an SDK function. A mismatch raises `dagger.StaleClientError` (an `ImportError`) with "run `dagger generate`". While the SDK registers a module's types, it logs a warning instead, so a module with a self client can regenerate (7.3). The SDK receives two strings, so it does not import core. | No | +| Load and query | A failed load raises `dagger.ClientLoadError` with the descriptor and the cause. An engine below the floor, which has no `serveModule`, fails there too: the cause names the field, and it is never a `StaleClientError`, because regenerating cannot give an engine a field it lacks. Once the module is loaded, a validation error naming a missing field, on a query that needs a descriptor, becomes `StaleClientError`. It must be the validator's own error: an internal error that merely quotes the phrase is the engine failing, not a stale client, and telling the user to regenerate would send them the wrong way. An engine/core mismatch is a warning in phase 1. | Yes | + +A CI check that runs `dagger generate` and asserts no diff catches the rest. + +## 11. Artifact graph rules in Python + +| Rule | Python meaning | Check | +| --- | --- | --- | +| Core names no client | `dagger_clients/core/` imports no other `dagger_clients` package. | AST scan; generate core with two client sets and compare digests. | +| No client names another | A client imports only `dagger`, `dagger_clients.core`, itself. | AST scan. | +| The SDK files depend on nothing generated | No `dagger/` module imports `dagger_clients`, `dagger.client.gen`, `dagger_gen`. One exception: the optional `dagger_global` import. | Import every `dagger.*` module in a venv with no generated code; AST scan with the one exception. | + +Where the SDK files depend on generated core today [confident]. +`dagger/telemetry.py` is clean; the trap is elsewhere: + +| Location | Dependency | Fix | +| --- | --- | --- | +| `sdk/src/dagger/__init__.py:10-16` | Star-imports `dagger_gen` or `dagger.client.gen`. | Replace with the optional `dagger_global` import; add PEP 562 message. | +| `sdk/src/dagger/client/_core.py:24`, `client/_session.py:11` | `from dagger import …` runs `dagger/__init__.py`, which loads core. The Python-specific trap. | Import from `dagger._exceptions`, `dagger.telemetry`. | +| `sdk/src/dagger/mod/_module.py`, `_converter.py`, `_exceptions.py`, `_describe.py`, `_entrypoint.py` | Type registration through generated `dag`, `TypeDef`, `TypeDefKind`, `FunctionCachePolicy`, `JSON`. | [decided] Raw query builder. The new `describe` command already emits the types as JSON (`_describe.py`, `describe_json`), which is half of the work. | +| `sdk/src/dagger/provisioning/_connection.py`, `_engine.py` | Imports `dag`, `Client`. | Return a `Session`. | +| `sdk/src/dagger/_engine/_version.py:3` | Generated `CLI_VERSION` in the SDK files, used to download a CLI. | Keep it there; revisit when the SDK files are published. | +| `sdk/codegen/src/codegen/generator.py:245-257` | Emits `Client`, `dag`. | Emit `core()`; emit `Client` and `dag` only into `dagger_global`. | +| `sdk/src/dagger/client/_guards.py:44` | Text names `dagger.client.gen`. | Take the name from the caller. | + +## 12. generateScope and findClientRoot + +**`generateScope` (`python-sdk.dang:108`) [provisional].** The function keeps its +signature. The scope directory (`ws.cwd`) is the workspace root of the scope. +One path serves a module and a plain project; `isModule` only decides whether a +module config is written. + +1. Scope file: read the scope `pyproject.toml`. A scope without one gets a new + file with the workspace and the sources. +2. SDK files: write `sdk/` with the hand-written SDK only. In an existing + module, this replaces the vendored `sdk/` and its `gen.py`. +3. Core: generate once from the client-facing schema into `clients/core`. The + schema is read through an empty stand-in module, because the engine serves a + client-facing schema to a module, not to nothing. That stand-in is then + stripped out, or the next step would see it as a client of its own. +4. Clients: for each declared client, read `clientSchemaIntrospectionJSON`, + partition by `@sourceMap`, write the descriptor (the workspace path, or the + ref and the pin), write `clients/`. A self client takes the same path. + Every schema, core's and each client's, is read **in the scope's engine-version + view** [confident]. The engine serves a module's schema in the view of that + module's own declared version, and two views give two different cores, which + the digest check would then refuse. The cost is that a client to an older + module is read at the scope's version, not at the version that module + declares. +5. Removed clients: delete each directory under `clients/` that carries + `[tool.dagger] generated = "client"` and is no longer declared. The scope + file loses only the entries of members whose **present** directory carries + the marker. A directory the user removed by hand keeps its entry, because + generation can no longer prove it owned it; `uv` then names the missing + member, and the user restores the directory or removes the entry. A visible + broken scope file is better than a silent edit to a line the SDK may not own. +6. SDK-owned entries: set the `members` entries, one `{ workspace = true }` + source per member, and one `[project] dependencies` entry per client. Keep + everything else, including members and dependencies the user added. + [provisional] This needs a TOML editor that keeps formatting; + `helpers/pyproject` reformats the file today. +7. Module scope: replace the old `dagger-io = { path = "sdk", editable = true }` + source with the workspace source; remove every `[[dependencies]]` entry from + `dagger-module.toml` (the manifest builder already has + `withoutLegacyRuntimeDependencies`, `python-sdk.dang:164`); handle the global + client flag. +8. Scope without a module: remove the refusal at `python-sdk.dang:109-114`. +9. Lock: refresh `uv.lock` if it exists. + +**Generation must never write a tree the runtime cannot build** [confident]. + +The module runtime reads `[tool.uv.workspace] members` and `[project] +dependencies` with **`tomllib`, in the SDK's pinned default image**, in one +cached exec per build. Those two are the user's own content, so they can hold +escapes, literal strings and multi-line strings, and only a real parser reads +them correctly. A hand-written reader failed this twice, in opposite +directions: first it read a marker out of a string that only looked like one, +then it refused `"tomli; python_version < \"3.11\""` as "not an array of +strings". **The parser boundary is where the user's content begins.** +Single-line keys that the templates and `mod config set` write — `name`, +`base-image`, `use-uv` — stay on regular expressions, because generation writes +them itself. + +The runtime cannot reach the Go helper, which does parse TOML: the shared +entrypoint carries a copy of this build, and there `currentModule` is the +user's module, so it can read none of its own non-Dang files. + +What is still refused, and by whom: + +- The runtime refuses an array holding anything but strings, rather than + reading strings out of a structure that is not a list of them. +- The SDK's editor refuses a scope file whose tables it cannot edit while + keeping the user's formatting, an inline workspace table among them. That + limit is the editor's, not the runtime's. + +Generation compares **both ways**: the members the runtime reads against the +members uv reads from the TOML. A member either one reads and the other does +not stops generation, naming the key and the difference. One way would not be +enough — "the runtime reads at least what I wrote" still lets it read a member +nobody named. This holds for a module scope only; a plain project is never +built by the runtime. + +**The module build installs what the project depends on** [confident], and the +members those depend on, in every path: locked uv, unlocked uv and pip. A +workspace member the project does not depend on is the user's business, and +installing it made an unrelated member's requirements break the module. + +Each scope writes only inside its own directory, so two scopes never write the +same file. + +Two side effects. With no `[[dependencies]]`, the refusal of clients on the +static path (`python-sdk.dang:191`) goes away. And a module's types already +travel as JSON through the shared Dang entrypoint, so the client set no longer +changes how the engine loads a module. + +**`findClientRoot` (`python-sdk.dang:42`) [provisional].** The marker stays +`pyproject.toml`. A `pyproject.toml` with `[tool.dagger] generated` belongs to a +generated member, so it is never a client root. The answer is the nearest +`pyproject.toml` above it, which is the scope. Today's `sdk/` lift stays for +modules that are not upgraded yet. This repo's own `sdk/` carries no marker, so +the check in `.dagger/modules/e2e/main.dang` still holds. + +## 13. How a client loads its module + +A client must load the module it targets, from a client session and from a +module session. The engine field exists and is merged, `dagger/dagger#14210` +at `284cd849`: + +```graphql +Query.serveModule(address: String!, refPin: String): Void +``` + +- A git address resolves through `moduleSource(address, refPin, requireKind: GIT)`. +- A workspace path resolves through `currentWorkspace`, absolute from the + workspace root and relative from the cwd. +- **A descriptor writes the absolute form, `/.dagger/modules/lib`, never + `./…`** [confident]. The relative form resolves against the cwd of whoever + runs the code, so a program started from its own scope directory would load a + different module, or none. The absolute form names one place in the workspace. +- A bare name is rejected, so a module cannot enumerate what its caller installed. +- Both end in `asModule().serve()`. + +**The descriptor keeps one shape**: `ref` is the address, `pin` is `refPin`. +Generated code never branches on which kind of reference it holds. + +**The SDK, however, sends one of two queries** [confident]. A module driven by a +Dang entrypoint runs its Python in an ordinary nested client, which the engine +gives no module context, so that process finds its workspace from its own +container rather than from the user's. An absolute workspace path then resolves +against a container and the load fails. + +The entrypoint therefore hands the process what it needs to load its clients — +and **only** that: + +| The target | The query | +| --- | --- | +| Local, and handed over by the entrypoint | `node(id: )` → `asModule` → `serve` | +| Everything else | `serveModule(address, refPin)` | + +What is handed over is one **module source per declared client**, built from the +files the engine has already loaded for it, detached from the workspace those +files came from. It is read from the caller's config at every call. A local +client that was not handed over fails by name, pointing at `dagger generate`; +the process never falls back to `serveModule` for a local target. + +**It must not be the workspace, and it must not be a handle that leads back to +one** [confident]. A module is third-party code, and in Dagger an ID is a +capability. Handing over the workspace lets module code read any file in it; a +probe did exactly that. Handing over `Workspace.moduleSource(path)` is no +better, because such a source keeps the workspace it came from — +`withIncludes(["../../../secret"])` reloads its context from there — and +`Module.source` leads back the same way. Both were tried, and both leaked. + +The branch lives in the load seam, `sdk/src/dagger/client/_load.py`, and nowhere +else. It is temporary: one query would serve both if the engine accepted a +least-privilege capability of this kind, which is filed against the engine. A +`serveModule` that took a *workspace* would not do: it would put the capability +back in the module's hands. + +**How far the evidence reaches** [confident]. A check runs module code that +walks every identifier the loader holds and every route that could rebuild a +directory from outside the files it was handed, at climb depths 1 to 12, one +level of recursion, under both entrypoint forms: 4305 routes, nothing read. It +distinguishes a refused capability from a field the engine does not have, and +asserts the second count is zero, so a route cannot pass by naming a field that +does not exist. What it does not cover: a field a later engine adds, a third +hop, and a route needing more than one level of recursion. The probe reads the +caller's file against the shape that leaked, so it can fail. + +**This protects a module that runs on this SDK's entrypoint, and nothing more.** +The engine hands every Dang entrypoint the caller's workspace, so a module that +brings its own entrypoint reads the caller's files whatever an SDK does. That is +the engine's to fix. + +**What the field still owes the caller's cache** [open]. A load at run time is +invisible to the cache key of the call that performed it. So a caller keeps its +cached result after its client's target changes, which `[[dependencies]]` used +to prevent. Every SDK that drops `[[dependencies]]` inherits this, so the answer +belongs to the field, not to a language. + +### `core.serveModule`, not `dag.serveModule` [provisional] + +Both names describe one field of `Query`, so the engine schema is the same. The +difference is what an SDK shows the user. + +- In this design `dag` is the session. It owns the connection and holds no API + field (property 2). A field named on `dag` would put an API call back on the + session. +- Core binds `Query`, so the field arrives as `core().serve_module(…)` with no + work on our side. That is also how a user would reach it by hand, for a module + the SDK did not generate a client for. +- Neither choice changes the SDK files: the load query goes out through the raw + query builder, so the SDK never imports core to send it. + +So: yes to `serveModule`, and please put it on `Query`. Python will read it as core. + +### What this SDK needs from the field + +| Need | Why | +| --- | --- | +| One argument for both kinds of reference: a workspace path and a git URL. | The SDK gets one code path and one descriptor shape. Today it needs two: `moduleSource(refString, refPin)` for git, and a path field for local. | +| A path resolves in the caller's own context. | This is the property that makes one client work in a module and in a plain program (`dagger/dagger#14148`). A module resolves against its own context root, a client session against its workspace. | +| A pin for a git reference. | The client records the commit it was generated against, and loads that commit. | +| Idempotent. | The SDK loads once per session and per client, but a second call must not fail. | +| The same call in both session kinds. | One code path in the SDK, for a module and for a plain program. | +| One round trip. | It replaces the chain `moduleSource` → `withName` → `asModule` → `serve`. A first use costs one query. | + +With that field, the descriptor holds one reference and an optional pin (8.3), +and the SDK has one load path for every client. + +**No name is pinned, and none is needed.** An earlier revision asked the field +for a `name:` argument, so that a client could state the name its bindings were +generated under. That was wrong: nobody chooses that name. `dagger module client +add` cannot set one, `asModule().serve()` has none, and the engine derives it +from the module's own config, by the same code, both when the SDK generates +against that module's schema and when `serveModule` resolves the same address. +The two agree unless the module renamed itself, and that client is stale by +definition. The first selection then fails with "cannot query field", which the +SDK turns into `StaleClientError` telling the user to run `dagger generate` +(section 10). The cost of not pinning: if the engine ever changes how it derives +a name, every client regenerates rather than keeping the old name. + +- The design does not use `[[dependencies]]`, and it does not use + `currentWorkspace` from module code. +- The floor is released `v1.0.0-beta.14`, which has the field and drives Dang + entrypoints. The SDK's load path for engines without `serveModule` is gone, so + an older engine fails the load with `ClientLoadError` naming the missing + field — never with advice to regenerate, which could not help. +- `entrypointClientCallCheck` runs a module through each entrypoint form, + calling a local client. Nothing did that before, which is why the entrypoint + defect above reached a user rather than a check. +- A path in a descriptor is written against the workspace root, absolute + [confident]. A module session resolves it against the same workspace, and + `/../tmp` is normalised back inside it rather than escaping. Symlinks are not + verified. +- [speculative] A module can load itself (7.3). + +## 14. Decisions + +| Question | Decision | +| --- | --- | +| Q1. Namespace name | `dagger_clients`. | +| Q2. Where the SDK files come from | A copy in each scope: `sdk/`, with no generated code. Published later, which removes the copy. | +| Q3. The session | An optional keyword-only `session=`. One default session per process, started on the first query and shared by every client. In a plain program, the SDK provisions the engine on the first query. | +| Q4. Contributed field shape | Module-level function: `as_linter(binding)`. | +| Q5. The global client | Flag `[tool.dagger] global-client = true`. Contributed fields are added to the core class at run time. A `DeprecationWarning` in phase 2; removal in a later release. It sits with the SDK files (section 9). | +| Q6. Module support and core | Rewrite the module support protocol on the raw query builder. | +| Q7. How a local client loads its module | Through an engine field that resolves a path in the caller's own context: `serveModule` (section 13). No `[[dependencies]]`. | +| Q8. Clients outside the scope | [closed] A client is generated inside the scope that uses it, so the case does not exist. Sharing between scopes may come later. | +| Q9. Who writes `[project] dependencies` | The SDK. A client declared on a scope is a client that scope uses, so generation writes the dependency, and removes it with the client. The user's code then works right after `dagger generate`. | +| Q10. Removing a client | `generateScope` deletes the member, its source and its `members` entry, then relocks. | +| Q11. Version check strictness | Client/core mismatch: import error, and a warning while the SDK registers types. Engine/core mismatch: warning in phase 1. | +| Q12. The tree inside a scope | `src/`, `sdk/`, `clients/core`, `clients/`. The same inside a module and outside one. No suggested workspace location. | +| Q13. Client types in a module's own signatures | Not supported. An exported signature names core types and the module's own types only. The SDK refuses a client type with a clear message (7.4). | +| Q14. A self client for every module | No. The user declares a client to the module when the module calls itself. | + +## 15. Checks + +Invert each assertion once and confirm that it fails. + +1. One structure: a generated module and a generated plain project have the same tree. +2. One artifact: the digest of `clients/linter` is the same in a module scope and in a plain project scope. +3. SDK isolation: import every `dagger.*` module in a venv with no generated code. +4. Graph rules: an AST scan of imports in core, in each client package and in the SDK files. +5. Call shape: module source that calls a client passes mypy and pyright, then `dagger call` runs it. +6. Standalone build: `uv build --wheel` for each member with no engine; the wheel exists and imports in a fresh venv. +7. Staleness: a changed `CORE_DIGEST` raises `StaleClientError`; during type registration it logs a warning. +8. Load memo: two calls on one client send one load; two sessions send two. +9. Shared session: `linter().lint(core().directory())` works. +10. End to end, module: a module with a git client and a local client loads both through the CLI. +11. End to end, plain project: a test project with a client runs its test with no engine handling of its own. +12. No `[[dependencies]]`: generating a module that has them removes them; the module still calls its clients. +13. Configuration drives generation: adding a client writes exactly one member, one source, one `members` entry and one dependency; removing it undoes all four. +14. Import after generate: a scope with a fresh client imports it with no edit by the user. +15. Name pinning: none. A client whose generated name differs from the name the engine serves fails on its first selection, and that becomes `StaleClientError` naming the client, its address and the remedy (13). +16. Generated code needs no configuration: delete the client entries from `dagger.toml`, keep the tree, and the module still calls its clients. +17. Global client, existing module: the flag is written; the unchanged source passes mypy and runs through `dagger call`; `type(dag.container()) is dagger_clients.core.Container`. +18. Global client, new module: no flag, no `dagger_global`; `dag.container` raises the migration message. +19. Global client, turned off: the global client and its dependency are gone. +20. Install one: a consumer that names one client gets only that client, core and the SDK files. +21. Remove a client: its member, source and `members` entry are gone; `uv sync --locked` passes. +22. User content kept: user tables, comments and a user member survive generation byte for byte. +23. Default session in a plain program: a program with no connection handling runs a client call and exits cleanly. **Passes**: exit 0, no session process left, 23.7s cold and 1.9s warm. +24. Self client: a module with a client to itself calls its own function through `dagger call`; after an API change, generation succeeds. +25. Signature rule: a function that returns a client type fails registration with a clear message; a core type and the module's own class pass. + +## 16. Phases + +**Phase 1: clients that load their own module, in every scope.** + +- Spikes first: a module loads itself, with and without `SELF_CALLS`, and + regenerates with a self client; core-alone schema; a workspace scope in the + module build (uv and pip modes); a format-keeping TOML editor; where the global + client lives; typing through `dagger_global`; the default session in a plain + program. +- Remove every import of generated core from the SDK files. +- SDK files: `Session` and the default session, `Target`, the load memo, the + core digest check, error types, the signature rule in `describe_type`. +- Generator: partition by `@sourceMap`; core and one member per client; entry + functions; descriptors; digests; the global client with the flag. +- `generateScope` writes the one structure in every scope, manages the members + and the sources, and removes `[[dependencies]]`. It generates for a scope + without a module too. +- `findClientRoot` lifts a member to its scope. `mod config` handles the flag. +- Checks 1–25. Local-client checks run once the engine has the field. + +**Phase 2: published SDK files, end of migration.** Publish the SDK files and +remove the `sdk/` copy from a scope. The global client warns on import; its +removal date is Yves's call. + +## 17. Not verified + +- [speculative] A module can load itself into its own session, and whether that + needs `SELF_CALLS`. +- [speculative] Regeneration of a module with a self client does not block. +- [speculative] The module build installs a scope that is a uv workspace root, + in uv and pip modes. The locked uv path is proved; the pip and unlocked paths + install every member, which is a defect being fixed. +- [speculative] A symlinked path in a descriptor. +- [speculative] A scope inside a tree that already has a uv workspace root above it. +- [speculative] The global client as a second import package of the `sdk/` + member, and whether mypy and pyright then type `dagger.dag` as the global + `Client`. +- **[blocker, not ours to fix] A changed client target returns a cached result.** + `[[dependencies]]` used to put the target module into the caller's identity, so + changing the target changed the caller's digest. A unified client records a + path and loads the module at run time, so nothing about the target reaches the + caller's cache key. Proved on the `284cd849` engine: call a local client, + change only the target's implementation, call the unchanged caller again, get + the old result. This is a question for the engine and the specification — what + `serveModule` contributes to the cache key of the call that used it. No SDK can + fix it from outside. +- [speculative] `Module.serve` from a module session on the engine this repo targets. +- [speculative] Spec decision 4: a client function whose signature names a type + from another client. + +Verified by experiment with uv 0.12.13, on hand-written stand-ins for generated +members: namespace build; shared namespace across editable installs; mypy and +pyright errors across the namespace; a scope root with and without `[project]`; +a scope that installs only the client it names, plus core; +`uv sync --locked --no-dev`; removal and relock; one member's files are identical +in two scopes. + +Verified while building it, against a live engine: + +- The engine's introspection JSON carries `@sourceMap(module:)`, and a partition + on that directive splits core from the module-owned types. **One gap** [known + limitation]: in an engine view before v1.0.0, a module's own `XID` scalar and + its `loadXFromID` field carry no `@sourceMap`, so the partition counts them as + core. Core then differs from the core the scope generated, and the client is + refused as skew. A client in a scope older than v1.0.0 cannot be generated + until the partition treats those two shapes as module-owned. +- Core alone: read the client-facing schema through an empty stand-in module, + then strip that module out of the schema. Core comes out byte for byte the + same as without the stand-in. +- The whole path works with the released CLI on a released engine, outside the + check harness (`hack/try-unified-clients.sh`): `dagger module init python` + twice, `dagger module client add ../lib`, then a module calling + `lib().greeting()` through `dagger_clients.lib`, and `core()` beside it. +- A plain program, in a scope with no `[project]` table, calls a client through + `dagger.connection()`. The engine is provisioned, the module is loaded, and + the call returns. +- **The SDK that generates a module must be the one that runs it.** A generated + manifest now names only a Dang entrypoint, so the danger is no longer the + builtin runtime but a *published* entrypoint older than the layout it is asked + to run: it refuses the module for want of `sdk/src/dagger/client/gen.py`. A + checkout under development therefore needs its own entrypoint reachable, and + an entrypoint source may name only a git ref or a path inside the module. +- Core's digest does not depend on which clients are generated beside it. The + digest comes from the client-facing schema. The module-facing schema gives a + different digest, so generation must always read the same view. +- The global client cannot be imported eagerly from `dagger/__init__.py`. Every + generated package imports `dagger`, so the eager import is a cycle. A + module-level `__getattr__` (PEP 562) removes it. +- An engine reports an unknown field with a GraphQL validation error, with no + path and with `extensions.code == "GRAPHQL_VALIDATION_FAILED"`. Checked on + beta.11 and beta.13. Only that exact signal marks a stale client (section 10). + It no longer decides how a client loads: the floor is beta.14, which has + `serveModule`, and the load path for engines without it is gone. +- A module driven by a Dang entrypoint runs its Python as an ordinary nested + client. The engine attaches module context only to execs it starts itself, so + that process reports `currentWorkspace` from its own container and no + `currentModule` at all. Whatever module context the code needs, the entrypoint + must hand over. diff --git a/hack/designs/2026-09-18-serve-module-cache.md b/hack/designs/2026-09-18-serve-module-cache.md new file mode 100644 index 0000000..107aee3 --- /dev/null +++ b/hack/designs/2026-09-18-serve-module-cache.md @@ -0,0 +1,120 @@ +# A unified client's target is not in its caller's cache key + +*2026-09-18. For the engine and the specification, not for one SDK.* + +## The question + +A unified client loads its target module at run time, through +`Query.serveModule(address, refPin)`. What should that load contribute to the +cache key of the call that performed it? + +Today it contributes nothing, so a caller keeps a stale result after its +client's target changes. + +## What we saw + +Two modules. `lib.greeting(name)` returns `hello, `. `demo.hello(name)` +calls it through a generated client. Then `lib` is changed to return +`CHANGED, `, and nothing else is touched. + +``` +$ dagger call -m demo hello --name probe # before the change +hello, probe + +# lib now returns "CHANGED, ..." + +$ dagger call -m demo hello --name probe # same arguments +hello, probe # stale + +$ dagger call -m lib greeting --name probe # the target itself +CHANGED, probe # the change is there + +$ dagger call -m demo hello --name other # different argument +CHANGED, other # a fresh key sees it +``` + +The third and fourth commands are what make the mechanism plain. The target's +change *is* visible to the engine. The caller is stale only where its own cache +key is unchanged. + +Seen on a released `v1.0.0-beta.13` engine through the SDK's fallback load path, +and on an engine built from `dagger/dagger` at `284cd849` through `serveModule` +itself. Seen again on released `v1.0.0-beta.14`, with the module driven by a +Dang entrypoint rather than a runtime. It is not a property of the new field, +nor of how the module is driven; it is a property of loading a module at run +time. + +## Why it used to work + +`[[dependencies]]` in the caller's manifest made the target part of the +caller's module identity. Change the target, and the caller's digest changed +with it, so the caller's functions were re-evaluated. + +This paragraph is an **inference from the behaviour**, not something read in +engine code. What is measured is below; the mechanism that used to prevent it is +the engine's to confirm. + +Unified clients remove `[[dependencies]]`. A generated local client records only +a path: + +```python +NAME = "lib" +REF = "/.dagger/modules/lib" +PIN = None +``` + +Nothing about the target's content reaches the caller. + +## Why an SDK cannot fix it + +The staleness is decided before any SDK code runs. + +1. The engine evaluates `Demo.hello`. Its cache key comes from the caller's + module identity and the arguments. +2. That key hits. The function body never runs. +3. So `serveModule` is never called, and the client's own subcall — which + *would* be keyed correctly on the target — never happens. + +An SDK can only act inside step 3. By then the answer has been returned. The +`--name other` run above confirms it: give the caller a key it has not seen, and +the body runs and reads the new target. + +## Three answers + +**(a) Record the client targets in the caller's own config.** The SDK already +knows them: the workspace file lists a scope's clients. Writing the target and +its pin into the module's manifest would put them back into the caller's +identity. Cheap, and it restores exactly the invariant that was lost. + +The cost: it is `[[dependencies]]` again in everything but name, and the point +of unified clients is that the artifact is self-sufficient. It also cannot +express a local target's *content*, only its path, unless the manifest carries a +digest that generation refreshes — and then a stale manifest is a new way to be +wrong. + +**(b) Make a served module part of the cache key of whatever served it.** The +engine records, for each cached function result, the modules served during its +evaluation, and invalidates that result when one of them changes. This is the +discovered-dependency problem that build systems already solve, and it is the +only answer that stays true when the set of clients a call uses depends on the +arguments. + +The cost: it is engine work, and it needs a story for a git target (the pin +settles it) against a local one (content). + +**(c) Document it.** A user learns that changing a client's target needs +`--no-cache`, or a touch of the caller. We consider this unacceptable: the first +time a user debugs a module by changing its dependency, the tool lies to them. + +## Recommendation + +(b) is the right answer, and it belongs to the field rather than to any +language. (a) would unblock end-to-end use sooner, at the price of re-creating +what this design set out to delete; take it only as a stopgap, and only if (b) +is far off. + +## What it blocks + +Nothing in generation, and nothing in the Python SDK's own checks. It blocks +using unified clients to replace real dependencies, because a module developer +who changes a dependency and re-runs the caller gets the previous answer. diff --git a/hack/designs/2026-09-18-workspace-sdk-runtime.md b/hack/designs/2026-09-18-workspace-sdk-runtime.md new file mode 100644 index 0000000..9e42591 --- /dev/null +++ b/hack/designs/2026-09-18-workspace-sdk-runtime.md @@ -0,0 +1,101 @@ +# The SDK that generates a module must be the one that runs it + +*2026-09-18, rewritten the same day. For the engine, not for one SDK.* + +## The question + +A workspace can name its own SDK: + +```toml +[modules.python-sdk] +source = "python-sdk" + +[sdks.python] +module = "python-sdk" +``` + +`dagger module init python` then generates the module with **that** SDK. What +runs the module afterwards is decided by the module's own manifest, and nothing +ties the two together. When they differ, the module is generated by one SDK and +refused by another. + +## How it looked first, and why that version is gone + +The generated manifest used to say: + +```toml +[runtime] +source = "python" +``` + +At run time `python` meant the engine's **builtin** Python SDK, not the one the +workspace named. A user saw: + +``` +$ dagger module init python -n my-module # generated by the workspace SDK +$ dagger api functions my-module +! module "my-module": generated file "sdk/src/dagger/client/gen.py" is missing; + run `dagger generate` and commit the generated files +``` + +Python modules no longer carry a `[runtime]` at all, so that particular sentence +cannot be written any more. **The problem it was an instance of did not go with +it.** + +## How it looks now + +A generated manifest names a Dang entrypoint: + +```toml +[entrypoint] +kind = "dang" +source = "dagger.io/sdk/python/entrypoint@v1" +``` + +That address is **published**. A workspace developing the SDK generates the new +layout and then hands the module to a published entrypoint that predates it, +which refuses the module with the same message as before. The failure moved from +the runtime to the entrypoint; the shape is identical. + +It is worse in one respect. An entrypoint source may name only a git ref or a +path **inside** the module: + +``` +! resolve module entrypoint source "../../../entrypoint": + entrypoint source path "../../../entrypoint" escapes the module directory +``` + +So a developer cannot point a module at the entrypoint in their own checkout. +The options are to publish a branch and name it by git ref, or to generate the +static entrypoint into the module, which copies the checkout's code inside. + +## Why an SDK cannot settle it + +The SDK knows it was named by the workspace — `Workspace.sdk(name:).ref` says +so. It could write that reference into every manifest it generates. We have not, +because a manifest is the user's file and it must stay portable: a published SDK +must keep naming the published entrypoint, so the same field would mean +different things depending on who generated it. The same workaround would also +have to be written once per SDK, in every language, for a mapping the engine +already holds. + +## The question for the engine + +**Should an entrypoint source be able to name the workspace's SDK?** + +The workspace already maps `python` to a module, and `dagger module init` and +`dagger generate` both honour that mapping. If an entrypoint source could name +it too, the SDK that generated a module would be the SDK that runs it, by +construction, in every phase. One fix would serve every language, and no path or +branch name would be baked into a user's manifest. + +## Until then + +`hack/try-unified-clients.sh` copies the checkout's own shared entrypoint into +each module, so the walkthrough runs on the SDK under test rather than on a +published release. A branch pushed to a fork and named by git ref works too, and +is what a developer testing this SDK does today. + +**Before releasing unified clients, tag `entrypoint/v1.x` from the branch that +carries them.** Until that tag exists, every module this SDK generates names a +published entrypoint that cannot run it. diff --git a/hack/e2e-floor.sh b/hack/e2e-floor.sh new file mode 100755 index 0000000..1398416 --- /dev/null +++ b/hack/e2e-floor.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# The floor job: the e2e checks on the engine of this SDK's floor release, the +# oldest the SDK claims to run on. An ordinary run (hack/e2e-local.sh) takes +# whatever engine it finds; this one provisions the floor's and asserts it +# answers before any check runs, so it cannot pass on another. +# +# hack/e2e-floor.sh # every check +# hack/e2e-floor.sh runtime-call-check # named checks, as e2e-local.sh +# +# The floor is floorVersion in .dagger/modules/e2e/main.dang. +set -eu +root=$(CDPATH= cd "$(dirname "$0")/.." && /bin/pwd) +floor=$(sed -n 's/^ *let floorVersion: String! = "\(.*\)"$/\1/p' "$root/.dagger/modules/e2e/main.dang") +[ -n "$floor" ] || { echo "no floorVersion in .dagger/modules/e2e/main.dang" >&2; exit 1; } +export DAGGER_ENGINE="image://registry.dagger.io/engine:v$floor" +export E2E_ASSERT_FLOOR=1 +echo "floor job on $DAGGER_ENGINE" >&2 +exec "$root/hack/e2e-local.sh" "$@" diff --git a/hack/e2e-local.sh b/hack/e2e-local.sh new file mode 100755 index 0000000..1b87a01 --- /dev/null +++ b/hack/e2e-local.sh @@ -0,0 +1,76 @@ +#!/bin/sh +# Run the e2e checks on the local engine, in a copy of this tree, so a check +# never writes into it. dagger.toml is the e2e workspace config: it registers +# this checkout as the python SDK and the fixture scopes to it. +# +# hack/e2e-local.sh # every check +# hack/e2e-local.sh find-client-root-check generate-scope-clients-check +# E2E_WITH=other-branch hack/e2e-local.sh runtime-client-call-check +# +# It runs on whatever engine the CLI finds. hack/e2e-floor.sh runs the same +# checks on the floor release's engine, and asserts it. +# +# E2E_SCRATCH names the copy; it defaults to a fresh temporary directory. +# Each named check logs to .logs/.log, next to the copy rather +# than in it, because the copy is synced with --delete; a failure prints the +# log's error lines. +# +# E2E_WITH names a branch that must land with this one: its changes since the +# two forked are applied to the copy, so the checks run on both together +# without a merge in either branch. +# +# E2E_TIMEOUT stops a named check after that many seconds, 900 by default: +# on a shared engine a stuck connection otherwise holds a check until the +# engine drops it, some twenty minutes later. +set -eu +root=$(CDPATH= cd "$(dirname "$0")/.." && /bin/pwd) +scratch=${E2E_SCRATCH:-$(mktemp -d)} +module=.dagger/modules/e2e +logs="$scratch.logs" +mkdir -p "$logs" +rsync -a --delete \ + --exclude .git --exclude .venv --exclude __pycache__ \ + --exclude .dagger/modules/e2e/out \ + "$root/" "$scratch/" +if [ -n "${E2E_WITH:-}" ]; then + base=$(git -C "$root" merge-base HEAD "$E2E_WITH") + git -C "$root" diff --binary "$base" "$E2E_WITH" | git -C "$scratch" apply - + echo "applied $E2E_WITH since $(git -C "$root" rev-parse --short "$base")" >&2 +fi +CDPATH= cd "$scratch" +[ -d .git ] || git init --quiet +echo "checks of $module in $scratch" >&2 +# Set by hack/e2e-floor.sh: the engine must be the floor before any check. +if [ -n "${E2E_ASSERT_FLOOR:-}" ]; then + dagger call -m "$module" assert-floor-engine >"$logs/assert-floor-engine.log" 2>&1 || { + echo "FAIL the engine is not the floor ($logs/assert-floor-engine.log)" >&2 + sed 's/\x1b\[[0-9;]*m//g' "$logs/assert-floor-engine.log" | grep -E '^ *! ' | awk '!seen[$0]++' | head -4 >&2 + exit 1 + } + echo "engine is the floor" >&2 +fi +if [ $# -eq 0 ]; then + exec dagger check -m "$module" +fi +status=0 +for check in "$@"; do + dagger call -m "$module" "$check" >"$logs/$check.log" 2>&1 & + call=$! + ( + trap 'kill "$nap" 2>/dev/null; exit 0' TERM + sleep "${E2E_TIMEOUT:-900}" & + nap=$! + wait "$nap" + kill "$call" 2>/dev/null && echo "TIMEOUT after ${E2E_TIMEOUT:-900}s" >>"$logs/$check.log" + ) & + watchdog=$! + if wait "$call"; then + echo "PASS $check" >&2 + else + echo "FAIL $check ($logs/$check.log)" >&2 + sed 's/\x1b\[[0-9;]*m//g' "$logs/$check.log" | grep -E '^ *! |^TIMEOUT' | awk '!seen[$0]++' | head -8 >&2 + status=1 + fi + kill "$watchdog" 2>/dev/null || true +done +exit $status diff --git a/hack/try-unified-clients.sh b/hack/try-unified-clients.sh new file mode 100755 index 0000000..15e6978 --- /dev/null +++ b/hack/try-unified-clients.sh @@ -0,0 +1,116 @@ +#!/bin/sh +# Try unified clients by hand, with the real CLI, in a throwaway workspace. +# +# hack/try-unified-clients.sh # a fresh temporary workspace +# TRY_DIR=/tmp/uc hack/try-unified-clients.sh +# TRY_SDK=/path/to/another/checkout hack/try-unified-clients.sh +# +# It builds this, from nothing: +# +# /dagger.toml declares this checkout as the python SDK +# /python-sdk/ a copy of this checkout +# /.dagger/modules/lib/ a module with one function +# /.dagger/modules/demo/ a module that calls lib through a client +# +# and then calls demo, which calls lib through serveModule. It needs an engine +# that runs Dang entrypoints and has serveModule: v1.0.0-beta.14 or later. +# +# Why each module's manifest is rewritten: generation names the shared Dang +# entrypoint this repository publishes, dagger.io/sdk/python/entrypoint@v1, +# which runs the SDK of its release, not this checkout. The engine loads an +# entrypoint from a git ref or from a path inside the module, never from a path +# above it, so each module gets a copy of this checkout's entrypoint/ and its +# manifest names that copy. Every generation names the published one again, +# so the copy is named again after each. A user of a published SDK never does +# this. +set -eu + +# TRY_SDK runs the walkthrough on another checkout, a branch under review say. +sdk=${TRY_SDK:-$(CDPATH= cd "$(dirname "$0")/.." && /bin/pwd)} +dir=${TRY_DIR:-$(mktemp -d)} +say() { printf '\n== %s\n' "$1" >&2; } + +say "workspace $dir" +rm -rf "$dir" +mkdir -p "$dir" +cd "$dir" +git init --quiet +rsync -a --exclude .git --exclude .venv --exclude __pycache__ "$sdk/" "$dir/python-sdk/" +cat >dagger.toml <<'TOML' +[modules.python-sdk] +source = "python-sdk" +check.skip = ["*"] + +[sdks.python] +module = "python-sdk" +TOML + +# The shared entrypoint of this checkout, copied into .dagger/modules/ +# and named by the module's manifest. +entrypoint() { + rm -rf ".dagger/modules/$1/checkout-entrypoint" + mkdir -p ".dagger/modules/$1/checkout-entrypoint" + cp python-sdk/entrypoint/*.dang ".dagger/modules/$1/checkout-entrypoint/" + cat >".dagger/modules/$1/dagger-module.toml" <.dagger/modules/lib/src/lib/__init__.py <<'PY' +from dagger import function, object_type + + +@object_type +class Lib: + @function + def greeting(self, name: str = "world") -> str: + return f"hello, {name}" +PY +dagger call -m lib greeting --name lib + +say "dagger module init python --name demo" +dagger module init python --name demo -y + +say "dagger module client add ../lib" +(cd .dagger/modules/demo && dagger module client add ../lib -y) +entrypoint demo + +say "what the scope looks like now" +cat .dagger/modules/demo/pyproject.toml +find .dagger/modules/demo -maxdepth 3 -name pyproject.toml | sort +cat .dagger/modules/demo/clients/lib/src/dagger_clients/lib/_target.py + +say "a module calling its client" +cat >.dagger/modules/demo/src/demo/__init__.py <<'PY' +from dagger import function, object_type +from dagger_clients.core import Container, core +from dagger_clients.lib import lib + + +@object_type +class Demo: + @function + async def hello(self, name: str = "unified clients") -> str: + return await lib().greeting(name=name) + + @function + def base(self) -> Container: + return core().container().from_("alpine:3.21") +PY +dagger call -m demo hello --name "unified clients" +dagger call -m demo base with-exec --args=echo,core-still-works stdout + +say "done: $dir" +cat >&2 < +""" + +import json +import sys + +MODULE = "core-schema" + + +def main(src: str, dst: str) -> None: + result = json.load(open(src)) + schema = result["__schema"] + gone = {t["name"] for t in schema["types"] if is_stand_in(t)} + if not gone: + msg = f"no type of the module {MODULE!r} in {src}" + raise SystemExit(msg) + gone |= {name + "ID" for name in gone} + loaders = {f"load{name}FromID" for name in gone} + schema["types"] = [t for t in schema["types"] if t["name"] not in gone] + for t in schema["types"]: + if t.get("fields") is not None: + t["fields"] = [ + f + for f in t["fields"] + if not is_stand_in(f) and f["name"] not in loaders + ] + for key in ("possibleTypes", "interfaces"): + if t.get(key) is not None: + t[key] = [ref for ref in t[key] if ref["name"] not in gone] + json.dump(result, open(dst, "w")) + + +def is_stand_in(node: dict) -> bool: + return any( + d["name"] == "sourceMap" + and any( + a["name"] == "module" and json.loads(a["value"]) == MODULE + for a in d.get("args") or [] + ) + for d in node.get("directives") or [] + ) + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2]) diff --git a/helpers/core-schema/dagger-module.toml b/helpers/core-schema/dagger-module.toml new file mode 100644 index 0000000..7fc8123 --- /dev/null +++ b/helpers/core-schema/dagger-module.toml @@ -0,0 +1,5 @@ +name = "core-schema" +engineVersion = "v1.0.0-0" + +[runtime] +source = "dang" diff --git a/helpers/core-schema/main.dang b/helpers/core-schema/main.dang new file mode 100644 index 0000000..ca0d4dd --- /dev/null +++ b/helpers/core-schema/main.dang @@ -0,0 +1,4 @@ +""" +A module with nothing in it, whose client-facing schema is core alone. +""" +type CoreSchema {} diff --git a/helpers/pyproject/main.go b/helpers/pyproject/main.go index 10dbf64..ed3147e 100644 --- a/helpers/pyproject/main.go +++ b/helpers/pyproject/main.go @@ -1,8 +1,10 @@ package main import ( + "flag" "fmt" "os" + "strings" ) func main() { @@ -13,19 +15,37 @@ func main() { } // run dispatches a subcommand. get-* commands print to stdout (no trailing -// newline). set-* commands edit the file in place. +// newline). set-* commands edit the file in place and re-emit it. +// edit-scope and set-global-client edit the file in place and keep its +// formatting: they touch a scope file that generation also owns. // -// usage: pyproject [value] +// get-member-kinds takes a scope directory instead of a file. +// +// usage: pyproject [value | flags] func run(args []string) error { if len(args) < 2 { - return fmt.Errorf("usage: pyproject [value]") + return fmt.Errorf("usage: pyproject [value | flags]") } cmd, path := args[0], args[1] + if cmd == "get-member-kinds" { + out, err := memberKinds(path) + if err != nil { + return err + } + fmt.Print(out) + return nil + } data, err := os.ReadFile(path) if err != nil { return err } + switch cmd { + case "edit-scope": + return runEditScope(path, data, args[2:]) + case "set-global-client": + return runSetGlobalClient(path, data, args) + } doc, err := load(data) if err != nil { return err @@ -45,6 +65,14 @@ func run(args []string) error { case "get-base-image": fmt.Print(getBaseImage(doc)) return nil + case "get-members": + fmt.Print(getMembers(doc)) + return nil + case "get-global-client": + if v, ok := getGlobalClient(doc); ok { + fmt.Print(boolStr(v)) + } + return nil case "set-python-version": v, err := value(args) if err != nil { @@ -74,6 +102,76 @@ func run(args []string) error { return os.WriteFile(path, out, 0o644) } +// runEditScope sets the SDK-owned entries of a scope pyproject.toml, and +// leaves the file alone when nothing changes. +// +// usage: pyproject edit-scope [--member M]... [--stale-member M]... [--source S]... +// [--dependency D]... [--global-client true|false] +func runEditScope(path string, data []byte, args []string) error { + flags := flag.NewFlagSet("edit-scope", flag.ContinueOnError) + flags.SetOutput(os.Stderr) + var edit scopeEdit + flags.Var((*repeated)(&edit.Members), "member", "a workspace member the SDK generates") + flags.Var((*repeated)(&edit.Stale), "stale-member", "a workspace member the SDK generated before and removes now") + flags.Var((*repeated)(&edit.Sources), "source", "a distribution that resolves to a workspace member") + flags.Var((*repeated)(&edit.Dependencies), "dependency", "a generated distribution the project depends on") + globalClient := flags.String("global-client", "", "write (true) or clear (false) the global client flag") + if err := flags.Parse(args); err != nil { + return err + } + if flags.NArg() > 0 { + return fmt.Errorf("edit-scope: unexpected argument: %s", flags.Arg(0)) + } + switch *globalClient { + case "": + case "true", "false": + on := *globalClient == "true" + edit.GlobalClient = &on + default: + return fmt.Errorf("edit-scope: --global-client takes true or false, not %q", *globalClient) + } + out, err := editScope(data, edit) + if err != nil { + return err + } + if string(out) == string(data) { + return nil + } + return os.WriteFile(path, out, 0o644) +} + +// runSetGlobalClient writes or clears the global client flag and nothing +// else, so clearing it gives back the file as it was before it was set. +// +// usage: pyproject set-global-client true|false +func runSetGlobalClient(path string, data []byte, args []string) error { + v, err := value(args) + if err != nil { + return err + } + if v != "true" && v != "false" { + return fmt.Errorf("set-global-client takes true or false, not %q", v) + } + out, err := editGlobalClient(data, v == "true") + if err != nil { + return err + } + if string(out) == string(data) { + return nil + } + return os.WriteFile(path, out, 0o644) +} + +// repeated collects every value of a flag given more than once. +type repeated []string + +func (r *repeated) String() string { return strings.Join(*r, ",") } + +func (r *repeated) Set(v string) error { + *r = append(*r, v) + return nil +} + func value(args []string) (string, error) { if len(args) < 3 { return "", fmt.Errorf("%s requires a value", args[0]) diff --git a/helpers/pyproject/main_test.go b/helpers/pyproject/main_test.go index 50517a7..d97d4f3 100644 --- a/helpers/pyproject/main_test.go +++ b/helpers/pyproject/main_test.go @@ -63,3 +63,133 @@ func TestRunRequiresValue(t *testing.T) { t.Error("expected error when value is missing") } } + +func TestRunEditScope(t *testing.T) { + p := writeTemp(t, sample) + err := run([]string{ + "edit-scope", p, + "--member", "sdk", "--member", "clients/core", "--member", "clients/linter", + "--source", "dagger-io", "--source", "dagger-clients-core", "--source", "dagger-clients-linter", + "--dependency", "dagger-clients-core", "--dependency", "dagger-clients-linter", + "--global-client", "true", + }) + if err != nil { + t.Fatalf("edit-scope: %v", err) + } + data, err := os.ReadFile(p) + if err != nil { + t.Fatalf("read: %v", err) + } + for _, want := range []string{ + `members = ["sdk", "clients/core", "clients/linter"]`, + "dagger-io = { workspace = true }", + "dagger-clients-linter = { workspace = true }", + `dependencies = ["dagger-io", "dagger-clients-core", "dagger-clients-linter"]`, + "global-client = true", + } { + if !strings.Contains(string(data), want) { + t.Errorf("missing %q in:\n%s", want, data) + } + } + if strings.Contains(string(data), `path = "sdk"`) { + t.Errorf("the vendored source survived:\n%s", data) + } + + err = run([]string{ + "edit-scope", p, + "--member", "sdk", "--member", "clients/core", "--stale-member", "clients/linter", + "--source", "dagger-io", "--source", "dagger-clients-core", + "--dependency", "dagger-clients-core", + }) + if err != nil { + t.Fatalf("edit-scope: %v", err) + } + if data, _ = os.ReadFile(p); strings.Contains(string(data), "linter") { + t.Errorf("the stale member survived:\n%s", data) + } +} + +func TestRunEditScopeRejectsAnUnknownFlag(t *testing.T) { + p := writeTemp(t, sample) + if err := run([]string{"edit-scope", p, "--bogus", "x"}); err == nil { + t.Error("expected an error for an unknown flag") + } +} + +func TestRunSetGlobalClient(t *testing.T) { + p := writeTemp(t, sample) + if err := run([]string{"set-global-client", p, "true"}); err != nil { + t.Fatalf("set: %v", err) + } + data, _ := os.ReadFile(p) + if !strings.Contains(string(data), "global-client = true") { + t.Errorf("flag not written:\n%s", data) + } + if err := run([]string{"set-global-client", p, "false"}); err != nil { + t.Fatalf("clear: %v", err) + } + data, _ = os.ReadFile(p) + if strings.Contains(string(data), "global-client") || strings.Contains(string(data), "[tool.dagger]") { + t.Errorf("clearing the flag left it or its table behind:\n%s", data) + } +} + +// set-global-client owns one key, not the file: set then cleared, the file +// comes back byte for byte, with its table order, quoting and inline sources. +func TestRunSetGlobalClientRoundTripsTheFile(t *testing.T) { + for name, src := range map[string]string{ + "no dagger table": `[project] +name = 'config' # single quotes +dependencies = [ + "dagger-io", + "dagger-clients-core", +] + +[tool.uv.sources] +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } + +[tool.uv.workspace] +members = ["sdk", "clients/core"] + +[build-system] +requires = ["uv_build>=0.8.4,<0.12.0"] +build-backend = "uv_build" +`, + "a dagger table in the middle": `[project] +name = "config" +requires-python = ">=3.12" + +[tool.dagger] +use-uv = false # a comment of the user's +base-image = "python:3.12-slim" + +[tool.uv.sources] +dagger-io = { workspace = true } +`, + } { + t.Run(name, func(t *testing.T) { + p := writeTemp(t, src) + if err := run([]string{"set-global-client", p, "true"}); err != nil { + t.Fatalf("set: %v", err) + } + data, _ := os.ReadFile(p) + if !strings.Contains(string(data), "global-client = true") { + t.Fatalf("flag not written:\n%s", data) + } + if err := run([]string{"set-global-client", p, "false"}); err != nil { + t.Fatalf("clear: %v", err) + } + if data, _ = os.ReadFile(p); string(data) != src { + t.Errorf("clearing the flag did not give the file back:\ngot:\n%s\nwant:\n%s", data, src) + } + }) + } +} + +func TestRunSetGlobalClientRejectsAnotherValue(t *testing.T) { + p := writeTemp(t, sample) + if err := run([]string{"set-global-client", p, "yes"}); err == nil { + t.Error("expected an error for a value that is not true or false") + } +} diff --git a/helpers/pyproject/members.go b/helpers/pyproject/members.go new file mode 100644 index 0000000..e307edc --- /dev/null +++ b/helpers/pyproject/members.go @@ -0,0 +1,59 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// generatedKinds are the values of [tool.dagger] generated the SDK writes. A +// member marked with anything else is a file the SDK does not understand, so +// it is never the SDK's to delete or to write over. +var generatedKinds = map[string]bool{"client": true, "core": true, "runtime": true} + +// generatedKind reads the marker of a member's pyproject.toml as TOML: a +// string that merely looks like the table is not the table. Empty when the +// file does not parse, has no marker, or has one of an unknown kind. +func generatedKind(data []byte) string { + doc, err := load(data) + if err != nil { + return "" + } + kind, _ := table(table(doc, "tool"), "dagger")["generated"].(string) + if !generatedKinds[kind] { + return "" + } + return kind +} + +// memberKinds lists the members of a scope that carry a known marker, one +// " " line each: sdk/ and every directory under clients/. +func memberKinds(scope string) (string, error) { + candidates := []string{"sdk"} + entries, err := os.ReadDir(filepath.Join(scope, "clients")) + if err != nil && !os.IsNotExist(err) { + return "", err + } + for _, e := range entries { + if e.IsDir() { + candidates = append(candidates, "clients/"+e.Name()) + } + } + sort.Strings(candidates) + var out strings.Builder + for _, member := range candidates { + data, err := os.ReadFile(filepath.Join(scope, member, "pyproject.toml")) + if err != nil { + if os.IsNotExist(err) { + continue + } + return "", err + } + if kind := generatedKind(data); kind != "" { + fmt.Fprintf(&out, "%s %s\n", member, kind) + } + } + return out.String(), nil +} diff --git a/helpers/pyproject/members_test.go b/helpers/pyproject/members_test.go new file mode 100644 index 0000000..5e91b79 --- /dev/null +++ b/helpers/pyproject/members_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestGeneratedKindReadsTheMarkerAsTOML(t *testing.T) { + for name, tc := range map[string]struct { + src string + want string + }{ + "client": {"[project]\nname = \"x\"\n\n[tool.dagger]\ngenerated = \"client\"\n", "client"}, + "core": {"[tool.dagger]\ngenerated = 'core'\n", "core"}, + "runtime": {"[tool.dagger] # the SDK files\ngenerated = \"runtime\"\n", "runtime"}, + "dotted": {"tool.dagger.generated = \"client\"\n", "client"}, + "none": {"[project]\nname = \"x\"\n", ""}, + "unknown kind": {"[tool.dagger]\ngenerated = \"hand-written\"\n", ""}, + "not a string": {"[tool.dagger]\ngenerated = true\n", ""}, + "unparsable": {"[tool.dagger]\ngenerated = \"client\"\n[tool.dagger]\n", ""}, + "inside a multiline string": {`[project] +name = "handmade" +description = """ +[tool.dagger] +generated = "client" +""" +`, ""}, + } { + t.Run(name, func(t *testing.T) { + if got := generatedKind([]byte(tc.src)); got != tc.want { + t.Errorf("got %q, want %q", got, tc.want) + } + }) + } +} + +func TestRunGetMemberKinds(t *testing.T) { + scope := t.TempDir() + write := func(rel, contents string) { + p := filepath.Join(scope, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(contents), 0o644); err != nil { + t.Fatal(err) + } + } + write("sdk/pyproject.toml", "[tool.dagger]\ngenerated = \"runtime\"\n") + write("clients/core/pyproject.toml", "[tool.dagger]\ngenerated = \"core\"\n") + write("clients/linter/pyproject.toml", "[tool.dagger]\ngenerated = \"client\"\n") + write("clients/handmade/pyproject.toml", "description = \"\"\"\n[tool.dagger]\ngenerated = \"client\"\n\"\"\"\n") + write("clients/odd/pyproject.toml", "[tool.dagger]\ngenerated = \"hand-written\"\n") + write("clients/notes/README.md", "no project\n") + + got, err := memberKinds(scope) + if err != nil { + t.Fatal(err) + } + want := "clients/core core\nclients/linter client\nsdk runtime\n" + if got != want { + t.Errorf("got:\n%s\nwant:\n%s", got, want) + } + + if got, err := memberKinds(t.TempDir()); err != nil || got != "" { + t.Errorf("an empty scope: got %q, %v", got, err) + } +} diff --git a/helpers/pyproject/pyproject.go b/helpers/pyproject/pyproject.go index ac74a06..9c5aaf0 100644 --- a/helpers/pyproject/pyproject.go +++ b/helpers/pyproject/pyproject.go @@ -65,6 +65,26 @@ func getUseUv(doc map[string]any) (value bool, ok bool) { return value, ok } +// getMembers lists the [tool.uv.workspace] members uv reads, one a line. +// What is not a string, such as a nested array, is not a member. +func getMembers(doc map[string]any) string { + members, _ := table(table(table(doc, "tool"), "uv"), "workspace")["members"].([]any) + var out strings.Builder + for _, m := range members { + if s, ok := m.(string); ok { + out.WriteString(s + "\n") + } + } + return out.String() +} + +// getGlobalClient reports the [tool.dagger].global-client flag and whether it +// was set at all, like getUseUv. +func getGlobalClient(doc map[string]any) (value bool, ok bool) { + value, ok = table(table(doc, "tool"), "dagger")["global-client"].(bool) + return value, ok +} + func getBaseImage(doc map[string]any) string { s, _ := table(table(doc, "tool"), "dagger")["base-image"].(string) return s diff --git a/helpers/pyproject/pyproject_test.go b/helpers/pyproject/pyproject_test.go index 29f1cb9..e5f3cad 100644 --- a/helpers/pyproject/pyproject_test.go +++ b/helpers/pyproject/pyproject_test.go @@ -135,3 +135,24 @@ func TestSetBaseImage(t *testing.T) { } } +func TestGetGlobalClient(t *testing.T) { + if _, ok := getGlobalClient(mustLoad(t, sample)); ok { + t.Error("sample: global-client should report unset when absent") + } + doc := mustLoad(t, "[tool.dagger]\nglobal-client = true\nuse-uv = false\n") + if v, ok := getGlobalClient(doc); !ok || !v { + t.Errorf("should report set and true, got value=%v ok=%v", v, ok) + } +} + +func TestGetMembers(t *testing.T) { + doc := mustLoad(t, `["tool"."uv"."workspace"] +members = ["sdk", "clients/core", ["clients/unowned"]] +`) + if got := getMembers(doc); got != "sdk\nclients/core\n" { + t.Errorf("got %q", got) + } + if got := getMembers(mustLoad(t, sample)); got != "" { + t.Errorf("a file without a workspace: got %q", got) + } +} diff --git a/helpers/pyproject/scope.go b/helpers/pyproject/scope.go new file mode 100644 index 0000000..cf89899 --- /dev/null +++ b/helpers/pyproject/scope.go @@ -0,0 +1,947 @@ +package main + +import ( + "fmt" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/pelletier/go-toml/v2" +) + +// scopeEdit is what the SDK owns in a scope pyproject.toml: the workspace +// members it generates, their sources, and the generated distributions the +// project depends on. Stale names the members generation made before and +// removes now; GlobalClient nil leaves the flag as it is. +type scopeEdit struct { + Members []string + Stale []string + Sources []string + Dependencies []string + GlobalClient *bool +} + +// ownedMember reports whether generation may remove a workspace member. The +// caller names them: a member under clients/ may be the user's, and only the +// marker in its directory says otherwise, which the editor cannot see. +// Sources and dependencies are owned by name, dagger-io and dagger-clients-* +// being the SDK's namespace. +func (e scopeEdit) ownedMember(path string) bool { + return contains(e.Stale, path, normalizeMember) +} + +func ownedSource(name string) bool { + return name == "dagger-io" || strings.HasPrefix(name, "dagger-clients-") +} + +func ownedDependency(name string) bool { + return strings.HasPrefix(name, "dagger-clients-") +} + +func normalizeMember(path string) string { + return strings.TrimSuffix(strings.TrimPrefix(path, "./"), "/") +} + +var nameSeparators = regexp.MustCompile(`[-_.]+`) + +// normalizeDistribution applies PEP 503 to a distribution name. +func normalizeDistribution(name string) string { + return nameSeparators.ReplaceAllString(strings.ToLower(name), "-") +} + +var requirementName = regexp.MustCompile(`^[A-Za-z0-9._-]+`) + +// dependencyName is the distribution a PEP 508 requirement names. +func dependencyName(requirement string) string { + return normalizeDistribution(requirementName.FindString(strings.TrimSpace(requirement))) +} + +// editScope rewrites the SDK-owned entries and nothing else: user tables, +// comments and user entries keep their bytes. The file is parsed before and +// after, so a layout the line editor does not understand is refused rather +// than corrupted. +func editScope(src []byte, edit scopeEdit) ([]byte, error) { + if _, err := load(src); err != nil { + return nil, err + } + d := &document{text: string(src)} + + d.editArray("tool.uv.workspace", "members", edit.Members, edit.ownedMember, normalizeMember, true) + d.editSources(edit.Sources) + if d.section("project") != nil { + d.editArray("project", "dependencies", edit.Dependencies, ownedDependency, dependencyName, false) + } + if edit.GlobalClient != nil { + d.editGlobalClient(*edit.GlobalClient) + } + + if err := checkScope([]byte(d.text), edit); err != nil { + return nil, err + } + return []byte(d.text), nil +} + +// The forms the editor asks for when it refuses a file. They are the +// editor's own limits: its line editor finds the workspace and sources +// tables by their headers, not as inline tables or dotted keys. The module +// runtime reads those tables with tomllib and has no such limit. +const ( + workspaceForm = "Write the workspace table as an unquoted [tool.uv.workspace] header with members as an array of strings" + sourcesForm = "the sources as an unquoted [tool.uv.sources] header with one key per source, such as dagger-io = { workspace = true }" + regenerate = ", then run dagger generate again." +) + +// checkScope reads the result back the way uv will. +func checkScope(out []byte, edit scopeEdit) error { + doc, err := load(out) + if err != nil { + // The edit broke a file that parsed before, so a table was written in + // a shape the line editor does not see; which one, the parser cannot say. + return fmt.Errorf("pyproject.toml has a layout the SDK cannot edit: %w. %s, and %s%s", err, workspaceForm, sourcesForm, regenerate) + } + fail := func(what string) error { + return fmt.Errorf("pyproject.toml has a layout the SDK cannot edit: %s", what) + } + uv := table(table(doc, "tool"), "uv") + members, _ := table(uv, "workspace")["members"].([]any) + if err := checkList(members, edit.Members, edit.ownedMember, normalizeMember); err != nil { + return fail("[tool.uv.workspace] members " + err.Error() + ". " + workspaceForm + regenerate) + } + sources := table(uv, "sources") + for _, name := range edit.Sources { + source, _ := sources[name].(map[string]any) + if workspace, _ := source["workspace"].(bool); !workspace || len(source) != 1 { + return fail("[tool.uv.sources] lacks " + name + " = { workspace = true }. Write " + sourcesForm + regenerate) + } + } + for name := range sources { + if ownedSource(normalizeDistribution(name)) && !contains(edit.Sources, name, normalizeDistribution) { + return fail("[tool.uv.sources] keeps " + name + ". Write " + sourcesForm + regenerate) + } + } + if project := table(doc, "project"); project != nil { + dependencies, _ := project["dependencies"].([]any) + if err := checkList(dependencies, edit.Dependencies, ownedDependency, dependencyName); err != nil { + return fail("[project] dependencies " + err.Error()) + } + } + if edit.GlobalClient != nil { + return checkGlobalClient(doc, *edit.GlobalClient) + } + return nil +} + +func checkGlobalClient(doc map[string]any, on bool) error { + value, ok := table(table(doc, "tool"), "dagger")["global-client"].(bool) + if on && !(ok && value) { + return fmt.Errorf("pyproject.toml has a layout the SDK cannot edit: [tool.dagger] lacks global-client = true") + } + if !on && ok { + return fmt.Errorf("pyproject.toml has a layout the SDK cannot edit: [tool.dagger] keeps global-client") + } + return nil +} + +// editGlobalClient writes or clears the global client flag alone, with the +// same care as editScope: `mod config set` owns that one key, not the file. +func editGlobalClient(src []byte, on bool) ([]byte, error) { + if _, err := load(src); err != nil { + return nil, err + } + d := &document{text: string(src)} + d.editGlobalClient(on) + doc, err := load([]byte(d.text)) + if err != nil { + return nil, fmt.Errorf("pyproject.toml has a layout the SDK cannot edit: %w", err) + } + if err := checkGlobalClient(doc, on); err != nil { + return nil, err + } + return []byte(d.text), nil +} + +func checkList(values []any, want []string, owned func(string) bool, normalize func(string) string) error { + var have []string + for _, v := range values { + if s, ok := v.(string); ok { + have = append(have, s) + } + } + for _, w := range want { + if !contains(have, w, normalize) { + return fmt.Errorf("lacks %q", w) + } + } + for _, h := range have { + if owned(normalize(h)) && !contains(want, h, normalize) { + return fmt.Errorf("keeps %q", h) + } + } + return nil +} + +func contains(values []string, want string, normalize func(string) string) bool { + for _, v := range values { + if normalize(v) == normalize(want) { + return true + } + } + return false +} + +// document is a TOML file edited by byte offsets. Every edit recomputes the +// sections, so offsets never go stale. +type document struct { + text string +} + +// section is one table of the document: its header line and the body up to +// the next header. The root section has no header. +type section struct { + name string + headerStart int + headerEnd int // after the header line's newline + bodyStart int + bodyEnd int +} + +func (d *document) sections() []section { + var found []section + current := section{name: "", headerStart: -1, headerEnd: 0, bodyStart: 0} + pos := 0 + for pos < len(d.text) { + end := strings.IndexByte(d.text[pos:], '\n') + next := len(d.text) + if end >= 0 { + next = pos + end + 1 + } + if name, ok := headerName(d.text[pos:next]); ok { + current.bodyEnd = pos + found = append(found, current) + current = section{name: name, headerStart: pos, headerEnd: next, bodyStart: next} + pos = next + continue + } + pos = d.statementEnd(pos, strings.TrimSuffix(d.text[pos:next], "\n")) + } + current.bodyEnd = len(d.text) + return append(found, current) +} + +func (d *document) section(name string) *section { + for _, s := range d.sections() { + if s.name == name { + s := s + return &s + } + } + return nil +} + +// headerName reads `[a.b]` or `[[a.b]]`, with any spacing and quoting of the +// parts and a trailing comment, into "a.b". +func headerName(line string) (string, bool) { + trimmed := strings.TrimSpace(line) + if !strings.HasPrefix(trimmed, "[") { + return "", false + } + inner := strings.TrimPrefix(strings.TrimPrefix(trimmed, "["), "[") + close := strings.IndexByte(inner, ']') + if close < 0 { + return "", false + } + var parts []string + for _, part := range splitOutsideQuotes(inner[:close], '.') { + parts = append(parts, strings.Trim(strings.TrimSpace(part), `"'`)) + } + return strings.Join(parts, "."), true +} + +func splitOutsideQuotes(s string, sep byte) []string { + var parts []string + start := 0 + for i := 0; i < len(s); { + switch s[i] { + case '"', '\'': + i = skipString(s, i) + case sep: + parts = append(parts, s[start:i]) + start = i + 1 + i++ + default: + i++ + } + } + return append(parts, s[start:]) +} + +// skipString returns the offset after the string that starts at i. +func skipString(s string, i int) int { + q := s[i] + if strings.HasPrefix(s[i:], strings.Repeat(string(q), 3)) { + end := strings.Index(s[i+3:], strings.Repeat(string(q), 3)) + if end < 0 { + return len(s) + } + return i + 3 + end + 3 + } + j := i + 1 + for j < len(s) && s[j] != q { + if q == '"' && s[j] == '\\' { + j++ + } + j++ + } + if j < len(s) { + j++ + } + return j +} + +// valueEnd is the offset after the value that starts at start: a bracketed +// value runs to its matching bracket across lines, any other to the end of +// its line before a comment. +func valueEnd(s string, start int) int { + if start >= len(s) { + return start + } + if s[start] == '[' || s[start] == '{' { + depth := 0 + for i := start; i < len(s); { + switch s[i] { + case '"', '\'': + i = skipString(s, i) + continue + case '#': + for i < len(s) && s[i] != '\n' { + i++ + } + continue + case '[', '{': + depth++ + case ']', '}': + depth-- + if depth == 0 { + return i + 1 + } + } + i++ + } + return len(s) + } + i := start + for i < len(s) && s[i] != '\n' && s[i] != '#' { + if s[i] == '"' || s[i] == '\'' { + i = skipString(s, i) + continue + } + i++ + } + for i > start && (s[i-1] == ' ' || s[i-1] == '\t') { + i-- + } + return i +} + +// keyValue locates `key = value` in a section: the line's start, the value's +// span, and the line's leading whitespace. +type keyValue struct { + lineStart int + valueStart int + valueEnd int + indent string +} + +func (d *document) findKey(sec *section, key string) *keyValue { + quoted := regexp.QuoteMeta(key) + pattern := regexp.MustCompile(`^([ \t]*)(?:"` + quoted + `"|'` + quoted + `'|` + quoted + `)[ \t]*=[ \t]*`) + for _, line := range d.lines(sec) { + m := pattern.FindStringSubmatchIndex(d.text[line.start:line.end]) + if m == nil { + continue + } + valueStart := line.start + m[1] + return &keyValue{ + lineStart: line.start, + valueStart: valueStart, + valueEnd: valueEnd(d.text, valueStart), + indent: d.text[line.start+m[2] : line.start+m[3]], + } + } + return nil +} + +type line struct { + start, end int // end excludes the newline +} + +// lines of a section body that start a statement: the lines inside a +// multi-line value are skipped, so an array element never passes for a key. +func (d *document) lines(sec *section) []line { + var found []line + pos := sec.bodyStart + for pos < sec.bodyEnd { + l := line{start: pos, end: sec.bodyEnd} + next := sec.bodyEnd + if nl := strings.IndexByte(d.text[pos:sec.bodyEnd], '\n'); nl >= 0 { + l.end = pos + nl + next = l.end + 1 + } + found = append(found, l) + pos = max(next, d.statementEnd(l.start, d.text[l.start:l.end])) + } + return found +} + +// statementEnd is the offset of the line after the statement on the line at +// start, past the lines of a multi-line value: a `[` that opens one of those +// lines is inside a string or an array, not a table header. +func (d *document) statementEnd(start int, text string) int { + next := start + len(text) + if next < len(d.text) { + next++ + } + eq := strings.IndexByte(stripComments(text), '=') + if isComment(text) || eq < 0 { + return next + } + valueStart := start + eq + 1 + for valueStart < start+len(text) && (d.text[valueStart] == ' ' || d.text[valueStart] == '\t') { + valueStart++ + } + after := valueEnd(d.text, valueStart) + if after <= next { + return next + } + if nl := strings.IndexByte(d.text[after:], '\n'); nl >= 0 { + return after + nl + 1 + } + return len(d.text) +} + +func isBlank(s string) bool { + return strings.TrimSpace(s) == "" +} + +func isComment(s string) bool { + return strings.HasPrefix(strings.TrimSpace(s), "#") +} + +// insertionPoint is where a new key line goes in a section: after its last +// non-blank line, or right after the header of an empty one. +func (d *document) insertionPoint(sec *section) (int, string) { + at := sec.headerEnd + indent := "" + if sec.headerStart >= 0 { + header := d.text[sec.headerStart:sec.headerEnd] + indent = header[:len(header)-len(strings.TrimLeft(header, " \t"))] + } + for _, l := range d.lines(sec) { + text := d.text[l.start:l.end] + if isBlank(text) { + continue + } + at = l.end + if at < len(d.text) && d.text[at] == '\n' { + at++ + } + if !isComment(text) { + indent = text[:len(text)-len(strings.TrimLeft(text, " \t"))] + } + } + return at, indent +} + +func (d *document) insertLine(sec *section, text string) { + at, indent := d.insertionPoint(sec) + // The line before the point may lack its newline, at the end of the file. + if at > 0 && d.text[at-1] != '\n' { + d.text = d.text[:at] + "\n" + d.text[at:] + at++ + } + d.text = d.text[:at] + indent + text + "\n" + d.text[at:] +} + +// ensureSection appends a table at the end when the document has none. +func (d *document) ensureSection(name string) *section { + if sec := d.section(name); sec != nil { + return sec + } + if d.text != "" && !strings.HasSuffix(d.text, "\n") { + d.text += "\n" + } + if d.text != "" { + d.text += "\n" + } + d.text += "[" + name + "]\n" + return d.section(name) +} + +func (d *document) deleteLine(lineStart int) { + end := strings.IndexByte(d.text[lineStart:], '\n') + if end < 0 { + d.text = d.text[:lineStart] + return + } + d.text = d.text[:lineStart] + d.text[lineStart+end+1:] +} + +func (d *document) replace(start, end int, with string) { + d.text = d.text[:start] + with + d.text[end:] +} + +func quote(s string) string { + return strconv.Quote(s) +} + +// editArray sets the owned entries of a string array, in the array's own +// style. A missing key is added to the section, which must exist unless +// create says otherwise. +func (d *document) editArray(sectionName, key string, want []string, owned func(string) bool, normalize func(string) string, create bool) { + sec := d.section(sectionName) + if sec == nil { + if !create { + return + } + sec = d.ensureSection(sectionName) + } + kv := d.findKey(sec, key) + if kv == nil { + var quoted []string + for _, w := range want { + quoted = append(quoted, quote(w)) + } + d.insertLine(sec, key+" = ["+strings.Join(quoted, ", ")+"]") + return + } + raw := d.text[kv.valueStart:kv.valueEnd] + if !strings.HasPrefix(raw, "[") { + return + } + if edited, changed := editArrayText(raw, want, owned, normalize); changed { + d.replace(kv.valueStart, kv.valueEnd, edited) + } +} + +// arrayElement is one element of an array with the bytes around it, so an +// element the SDK does not own goes back exactly as it came. +type arrayElement struct { + raw string + value string + isStr bool +} + +func editArrayText(raw string, want []string, owned func(string) bool, normalize func(string) string) (string, bool) { + inner := raw[1 : len(raw)-1] + elements, tail := splitArray(inner) + multiline := strings.Contains(inner, "\n") + + keep := make([]bool, len(elements)) + changed := false + for i, e := range elements { + keep[i] = !(e.isStr && owned(normalize(e.value)) && !contains(want, e.value, normalize)) + changed = changed || !keep[i] + } + var added []string + for _, w := range want { + present := false + for i, e := range elements { + if keep[i] && e.isStr && normalize(e.value) == normalize(w) { + present = true + break + } + } + if !present { + added = append(added, w) + } + } + if len(added) == 0 && !changed { + return raw, false + } + + if multiline { + return editMultilineArray(elements, keep, tail, added), true + } + var parts []string + for i, e := range elements { + if keep[i] { + parts = append(parts, e.raw) + } + } + for _, a := range added { + parts = append(parts, " "+quote(a)) + } + if len(parts) > 0 { + parts[0] = strings.TrimLeft(parts[0], " \t") + } + return "[" + strings.Join(parts, ",") + "]", true +} + +// editMultilineArray writes an array one element a line, in its own style. +// A comment after an element's comma is cut into the next segment, or into +// the tail, but it belongs to the element's line: it goes with the element +// when that is removed, and stays when the element does. +func editMultilineArray(elements []arrayElement, keep []bool, tail string, added []string) string { + indent := elementIndent(elements) + if tail == "" && len(elements) > 0 { + // No trailing comma: what follows the last value is the tail. + last := &elements[len(elements)-1] + value := strings.TrimRight(last.raw, " \t\r\n") + last.raw, tail = value, last.raw[len(value):] + } + leads := make([]string, len(elements)) + bodies := make([]string, len(elements)) + for i, e := range elements { + leads[i], bodies[i] = lineComment(e.raw) + } + tailLead, rest := lineComment(tail) + // The comment of the line the bracket opens on is the bracket's. + text := "[" + if len(elements) > 0 { + text += leads[0] + } + for i := range elements { + if !keep[i] { + continue + } + own := tailLead + if i+1 < len(elements) { + own = leads[i+1] + } + text += bodies[i] + "," + own + } + if len(added) == 0 { + return text + rest + "]" + } + // New elements go after any comment lines, before the bracket's line. + head, closing := "", rest + if nl := strings.LastIndexByte(rest, '\n'); nl >= 0 { + head, closing = rest[:nl], rest[nl+1:] + } + for _, a := range added { + text += head + "\n" + indent + quote(a) + "," + head = "" + } + return text + "\n" + closing + "]" +} + +// lineComment splits off what a segment holds before its first newline when +// that is only blanks and a comment: the rest of the line before it. +func lineComment(segment string) (string, string) { + nl := strings.IndexByte(segment, '\n') + if nl < 0 || !isBlank(stripComments(segment[:nl])) { + return "", segment + } + return segment[:nl], segment[nl:] +} + +// elementIndent is the indentation of the elements, for a new one to match. +func elementIndent(elements []arrayElement) string { + for i := len(elements) - 1; i >= 0; i-- { + raw := elements[i].raw + if nl := strings.LastIndexByte(raw, '\n'); nl >= 0 { + rest := raw[nl+1:] + return rest[:len(rest)-len(strings.TrimLeft(rest, " \t"))] + } + } + return " " +} + +// splitArray cuts the inside of an array at its top-level commas. What +// follows the last comma is the tail unless it holds a value. +func splitArray(inner string) ([]arrayElement, string) { + var segments []string + start := 0 + depth := 0 + for i := 0; i < len(inner); { + switch inner[i] { + case '"', '\'': + i = skipString(inner, i) + continue + case '#': + for i < len(inner) && inner[i] != '\n' { + i++ + } + continue + case '[', '{': + depth++ + case ']', '}': + depth-- + case ',': + if depth == 0 { + segments = append(segments, inner[start:i]) + start = i + 1 + } + } + i++ + } + last := inner[start:] + tail := "" + if isBlank(stripComments(last)) { + tail = last + } else { + segments = append(segments, last) + } + var elements []arrayElement + for _, seg := range segments { + value, ok := stringValue(seg) + elements = append(elements, arrayElement{raw: seg, value: value, isStr: ok}) + } + return elements, tail +} + +func stripComments(s string) string { + var out strings.Builder + for i := 0; i < len(s); { + switch s[i] { + case '"', '\'': + end := skipString(s, i) + out.WriteString(s[i:end]) + i = end + case '#': + for i < len(s) && s[i] != '\n' { + i++ + } + default: + out.WriteByte(s[i]) + i++ + } + } + return out.String() +} + +// stringValue reads a segment that holds one string literal. +func stringValue(seg string) (string, bool) { + s := strings.TrimSpace(stripComments(seg)) + if len(s) < 2 { + return "", false + } + switch { + case s[0] == '"' && s[len(s)-1] == '"' && skipString(s, 0) == len(s): + value, err := strconv.Unquote(s) + if err != nil { + return "", false + } + return value, true + case s[0] == '\'' && s[len(s)-1] == '\'' && skipString(s, 0) == len(s): + return s[1 : len(s)-1], true + } + return "", false +} + +// editSources points every owned source at the workspace, drops the owned +// ones that are gone, and keeps the user's. A source is a key of +// [tool.uv.sources] or a table of its own, [tool.uv.sources.], which is +// how a file re-marshaled by a TOML library writes it; a new source follows +// the file's style. +func (d *document) editSources(want []string) { + tables := d.editSourceTables(want) + + var stale []int + var keep, present []string + if sec := d.section(sourcesTable); sec != nil { + // A source may also be written as dotted keys, one line a field. + dotted := map[string][]line{} + var dottedNames []string + for _, l := range d.lines(sec) { + key, field, ok := lineKey(d.text[l.start:l.end]) + if !ok || !ownedSource(normalizeDistribution(key)) { + continue + } + switch { + case !contains(want, key, normalizeDistribution): + stale = append(stale, l.start) + case field: + name := normalizeDistribution(key) + if _, seen := dotted[name]; !seen { + dottedNames = append(dottedNames, key) + } + dotted[name] = append(dotted[name], l) + default: + keep = append(keep, key) + } + } + for _, key := range dottedNames { + lines := dotted[normalizeDistribution(key)] + if len(lines) == 1 && isWorkspaceLine(d.text[lines[0].start:lines[0].end]) { + present = append(present, key) + continue + } + // Any other field: the source is written again, inline. + for _, l := range lines { + stale = append(stale, l.start) + } + } + } + sort.Ints(stale) + // Last first, so the offsets before each deletion hold. + for i := len(stale) - 1; i >= 0; i-- { + d.deleteLine(stale[i]) + } + for _, key := range keep { + kv := d.findKey(d.section(sourcesTable), key) + if !isWorkspaceSource(d.text[kv.valueStart:kv.valueEnd]) { + d.replace(kv.valueStart, kv.valueEnd, "{ workspace = true }") + } + } + present = append(present, keep...) + for _, w := range want { + if contains(present, w, normalizeDistribution) || contains(tables, w, normalizeDistribution) { + continue + } + if d.section(sourcesTable) == nil && len(tables) > 0 { + d.appendSourceTable(w) + continue + } + d.insertLine(d.ensureSection(sourcesTable), w+" = { workspace = true }") + } +} + +const sourcesTable = "tool.uv.sources" + +// sourceTableName is the source a [tool.uv.sources.] table is for. +func sourceTableName(sec section) (string, bool) { + return strings.CutPrefix(sec.name, sourcesTable+".") +} + +// editSourceTables applies editSources to the sources written as tables, one +// edit at a time because each moves the offsets after it, and returns the +// names of every source table left. +func (d *document) editSourceTables(want []string) []string { + for d.editOneSourceTable(want) { + } + var names []string + for _, sec := range d.sections() { + if name, ok := sourceTableName(sec); ok { + names = append(names, name) + } + } + return names +} + +func (d *document) editOneSourceTable(want []string) bool { + for _, sec := range d.sections() { + name, ok := sourceTableName(sec) + if !ok || !ownedSource(normalizeDistribution(name)) { + continue + } + if !contains(want, name, normalizeDistribution) { + d.text = d.text[:sec.headerStart] + d.text[sec.bodyEnd:] + return true + } + body := d.text[sec.bodyStart:sec.bodyEnd] + if !isWorkspaceTable(body) { + // The blank lines that part it from the next table stay. + end := sec.bodyStart + len(strings.TrimRight(body, " \t\r\n")) + d.replace(sec.bodyStart, end, "workspace = true") + return true + } + } + return false +} + +// appendSourceTable adds a source table after the last one, before the blank +// lines that part it from what follows. +func (d *document) appendSourceTable(name string) { + var last section + for _, sec := range d.sections() { + if _, ok := sourceTableName(sec); ok { + last = sec + } + } + body := d.text[last.bodyStart:last.bodyEnd] + at := last.bodyStart + len(strings.TrimRight(body, " \t\r\n")) + if at < len(d.text) && d.text[at] == '\n' { + at++ + } else { + d.text = d.text[:at] + "\n" + d.text[at:] + at++ + } + d.text = d.text[:at] + "\n[" + sourcesTable + "." + name + "]\nworkspace = true\n" + d.text[at:] +} + +func isWorkspaceTable(body string) bool { + var source map[string]any + if err := toml.Unmarshal([]byte(body), &source); err != nil { + return false + } + workspace, _ := source["workspace"].(bool) + return workspace && len(source) == 1 +} + +const keyPart = `(?:"[^"]*"|'[^']*'|[A-Za-z0-9_-]+)` + +var keyLine = regexp.MustCompile(`^[ \t]*(` + keyPart + `)((?:[ \t]*\.[ \t]*` + keyPart + `)*)[ \t]*=`) + +// lineKey reads the key a line sets, and whether it sets a field of it with +// a dotted key, as in `dagger-io.workspace = true`. +func lineKey(text string) (string, bool, bool) { + m := keyLine.FindStringSubmatch(text) + if m == nil { + return "", false, false + } + return strings.Trim(m[1], `"'`), m[2] != "", true +} + +// isWorkspaceLine reports whether one line alone makes a workspace source. +func isWorkspaceLine(text string) bool { + var parsed map[string]any + if err := toml.Unmarshal([]byte(text), &parsed); err != nil || len(parsed) != 1 { + return false + } + for _, v := range parsed { + source, _ := v.(map[string]any) + workspace, _ := source["workspace"].(bool) + return workspace && len(source) == 1 + } + return false +} + +func isWorkspaceSource(value string) bool { + var parsed map[string]any + if err := toml.Unmarshal([]byte("x = "+value), &parsed); err != nil { + return false + } + source, _ := parsed["x"].(map[string]any) + workspace, _ := source["workspace"].(bool) + return workspace && len(source) == 1 +} + +// editGlobalClient writes or clears [tool.dagger] global-client, pruning a +// table the flag alone kept. +func (d *document) editGlobalClient(on bool) { + sec := d.section("tool.dagger") + if on { + if sec == nil { + sec = d.ensureSection("tool.dagger") + } + if kv := d.findKey(sec, "global-client"); kv != nil { + if strings.TrimSpace(d.text[kv.valueStart:kv.valueEnd]) != "true" { + d.replace(kv.valueStart, kv.valueEnd, "true") + } + return + } + d.insertLine(sec, "global-client = true") + return + } + if sec == nil { + return + } + kv := d.findKey(sec, "global-client") + if kv == nil { + return + } + d.deleteLine(kv.lineStart) + sec = d.section("tool.dagger") + // A comment left in the table is the user's, and keeps the table. + if !isBlank(d.text[sec.bodyStart:sec.bodyEnd]) { + return + } + before := d.text[:sec.headerStart] + // ensureSection opened a last table with one blank line; take it back. + if sec.bodyEnd == len(d.text) && strings.HasSuffix(before, "\n\n") { + before = strings.TrimSuffix(before, "\n") + } + d.text = before + d.text[sec.bodyEnd:] +} diff --git a/helpers/pyproject/scope_test.go b/helpers/pyproject/scope_test.go new file mode 100644 index 0000000..07ffd90 --- /dev/null +++ b/helpers/pyproject/scope_test.go @@ -0,0 +1,596 @@ +package main + +import ( + "strings" + "testing" +) + +func boolPtr(b bool) *bool { return &b } + +const scopeTemplate = `[project] +name = "my-module" +version = "0.1.0" +requires-python = ">=3.14" +dependencies = ["dagger-io", "dagger-clients-core"] + +[build-system] +requires = ["uv_build>=0.8.4,<0.12.0"] +build-backend = "uv_build" + +[tool.uv.workspace] +members = ["sdk", "clients/core"] + +[tool.uv.sources] +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } +` + +func coreOnly() scopeEdit { + return scopeEdit{ + Members: []string{"sdk", "clients/core"}, + Sources: []string{"dagger-io", "dagger-clients-core"}, + Dependencies: []string{"dagger-clients-core"}, + } +} + +func withLinter() scopeEdit { + e := coreOnly() + e.Members = append(e.Members, "clients/linter") + e.Sources = append(e.Sources, "dagger-clients-linter") + e.Dependencies = append(e.Dependencies, "dagger-clients-linter") + return e +} + +func mustEdit(t *testing.T, src string, edit scopeEdit) string { + t.Helper() + out, err := editScope([]byte(src), edit) + if err != nil { + t.Fatalf("editScope: %v\ninput:\n%s", err, src) + } + return string(out) +} + +func TestEditScopeKeepsAMatchingFileByteForByte(t *testing.T) { + if got := mustEdit(t, scopeTemplate, coreOnly()); got != scopeTemplate { + t.Errorf("a file that already matches was rewritten:\n%s", got) + } +} + +func TestEditScopeAddsAClient(t *testing.T) { + got := mustEdit(t, scopeTemplate, withLinter()) + want := `[project] +name = "my-module" +version = "0.1.0" +requires-python = ">=3.14" +dependencies = ["dagger-io", "dagger-clients-core", "dagger-clients-linter"] + +[build-system] +requires = ["uv_build>=0.8.4,<0.12.0"] +build-backend = "uv_build" + +[tool.uv.workspace] +members = ["sdk", "clients/core", "clients/linter"] + +[tool.uv.sources] +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } +dagger-clients-linter = { workspace = true } +` + if got != want { + t.Errorf("got:\n%s\nwant:\n%s", got, want) + } +} + +func TestEditScopeRemovesAClientAndKeepsTheUsersEntries(t *testing.T) { + src := `# the user's comment stays +[project] +name = "my-module" +dependencies = [ + "dagger-io", + "dagger-clients-core", + "dagger-clients-old", + "httpx>=0.27", # the user's own dependency +] + +[tool.uv.workspace] +members = ["sdk", "clients/core", "clients/old", "tools/mine", "clients/tools"] + +[tool.uv.sources] +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } +dagger-clients-old = { workspace = true } +mine = { workspace = true } +tools = { workspace = true } + +[tool.mine] +answer = 42 +` + want := `# the user's comment stays +[project] +name = "my-module" +dependencies = [ + "dagger-io", + "dagger-clients-core", + "httpx>=0.27", # the user's own dependency +] + +[tool.uv.workspace] +members = ["sdk", "clients/core", "tools/mine", "clients/tools"] + +[tool.uv.sources] +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } +mine = { workspace = true } +tools = { workspace = true } + +[tool.mine] +answer = 42 +` + // clients/old carried the generated marker; clients/tools is the user's + // own member, in the same directory, and stays. + edit := coreOnly() + edit.Stale = []string{"clients/old"} + if got := mustEdit(t, src, edit); got != want { + t.Errorf("got:\n%s\nwant:\n%s", got, want) + } +} + +// A member under clients/ is removed only when the caller names it stale: +// the marker in its directory is what makes it generation's. +func TestEditScopeKeepsAnUnnamedMemberUnderClients(t *testing.T) { + src := `[project] +name = "my-module" +dependencies = ["dagger-io", "dagger-clients-core"] + +[tool.uv.workspace] +members = ["sdk", "clients/core", "clients/tools", "clients/*"] + +[tool.uv.sources] +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } +` + if got := mustEdit(t, src, coreOnly()); got != src { + t.Errorf("a member generation did not make was touched:\n%s", got) + } +} + +func TestEditScopeAppendsToAMultiLineArrayInItsStyle(t *testing.T) { + src := `[project] +name = "my-module" +dependencies = [ + "dagger-io", + "dagger-clients-core", +] + +[tool.uv.workspace] +members = [ + "sdk", + "clients/core", +] + +[tool.uv.sources] +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } +` + want := `[project] +name = "my-module" +dependencies = [ + "dagger-io", + "dagger-clients-core", + "dagger-clients-linter", +] + +[tool.uv.workspace] +members = [ + "sdk", + "clients/core", + "clients/linter", +] + +[tool.uv.sources] +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } +dagger-clients-linter = { workspace = true } +` + if got := mustEdit(t, src, withLinter()); got != want { + t.Errorf("got:\n%s\nwant:\n%s", got, want) + } +} + +func TestEditScopeReplacesTheVendoredSourceAndAddsTheTables(t *testing.T) { + src := `[project] +name = "my-module" +version = "0.1.0" +requires-python = ">=3.14" +dependencies = ["dagger-io"] + +[build-system] +requires = ["uv_build>=0.8.4,<0.12.0"] +build-backend = "uv_build" + +[tool.uv.sources] +dagger-io = { path = "sdk", editable = true } +` + want := `[project] +name = "my-module" +version = "0.1.0" +requires-python = ">=3.14" +dependencies = ["dagger-io", "dagger-clients-core"] + +[build-system] +requires = ["uv_build>=0.8.4,<0.12.0"] +build-backend = "uv_build" + +[tool.uv.sources] +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } + +[tool.uv.workspace] +members = ["sdk", "clients/core"] +` + if got := mustEdit(t, src, coreOnly()); got != want { + t.Errorf("got:\n%s\nwant:\n%s", got, want) + } +} + +func TestEditScopeWritesAnEmptyFile(t *testing.T) { + want := `[tool.uv.workspace] +members = ["sdk", "clients/core", "clients/linter"] + +[tool.uv.sources] +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } +dagger-clients-linter = { workspace = true } +` + if got := mustEdit(t, "", withLinter()); got != want { + t.Errorf("got:\n%s\nwant:\n%s", got, want) + } +} + +func TestEditScopeWritesNoDependencyWithoutAProject(t *testing.T) { + src := `[tool.uv.workspace] +members = ["sdk", "clients/core"] + +[tool.uv.sources] +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } +` + got := mustEdit(t, src, withLinter()) + if strings.Contains(got, "[project]") || strings.Contains(got, "dependencies") { + t.Errorf("a project table was invented:\n%s", got) + } + if !strings.Contains(got, `"clients/linter"`) || !strings.Contains(got, "dagger-clients-linter = { workspace = true }") { + t.Errorf("the member or its source is missing:\n%s", got) + } +} + +func TestEditScopeAddsAMissingKeyToAnExistingTable(t *testing.T) { + src := `[project] +name = "my-module" + +[tool.uv.workspace] +# nothing yet + +[tool.uv.sources] +` + want := `[project] +name = "my-module" +dependencies = ["dagger-clients-core"] + +[tool.uv.workspace] +# nothing yet +members = ["sdk", "clients/core"] + +[tool.uv.sources] +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } +` + if got := mustEdit(t, src, coreOnly()); got != want { + t.Errorf("got:\n%s\nwant:\n%s", got, want) + } +} + +func TestEditScopeGlobalClientFlag(t *testing.T) { + on := coreOnly() + on.GlobalClient = boolPtr(true) + got := mustEdit(t, scopeTemplate, on) + if !strings.HasSuffix(got, "\n[tool.dagger]\nglobal-client = true\n") { + t.Errorf("the flag was not appended:\n%s", got) + } + if got != mustEdit(t, got, on) { + t.Errorf("setting the flag twice changed the file") + } + + indented := `[project] +name = "my-module" +dependencies = ["dagger-clients-core"] + + [tool.dagger] # indented, with a trailing comment + use-uv = false + +[tool.uv.workspace] +members = ["sdk", "clients/core"] + +[tool.uv.sources] +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } +` + got = mustEdit(t, indented, on) + if !strings.Contains(got, " use-uv = false\n global-client = true\n") { + t.Errorf("the flag did not join the existing table:\n%s", got) + } + + off := coreOnly() + off.GlobalClient = boolPtr(false) + if got = mustEdit(t, got, off); got != indented { + t.Errorf("clearing the flag did not restore the file:\n%s", got) + } + // Clearing prunes a table the flag alone kept. + if got = mustEdit(t, mustEdit(t, scopeTemplate, on), off); got != scopeTemplate { + t.Errorf("clearing the flag left the table behind:\n%s", got) + } + + // nil leaves whatever is there. + if got = mustEdit(t, mustEdit(t, scopeTemplate, on), coreOnly()); !strings.Contains(got, "global-client = true") { + t.Errorf("an unset flag touched the file:\n%s", got) + } +} + +func TestEditScopeRefusesALayoutItCannotEdit(t *testing.T) { + src := `[project] +name = "my-module" + +[tool.uv] +workspace = { members = ["sdk"] } +` + _, err := editScope([]byte(src), coreOnly()) + if err == nil { + t.Error("an inline workspace table was edited silently") + } else if !strings.Contains(err.Error(), workspaceForm) { + t.Errorf("the refusal does not say what to write: %v", err) + } + for name, src := range map[string]string{ + "inline sources": "[project]\nname = \"x\"\n\n[tool.uv]\nsources = { dagger-io = { path = \"sdk\" } }\n", + "dotted members": "[project]\nname = \"x\"\n\n[tool]\nuv.workspace.members = [\"sdk\"]\n", + } { + if _, err := editScope([]byte(src), coreOnly()); err == nil { + t.Errorf("%s: edited silently", name) + } else if !strings.Contains(err.Error(), workspaceForm) || !strings.Contains(err.Error(), sourcesForm) { + t.Errorf("%s: the refusal does not say what to write: %v", name, err) + } + } + if _, err := editScope([]byte("not = toml = at all\n"), coreOnly()); err == nil { + t.Error("an unparsable file was edited") + } +} + +func TestEditScopeOwnsOnlyItsNames(t *testing.T) { + src := `[project] +name = "my-module" +dependencies = ["dagger-io", "dagger-clients-core", "dagger-clients-linter>=0.0.0"] + +[tool.uv.workspace] +members = ["sdk", "clients/core", "clients/linter/"] + +[tool.uv.sources] +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } +"dagger-clients-linter" = { workspace = true } +` + // A pinned dependency and a quoted or slash-terminated entry are the + // same member, so nothing is added twice. + if got := mustEdit(t, src, withLinter()); got != src { + t.Errorf("entries were duplicated:\n%s", got) + } +} + +func TestEditScopeReadsNoHeaderInsideAValue(t *testing.T) { + src := `[project] +name = "my-module" +dependencies = ["dagger-clients-core"] + +[tool.mine] +snippet = """ +[tool.uv.sources] +dagger-clients-fake = { workspace = true } +""" +matrix = [ + ["a", "b"], +] + +[tool.uv.workspace] +members = ["sdk", "clients/core"] + +[tool.uv.sources] +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } +` + got := mustEdit(t, src, withLinter()) + want := strings.Replace(src, "members = [\"sdk\", \"clients/core\"]", "members = [\"sdk\", \"clients/core\", \"clients/linter\"]", 1) + want = strings.Replace(want, "dependencies = [\"dagger-clients-core\"]", "dependencies = [\"dagger-clients-core\", \"dagger-clients-linter\"]", 1) + want += "dagger-clients-linter = { workspace = true }\n" + if got != want { + t.Errorf("got:\n%s\nwant:\n%s", got, want) + } +} + +func TestEditScopeClearingTheFlagKeepsTheUsersComment(t *testing.T) { + src := scopeTemplate + "\n[tool.dagger]\n# keep the global client until the lint module migrates\n" + on := coreOnly() + on.GlobalClient = boolPtr(true) + off := coreOnly() + off.GlobalClient = boolPtr(false) + if got := mustEdit(t, mustEdit(t, src, on), off); got != src { + t.Errorf("clearing the flag did not restore the file:\n%s", got) + } +} + +func TestEditScopeClearingTheFlagKeepsWhatFollowsTheTable(t *testing.T) { + src := `[project] +name = "my-module" +dependencies = ["dagger-clients-core"] + +[tool.dagger] +global-client = true + +[tool.uv.workspace] +members = ["sdk", "clients/core"] + +[tool.uv.sources] +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } + + +` + off := coreOnly() + off.GlobalClient = boolPtr(false) + want := strings.Replace(src, "[tool.dagger]\nglobal-client = true\n\n", "", 1) + if got := mustEdit(t, src, off); got != want { + t.Errorf("got:\n%q\nwant:\n%q", got, want) + } +} + +func TestEditScopeEditsSourcesWrittenAsTables(t *testing.T) { + src := `[project] +name = "my-module" +dependencies = ["dagger-io", "dagger-clients-core", "dagger-clients-old"] + +[tool.uv.workspace] +members = ['sdk', 'clients/core', 'clients/old'] + +[tool.uv.sources.dagger-io] +path = "sdk" +editable = true + +[tool.uv.sources."dagger-clients-core"] +workspace = true + +[tool.uv.sources.dagger-clients-old] +workspace = true + +[tool.uv.sources.mine] +path = "../mine" + +[tool.mine] +answer = 42 +` + want := `[project] +name = "my-module" +dependencies = ["dagger-io", "dagger-clients-core", "dagger-clients-linter"] + +[tool.uv.workspace] +members = ['sdk', 'clients/core', "clients/linter"] + +[tool.uv.sources.dagger-io] +workspace = true + +[tool.uv.sources."dagger-clients-core"] +workspace = true + +[tool.uv.sources.mine] +path = "../mine" + +[tool.uv.sources.dagger-clients-linter] +workspace = true + +[tool.mine] +answer = 42 +` + edit := withLinter() + edit.Stale = []string{"clients/old"} + got := mustEdit(t, src, edit) + if got != want { + t.Errorf("got:\n%s\nwant:\n%s", got, want) + } + if again := mustEdit(t, got, withLinter()); again != got { + t.Errorf("a second edit changed the file:\n%s", again) + } +} + +// A comment after an element's comma is on that element's line: removing the +// next element keeps it, and removing the element takes it along. +func TestEditScopeKeepsACommentWithItsLine(t *testing.T) { + for name, tc := range map[string]struct{ src, want string }{ + "the next element goes": { + src: `dependencies = [ + "httpx", # keep: this documents the user's dependency + "dagger-clients-old", +]`, + want: `dependencies = [ + "httpx", # keep: this documents the user's dependency +]`, + }, + "the element goes with its comment": { + src: `dependencies = [ + "httpx", # the user's + "dagger-clients-old", # generated + "rich", # the user's too +]`, + want: `dependencies = [ + "httpx", # the user's + "rich", # the user's too +]`, + }, + "the last element, without a trailing comma": { + src: `dependencies = [ # the bracket's + "httpx", # the user's + "dagger-clients-old" # generated +]`, + want: `dependencies = [ # the bracket's + "httpx", # the user's +]`, + }, + } { + t.Run(name, func(t *testing.T) { + src := "[project]\nname = \"x\"\n" + tc.src + "\n" + edit := scopeEdit{Members: []string{"sdk"}, Sources: []string{"dagger-io"}} + got := mustEdit(t, src, edit) + want := "[project]\nname = \"x\"\n" + tc.want + "\n" + if !strings.HasPrefix(got, want) { + t.Errorf("got:\n%s\nwant it to start with:\n%s", got, want) + } + }) + } +} + +func TestEditScopeAddsToAnEmptyMultiLineArray(t *testing.T) { + src := "[project]\nname = \"x\"\ndependencies = [\n]\n" + edit := scopeEdit{Members: []string{"sdk"}, Sources: []string{"dagger-io"}, Dependencies: []string{"dagger-clients-core"}} + got := mustEdit(t, src, edit) + if !strings.Contains(got, "dependencies = [\n \"dagger-clients-core\",\n]\n") { + t.Errorf("got:\n%s", got) + } +} + +func TestEditScopeReadsDottedSourceKeys(t *testing.T) { + head := `[project] +name = "my-module" +dependencies = ["dagger-io", "dagger-clients-core"] + +[tool.uv.workspace] +members = ["sdk", "clients/core"] + +[tool.uv.sources] +` + kept := head + `dagger-io.workspace = true +"dagger-clients-core" . workspace = true # spaced and quoted +mine.path = "../mine" +` + if got := mustEdit(t, kept, coreOnly()); got != kept { + t.Errorf("a dotted workspace source was rewritten:\n%s", got) + } + + vendored := head + `dagger-io.path = "sdk" +dagger-io.editable = true +dagger-clients-core.workspace = true +dagger-clients-old.workspace = true +mine.path = "../mine" +` + want := head + `dagger-clients-core.workspace = true +mine.path = "../mine" +dagger-io = { workspace = true } +` + if got := mustEdit(t, vendored, coreOnly()); got != want { + t.Errorf("got:\n%s\nwant:\n%s", got, want) + } +} diff --git a/helpers/vendor-pyproject/strip_dev_sections.py b/helpers/vendor-pyproject/strip_dev_sections.py index a1cb3e7..3979861 100644 --- a/helpers/vendor-pyproject/strip_dev_sections.py +++ b/helpers/vendor-pyproject/strip_dev_sections.py @@ -3,17 +3,25 @@ As published, the Python SDK's pyproject.toml declares the code generator as a uv workspace member and a dev dependency. A module vendors the library without the generator, and uv refuses to install a project whose workspace member is -missing, so those sections are dropped on the way in. +missing, so those sections are dropped on the way in. The copy is a member of +the scope that holds it, and gets the marker every generated member carries. -Usage: strip_dev_sections.py +With the temporary global client, the copy also carries the dagger_global +package, which imports core and the scope's clients, so it depends on them. + +Usage: strip_dev_sections.py [...] """ import sys DROP = {"dependency-groups", "tool.uv.sources", "tool.uv.workspace"} +MARKER = '[tool.dagger]\ngenerated = "runtime"\n' +MODULE_NAME = 'module-name = "dagger"\n' +GLOBAL_MODULE_NAME = 'module-name = ["dagger", "dagger_global"]\n' +DEPENDENCIES = "dependencies = [\n" -def main(src: str, dst: str) -> None: +def main(src: str, dst: str, global_dependencies: list[str]) -> None: kept: list[str] = [] keep = True for line in open(src): @@ -22,8 +30,28 @@ def main(src: str, dst: str) -> None: keep = stripped.strip("[]") not in DROP if keep: kept.append(line) - open(dst, "w").write("".join(kept).rstrip() + "\n") + if global_dependencies: + kept = with_global_client(kept, global_dependencies) + open(dst, "w").write("".join(kept).rstrip() + "\n\n" + MARKER) + + +def with_global_client(lines: list[str], dependencies: list[str]) -> list[str]: + # The SDK's own file is the only input, so a changed layout is a bug here, + # not a user's file to be tolerant of. + for expected in (MODULE_NAME, DEPENDENCIES): + if lines.count(expected) != 1: + msg = f"expected one line {expected.strip()!r} in the SDK's pyproject.toml" + raise SystemExit(msg) + out: list[str] = [] + for line in lines: + if line == MODULE_NAME: + out.append(GLOBAL_MODULE_NAME) + continue + out.append(line) + if line == DEPENDENCIES: + out.extend(f' "{name}",\n' for name in dependencies) + return out if __name__ == "__main__": - main(sys.argv[1], sys.argv[2]) + main(sys.argv[1], sys.argv[2], sys.argv[3:]) diff --git a/mod-config.dang b/mod-config.dang index f5203c9..b554ce9 100644 --- a/mod-config.dang +++ b/mod-config.dang @@ -21,10 +21,18 @@ type ModConfigValues { """ pub baseImage: String - new(pythonVersion: String = null, useUv: Boolean = null, baseImage: String = null) { + """ + Whether generation emits the temporary global client, `dagger.dag` with one + method per core field and per client, or null when unset. When null, no + global client is generated. + """ + pub globalClient: Boolean + + new(pythonVersion: String = null, useUv: Boolean = null, baseImage: String = null, globalClient: Boolean = null) { self.pythonVersion = pythonVersion self.useUv = useUv self.baseImage = baseImage + self.globalClient = globalClient self } } @@ -54,6 +62,7 @@ type ModConfig { pythonVersion: readPythonVersion, useUv: readUseUv, baseImage: readBaseImage, + globalClient: readGlobalClient, ) } @@ -61,9 +70,10 @@ type ModConfig { Set one or more configuration values in pyproject.toml at once. Each flag is optional; omitting it leaves that setting untouched. Setting - useUv to true (the SDK default) writes nothing, keeping the file minimal. + useUv to true or globalClient to false (the SDK defaults) writes nothing, + keeping the file minimal. """ - pub set(pythonVersion: String = null, useUv: Boolean = null, baseImage: String = null): Changeset! { + pub set(pythonVersion: String = null, useUv: Boolean = null, baseImage: String = null, globalClient: Boolean = null): Changeset! { let withPython = if (pythonVersion != null) { tool.withExec(["pyproject", "set-python-version", toolPath, pythonVersion]) } else { @@ -82,7 +92,13 @@ type ModConfig { withUv } - let edited = withImage.file(toolPath).contents + let withGlobal = if (globalClient != null) { + withImage.withExec(["pyproject", "set-global-client", toolPath, if (globalClient) { "true" } else { "false" }]) + } else { + withImage + } + + let edited = withGlobal.file(toolPath).contents ws.withNewFile("/" + pyprojectPath, edited).changes(ws) } @@ -110,6 +126,14 @@ type ModConfig { if (value == "") { null } else { value } } + """ + Whether generation emits the global client, or null when unset. + """ + let readGlobalClient: Boolean { + let value = tool.withExec(["pyproject", "get-global-client", toolPath]).stdout + if (value == "true") { true } else if (value == "false") { false } else { null } + } + let pyprojectPath: String! { if (path == ".") { "pyproject.toml" } else { path + "/pyproject.toml" } } diff --git a/mod.dang b/mod.dang index 5caee36..ea59e93 100644 --- a/mod.dang +++ b/mod.dang @@ -18,6 +18,17 @@ type Mod { """ pub dangEntrypoint: Boolean! + """ + The module scope's clients, as the engine hands them to generation. Null + reads them from the workspace config, for a module generated directly. + """ + let clients: [ModuleSource!] + + """ + Write the commit a Git client was generated against into its descriptor. + """ + let lock: Boolean! + """ Module root relative to the client's cwd. """ @@ -52,35 +63,62 @@ type Mod { } """ - The workspace with this module's generated files merged in. On the static - path the entrypoint manifest is written here too, so a module generated - directly gets the same files as one generated through the SDK scope. + The workspace with this module's scope generated. On the static path the + entrypoint manifest and the entrypoint are written here too, so a module + generated directly gets the same files as one generated through the SDK + scope. """ pub generated: Workspace! { - let files = if (isModern) { - # Generated here: the runtime generates nothing, so the engine's - # generated context would be empty. - vendoredDir - } else { + if (isModern == false) { # A pre-1.0 module is still generated by the engine's builtin Python SDK. - ws + # Merge, don't replace: the generated context holds only generated files. + ws.withDirectory("/" + rootPath, ws .moduleSource("/" + rootPath) .generatedContextDirectory - .directory(rootPath) - } - - # Merge, don't replace: the generated context holds only generated files. - let merged = ws.withDirectory("/" + rootPath, files) - let entrypointPath = vendorDirName + "/" + entrypointDirName - if (dangEntrypoint) { - entrypointManifest(merged, entrypointPath) - } else if (ws.directory("/" + rootPath).exists(entrypointPath)) { - merged.withoutDirectory("/" + rootPath + "/" + entrypointPath) + .directory(rootPath)) } else { - merged + # The scope writes sdk/ whole, so an entrypoint from an earlier static + # generation is gone unless it is written again. + let scoped = Scope( + path: rootPath, + ws: ws, + clients: clients ?? declaredClients, + lock: lock, + isModule: true, + engineVersion: engineVersion, + ).generated + if (dangEntrypoint) { + let entrypointPath = vendorDirName + "/" + entrypointDirName + entrypointManifest(scoped.withDirectory("/" + rootPath + "/" + entrypointPath, entrypointDir(scoped)), entrypointPath) + } else { + scoped + } } } + """ + The clients the workspace config declares on this module's scope. The + config names a local client by its path from the workspace root, starting + with "./", and a Git client by its ref. + """ + let declaredClients: [ModuleSource!]! { + ws.sdk(name: currentModule.name) + .clients.{{name, source}} + .filter { client => scopeOf(client.name) == rootPath } + .map { client => + if (client.source.hasPrefix(".") or client.source.hasPrefix("/")) { + ws.moduleSource("/" + scopeOf(client.source)) + } else { + moduleSource(client.source) + } + } + } + + let scopeOf(path: String!): String! { + let trimmed = path.trimPrefix("./").trimPrefix("/").trimSuffix("/") + if (trimmed == "") { "." } else { trimmed } + } + """ Write the manifest of a static module: the name and the Dang entrypoint, and nothing the runtime manifest keeps. sdk-helpers owns the manifest's @@ -118,18 +156,20 @@ type Mod { } } + let engineVersion: String! { + let m = ws.file("/" + manifestPath).contents.match("(?m)^\\s*engineVersion\\s*=\\s*\"([^\"]*)\"") + if (m == null) { "" } else { m.captures[0] ?? "" } + } + """ - The workspace the module's schema is read from and its container is built - in: the module's own manifest with the entrypoint table taken out. + The generated workspace with the module's own manifest, entrypoint table + taken out, which the module's container is built from. - An engine that does not know the table serves a static entrypoint manifest - the oldest schema view. An engine that loads manifest version 2 would resolve - the entrypoint here, shared or static, and build the module from source it - has not generated yet. Either way the runtime manifest points back at the - module being built. Everything else the manifest holds stays, the - dependencies above all: the bindings are generated from this schema. + An engine that loads manifest version 2 would resolve the entrypoint here, + shared or static, and build the module from source it has not generated + yet. So the runtime manifest points back at the module being built. """ - let stagingWs: Workspace! { + let staging(target: Workspace!): Workspace! { if (hasEntrypointManifest) { let staged = ManifestToml(ws.file("/" + manifestPath).contents).withoutEntrypoint let loaded = sdkHelpers.moduleManifest(loadToml: staged) @@ -140,40 +180,24 @@ type Mod { loaded.withLegacyPythonRuntime } runnable.withName(name: moduleName) - .generate(ws.withWorkdir(rootPath), lock: false, legacyJson: false) + .generate(target.withWorkdir(rootPath), lock: false, legacyJson: false) .withWorkdir(".") } else { - ws - } - } - - """ - The client library with the module's bindings, in a module-rooted directory - holding only `sdk/` so it merges onto the module without touching anything else. - """ - let vendoredDir: Directory! { - let schemaJSON = stagingWs.moduleSource("/" + rootPath).introspectionSchemaJSON - let vendored = library.withFile(generatedBindingsPath, bindings(schemaJSON)) - let files = if (dangEntrypoint) { - vendored.withDirectory(entrypointDirName, entrypointDir(vendored)) - } else { - vendored + target } - - directory.withDirectory(vendorDirName, files) } """ The entrypoint the engine loads instead of calling a runtime: the module's own container renders its types, and the container build travels with it. """ - let entrypointDir(vendored: Directory!): Directory! { - let staged = stagingWs.withDirectory("/" + rootPath + "/" + vendorDirName, vendored) + let entrypointDir(generated: Workspace!): Directory! { pythonSdkRuntime - .moduleRuntime(modSource: staged.moduleSource("/" + rootPath), introspectionJson: null) - .withExec(["python", "-m", "dagger.mod", "entrypoint", "--name", moduleName, "--path", rootPath, "--output", entrypointOutput]) + .moduleRuntime(modSource: staging(generated).moduleSource("/" + rootPath), introspectionJson: null) + .withExec(["python", "-m", "dagger.mod", "entrypoint", "--name", moduleName, "--output", entrypointOutput]) .directory(entrypointOutput) .withNewFile("build.dang", buildDang) + .withFile("handover.dang", currentModule.source.file("entrypoint/handover.dang")) } """ @@ -201,89 +225,7 @@ type Mod { } } - """ - Bindings generated from a module's schema, straight from the synced - environment: `uv run --isolated` built a throwaway one per module. - """ - let bindings(schemaJSON: File!): File! { - codegenEnv - .withMountedFile(schemaPath, schemaJSON) - .withExec([ - codegenPython, "-m", "codegen", "generate", "-i", schemaPath, "-o", "/gen.py", - ]) - .file("/gen.py") - } - - """ - The code generator's environment, synced once per SDK version and shared by - every module that generates. - """ - let codegenEnv: Container! { - codegenBase.withExec(["uv", "sync", "--frozen", "--no-dev", "--package", "codegen"]) - } - - let codegenBase: Container! { - container - .from(codegenImage) - .withoutEntrypoint - .withMountedCache("/root/.cache/uv", cacheVolume("python-sdk-uv")) - .withEnvVariable("UV_LINK_MODE", "copy") - .withEnvVariable("UV_COMPILE_BYTECODE", "1") - .withDirectory("/sdk", codegenSource) - .withWorkdir("/sdk") - } - - let codegenPython: String! = "/sdk/.venv/bin/python" - - let codegenSource: Directory! { - currentModule.source.directory("sdk").filter(include: [ - "pyproject.toml", - "uv.lock", - "src/**/*.py", - "src/**/*.typed", - "codegen/pyproject.toml", - "codegen/**/*.py", - ]) - } - - """ - What a module vendors: the importable library and its license. The generator - runs here, never in a module, and the lock pins the generator's environment. - """ - let library: Directory! { - currentModule.source - .directory("sdk") - .filter(include: [ - "LICENSE", - "README.md", - "src/**/*.py", - "src/**/*.typed", - # An optional import that provisions an engine; a module already has one. - "!src/dagger/provisioning/**", - ]) - .withFile("pyproject.toml", libraryPyproject) - } - - """ - pyproject.toml without its development sections: vendored verbatim, it names - the absent codegen workspace member and uv refuses to install the library. - """ - let libraryPyproject: File! { - codegenBase - .withFile(stripScriptPath, currentModule.source.file("helpers/vendor-pyproject/strip_dev_sections.py")) - .withExec(["python", stripScriptPath, "pyproject.toml", "/library-pyproject.toml"]) - .file("/library-pyproject.toml") - } - - let stripScriptPath: String! = "/strip-dev-sections.py" - let vendorDirName: String! = "sdk" let entrypointDirName: String! = "entrypoint" let entrypointOutput: String! = "/dagger/entrypoint" - let generatedBindingsPath: String! = "src/dagger/client/gen.py" - let schemaPath: String! = "/schema.json" - - # musl runs the generator ~0.3s slower than glibc, but the glibc image is - # 25 MiB larger to pull, which costs more on the first generate. - let codegenImage: String! = "ghcr.io/astral-sh/uv:python3.14-alpine" } diff --git a/python-sdk.dang b/python-sdk.dang index fe89b80..381da11 100644 --- a/python-sdk.dang +++ b/python-sdk.dang @@ -34,17 +34,36 @@ type PythonSdk { """ Find the Python client root containing the workspace cwd: the directory of - the nearest pyproject.toml, relative to the workspace root. A module's - vendored client library carries its own pyproject.toml, so the Python module - owning it answers instead; a directory with a module config of its own is a - module, never a vendored library. Null when there is none. + the nearest pyproject.toml, relative to the workspace root. A generated + member, marked `[tool.dagger] generated` in its pyproject.toml, belongs to + the scope above it, so that scope answers. A module's vendored client + library from before the marker carries its own pyproject.toml too, so the + Python module owning it answers instead; a directory with a module config + of its own is a module, never a vendored library. Null when there is none. """ pub findClientRoot(ws: Workspace!): String { let found = ws.findUp("pyproject.toml") if (found == null) { null } else { - let scope = normalizePath(found.trimSuffix("pyproject.toml")) + clientRootOf(ws, normalizePath(found.trimSuffix("pyproject.toml"))) + } + } + + """ + The scope that a pyproject.toml belongs to. + """ + let clientRootOf(ws: Workspace!, scope: String!): String! { + if (scope != "." and isGeneratedMember(ws, scope)) { + # Members sit at most two levels below their scope, but a scope can be + # a member of something else, so lift until the file is not a member's. + let above = ws.findUp("pyproject.toml", from: "/" + parentPath(scope)) + if (above == null) { + scope + } else { + clientRootOf(ws, normalizePath(above.trimSuffix("pyproject.toml"))) + } + } else { let vendorParent = if (scope == vendorDirName) { "." } else if (scope.trimSuffix("/" + vendorDirName) != scope) { @@ -63,17 +82,34 @@ type PythonSdk { } } + """ + Whether a scope's pyproject.toml is one this SDK generated: sdk/, core or a + client. Not a TOML parser: the marker is a key of one table, on its own line. + """ + let isGeneratedMember(ws: Workspace!, scope: String!): Boolean! { + scopeHasFile(ws, scope, "pyproject.toml") and + hasGeneratedMarker(ws.file("/" + scopePath(scope, "pyproject.toml")).contents) + } + + let hasGeneratedMarker(toml: String!): Boolean! { + toml.containsMatch("(?ms)^\\s*\\[tool\\.dagger\\][^\\[]*^\\s*generated\\s*=") + } + + let parentPath(path: String!): String! { + let parts = path.split("/") + if (parts.length < 2) { "." } else { parts.takeFirst(parts.length - 1).join("/") } + } + """ Directory a module's generated client library is vendored into. """ let vendorDirName: String! = "sdk" """ - The shared Dang entrypoint every module on the runtime path names, served - from this repository's entrypoint/ directory. An engine that loads manifest - version 2 drives the module through it; an older engine ignores the table - and uses the builtin runtime named by [runtime]. `@v1` selects the greatest - `entrypoint/v1.*` tag of this repository, then the greatest plain `v1.*` one. + The shared Dang entrypoint every module names unless it has a static one, + served from this repository's entrypoint/ directory. `@v1` selects the + greatest `entrypoint/v1.*` tag of this repository, then the greatest plain + `v1.*` one. """ let sharedEntrypointSource: String! = "dagger.io/sdk/python/entrypoint@v1" @@ -95,75 +131,252 @@ type PythonSdk { } """ - Generate one SDK scope: the module at the workspace cwd, when the scope has - one. A module without a config file is initialized from the configured - template. Every module receives a dagger-module.toml from the manifest - builder module and is then generated. A pre-1.0 dagger.json is migrated - and removed. The scope's module clients become the module's dependencies, - so the generated bindings include their types. With dangEntrypoint, the - module gets an entrypoint manifest and a generated entrypoint instead of a - runtime. - Standalone clients, in a scope without a module, are not generated yet. + Generate one SDK scope, the directory at the workspace cwd: sdk/, core and + one member per client, with the SDK's entries in its pyproject.toml. A scope + without a module gets the same tree and no module config. In a module scope, + a module without a config file is initialized from the configured template, + every module receives a dagger-module.toml from the manifest builder, and a + pre-1.0 dagger.json is migrated and removed. The manifest names the shared + entrypoint, or with dangEntrypoint one generated into the module. """ pub generateScope(ws: Workspace!, isModule: Boolean!, name: String!, clients: [ModuleSource!]!): Workspace! { + let scope = normalizePath(ws.cwd) + # Resolve local client paths from the workspace root. + let rooted = ws.withWorkdir(".") if (isModule == false) { - if (clients.length > 0) { - raise "python-sdk does not generate standalone module clients yet" - } else { - ws - } + Scope(path: scope, ws: rooted, clients: clients, lock: lock, isModule: false, engineVersion: "").generated.withWorkdir(scope) } else { - let scope = normalizePath(ws.cwd) let hadConfig = hasModuleConfig(ws, scope) - # Resolve local dependency paths from the workspace root. - let rooted = ws.withWorkdir(".") let initialized = if (hadConfig) { rooted } else { rooted.withDirectory("/" + scope, moduleTemplate(name, template, pythonVersion, useUv, baseImage)) } if (dangEntrypoint) { - checkStaticScope(initialized, scope, hadConfig, clients) + checkStaticScope(initialized, scope, hadConfig) } - # The runtime manifest is what every engine serves a schema for; the - # static path swaps it for the entrypoint manifest as it generates. - let configured = generateScopeManifest(initialized, scope, name, clients) - mod(configured, path: scope, findUp: false).generated.withWorkdir(scope) + # The shared entrypoint's manifest; the static path swaps it for its own + # as it generates. + let configured = generateScopeManifest(initialized, scope, name) + if (dangEntrypoint == false) { + checkHandover(configured, scope, clients) + } + Mod( + rootPath: scope, + ws: configured, + dangEntrypoint: dangEntrypoint or hasEntrypointManifest(configured, scope), + clients: clients, + lock: lock, + ).generated.withWorkdir(scope) } } """ - Generate dagger-module.toml from the existing manifest and complete client - set. Preserve fields that the SDK does not own. Remove dagger.json so two - manifest files cannot contain different state. + Generate dagger-module.toml from the existing manifest. Preserve fields that + the SDK does not own. A client is a member of the scope, loaded by the code + that uses it, so the manifest lists none, and the [[dependencies]] of the + layout before go. Remove dagger.json so two manifest files cannot contain + different state. + + The module runs on a Dang entrypoint and on nothing else. The runtime this + SDK used to write, `[runtime] source = "python"`, goes, and so does what + only a runtime reads: engineVersion, since an entrypoint runs a module on + the engine's own version. Any other runtime is one the user chose, and the + manifest keeps it with no entrypoint added, because the engine follows an + entrypoint over a runtime; for the same reason a manifest with both is + refused. """ - let generateScopeManifest(ws: Workspace!, scope: String!, name: String!, clients: [ModuleSource!]!): Workspace! { + let generateScopeManifest(ws: Workspace!, scope: String!, name: String!): Workspace! { let root = "/" + scope let hasToml = scopeHasFile(ws, scope, "dagger-module.toml") let hasJson = scopeHasFile(ws, scope, "dagger.json") + if (hasToml) { + checkOneRunner(ws.file(root + "/dagger-module.toml").contents) + } let base = if (hasToml and isEntrypointManifest(ws.file(root + "/dagger-module.toml").contents)) { - # A static entrypoint manifest holds nothing the runtime manifest keeps. - sdkHelpers.moduleManifest.withLegacyPythonRuntime - .withDangEntrypoint(source: sharedEntrypointSource) + # This SDK's static entrypoint manifest holds nothing the shared one keeps. + sdkHelpers.moduleManifest.withDangEntrypoint(source: sharedEntrypointSource) } else if (hasToml) { let manifest = ws.file(root + "/dagger-module.toml") + let config = tomlConfig(manifest.contents) let loaded = sdkHelpers.moduleManifest(loadToml: withoutStaleEntrypoint(manifest)) - # A Dang entrypoint already there is the user's: a pinned version, or a fork. - if (ManifestToml(manifest.contents).entrypointKind == "dang") { + if (keepsItsRuntime(config)) { loaded } else { - loaded.withDangEntrypoint(source: sharedEntrypointSource) + checkEntrypointOnly(config, "dagger-module.toml") + # A Dang entrypoint already there is kept: the shared one, or one the + # user chose, a pinned version, a fork or one of their own. + let entrypointed = if (ManifestToml(manifest.contents).entrypointKind == "dang") { + loaded + } else { + loaded.withDangEntrypoint(source: sharedEntrypointSource) + } + entrypointed.withoutLegacyFields } } else if (hasJson) { - sdkHelpers.moduleManifest(loadJson: ws.file(root + "/dagger.json")) - .withDangEntrypoint(source: sharedEntrypointSource) + let manifest = ws.file(root + "/dagger.json") + let config = json.withContents((manifest.contents :: Dagger.JSON!)) + let loaded = sdkHelpers.moduleManifest(loadJson: manifest) + if (keepsItsRuntime(config)) { + loaded + } else { + checkEntrypointOnly(config, "dagger.json") + loaded.withDangEntrypoint(source: sharedEntrypointSource).withoutLegacyFields + } + } else { + sdkHelpers.moduleManifest.withDangEntrypoint(source: sharedEntrypointSource) + } + base.withName(name: name).withoutLegacyRuntimeDependencies + .generate(ws.withWorkdir(scope), lock: lock, legacyJson: false) + .withWorkdir(".") + } + + """ + Whether a manifest names a runtime of the user's own: anything but the + builtin `python` this SDK wrote. Read with a TOML or JSON parser, because + the manifest is the user's content; a dagger.json names it as `sdk`. + """ + let keepsItsRuntime(config: JSONValue!): Boolean! { + let fields = config.fields + let key = if (fields.contains("runtime")) { "runtime" } else if (fields.contains("sdk")) { "sdk" } else { "" } + if (key == "") { + false } else { - sdkHelpers.moduleManifest.withLegacyPythonRuntime - .withDangEntrypoint(source: sharedEntrypointSource) + let value = config.field([key]) + let source = value.asString rescue { + err: Error => value.field(["source"]).asString + } + source != "" and source != "python" + } + } + + """ + Refuse a manifest that names both a runtime of the user's own and an + entrypoint. The engine runs the entrypoint and never calls the runtime, + while generation would take the core it generates against from the + runtime's engineVersion: a core that is not the session's. Both tables are + the user's, so the user picks one. + """ + let checkOneRunner(toml: String!): Void { + let manifest = ManifestToml(toml) + let config = tomlConfig(toml) + if (manifest.hasEntrypoint and keepsItsRuntime(config)) { + raise "dagger-module.toml names both a runtime (" + runtimeSource(config) + ") and an entrypoint (" + + manifest.entrypointSource + "); the engine runs the entrypoint and never calls the runtime. " + + "Remove [entrypoint] to run on your runtime, or [runtime] to run on the entrypoint" + } + null + } + + let runtimeSource(config: JSONValue!): String! { + let value = config.field(["runtime"]) + value.asString rescue { + err: Error => value.field(["source"]).asString + } + } + + """ + Refuse what a runtime manifest can say and an entrypoint manifest cannot, + before anything is written: which files are the module (include, exclude) + and where its source is. Dropping either would change the module silently. + """ + let checkEntrypointOnly(config: JSONValue!, file: String!): Void { + let fields = config.fields + let listed = ["include", "exclude"].filter { key => fields.contains(key) } + let source = if (fields.contains("source") and normalizePath(config.field(["source"]).asString) != ".") { + ["source"] + } else { + [] :: [String!]! + } + let refused = listed + source + if (refused.length > 0) { + raise file + " has settings an entrypoint manifest cannot carry (" + refused.join(", ") + "); remove them, or keep the module on a runtime of your own" + } + null + } + + """ + Refuse a manifest that cannot run the module, before anything is written: a + Dang entrypoint this SDK does not write, in a scope with a local client, + unless that entrypoint hands clients over the way this SDK's does. + + The module's code loads a local client only through what the entrypoint + hands it with each call (entrypoint/handover.dang). An entrypoint the user + chose, a pinned version or a fork, may not send it, and the module would + fail at its first call to the client. Replacing the entrypoint would + destroy a table the user wrote, so the choice is theirs; one of the choices + is to make their entrypoint hand clients over. Without a local client any + entrypoint that runs the module will do. + """ + let checkHandover(ws: Workspace!, scope: String!, clients: [ModuleSource!]!): Void { + let manifest = ManifestToml(ws.file("/" + scopePath(scope, "dagger-module.toml")).contents) + let source = manifest.entrypointSource + if (manifest.entrypointKind == "dang" and source != sharedEntrypointSource) { + let local = clients.{{kind, moduleName}} + .filter { client => client.kind != ModuleSourceKind.GIT_SOURCE } + .map { client => client.moduleName } + if (local.length > 0 and handsOverClients(ws, scope, source) == false) { + raise "dagger-module.toml names the entrypoint \"" + source + "\", which this SDK does not write, " + + "and the scope has local clients (" + local.join(", ") + "); a local client loads only through " + + "the clients an entrypoint hands the module with each call, and this one does not, so the module could not run. " + + "Either update the entrypoint to hand them over: carry this SDK's entrypoint/handover.dang in it unchanged, " + + "and add `clients: ClientHandover(workspace: workspace).clients.map { client => {{name: client.name, source: client.source}} }` " + + "to the request its call sends, as the SDK's own entrypoint/main.dang does; " + + "or remove the [entrypoint] table to run on the SDK's, \"" + sharedEntrypointSource + "\"; " + + "or remove the local clients" + } + } + null + } + + """ + Whether an entrypoint hands clients over the way this SDK's does: it carries + this SDK's handover.dang unchanged, and another of its files calls it. Read + from where the engine reads the entrypoint; one that cannot be read does + not. + """ + let handsOverClients(ws: Workspace!, scope: String!, source: String!): Boolean! { + handsOverClientsIn(entrypointDirectory(ws, scope, source)) rescue { + err: Error => false + } + } + + let handsOverClientsIn(dir: Directory!): Boolean! { + let ours = currentModule.source.file("entrypoint/handover.dang").contents + let carried = dir.file("handover.dang").contents == ours + let calling = dir.entries + .filter { entry => entry.hasSuffix(".dang") and entry != "handover.dang" } + .filter { entry => dir.file(entry).contents.contains("ClientHandover(workspace: workspace).clients") } + carried and calling.length > 0 + } + + """ + The directory an entrypoint source names, resolved as the engine resolves + it: a path in the module, an address, or a module reference. + """ + let entrypointDirectory(ws: Workspace!, scope: String!, source: String!): Directory! { + if (isLocalEntrypoint(source)) { + ws.directory("/" + scopePath(scope, normalizePath(source))) + } else if (source.contains("://") == false and source.contains(":")) { + address(source).directory + } else { + let ref = moduleSource(source, disableFindUp: true, allowNotExists: true) + ref.contextDirectory.directory(ref.sourceRootSubpath) + } + } + + """ + The engine's rule: a path prefix or a dot-free first segment is local, a + URL or a host name is not. + """ + let isLocalEntrypoint(source: String!): Boolean! { + if (source.hasPrefix(".") or source.hasPrefix("/")) { + true + } else if (source.contains(":")) { + false + } else { + (source.split("/")[0] ?? "").contains(".") == false } - clients.reduce(base.withName(name: name).withoutLegacyRuntimeDependencies) { manifest, client => - manifest.withLegacyRuntimeDependency(module: client) - }.generate(ws.withWorkdir(scope), lock: lock, legacyJson: false).withWorkdir(".") } """ @@ -186,10 +399,7 @@ type PythonSdk { """ What a static entrypoint cannot carry, refused before anything is written. """ - let checkStaticScope(ws: Workspace!, scope: String!, hadConfig: Boolean!, clients: [ModuleSource!]!): Void { - if (clients.length > 0) { - raise "a static entrypoint cannot use other modules yet: an entrypoint manifest has no dependencies; set dangEntrypoint = false" - } + let checkStaticScope(ws: Workspace!, scope: String!, hadConfig: Boolean!): Void { if (hadConfig == false and template == "legacy") { raise "the legacy template is not available with a static entrypoint" } @@ -203,35 +413,46 @@ type PythonSdk { } """ - Manifest keys and tables that an entrypoint manifest has no place for. + Manifest keys and tables that an entrypoint manifest has no place for, read + with a TOML parser. """ let staticObjections(toml: String!): [String!]! { - let head = toml.split("\n[").takeFirst(1).join("") - let keys = ["include", "disableDefaultFunctionCaching"].filter { key => - head.containsMatch(Regexp("(?m)^\\s*" + key + "\\s*=")) - } + let config = tomlConfig(toml) + let fields = config.fields + let keys = ["include", "disableDefaultFunctionCaching"].filter { key => fields.contains(key) } # An entrypoint manifest roots a module at its own directory, which is what # `source = "."` says; any other source is what it cannot carry. - let sourceMatch = head.match("(?m)^\\s*source\\s*=\\s*\"([^\"]*)\"") - let sourceValue = if (sourceMatch == null) { "." } else { sourceMatch.captures[0] ?? "." } - let source = if (sourceValue == ".") { [] :: [String!]! } else { ["source"] } - let tables = ["codegen", "clients", "dependencies"].filter { name => - toml.containsMatch(Regexp("(?m)^\\s*\\[\\[?" + name + "\\]\\]?")) + let source = if (fields.contains("source") and normalizePath(config.field(["source"]).asString) != ".") { + ["source"] + } else { + [] :: [String!]! } - let pythonRuntime = toml.containsMatch("(?m)^\\s*\\[runtime\\]\\s*\\n\\s*source\\s*=\\s*\"python\"") - let runtime = if (toml.containsMatch("(?m)^\\s*\\[runtime\\]") and pythonRuntime == false) { ["runtime"] } else { [] :: [String!]! } + # [[dependencies]] is not among them: generation removes it on every path. + let tables = ["codegen", "clients"].filter { name => fields.contains(name) } + let runtime = if (keepsItsRuntime(config)) { ["runtime"] } else { [] :: [String!]! } keys + source + tables + runtime } + let tomlConfig(toml: String!): JSONValue! { + json.withContents((JSON.encode(TOML.decode(toml)) :: Dagger.JSON!)) + } + """ - Whether the manifest names an entrypoint generated into the module. The - shared entrypoint is a Dang entrypoint too, so the kind alone does not tell: - a static entrypoint is the one whose source is a path inside the module. + Whether the manifest names the static entrypoint this SDK generates into the + module: a Dang entrypoint at the path generation writes. Any other + entrypoint, a path of the user's own among them, is not this SDK's to + replace. """ let isEntrypointManifest(toml: String!): Boolean! { - ManifestToml(toml).isStatic + let manifest = ManifestToml(toml) + manifest.entrypointKind == "dang" and normalizePath(manifest.entrypointSource) == staticEntrypointPath } + """ + Where the static entrypoint lives in a module, as generation names it. + """ + let staticEntrypointPath: String! = vendorDirName + "/entrypoint" + let hasEntrypointManifest(ws: Workspace!, scope: String!): Boolean! { scopeHasFile(ws, scope, "dagger-module.toml") and isEntrypointManifest(ws.file("/" + scopePath(scope, "dagger-module.toml")).contents) } @@ -308,6 +529,8 @@ type PythonSdk { rootPath: modPath, ws: ws, dangEntrypoint: dangEntrypoint or hasEntrypointManifest(ws, modPath), + clients: null, + lock: lock, ) } @@ -393,10 +616,12 @@ type PythonSdk { """ A dagger-module.toml, read for its entrypoint table. -Not a TOML parser. A table runs to the next table header, so one pass over the -lines separates the entrypoint table from the rest. A header is recognised with +Values are read with a TOML parser, because the manifest is the user's +content: quoting, spacing and key order are theirs. Only removing the table +works on lines, so that everything else is carried over exactly as it was +read: a table runs to the next table header, and a header is recognised with its trailing comment removed, so a "[" inside a comment or a value never ends -the table; every line is carried over exactly as it was read. +the table. """ type ManifestToml { pub toml: String! @@ -407,7 +632,7 @@ type ManifestToml { } pub hasEntrypoint: Boolean! { - split.entrypoint.isEmpty == false + config.fields.contains("entrypoint") } """The entrypoint's kind, or empty when the manifest has no entrypoint.""" @@ -419,14 +644,6 @@ type ManifestToml { entrypointValue("source") } - """ - Whether the entrypoint is one generated into the module: a Dang entrypoint - whose source is a path inside the module, read the way the engine reads it. - """ - pub isStatic: Boolean! { - entrypointKind == "dang" and isLocalSource(entrypointSource) - } - """The manifest with its entrypoint table removed.""" pub withoutEntrypoint: File! { directory.withNewFile(fileName, split.kept.join("\n")).file(fileName) @@ -434,11 +651,25 @@ type ManifestToml { let fileName: String! = "dagger-module.toml" + let config: JSONValue! { + json.withContents((JSON.encode(TOML.decode(toml)) :: Dagger.JSON!)) + } + + let entrypointValue(key: String!): String! { + if (hasEntrypoint == false) { + "" + } else { + let table = config.field(["entrypoint"]) + if (table.fields.contains(key)) { table.field([key]).asString } else { "" } + } + } + let split: TomlSplit! { toml.split("\n").reduce(TomlSplit()) { state, line => let header = line.replaceMatches("#.*", "").trimSpace if (header.hasPrefix("[")) { - let isEntrypoint = header == "[entrypoint]" + # [entrypoint], [ entrypoint ] and ["entrypoint"] are one table. + let isEntrypoint = header.replaceMatches("[\\s\"']", "") == "[entrypoint]" TomlSplit( kept: if (isEntrypoint) { state.kept } else { state.kept + [line] }, entrypoint: if (isEntrypoint) { state.entrypoint + [line] } else { state.entrypoint }, @@ -451,25 +682,6 @@ type ManifestToml { } } } - - let entrypointValue(key: String!): String! { - let m = split.entrypoint.join("\n").match(Regexp("(?m)^\\s*" + Regexp.escape(key) + "\\s*=\\s*\"([^\"]*)\"")) - if (m == null) { "" } else { m.captures[0] ?? "" } - } - - """ - The engine's own rule: a path prefix or a dot-free first segment is local, a - URL or a host name is not. - """ - let isLocalSource(source: String!): Boolean! { - if (source.hasPrefix(".") or source.hasPrefix("/")) { - true - } else if (source.contains(":")) { - false - } else { - (source.split("/")[0] ?? "").contains(".") == false - } - } } """ diff --git a/runtime/build.dang b/runtime/build.dang index e469960..23c6c85 100644 --- a/runtime/build.dang +++ b/runtime/build.dang @@ -27,8 +27,8 @@ type PythonModuleBuild { let modSourceDirPath: String! = "/src" let venvPath: String! = "/opt/venv" let genDir: String! = "sdk" + let clientsDir: String! = "clients" let sdkGenPath: String! = "src/dagger/client/gen.py" - let userGenPath: String! = "src/dagger_gen.py" let projectCfg: String! = "pyproject.toml" let pipCompileLock: String! = "requirements.lock" let uvLock: String! = "uv.lock" @@ -68,8 +68,7 @@ type PythonModuleBuild { raise "no python files found in module source" } else { let cfg = pyConfig(source) - let vendorPath = vendorPathFor(source, cfg) - checkGeneratedFiles(source, moduleName, vendorPath) + checkGeneratedFiles(source, moduleName, requiredGeneratedFiles(source, cfg)) let baseImage = baseImageFor(source, cfg) let uvImage = uvImageFor(cfg) @@ -89,6 +88,13 @@ type PythonModuleBuild { } } + """ + The workspace members of a scope's pyproject.toml, read as the build does. + """ + pub scopeMembers(pyproject: File!): [String!]! { + pyConfig(directory.withFile(projectCfg, pyproject)).members + } + """ The class name of a module's main object, converted like the engine's builtin Python runtime (strcase.ToCamel) so a module loads the same on both. @@ -161,15 +167,18 @@ type PythonModuleBuild { } """ - The vendored library is installed non-editable so uv compiles its bytecode - once, not on every call into a throwaway mount; the module's own package - stays editable. + The SDK files and the generated clients are installed non-editable so uv + compiles their bytecode once, not on every call into a throwaway mount; the + module's own package stays editable. """ let install(ctr: Container!, source: Directory!, cfg: PyConfig!): Container! { let compiled = ctr.withEnvVariable("UV_COMPILE_BYTECODE", "1") + let packages = localPackages(source, cfg) if (cfg.useUv and source.exists(uvLock)) { # --locked: fail loudly on a stale lockfile instead of re-resolving. + # In a scope, sync installs the members the project depends on, which + # --no-install-project leaves in. compiled .withExec(["uv", "sync", "--no-dev", "--locked", "--no-editable", "--no-install-project"]) .withEnvVariable("VIRTUAL_ENV", "$UV_PROJECT_ENVIRONMENT", expand: true) @@ -184,22 +193,139 @@ type PythonModuleBuild { ["-r", projectCfg] } compiled - .withExec(["uv", "pip", "install", "--no-editable", "./" + genDir] + deps) + .withExec(["uv", "pip", "install", "--no-editable"] + packages + deps) .withExec(["uv", "pip", "install", "--no-deps", "-e", "."]) } else { - compiled.withExec(["pip", "install", "./" + genDir, "-e", "."]) + # pip reads no uv sources, so every local package goes in by path, in + # the same command that resolves the project's dependencies on them. + compiled.withExec(["pip", "install"] + packages + ["-e", "."]) } } """ - Fail early when the committed generated files are missing. + What the module installs from its own tree: the members of a scope's + workspace that the project depends on, directly or through another member, + the way `uv sync` picks them; or the vendored library of the layout before. + A member nothing depends on is not installed: it may not even install here. + """ + let localPackages(source: Directory!, cfg: PyConfig!): [String!]! { + if (cfg.isScope) { + workspaceRead(source) + .filter { line => line.hasPrefix("local\t") } + .map { line => line.trimPrefix("local\t") } + } else { + ["./" + genDir] + } + } + + """ + The workspace as TOML defines it, read by tomllib in the pinned default + image: `member\t` for each member, then `local\t./` for each + member directory to install. [project] dependencies and a member's are the + user's own content, with escapes, literal and multi-line strings, so the + regular expressions of pyConfig must not read them. Only the + pyproject.toml files that can be a member's go in, so the read is cached + until one of them changes. A file that does not read is named. """ - let checkGeneratedFiles(source: Directory!, modName: String!, vendorPath: String!): Void { - let required = if (vendorPath == "") { - [userGenPath] + let workspaceRead(source: Directory!): [String!]! { + let lines = container + .from(defaultBaseImage) + .withMountedDirectory("/scope", source.filter( + include: ["**/" + projectCfg], + # No member lives in these, and a virtualenv alone holds hundreds of + # pyproject.toml files that would reach the read and its cache key. + exclude: ["**/.venv", "**/__pycache__", "**/node_modules", "**/.git"], + )) + .withExec(["python", "-c", workspaceReader, "/scope"]) + .stdout + .split("\n") + .filter { line => line != "" } + let errors = lines.filter { line => line.hasPrefix("error\t") } + if (errors.length > 0) { + raise errors.map { line => line.trimPrefix("error\t") }.join("; ") } else { - [vendorPath + "/" + projectCfg, vendorPath + "/" + sdkGenPath] + lines } + } + + let workspaceReader: String! = + "import glob, os, re, sys, tomllib\n" + + "root = sys.argv[1]\n" + + "def fail(message):\n" + + " print('error\\t' + message)\n" + + " sys.exit(0)\n" + + "def load(rel):\n" + + " path = os.path.normpath(os.path.join(rel, 'pyproject.toml'))\n" + + " try:\n" + + " with open(os.path.join(root, path), 'rb') as f:\n" + + " return tomllib.load(f)\n" + + " except (tomllib.TOMLDecodeError, UnicodeDecodeError, OSError) as e:\n" + + " fail(f'{path} is not valid TOML: {e}')\n" + + "def table(doc, *keys):\n" + + " for key in keys:\n" + + " doc = doc.get(key) if isinstance(doc, dict) else None\n" + + " return doc if isinstance(doc, dict) else {}\n" + + "def strings(value, key, where):\n" + + " if value is None:\n" + + " return []\n" + + " if not isinstance(value, list) or not all(isinstance(v, str) for v in value):\n" + + " fail(f'{where}: {key} is not an array of strings; write it as one, such as {key} = [\"a\", \"b\"]')\n" + + " return value\n" + + "def name(requirement):\n" + + " m = re.match(r'\\s*([A-Za-z0-9][A-Za-z0-9._-]*)', requirement if isinstance(requirement, str) else '')\n" + + " return re.sub(r'[-_.]+', '-', m.group(1).lower()) if m else ''\n" + + "doc = load('.')\n" + + "members = strings(table(doc, 'tool', 'uv', 'workspace').get('members'), 'members', 'pyproject.toml')\n" + + "wanted = {name(r) for r in strings(table(doc, 'project').get('dependencies'), 'dependencies', 'pyproject.toml')}\n" + + "dirs = []\n" + + "for member in members:\n" + + " print('member\\t' + member)\n" + + " for found in sorted(glob.glob(os.path.join(root, member, 'pyproject.toml'))):\n" + + " rel = os.path.relpath(os.path.dirname(found), root)\n" + + " if rel not in dirs:\n" + + " dirs.append(rel)\n" + + "projects = {}\n" + + "for rel in dirs:\n" + + " project = table(load(rel), 'project')\n" + + " projects[rel] = (name(project.get('name')), {name(r) for r in strings(project.get('dependencies'), 'dependencies', rel + '/pyproject.toml')})\n" + + "needed = []\n" + + "while True:\n" + + " found = [rel for rel in dirs if rel not in needed and projects[rel][0] in wanted]\n" + + " if not found:\n" + + " break\n" + + " needed += found\n" + + " wanted = set().union(*(projects[rel][1] for rel in found))\n" + + "for rel in dirs:\n" + + " if rel in needed:\n" + + " print('local\\t./' + rel)\n" + + """ + The generated files a module needs before it builds. A scope needs each of + the SDK's members it lists; a user's member is the user's to provide. + """ + let requiredGeneratedFiles(source: Directory!, cfg: PyConfig!): [String!]! { + if (cfg.isScope) { + cfg.members + .filter { member => + (member == genDir or member.hasPrefix(clientsDir + "/")) and member.contains("*") == false + } + .map { member => member + "/" + projectCfg } + } else { + # A published dagger-io brings its own files, and the SDK files no longer + # read bindings from src/dagger_gen.py, so nothing generated is required. + let vendorPath = vendorPathFor(source, cfg) + if (vendorPath == "") { + [] :: [String!]! + } else { + [vendorPath + "/" + projectCfg, vendorPath + "/" + sdkGenPath] + } + } + } + + """ + Fail early when the committed generated files are missing. + """ + let checkGeneratedFiles(source: Directory!, modName: String!, required: [String!]!): Void { required.each { rel => if (source.exists(rel, expectedType: ExistsType.REGULAR_TYPE) == false) { raise "module \"" + modName + "\": generated file \"" + rel + "\" is missing; run `dagger generate` and commit the generated files" @@ -297,7 +423,9 @@ type PythonModuleBuild { """ Not a TOML parser: regular expressions scoped to a table, for the keys the templates and `mod config set` write. Single-line booleans and quoted strings - only; arrays, multi-line strings and dotted keys are not seen. + only; arrays, multi-line strings and dotted keys are not seen. The workspace + members, and the dependencies that pick what to install, are the user's + arrays, so tomllib reads them (workspaceRead). """ let pyConfig(source: Directory!): PyConfig! { let toml = source.file(projectCfg).contents @@ -314,6 +442,9 @@ type PythonModuleBuild { useUv: tomlBool(dagger, "use-uv", true), uvVersion: tomlString(dagger, "uv-version"), vendorPath: inlineTableString(sources, "dagger-io", "path"), + members: workspaceRead(source) + .filter { line => line.hasPrefix("member\t") } + .map { line => line.trimPrefix("member\t") }, indexURL: indexes .filter { table => tomlBool(table, "default", false) } .map { table => tomlString(table, "url") } @@ -377,6 +508,11 @@ type PyConfig { pub useUv: Boolean! pub uvVersion: String! pub vendorPath: String! + + """ + The `[tool.uv.workspace] members` of a scope; empty in the layout before. + """ + pub members: [String!]! pub indexURL: String! pub extraIndexURL: String! @@ -387,6 +523,7 @@ type PyConfig { useUv: Boolean! = true, uvVersion: String! = "", vendorPath: String! = "", + members: [String!]! = [], indexURL: String! = "", extraIndexURL: String! = "", ) { @@ -396,10 +533,19 @@ type PyConfig { self.useUv = useUv self.uvVersion = uvVersion self.vendorPath = vendorPath + self.members = members self.indexURL = indexURL self.extraIndexURL = extraIndexURL self } + + """ + Whether the module is a scope of the unified layout, whose SDK files are + the workspace member sdk/ rather than a library vendored by path. + """ + pub isScope: Boolean! { + members.filter { member => member == "sdk" }.length > 0 + } } """ diff --git a/runtime/main.dang b/runtime/main.dang index a2159aa..9d45fa2 100644 --- a/runtime/main.dang +++ b/runtime/main.dang @@ -30,6 +30,14 @@ type PythonSdkRuntime { } } + """ + The workspace members this runtime reads from a scope's pyproject.toml. + Generation checks the file it writes with it. + """ + pub scopeMembers(pyproject: File!): [String!]! { + PythonModuleBuild().scopeMembers(pyproject) + } + """ The class name of a module's main object. """ diff --git a/scope.dang b/scope.dang new file mode 100644 index 0000000..f64a474 --- /dev/null +++ b/scope.dang @@ -0,0 +1,659 @@ +""" +One SDK scope: a directory whose pyproject.toml is a uv workspace root, with +the SDK files and the generated clients as its members. A module is a scope; +so is a plain project. Both get the same tree: + + pyproject.toml the scope file, edited in place + src/ the user's code, never written here + sdk/ the SDK files, no generated code + clients/core/ the core bindings + clients// one member per declared client +""" +type Scope { + """ + Workspace-root-relative path of the scope. + """ + pub path: String! + + """ + The workspace the scope belongs to. + """ + let ws: Workspace! + + """ + The clients declared on the scope, in dagger.toml. + """ + let clients: [ModuleSource!]! + + """ + Write the commit a Git client was generated against into its descriptor. + """ + let lock: Boolean! + + """ + Whether the scope is a module's, which the module runtime builds. + """ + let isModule: Boolean! + + """ + The engine version the scope's module declares for a runtime of its own. + Empty for a module on an entrypoint, which runs on the engine's own + version, and for a scope without a module. + """ + let engineVersion: String! + + """ + The workspace with the scope generated: sdk/, clients/core and one member + per client written whole, a client that is no longer declared deleted, and + the SDK-owned entries of the scope file set. Everything else is left as it + is. + """ + pub generated: Workspace! { + let existing = ws.directory(root) + let members = clientMembers + checkNames(members) + checkMembers(existing, members) + let stale = staleClients(existing, members) + let withoutStale = stale.reduce(ws) { target, dir => + target.withoutDirectory(memberPath(clientsDirName + "/" + dir)) + } + let cleared = if (hasGeneratedUserBindings(existing)) { + withoutStale.withoutFile(memberPath(legacyUserBindingsPath)) + } else { + withoutStale + } + let withSdk = replaced(cleared, sdkDirName, sdkMember(members)) + let withCore = replaced(withSdk, clientsDirName + "/" + coreDirName, coreMember) + let withClients = members.reduce(withCore) { target, member => + replaced(target, clientsDirName + "/" + member.dir, clientMember(member)) + } + let file = scopeFile(existing, members, stale) + checkRuntimeReads(file, members) + relocked(existing, withClients.withNewFile(memberPath(scopeFileName), file.contents)) + } + + """ + A module's scope file must say to the runtime what it says to uv, or the + module generates and then fails to build, or installs a member no one + named. Generation asks the runtime which members it reads, asks uv's + reading of the TOML, and refuses a file where the two differ either way, + or where the runtime refuses to read the members at all. + """ + let checkRuntimeReads(file: File!, members: [ClientMember!]!): Void { + if (isModule) { + # Members are never empty, so a leading newline marks the refusal. + let read = pythonSdkRuntime.scopeMembers(file).join("\n") rescue { + err: Error => "\n" + err.message + } + let uvReads = pyprojectTool + .withFile(toolPath, file) + .withExec(["pyproject", "get-members", toolPath]) + .stdout + .split("\n") + .filter { line => line != "" } + let form = "Write members as an array of strings, not a nested array or other values, then run dagger generate again" + # The runtime reads [project] dependencies with the members, and names + # the key it refuses. + if (read.hasPrefix("\n")) { + # Its own diagnosis, not the members it then could not read. + let m = read.match("(\\S+) is not an array of strings") + let key = if (m == null) { "" } else { m.captures[0] ?? "" } + if (key == "") { + raise scopeFileName + " of " + path + " cannot be read by the module runtime: " + read.trimSpace + } else { + raise scopeFileName + " of " + path + " has " + key + " the module runtime cannot read: " + + key + " is not an array of strings, so the module would not build as written. Write " + key + + " as an array of strings, not a nested array or other values, then run dagger generate again" + } + } else { + let runtimeReads = read.split("\n").filter { line => line != "" } + let missed = uvReads.filter { member => runtimeReads.filter { r => r == member }.length == 0 } + let invented = runtimeReads.filter { member => uvReads.filter { u => u == member }.length == 0 } + if (missed.length > 0 or invented.length > 0) { + let misses = if (missed.length > 0) { ["misses " + missed.join(", ")] } else { [] :: [String!]! } + let invents = if (invented.length > 0) { ["reads " + invented.join(", ") + ", which uv does not"] } else { [] :: [String!]! } + raise scopeFileName + " of " + path + " lists workspace members the module runtime does not read the way uv does " + + "(it " + (misses + invents).join("; ") + "), so the module would not build as written. " + form + } + } + } + null + } + + let sdkDirName: String! = "sdk" + let clientsDirName: String! = "clients" + let coreDirName: String! = "core" + let scopeFileName: String! = "pyproject.toml" + let lockFileName: String! = "uv.lock" + let sdkDistribution: String! = "dagger-io" + let coreDistribution: String! = "dagger-clients-core" + + """ + The bindings of the layout before this one: vendored with the library, or + next to the user's code for a module on a published dagger-io. A module + that still has them is being upgraded, and gets the global client so its + code keeps working. + """ + let legacyBindingsPath: String! = "sdk/src/dagger/client/gen.py" + let legacyUserBindingsPath: String! = "src/dagger_gen.py" + + let globalPackage: String! = "dagger_global" + + """ + Whether src/ holds bindings the layout before generated, which the SDK + files no longer read and warn about until they are gone. Only a file with + the generator's header goes: the SDK once invited hand-written bindings + under the same name. + """ + let hasGeneratedUserBindings(existing: Directory!): Boolean! { + existing.exists(legacyUserBindingsPath) and + existing.file(legacyUserBindingsPath).contents.hasPrefix(generatedHeader) + } + + """ + Whether sdk/ is the one an earlier SDK vendored. It carries no marker, so + its bindings say so: only a gen.py with the generator's header counts, and + a hand-written sdk/ is never taken for it. + """ + let hasLegacySdk(existing: Directory!): Boolean! { + existing.exists(legacyBindingsPath) and + existing.file(legacyBindingsPath).contents.hasPrefix(generatedHeader) + } + + let generatedHeader: String! = "# Code generated by dagger. DO NOT EDIT." + + """ + Whether the scope gets the temporary global client: its scope file asks for + it, or it is a module upgraded from the bindings of the layout before, + whose code still calls `dag`. Only generation reads the flag. + """ + let globalClient: Boolean! { + let existing = ws.directory(root) + upgrading or (existing.exists(scopeFileName) and hasGlobalClientFlag(existing.file(scopeFileName).contents)) + } + + """ + Whether this generation upgrades a module from the layout before. The + vendored bindings go with sdk/ and generated ones in src/ are deleted, but + a hand-written dagger_gen.py stays, so it counts only until the scope has + its core: a flag turned off afterwards stays off. + """ + let upgrading: Boolean! { + let existing = ws.directory(root) + hasLegacySdk(existing) or + (existing.exists(legacyUserBindingsPath) and generatedKind(existing, clientsDirName + "/" + coreDirName) != "core") + } + + let root: String! { + if (path == ".") { "/" } else { "/" + path } + } + + let memberPath(rel: String!): String! { + if (path == ".") { "/" + rel } else { "/" + path + "/" + rel } + } + + """ + A member is written whole, so nothing of an earlier generation survives in + it; merging would keep a file the generator no longer writes. + """ + let replaced(target: Workspace!, rel: String!, files: Directory!): Workspace! { + let abs = memberPath(rel) + let cleared = if (ws.directory(root).exists(rel)) { target.withoutDirectory(abs) } else { target } + cleared.withDirectory(abs, files) + } + + """ + A directory where a member goes, that is not a member, is the user's: the + SDK refuses to write over it. sdk/ from the previous layout has no marker + either; it is replaced, which is the upgrade, only when its bindings carry + the generator's header. + """ + let checkMembers(existing: Directory!, members: [ClientMember!]!): Void { + let sdkIsUsers = existing.exists(sdkDirName) and + generatedKind(existing, sdkDirName) != "runtime" and + hasLegacySdk(existing) == false + if (sdkIsUsers) { + raise sdkDirName + "/ exists in " + path + " and was not written by dagger generate: " + + "it has neither [tool.dagger] generated = \"runtime\" in " + sdkDirName + "/" + scopeFileName + + " nor the generated bindings of an earlier SDK in " + legacyBindingsPath + + ". Generation replaces " + sdkDirName + "/ whole, so move or rename it, then run dagger generate again" + } + checkMember(existing, coreDirName, "core") + members.each { member => checkMember(existing, member.dir, "client") } + null + } + + let checkMember(existing: Directory!, dir: String!, kind: String!): Void { + let rel = clientsDirName + "/" + dir + if (existing.exists(rel) and generatedKind(existing, rel) != kind) { + raise rel + "/ exists in " + path + " and is not a generated " + kind + "; move it away" + } + null + } + + """ + Two clients whose names differ only in case, "-", "_" or "." would share + one member and one import package. + """ + let checkNames(members: [ClientMember!]!): Void { + members.each { member => + let same = members.filter { other => other.dir == member.dir } + if (same.length > 1) { + raise "clients " + same.map { other => "\"" + other.name + "\"" }.join(" and ") + " of " + path + " would both be generated into " + clientsDirName + "/" + member.dir + } + null + } + null + } + + """ + A committed uv.lock names the members, so it is locked again against the + new ones; `uv sync --locked` in the module build refuses a stale lock. The + uv is the one the module build runs, so the build can read the lock. A + scope without a lock gets none. + """ + let relocked(existing: Directory!, target: Workspace!): Workspace! { + if (existing.exists(lockFileName)) { + let scopeDir = target.directory(root).filter(exclude: [".venv", "**/.venv", "**/__pycache__"]) + let lockFile = codegenBase + .withMountedFile("/usr/local/bin/uv", container.from(buildUvImage).rootfs.file("uv")) + .withDirectory("/scope", scopeDir) + .withWorkdir("/scope") + .withExec(["uv", "lock"]) + .file("/scope/" + lockFileName) + target.withFile(memberPath(lockFileName), lockFile) + } else { + target + } + } + + let buildUvImage: String! { + let line = currentModule.source.file("runtime/images/uv/Dockerfile").contents.match("(?m)^FROM\\s+(\\S+)") + if (line == null) { + raise "no FROM line in runtime/images/uv/Dockerfile" + } else { + line.captures[0] ?? "" + } + } + + """ + Generated clients under clients/ that the scope no longer declares. Only a + marked member is deleted; anything else there is left alone. + """ + let staleClients(existing: Directory!, members: [ClientMember!]!): [String!]! { + if (existing.exists(clientsDirName) == false) { + [] :: [String!]! + } else { + let declared = members.map { member => member.dir } + existing.directory(clientsDirName).entries + .map { entry => entry.trimSuffix("/") } + .filter { dir => + dir != coreDirName and + declared.filter { name => name == dir }.length == 0 and + generatedKind(existing, clientsDirName + "/" + dir) == "client" + } + } + } + + """ + What `[tool.dagger] generated` says in a member's pyproject.toml, or empty. + The helper reads it as TOML, so text that only looks like the marker, in a + string, is not one; and only the kinds the SDK writes count, so a member + marked with anything else is the user's, never deleted or written over. + """ + let generatedKind(existing: Directory!, rel: String!): String! { + let prefix = rel + " " + memberKinds(existing) + .filter { line => line.hasPrefix(prefix) } + .takeFirst(1) + .join("") + .trimPrefix(prefix) + } + + """ + One " " line for sdk/ and each directory under clients/ that + carries a known marker. + """ + let memberKinds(existing: Directory!): [String!]! { + pyprojectTool + .withDirectory("/scope", existing, include: [ + sdkDirName + "/" + scopeFileName, + clientsDirName + "/*/" + scopeFileName, + ]) + .withExec(["pyproject", "get-member-kinds", "/scope"]) + .stdout + .split("\n") + .filter { line => line != "" } + } + + """ + One member per declared client. A local module is loaded by its path from + the workspace root; a Git one by its ref, pinned when locked. + """ + let clientMembers: [ClientMember!]! { + clients.map { client => + let isGit = client.kind == ModuleSourceKind.GIT_SOURCE + ClientMember( + name: client.moduleName, + ref: if (isGit) { client.asString } else { localRef(client.sourceRootSubpath) }, + pin: if (isGit and lock and client.pin != "") { client.pin } else { null }, + source: client, + ) + } + } + + """ + A descriptor path starts with "/": serveModule reads it from the workspace + root, where "./" would be read from the cwd of whatever code runs the + client. The leading "/" also tells it from a Git ref. + """ + let localRef(subpath: String!): String! { + let trimmed = subpath.trimPrefix("./").trimPrefix("/").trimSuffix("/") + if (trimmed == "" or trimmed == ".") { "/" } else { "/" + trimmed } + } + + """ + The SDK files: the importable library and its license, with the project + file stripped down to what a member needs. No generated code goes in but + the temporary global client, the one generated package the SDK files may + import. + """ + let sdkMember(members: [ClientMember!]!): Directory! { + let files = currentModule.source + .directory("sdk") + .filter(include: [ + "LICENSE", + "README.md", + "src/**/*.py", + "src/**/*.typed", + "!" + "src/dagger/client/gen.py", + ]) + .withFile(scopeFileName, sdkPyproject(members)) + if (globalClient) { + files.withDirectory("src/" + globalPackage, globalSource(members)) + } else { + files + } + } + + """ + pyproject.toml without its development sections: vendored verbatim, it names + the absent codegen workspace member and uv refuses to install the library. + With the global client, which imports them, it depends on core and on each + client. + """ + let sdkPyproject(members: [ClientMember!]!): File! { + let globalDependencies = if (globalClient) { + [coreDistribution] + members.map { member => member.distribution } + } else { + [] :: [String!]! + } + codegenBase + .withFile(stripScriptPath, currentModule.source.file("helpers/vendor-pyproject/strip_dev_sections.py")) + .withExec(["python", stripScriptPath, "pyproject.toml", "/member-pyproject.toml"] + globalDependencies) + .file("/member-pyproject.toml") + } + + """ + The global client, from the schema each client is generated from, or from + core's for a scope without a client. The generator refuses schemas whose + cores differ, as it refuses a client generated against another core. + """ + let globalSource(members: [ClientMember!]!): Directory! { + let mounted = if (members.length == 0) { + codegenEnv.withMountedFile(schemasPath + "/" + coreDirName + ".json", coreSchema) + } else { + members.reduce(codegenEnv) { ctr, member => + ctr.withMountedFile(schemasPath + "/" + member.dir + ".json", clientSchema(member)) + } + } + let names = if (members.length == 0) { [coreDirName] } else { members.map { member => member.dir } } + let inputs = names.reduce([] :: [String!]!) { args, name => args + ["-i", schemasPath + "/" + name + ".json"] } + mounted + .withExec([codegenPython, "-m", "codegen", "generate-global"] + inputs + ["-o", outputPath]) + .directory(outputPath + "/" + globalPackage) + } + + let coreMember: Directory! { + directory + .withNewFile(scopeFileName, memberPyproject(coreDistribution, coreDirName, [sdkDistribution], "core")) + .withDirectory("src", coreSource) + } + + """ + Core is generated from a module with nothing in it, so its schema is core + and nothing else. Every client's schema holds the same core, read in the + same view, and the generator checks that when it writes the client. + """ + let coreSource: Directory! { + codegenEnv + .withMountedFile(schemaPath, coreSchema) + .withExec([codegenPython, "-m", "codegen", "generate-core", "-i", schemaPath, "-o", outputPath]) + .directory(outputPath) + } + + """ + The schema of core alone: the empty module it is read through would be a + client of its own to the global client. + """ + let coreSchema: File! { + codegenBase + .withFile(coreOnlyScriptPath, currentModule.source.file("helpers/core-schema/core_only.py")) + .withMountedFile(schemaPath, coreSchemaSource.withEngineVersion(view).clientSchemaIntrospectionJSON) + .withExec(["python", coreOnlyScriptPath, schemaPath, coreOnlyPath]) + .file(coreOnlyPath) + } + + let coreSchemaSource: ModuleSource! { + currentModule.source.directory("helpers/core-schema").asModuleSource + } + + """ + The engine version the scope's code runs against: the one its module's own + runtime declares, or else the version core is generated at, which is the + engine's own view, as on an entrypoint. The engine + serves a schema in the view of the module it describes, so a client to a + module that declares an older version would otherwise name core types that + its scope does not have. + """ + let view: String! { + if (engineVersion == "") { coreSchemaSource.engineVersion } else { engineVersion } + } + + """ + The digest a client must be generated against, read from core itself so + the two can never disagree. + """ + let coreDigest: String! { + let m = coreSource.file("dagger_clients/" + coreDirName + "/__init__.py").contents + .match("(?m)^CORE_DIGEST = \"([^\"]*)\"") + if (m == null) { + raise "the generated core carries no CORE_DIGEST" + } else { + m.captures[0] ?? "" + } + } + + let clientMember(member: ClientMember!): Directory! { + directory + .withNewFile(scopeFileName, memberPyproject(member.distribution, member.package, [sdkDistribution, coreDistribution], "client")) + .withDirectory("src", clientSource(member)) + } + + let clientSchema(member: ClientMember!): File! { + member.source.withEngineVersion(view).clientSchemaIntrospectionJSON + } + + let clientSource(member: ClientMember!): Directory! { + let pin = if (member.pin == null) { [] :: [String!]! } else { ["--pin", member.pin ?? ""] } + codegenEnv + .withMountedFile(schemaPath, clientSchema(member)) + .withExec([ + codegenPython, "-m", "codegen", "generate-client", + "-i", schemaPath, "-o", outputPath, + "--name", member.name, "--ref", member.ref, "--core-digest", coreDigest, + ] + pin) + .directory(outputPath) + } + + """ + A member holds no path and no source entry, so its files do not depend on + the scope. The marker is how the SDK finds its own members again. + """ + let memberPyproject(distribution: String!, package: String!, dependencies: [String!]!, kind: String!): String! { + "[project]\n" + + "name = \"" + distribution + "\"\n" + + "version = \"0.0.0\"\n" + + "dependencies = [" + dependencies.map { name => "\"" + name + "\"" }.join(", ") + "]\n" + + "\n" + + "[build-system]\n" + + "requires = [\"uv_build>=0.8.4,<0.12.0\"]\n" + + "build-backend = \"uv_build\"\n" + + "\n" + + "[tool.uv.build-backend]\n" + + "module-name = \"dagger_clients." + package + "\"\n" + + "\n" + + "[tool.dagger]\n" + + "generated = \"" + kind + "\"\n" + } + + """ + The scope file with the SDK-owned entries set: the members, one workspace + source per member, and a dependency on core and on each client. A scope + without one starts from an empty file. Being upgraded writes the global + client flag once, so turning it off later sticks. + + The editor removes only the stale members named here, the ones whose + directory carries the generated marker: any other member under clients/ + is the user's, and stays in the workspace with its directory. + """ + let scopeFile(existing: Directory!, members: [ClientMember!]!, stale: [String!]!): File! { + let current = if (existing.exists(scopeFileName)) { + existing.file(scopeFileName) + } else { + directory.withNewFile(scopeFileName, "").file(scopeFileName) + } + let base = [ + "--member", sdkDirName, + "--member", clientsDirName + "/" + coreDirName, + "--source", sdkDistribution, + "--source", coreDistribution, + "--dependency", coreDistribution, + ] + let perClient = members.reduce([] :: [String!]!) { flags, member => + flags + [ + "--member", clientsDirName + "/" + member.dir, + "--source", member.distribution, + "--dependency", member.distribution, + ] + } + let staleFlags = stale.reduce([] :: [String!]!) { flags, dir => + flags + ["--stale-member", clientsDirName + "/" + dir] + } + let upgrade = if (upgrading) { ["--global-client", "true"] } else { [] :: [String!]! } + pyprojectTool + .withFile(toolPath, current) + .withExec(["pyproject", "edit-scope", toolPath] + base + perClient + staleFlags + upgrade) + .file(toolPath) + } + + let hasGlobalClientFlag(toml: String!): Boolean! { + toml.containsMatch("(?ms)^\\s*\\[tool\\.dagger\\][^\\[]*^\\s*global-client\\s*=\\s*true") + } + + let pyprojectTool: Container! { + container + .from("golang:1.25-alpine") + .withoutEntrypoint + .withMountedCache("/go/pkg/mod", cacheVolume("go-mod")) + .withMountedCache("/root/.cache/go-build", cacheVolume("go-build")) + .withDirectory("/helper", currentModule.source.directory("helpers/pyproject")) + .withWorkdir("/helper") + .withExec(["go", "build", "-o", "/usr/local/bin/pyproject", "."]) + } + + let toolPath: String! = "/work/pyproject.toml" + + """ + The code generator's environment, synced once per SDK version and shared by + every scope that generates. + """ + let codegenEnv: Container! { + codegenBase.withExec(["uv", "sync", "--frozen", "--no-dev", "--package", "codegen"]) + } + + let codegenBase: Container! { + container + .from(codegenImage) + .withoutEntrypoint + .withMountedCache("/root/.cache/uv", cacheVolume("python-sdk-uv")) + .withEnvVariable("UV_LINK_MODE", "copy") + .withEnvVariable("UV_COMPILE_BYTECODE", "1") + .withDirectory("/sdk", codegenSource) + .withWorkdir("/sdk") + } + + let codegenPython: String! = "/sdk/.venv/bin/python" + + let codegenSource: Directory! { + currentModule.source.directory("sdk").filter(include: [ + "pyproject.toml", + "uv.lock", + "src/**/*.py", + "src/**/*.typed", + "codegen/pyproject.toml", + "codegen/**/*.py", + ]) + } + + let stripScriptPath: String! = "/strip-dev-sections.py" + let schemaPath: String! = "/schema.json" + let schemasPath: String! = "/schemas" + let coreOnlyScriptPath: String! = "/core-only.py" + let coreOnlyPath: String! = "/core-only.json" + let outputPath: String! = "/generated" + + # musl runs the generator ~0.3s slower than glibc, but the glibc image is + # 25 MiB larger to pull, which costs more on the first generate. + let codegenImage: String! = "ghcr.io/astral-sh/uv:python3.14-alpine" +} + +""" +One client of a scope, as the generator and the scope file see it. +""" +type ClientMember { + """ + The module's name in the schema, which is the client's name. + """ + pub name: String! + + """ + What the client loads: a path from the workspace root, or a Git ref. + """ + pub ref: String! + + """ + The commit a Git ref is pinned to, or null. + """ + pub pin: String + + let source: ModuleSource! + + """ + Directory under clients/, and the tail of the distribution name. + """ + pub dir: String! { + name.toLower.replace("_", "-").replace(".", "-") + } + + pub distribution: String! { + "dagger-clients-" + dir + } + + """ + Import package under dagger_clients. + """ + pub package: String! { + name.toLower.replace("-", "_").replace(".", "_") + } +} diff --git a/sdk/README.md b/sdk/README.md index d499a57..284670b 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -55,14 +55,14 @@ import sys import anyio import dagger -from dagger import dag +from dagger_clients.core import core async def main(args: list[str]): async with dagger.connection(): # build container with cowsay entrypoint ctr = ( - dag.container() + core().container() .from_("python:alpine") .with_exec(["pip", "install", "cowsay"]) ) diff --git a/sdk/codegen/src/codegen/cli.py b/sdk/codegen/src/codegen/cli.py index f61de09..42b53f0 100644 --- a/sdk/codegen/src/codegen/cli.py +++ b/sdk/codegen/src/codegen/cli.py @@ -5,14 +5,13 @@ import graphql -from codegen import ast, generator +from codegen import ast, generator, packages, partition -parser = argparse.ArgumentParser( - prog="python -m codegen", description="Dagger Python SDK" -) - -def main(): +def main(argv: list[str] | None = None): + parser = argparse.ArgumentParser( + prog="python -m codegen", description="Dagger Python SDK" + ) subparsers = parser.add_subparsers( title="additional commands", required=True, @@ -21,36 +20,162 @@ def main(): "generate", help="generate a Python client for the API", ) + add_introspection_argument(gen_parser) gen_parser.add_argument( + "-o", + "--output", + type=pathlib.Path, + help=( + "path to save the generated python module " + "(defaults to printing it to stdout)" + ), + ) + gen_parser.set_defaults(run=lambda args: codegen(args.introspection, args.output)) + + core_parser = subparsers.add_parser( + "generate-core", + help=f"generate the {packages.NAMESPACE}.core package", + ) + add_introspection_argument(core_parser) + add_package_output_argument(core_parser) + core_parser.set_defaults(run=lambda args: core(args.introspection, args.output)) + + client_parser = subparsers.add_parser( + "generate-client", + help=f"generate the {packages.NAMESPACE} package of one client", + ) + add_introspection_argument(client_parser) + add_package_output_argument(client_parser) + client_parser.add_argument( + "--name", + required=True, + type=non_empty, + help="name of the client, which is the module's name in the schema", + ) + client_parser.add_argument( + "--ref", + required=True, + type=non_empty, + help="where the client loads its module from: a workspace path or a git ref", + ) + client_parser.add_argument("--pin", help="commit that a git ref is pinned to") + client_parser.add_argument( + "--core-digest", + help=( + "CORE_DIGEST of the generated core package, which the client must " + "match to import (defaults to the digest of the given schema's core)" + ), + ) + client_parser.set_defaults(run=client) + + global_parser = subparsers.add_parser( + "generate-global", + help=f"generate the temporary {packages.GLOBAL} package", + ) + global_parser.add_argument( "-i", "--introspection", type=pathlib.Path, required=True, - help="path to a .json file holding the introspection result", + action="append", + help=( + "path to a .json file holding an introspection result; " + "repeat it for the schema of each client" + ), ) - gen_parser.add_argument( + global_parser.add_argument( "-o", "--output", type=pathlib.Path, - help=( - "path to save the generated python module " - "(defaults to printing it to stdout)" - ), + required=True, + help=f"directory to write the {packages.GLOBAL} package into", ) - args = parser.parse_args() + global_parser.set_defaults( + run=lambda args: global_client(args.introspection, args.output) + ) + + args = parser.parse_args(argv) # TODO: Add argument for module init. - codegen(args.introspection, args.output) + try: + args.run(args) + except partition.ClientError as e: + parser.error(str(e)) -def codegen(introspection: pathlib.Path, output: pathlib.Path | None): +def non_empty(value: str) -> str: + if not value: + msg = "must not be empty" + raise argparse.ArgumentTypeError(msg) + return value + + +def add_introspection_argument(subparser: argparse.ArgumentParser): + subparser.add_argument( + "-i", + "--introspection", + type=pathlib.Path, + required=True, + help="path to a .json file holding the introspection result", + ) + + +def add_package_output_argument(subparser: argparse.ArgumentParser): + subparser.add_argument( + "-o", + "--output", + type=pathlib.Path, + required=True, + help=f"directory to write the {packages.NAMESPACE} namespace package into", + ) + + +def read_schema(introspection: pathlib.Path) -> tuple[graphql.GraphQLSchema, str]: result = json.loads(introspection.read_text()) schema = graphql.build_client_schema(result) ast.insert_stubs(result["__schema"], schema) - code = generator.generate(schema, schema_version=result.get("__schemaVersion", "")) + return schema, result.get("__schemaVersion", "") + + +def codegen(introspection: pathlib.Path, output: pathlib.Path | None): + schema, schema_version = read_schema(introspection) + code = generator.generate(schema, schema_version=schema_version) if output: output.write_text(code) sys.stdout.write(f"Client generated successfully to {output}\n") else: sys.stdout.write(f"{code}\n") + + +def core(introspection: pathlib.Path, output: pathlib.Path): + schema, schema_version = read_schema(introspection) + files = packages.core_package(schema, schema_version) + root = packages.write_package(output, partition.CORE, files) + sys.stdout.write(f"Core generated successfully to {root}\n") + + +def client(args: argparse.Namespace): + schema, schema_version = read_schema(args.introspection) + package, files = packages.client_package( + schema, + args.name, + args.ref, + args.pin, + core_digest=args.core_digest, + schema_version=schema_version, + ) + root = packages.write_package(args.output, package, files) + sys.stdout.write(f"Client generated successfully to {root}\n") + + +def global_client(introspections: list[pathlib.Path], output: pathlib.Path): + schemas, versions = zip( + *(read_schema(path) for path in introspections), strict=True + ) + if len(set(versions)) > 1: + msg = f"the schemas have different versions: {', '.join(sorted(set(versions)))}" + raise partition.ClientError(msg) + files = packages.global_package(schemas, versions[0]) + root = packages.write_global(output, files) + sys.stdout.write(f"Global client generated successfully to {root}\n") diff --git a/sdk/codegen/src/codegen/generator.py b/sdk/codegen/src/codegen/generator.py index f17d438..a9af35c 100644 --- a/sdk/codegen/src/codegen/generator.py +++ b/sdk/codegen/src/codegen/generator.py @@ -30,7 +30,6 @@ GraphQLArgument, GraphQLEnumType, GraphQLField, - GraphQLFieldMap, GraphQLInputField, GraphQLInputFieldMap, GraphQLInputObjectType, @@ -53,6 +52,8 @@ from graphql.pyutils import camel_to_snake, snake_to_camel from graphql.type.schema import TypeMap +from codegen.partition import own_fields + ACRONYM_RE = re.compile(r"([A-Z\d]+)(?=[A-Z\d]|$)") """Pattern for grouping initialisms.""" @@ -126,11 +127,26 @@ class Context: remaining: set[str] = field(default_factory=set) """Remaining type names that haven't been defined yet.""" + partitioned: bool = False + """Render one package of a partition, rather than the whole schema.""" + @property def legacy_sdk_compat(self) -> bool: """Generate the pre-v0.21 ID/load helper source facade.""" return legacy_sdk_compat(self.schema_version) + def fields( + self, t: GraphQLObjectType | GraphQLInterfaceType + ) -> dict[str, GraphQLField]: + """Fields of a type that belong to the package being rendered.""" + return own_fields(t) if self.partitioned else t.fields + + def helper(self, name: str) -> str: + """Name that a runtime or generator helper goes by in the code.""" + # A package imports every helper under a private alias, so that no + # generated type can shadow it. The one-file client keeps plain names. + return f"_{name}" if self.partitioned else name + def process_type(self, name: str): # This is only needed to keep track of remaining types because # of forward references. @@ -170,7 +186,7 @@ class Handler(ABC, Generic[_H]): """Does this handler render the given type?""" def supertype_name(self, t: _H) -> str: - return self.__class__.__name__ + return self.ctx.helper(self.__class__.__name__) def type_name(self, t: _H) -> str: return t.name @@ -179,7 +195,8 @@ def type_name(self, t: _H) -> str: def render(self, t: _H) -> Iterator[str]: yield "" yield self.render_head(t) - yield indent(self.render_body(t)) + # A part of a schema can leave a class with nothing of its own. + yield indent(self.render_body(t) or "...") yield "" def render_head(self, t: _H) -> str: @@ -211,12 +228,47 @@ def generate(schema: GraphQLSchema, schema_version: str = "") -> Iterator[str]: """, ) + ctx = new_context(schema, schema_version) + yield from render_types(ctx, schema.type_map) + + yield "" + yield "" + yield "class Client(Query):" + yield indent( + '"""The Dagger client.\n' + "\n" + "Inherits all Query API methods and adds connection management.\n" + '"""' + ) + ctx.defined.add("Client") + + yield "" + yield "dag = Client()" + yield '"""The global client instance."""' + ctx.defined.add("dag") + + yield "" + yield "__all__ = [" + yield from (indent(f"{quote(name)},") for name in sorted(ctx.defined)) + yield "]" + + +def new_context( + schema: GraphQLSchema, schema_version: str = "", partitioned: bool = False +) -> Context: + """Shared state between all handler instances.""" # Pre-create handy maps to make handler code simpler. ids = frozenset(n for n, t in schema.type_map.items() if is_id_type(t)) + return Context( + ids=ids, + schema=schema, + schema_version=schema_version, + partitioned=partitioned, + ) - # shared state between all handler instances - ctx = Context(ids=ids, schema=schema, schema_version=schema_version) +def render_types(ctx: Context, type_map: TypeMap) -> Iterator[str]: + """Render the given types, which may be a part of the schema only.""" handlers: tuple[Handler, ...] = ( Scalar(ctx), Enum(ctx), @@ -226,12 +278,12 @@ def generate(schema: GraphQLSchema, schema_version: str = "") -> Iterator[str]: ) if ctx.legacy_sdk_compat: - for type_name in legacy_id_names(schema): - yield legacy_id_class(type_name) + for type_name in legacy_id_names(type_map): + yield legacy_id_class(type_name, ctx.helper("Scalar")) ctx.defined.add(type_name) # Split into two iterators to update ctx.remaining. - types_n, types_g = itertools.tee(get_grouped_types(handlers, schema.type_map)) + types_n, types_g = itertools.tee(get_grouped_types(handlers, type_map)) # Track types that haven't been defined yet, to format as a forward reference. ctx.remaining.update(name for _, name, _ in types_n) @@ -240,27 +292,6 @@ def generate(schema: GraphQLSchema, schema_version: str = "") -> Iterator[str]: yield handler.render(named_type) ctx.process_type(type_name) - yield "" - yield "" - yield "class Client(Query):" - yield indent( - '"""The Dagger client.\n' - "\n" - "Inherits all Query API methods and adds connection management.\n" - '"""' - ) - ctx.defined.add("Client") - - yield "" - yield "dag = Client()" - yield '"""The global client instance."""' - ctx.defined.add("dag") - - yield "" - yield "__all__ = [" - yield from (indent(f"{quote(name)},") for name in sorted(ctx.defined)) - yield "]" - def get_grouped_types(handlers: tuple[Handler, ...], type_map: TypeMap): """Group types by handler and sorted by their name.""" @@ -339,11 +370,11 @@ def is_enum_type(t: GraphQLNamedType) -> TypeGuard[GraphQLEnumType]: return isinstance(t, GraphQLEnumType) -def is_self_chainable(t: GraphQLObjectType) -> bool: +def is_self_chainable(t: GraphQLObjectType, fields: Iterable[GraphQLField]) -> bool: """Checks if an object type has any fields that return that same type.""" return any( f - for f in t.fields.values() + for f in fields # Only consider fields that return a non-null object. if is_required_type(f.type) and is_object_type(f.type.of_type) @@ -391,10 +422,10 @@ def legacy_id_name(type_name: TypeName) -> IDName: def legacy_idable_types( - schema: GraphQLSchema, + type_map: TypeMap, ) -> list[GraphQLObjectType | GraphQLInterfaceType]: types = [] - for t in schema.type_map.values(): + for t in type_map.values(): if not (is_object_type(t) or is_interface_type(t)): continue if t.name.startswith("_") or t.name == "Node": @@ -406,18 +437,20 @@ def legacy_idable_types( return sorted(types, key=lambda t: t.name) -def legacy_id_names(schema: GraphQLSchema) -> Iterator[IDName]: - for t in legacy_idable_types(schema): +def legacy_id_names(type_map: TypeMap) -> Iterator[IDName]: + # Only the part being rendered counts: a client's type must not take a + # compatibility class out of core, whose digest doesn't see clients. + for t in legacy_idable_types(type_map): name = legacy_id_name(t.name) - if schema.get_type(name) is None: + if name not in type_map: yield name -def legacy_id_class(type_name: IDName) -> str: +def legacy_id_class(type_name: IDName, scalar: str = "Scalar") -> str: return textwrap.dedent( f'''\ - class {type_name}(Scalar): + class {type_name}({scalar}): """Legacy typed ID alias for the unified ID scalar.""" ''' ) @@ -484,8 +517,12 @@ def format_input_type( convert_id=True, expected_type: TypeName | None = None, legacy_ids: bool = False, + any_object: str = "Type", ) -> str: - """May be used in an input object field or an object field parameter.""" + """May be used in an input object field or an object field parameter. + + `any_object` is what a generic ID stands for, as the package names it. + """ if is_required_type(t): t = t.of_type fmt = "%s" @@ -493,7 +530,9 @@ def format_input_type( fmt = "%s | None" if is_list_type(t): - inner = format_input_type(t.of_type, convert_id, expected_type, legacy_ids) + inner = format_input_type( + t.of_type, convert_id, expected_type, legacy_ids, any_object + ) return fmt % f"list[{inner}]" if is_id_type(t): @@ -501,7 +540,7 @@ def format_input_type( if expected_type is not None: return fmt % expected_type # Generic ID scalar — accept any Type (Dagger object) - return fmt % "Type" + return fmt % any_object if legacy_ids and expected_type is not None: return fmt % legacy_id_name(expected_type) @@ -592,6 +631,7 @@ def __init__( convert_id, self.expected_type, ctx.legacy_sdk_compat, + ctx.helper("Type"), ) self.is_self = self.type == self.parent_object_name self.description = graphql.description @@ -636,7 +676,7 @@ def __str__(self) -> Iterator[str]: def as_param(self) -> str: """As a parameter in a function signature.""" - type_ = "Self" if self.is_self else self.type + type_ = self.ctx.helper("Self") if self.is_self else self.type out = f"{self.name}: {type_}" if self.default_is_mutable: if not out.endswith("| None"): @@ -666,7 +706,7 @@ def as_arg(self) -> str: params[1] = f"{self.default_value} if {self.name} is None else {self.name}" if self.has_default: params.append(self.default_value) - return f"Arg({', '.join(params)})," + return f"{self.ctx.helper('Arg')}({', '.join(params)})," def check_expr(self, var: str) -> str: """Expression that is true when `var` matches the type as_param declares.""" @@ -691,7 +731,8 @@ def _check_non_null(self, var: str, t: GraphQLInputType, depth: int) -> str: if is_id_type(t): if self.convert_id: - return f"isinstance({var}, {self.expected_type or 'Type'})" + any_object = self.expected_type or self.ctx.helper("Type") + return f"isinstance({var}, {any_object})" if self.ctx.legacy_sdk_compat and self.expected_type is not None: return f"isinstance({var}, {legacy_id_name(self.expected_type)})" return f"isinstance({var}, str)" @@ -721,12 +762,15 @@ def as_check(self, method: str, var: str | None = None) -> str: class _ObjectField: """Field of an object type.""" + receiver = "self" + """Name of the object that the field is selected on.""" + def __init__( self, ctx: Context, name: str, field: GraphQLField, - parent: GraphQLObjectType, + parent: GraphQLObjectType | GraphQLInterfaceType, ) -> None: self.ctx = ctx self.graphql_name = name @@ -804,27 +848,67 @@ def __str__(self) -> Iterator[str]: indent("return self.sync().__await__()"), ) - def func_signature(self) -> str: - params = ", ".join( - chain( - ("self",), - (a.as_param() for a in self.required_args), - ("*",) if self.default_args else (), - (a.as_param() for a in self.default_args), - ) - ) + @property + def label(self) -> str: + """The name a rejected argument is reported against.""" + return f"{self.parent_name}.{self.name}" + + @property + def return_type(self) -> str: + if self.type == self.parent_name: + return self.ctx.helper("Self") + return self.type + + def params(self) -> Iterator[str]: + yield self.receiver + yield from (a.as_param() for a in self.required_args) + if self.default_args: + yield "*" + yield from (a.as_param() for a in self.default_args) + + def func_signature(self, name: str | None = None) -> str: + params = ", ".join(self.params()) # arbitrary heuristic to force trailing comma in long signatures if len(params) > 40: # noqa: PLR2004 params = f"{params}," - ret_type = "Self" if self.type == self.parent_name else self.type - sig = self.ctx.render_types(f"def {self.name}({params}) -> {ret_type}:") + sig = self.ctx.render_types( + f"def {name or self.name}({params}) -> {self.return_type}:" + ) if self.is_exec: sig = f"async {sig}" return sig + def select_expr(self) -> str: + return f'{self.receiver}._select("{self.graphql_name}", _args)' + @joiner def func_body(self) -> Iterator[str]: + yield from self.func_prelude() + + if self.convert_id: + args = (self.receiver, f'"{self.graphql_name}"', "_args") + call = f"execute_sync({', '.join(args)})" + yield f"return await {self.receiver}._ctx.{call}" + return + + yield f"_ctx = {self.select_expr()}" + + if not self.is_exec: + # Use the concrete client class for interface types + t = self._iface_client_name(self.type) + yield f"return {t}(_ctx)" + elif self.is_list: + n = self.named_type.name + t = self._iface_client_name(n) + yield f"return await _ctx.execute_object_list({t})" + elif self.is_void: + yield "await _ctx.execute()" + else: + yield f"return await _ctx.execute({self.type})" + + def func_prelude(self) -> Iterator[str]: + """Everything in the body that comes before the selection.""" if docstring := self.func_doc(): yield doc(docstring) @@ -834,7 +918,7 @@ def func_body(self) -> Iterator[str]: ) yield textwrap.dedent( f"""\ - warnings.warn( + {self.ctx.helper("warnings")}.warn( "{msg}", DeprecationWarning, stacklevel=4, @@ -843,34 +927,14 @@ def func_body(self) -> Iterator[str]: ) for arg in self.args: - yield arg.as_check(f"{self.parent_name}.{self.name}") + yield arg.as_check(self.label) if self.args: yield "_args = [" yield from (indent(arg.as_arg()) for arg in self.args) yield "]" else: - yield "_args: list[Arg] = []" - - if self.convert_id: - args = ("self", f'"{self.graphql_name}"', "_args") - yield f"return await self._ctx.execute_sync({', '.join(args)})" - return - - yield f'_ctx = self._select("{self.graphql_name}", _args)' - - if not self.is_exec: - # Use the concrete client class for interface types - t = self._iface_client_name(self.type) - yield f"return {t}(_ctx)" - elif self.is_list: - n = self.named_type.name - t = self._iface_client_name(n) - yield f"return await _ctx.execute_object_list({t})" - elif self.is_void: - yield "await _ctx.execute()" - else: - yield f"return await _ctx.execute({self.type})" + yield f"_args: list[{self.ctx.helper('Arg')}] = []" def _iface_client_name(self, name: str) -> str: """Return concrete client class name for interface types.""" @@ -1041,7 +1105,8 @@ class Input(ObjectHandler[GraphQLInputObjectType]): predicate: ClassVar[Predicate] = staticmethod(is_input_object_type) def render_head(self, t: GraphQLInputObjectType) -> str: - return f"@dataclass(slots=True)\n{super().render_head(t)}" + dataclass = self.ctx.helper("dataclass") + return f"@{dataclass}(slots=True)\n{super().render_head(t)}" @joiner def render_body(self, t: GraphQLInputObjectType) -> Iterator[str]: @@ -1088,20 +1153,21 @@ def type_name(self, t: GraphQLInterfaceType) -> str: return t.name def render_head(self, t: GraphQLInterfaceType) -> str: - return f"@runtime_checkable\nclass {t.name}(Protocol):" + checkable = self.ctx.helper("runtime_checkable") + return f"@{checkable}\nclass {t.name}({self.ctx.helper('Protocol')}):" @joiner def render(self, t: GraphQLInterfaceType) -> Iterator[str]: # First: the Protocol class (for type annotations and isinstance checks) yield "" yield self.render_head(t) - yield indent(self.render_body(t)) + yield indent(self.render_body(t) or "...") yield "" # Second: a concrete client class for query builder instantiation client_name = f"_{t.name}Client" yield "" - yield f"class {client_name}(Type):" + yield f"class {client_name}({self.ctx.helper('Type')}):" yield indent(f'"""Concrete client for {t.name} interface."""') yield "" # Override _graphql_name to return the interface name @@ -1110,7 +1176,7 @@ def render(self, t: GraphQLInterfaceType) -> Iterator[str]: yield indent(indent(f'return "{t.name}"')) # Generate method implementations using the Object handler's field rendering - for name, ifield in sorted(t.fields.items()): + for name, ifield in sorted(self.ctx.fields(t).items()): obj_field = _ObjectField(self.ctx, name, ifield, t) yield indent(str(obj_field)) @@ -1121,7 +1187,7 @@ def render_body(self, t: GraphQLInterfaceType) -> Iterator[str]: if t.description: yield from wrap(doc(t.description)) - for name, ifield in sorted(t.fields.items()): + for name, ifield in sorted(self.ctx.fields(t).items()): if name == "id": # id is available on all Type objects continue @@ -1140,26 +1206,24 @@ class Object(ObjectHandler[GraphQLObjectType]): predicate: ClassVar[Predicate] = staticmethod(is_object_type) def supertype_name(self, t: GraphQLObjectType) -> str: - return "Root" if t.name == "Query" else "Type" + return self.ctx.helper("Root" if t.name == "Query" else "Type") def type_name(self, t: GraphQLObjectType) -> str: return super().type_name(t) def fields(self, t: GraphQLObjectType) -> Iterator[_ObjectField]: - return ( - _ObjectField(self.ctx, *args, t) - for args in cast(GraphQLFieldMap, t.fields).items() - ) + return (_ObjectField(self.ctx, *args, t) for args in self.ctx.fields(t).items()) @joiner def render_body(self, t: GraphQLObjectType) -> Iterator[str]: yield super().render_body(t) - if is_self_chainable(t): + if is_self_chainable(t, self.ctx.fields(t).values()): self_name = self.type_name(t) + callable_ = self.ctx.helper("Callable") yield textwrap.dedent( f''' - def with_(self, cb: Callable[["{self_name}"], "{self_name}"]) -> "{self_name}": + def with_(self, cb: {callable_}[["{self_name}"], "{self_name}"]) -> "{self_name}": """Call the provided callable with current {self_name}. This is useful for reusability and readability by not breaking the calling chain. diff --git a/sdk/codegen/src/codegen/packages.py b/sdk/codegen/src/codegen/packages.py new file mode 100644 index 0000000..3301cb3 --- /dev/null +++ b/sdk/codegen/src/codegen/packages.py @@ -0,0 +1,714 @@ +"""One package per client, plus one for core, in `dagger_clients`. + +And, only when asked, the temporary global client, `dagger_global`. +""" + +import json +import pathlib +import textwrap +from collections.abc import Iterator, Sequence +from itertools import groupby + +from graphql import ( + GraphQLField, + GraphQLInterfaceType, + GraphQLObjectType, + GraphQLSchema, + GraphQLType, + Node, + get_named_type, +) +from graphql.type.schema import TypeMap + +from codegen import partition +from codegen.generator import ( + Context, + _ObjectField, + expected_type_name, + format_name, + indent, + is_builtin_scalar_type, + is_input_object_type, + is_interface_type, + is_object_type, + joiner, + legacy_id_names, + legacy_sdk_compat, + new_context, + quote, + render_types, +) + +NAMESPACE = "dagger_clients" +"""Namespace package that every generated package goes into.""" + +GLOBAL = "dagger_global" +"""Package of the temporary global client, next to the SDK files.""" + +Files = dict[str, str] +"""Content of a package, by file name.""" + +SESSION_NAMES = frozenset( + { + "_as_session", + "_loads", + "_ready", + "aclose", + "close", + "connect", + "connection", + "execute", + "forget", + "load", + "session", + } +) +"""What a `dagger.Session` has, which a method of the global client would hide.""" + +_HEADER = """\ +# Code generated by dagger. DO NOT EDIT. + +# Every helper goes by a private alias, so that no generated type shadows it. +import warnings as _warnings # noqa: F401 +from collections.abc import Callable as _Callable +from dataclasses import dataclass as _dataclass +from typing import Protocol as _Protocol +from typing import overload as _overload +from typing import runtime_checkable as _runtime_checkable + +from typing_extensions import Self as _Self + +%s +from dagger.client._core import Arg as _Arg +from dagger.client._guards import type_error as _type_error +from dagger.client.base import Enum as _Enum +from dagger.client.base import Input as _Input +from dagger.client.base import Root as _Root +from dagger.client.base import Scalar as _Scalar +from dagger.client.base import Type as _Type +""" + + +def _aliased(*names: str) -> str: + return "\n".join(f"from dagger.client import {name} as _{name}" for name in names) + + +class _EntryField(_ObjectField): + """Field of `Query` that is the way into a client.""" + + def __init__( + self, + ctx: Context, + name: str, + field: GraphQLField, + parent: GraphQLObjectType | GraphQLInterfaceType, + ) -> None: + super().__init__(ctx, name, field, parent) + for arg in self.args: + if arg.name == "session": + arg.name = "session_" + + @property + def label(self) -> str: + return self.name + + @property + def return_type(self) -> str: + return self.type + + def params(self) -> Iterator[str]: + yield from (a.as_param() for a in self.required_args) + yield "*" + yield from (a.as_param() for a in self.default_args) + yield "session: _Session | None = None" + + @joiner + def func_body(self) -> Iterator[str]: + yield from self.func_prelude() + yield ( + f"return _client_root({self.type}, _TARGET, " + f'"{self.graphql_name}", _args, session=session)' + ) + + def render(self) -> str: + return _block(self.func_signature(), indent(self.func_body())) + + +class _ContributedField(_ObjectField): + """Field that a client contributes to a core type, as a function.""" + + def __init__( + self, + ctx: Context, + name: str, + field: GraphQLField, + parent: GraphQLObjectType | GraphQLInterfaceType, + ) -> None: + super().__init__(ctx, name, field, parent) + self.receiver = format_name(parent.name) + self.receiver_is_interface = is_interface_type(parent) + self.receiver_class = ( + f"_{parent.name}Client" if self.receiver_is_interface else parent.name + ) + if self.receiver in {arg.name for arg in self.args}: + self.receiver += "_" + for arg in self.args: + # A function has no `Self` to name the receiver's type with. + arg.is_self = False + + @property + def label(self) -> str: + return self.name + + @property + def return_type(self) -> str: + return self.type + + @property + def private_name(self) -> str: + """Name of the function when the public one dispatches on the receiver.""" + # The GraphQL type name, not the snake-cased receiver: with its capital + # letter, the name can't spell the alias of a helper, like _client_select. + return f"_{self.parent_name}_{self.name}" + + def params(self) -> Iterator[str]: + # Positional only, because the name changes with the receiver's type + # when the function is overloaded. + yield f"{self.receiver}: {self.parent_name}" + yield "/" + yield from list(super().params())[1:] + + def select_expr(self) -> str: + return f'_client_select({self.receiver}, _TARGET, "{self.graphql_name}", _args)' + + @joiner + def func_body(self) -> Iterator[str]: + if not self.convert_id: + yield super().func_body() + return + # Never straight through the receiver's context, which would run the + # query before the module is loaded: the selection needs the target. + yield from self.func_prelude() + yield f"_ctx = {self.select_expr()}" + yield "_id = await _ctx.execute(_Scalar)" + yield f'return {self.receiver_class}(_ctx.select_id("{self.parent_name}", _id))' + + def render(self, name: str | None = None) -> str: + return _block(self.func_signature(name), indent(self.func_body())) + + +def _block(*lines: str) -> str: + """A top level statement, with the blank lines that set it apart.""" + return "\n".join(("", *lines, "")) + + +def _render_overloaded(name: str, fields: list[_ContributedField]) -> Iterator[str]: + yield from (f.render(f.private_name) for f in fields) + yield from (_block("@_overload", f"{f.func_signature()} ...") for f in fields) + + # By the exact GraphQL type first: an interface is a runtime-checkable + # protocol, so an object that implements it would match structurally and + # take the interface's hook, with the wrong arguments. The structural + # match is only the fallback for an object that has no hook of its own. + def _dispatch(f: _ContributedField, condition: str) -> str: + return indent( + f"if {condition}:\n return {f.private_name}(receiver, *args, **kwargs)" + ) + + expected = " | ".join(f.parent_name for f in fields) + yield _block( + f"def {name}(receiver, /, *args, **kwargs):", + indent( + "_name = receiver._graphql_name() if isinstance(receiver, _Type) else None" + ), + *(_dispatch(f, f'_name == "{f.parent_name}"') for f in fields), + *( + _dispatch(f, f"isinstance(receiver, {f.parent_name})") + for f in fields + if f.receiver_is_interface + ), + indent(f'raise _type_error("{name}", "receiver", receiver, "{expected}")'), + ) + + +def _owned(schema: GraphQLSchema, module: str | None) -> TypeMap: + return { + name: t + for name, t in schema.type_map.items() + if partition.source_module(t.ast_node) == module + } + + +def _all(names: list[str]) -> str: + return _block( + "__all__ = [", + *(indent(f"{quote(name)},") for name in sorted(names)), + "]", + ) + + +@joiner +def _core_init(schema: GraphQLSchema, schema_version: str) -> Iterator[str]: + ctx = new_context(schema, schema_version, partitioned=True) + digest = partition.core_digest(schema, legacy_sdk_compat=ctx.legacy_sdk_compat) + + yield _HEADER % _aliased("Session", "client_root") + yield from render_types(ctx, _owned(schema, None)) + yield _block( + "def core(*, session: _Session | None = None) -> Query:", + indent('"""The way into the core API."""'), + indent("return _client_root(Query, None, None, [], session=session)"), + ) + yield _block( + f"CORE_DIGEST = {quote(digest)}", + '"""Digest of the core schema that this package was generated from."""', + ) + yield _all([*ctx.defined, "CORE_DIGEST", "core"]) + + +def _type_names(schema: GraphQLSchema, module: str | None, *, legacy: bool) -> set[str]: + """Public names that the package of a module, or of core, defines.""" + owned = _owned(schema, module) + names = set(legacy_id_names(owned)) if legacy else set() + names |= { + name + for name, t in owned.items() + if not name.startswith("_") and not is_builtin_scalar_type(t) + } + return names + + +def _core_names(ctx: Context) -> set[str]: + """Every name that a client may import from the core package.""" + core = _owned(ctx.schema, None) + return _type_names(ctx.schema, None, legacy=ctx.legacy_sdk_compat) | { + f"_{name}Client" for name, t in core.items() if is_interface_type(t) + } + + +def _find_module(schema: GraphQLSchema, name: str) -> tuple[str, str]: + """The module of the schema that a client name stands for, and its package.""" + package = partition.package_name(name) + packages = partition.package_names(partition.modules(schema)) + for module, module_package in packages.items(): + if module_package == package: + return module, package + msg = ( + f'the schema attributes nothing to the client "{name}"; ' + f"it has: {', '.join(packages) or 'no module'}" + ) + raise partition.ClientError(msg) + + +def _references(schema: GraphQLSchema, module: str, owned: TypeMap) -> dict[str, str]: + """Types that the code of a client names, and one place that names each.""" + places: list[tuple[str, GraphQLType, Node | None]] = [] + members = [ + (t, name, f) + for t in owned.values() + if is_object_type(t) or is_interface_type(t) + for name, f in t.fields.items() + ] + for parent, name, f in [*partition.contributed_fields(schema, module), *members]: + place = f"{parent.name}.{name}" + places.append((place, f.type, f.ast_node)) + places += [(place, a.type, a.ast_node) for a in f.args.values()] + for t in owned.values(): + if is_input_object_type(t): + places += [ + (f"{t.name}.{name}", i.type, i.ast_node) for name, i in t.fields.items() + ] + + names: dict[str, str] = {} + for place, type_, node in places: + names.setdefault(get_named_type(type_).name, place) + if expected := expected_type_name(schema, node): + names.setdefault(expected, place) + return names + + +def _core_imports(ctx: Context, module: str, references: dict[str, str]) -> list[str]: + """Names to import from core, refusing a name that another client owns.""" + available = _core_names(ctx) + imports = set() + for name, place in references.items(): + t = ctx.schema.get_type(name) + owner = partition.source_module(t.ast_node) if t else None + if owner not in (None, module): + msg = ( + f'"{place}" of the client "{module}" names ' + f'"{name}" of the client "{owner}"' + ) + raise partition.ClientError(msg) + # The legacy ID and the concrete class of an interface go along with + # the type, because the code that uses the type may use them too. + imports |= {name, f"{name}ID", f"_{name}Client"} & available + return sorted(imports) + + +@joiner +def _client_init( + schema: GraphQLSchema, module: str, package: str, schema_version: str +) -> Iterator[str]: + ctx = new_context(schema, schema_version, partitioned=True) + owned = _owned(schema, module) + + entry: _EntryField | None = None + contributed: list[_ContributedField] = [] + for parent, name, field in partition.contributed_fields(schema, module): + # The entry is the `Query` field that the module contributes under its + # own name, because that is the constructor the engine makes for it. + # Any other `Query` field it contributes is a plain function. + if parent.name == "Query" and name.lower() == package.replace("_", ""): + entry = _EntryField(ctx, name, field, parent) + else: + contributed.append(_ContributedField(ctx, name, field, parent)) + + if entry is None or not is_object_type(entry.named_type): + msg = f'the schema has no constructor for the client "{module}"' + raise partition.ClientError(msg) + + references = _references(schema, module, owned) + references |= {f.parent_name: f.label for f in contributed} + + yield _HEADER.rstrip() % _aliased( + "Session", "Target", "check_core", "client_root", "client_select" + ) + yield f"from {NAMESPACE}.{partition.CORE} import CORE_DIGEST as _installed_core" + yield "" + yield "from ._target import CORE_DIGEST, NAME, PIN, REF" + yield "" + # Before any core symbol is imported: a stale core that dropped one would + # otherwise raise a plain ImportError, and the message to regenerate would + # never be seen. + yield "_check_core(NAME, CORE_DIGEST, _installed_core)" + # A client whose types name nothing of core has no block: an empty one + # is a syntax error. + if imports := _core_imports(ctx, module, references): + yield "" + yield f"from {NAMESPACE}.{partition.CORE} import ( # noqa: E402" + yield from (indent(f"{name},") for name in imports) + yield ")" + # Private like the helpers: a client type named TARGET would otherwise + # replace the descriptor, and the class itself would reach client_root. + yield _block( + "_TARGET = _Target(name=NAME, ref=REF, pin=PIN)", + '"""The module that this client loads on first use."""', + ) + yield from _client_body(ctx, owned, entry, contributed) + + +def _client_body( + ctx: Context, + owned: TypeMap, + entry: _EntryField, + contributed: list[_ContributedField], +) -> Iterator[str]: + yield from render_types(ctx, owned) + yield entry.render() + + functions = [entry.name] + by_name = sorted(contributed, key=lambda f: (f.name, f.parent_name)) + for name, group in groupby(by_name, key=lambda f: f.name): + fields = list(group) + functions.append(name) + if len(fields) == 1: + yield fields[0].render() + else: + yield from _render_overloaded(name, fields) + + yield _all([*ctx.defined, *functions]) + + +def _target(name: str, ref: str, pin: str | None, core_digest: str) -> str: + # Plain data, so that reading the descriptor never needs the SDK. + return textwrap.dedent( + f"""\ + # Code generated by dagger. DO NOT EDIT. + + NAME = {json.dumps(name)} + REF = {json.dumps(ref)} + PIN = {json.dumps(pin) if pin else None} + CORE_DIGEST = {json.dumps(core_digest)} + """ + ) + + +def core_package(schema: GraphQLSchema, schema_version: str = "") -> Files: + """Files of `dagger_clients.core`, whatever clients the schema holds.""" + partition.check_attribution(schema) + return { + "__init__.py": _core_init(schema, schema_version), + "py.typed": "", + } + + +def client_package( # noqa: PLR0913 + schema: GraphQLSchema, + name: str, + ref: str, + pin: str | None = None, + core_digest: str | None = None, + schema_version: str = "", +) -> tuple[str, Files]: + """Package name and files of one client, whatever the other clients are.""" + partition.check_attribution(schema) + module, package = _find_module(schema, name) + # The client's schema holds core, so the digest it must match is known. + # A given one that differs would bless a client generated against other + # core types, which is the skew that check_core is there to catch. + expected = partition.core_digest( + schema, legacy_sdk_compat=legacy_sdk_compat(schema_version) + ) + if core_digest is None: + core_digest = expected + elif core_digest != expected: + msg = ( + f'the core digest "{core_digest}" given for the client "{name}" ' + f"is not that of its schema's core, {expected}" + ) + raise partition.ClientError(msg) + return package, { + "__init__.py": _client_init(schema, module, package, schema_version), + "_target.py": _target(name, ref, pin, core_digest), + "py.typed": "", + } + + +_GLOBAL_HEADER = """\ +# Code generated by dagger. DO NOT EDIT. +# +# Temporary. This global client keeps dag.container(), dag.linter() and +# dagger.Container working while a module moves to dagger_clients: one +# method per root field and per client, and every field a client +# contributes put back on its core class at import. It exists only with +# `global-client = true` under [tool.dagger], and the next `dagger generate` +# without the flag removes it. + +from dagger.client import Session as _Session +""" + + +class _GlobalField(_ObjectField): + """A method of the global client: the field, delegated to the new API.""" + + def __init__( # noqa: PLR0913 + self, + ctx: Context, + name: str, + field: GraphQLField, + parent: GraphQLObjectType | GraphQLInterfaceType, + target: str, + *, + owner: str, + receiver: str | None = None, + entry: bool = False, + ) -> None: + super().__init__(ctx, name, field, parent) + if self.name in SESSION_NAMES: + # The method would replace what the session itself runs on, and + # dag would then fail far from the field that caused it. + msg = ( + f'the global client cannot have a method for "{parent.name}.{name}" ' + f"of {owner}: it would hide Session.{self.name}" + ) + raise partition.ClientError(msg) + self.target = target + self.receiver_expr = receiver + self.entry = entry + for arg in self.args: + # A method of the global client has no Self of the root type. + arg.is_self = False + + @property + def return_type(self) -> str: + return self.type + + def params(self) -> Iterator[str]: + yield "self" + yield from (a.as_param() for a in self.required_args) + if self.default_args: + yield "*" + yield from (a.as_param() for a in self.default_args) + + def _keyword(self, name: str) -> str: + # The entry function renamed its `session` argument. The method keeps + # the name the legacy client had, and passes it on under the new one. + return "session_" if self.entry and name == "session" else name + + def render(self) -> str: + args = [self.receiver_expr] if self.receiver_expr else [] + args += [a.name for a in self.required_args] + args += [f"{self._keyword(a.name)}={a.name}" for a in self.default_args] + if self.entry: + args.append("session=self") + call = f"{self.target}({', '.join(args)})" + if self.is_void: + body = f"await {call}" + elif self.is_exec: + body = f"return await {call}" + else: + body = f"return {call}" + return "\n".join((self.func_signature(), indent(body))) + + +def _global_clients( + schemas: Sequence[GraphQLSchema], +) -> dict[str, tuple[str, GraphQLSchema]]: + """Module and schema of each client package, over every given schema.""" + clients: dict[str, tuple[str, GraphQLSchema]] = {} + for schema in schemas: + for module, package in partition.package_names( + partition.modules(schema) + ).items(): + known = clients.setdefault(package, (module, schema)) + if known[0] != module: + msg = ( + f'clients "{known[0]}" and "{module}" ' + f'both become the package "{package}"' + ) + raise partition.ClientNameError(msg) + return clients + + +@joiner +def _global_init( + schemas: Sequence[GraphQLSchema], schema_version: str +) -> Iterator[str]: + core_schema = schemas[0] + ctx = new_context(core_schema, schema_version, partitioned=True) + legacy = ctx.legacy_sdk_compat + clients = _global_clients(schemas) + + core_alias = f"_{partition.CORE}" + core_root = f"{core_alias}.{partition.CORE}(session=self)" + methods: list[_GlobalField] = [] + patches: list[tuple[str, str, str]] = [] + exported: set[str] = _type_names(core_schema, None, legacy=legacy) + imports: list[str] = [] + + query = core_schema.get_type("Query") + if isinstance(query, GraphQLObjectType): + methods += [ + _GlobalField( + ctx, + name, + f, + query, + f"{core_root}.{format_name(name)}", + owner=partition.CORE, + ) + for name, f in partition.own_fields(query).items() + ] + + for package, (module, schema) in sorted(clients.items()): + alias = f"_{package}" + names = sorted(_type_names(schema, module, legacy=legacy)) + exported |= set(names) + if names: + imports.append(f"from {NAMESPACE}.{package} import ({_lines(names)})") + client_ctx = new_context(schema, schema_version, partitioned=True) + owner = f'the client "{module}"' + for parent, name, f in partition.contributed_fields(schema, module): + function = format_name(name) + if parent.name == "Query" and name.lower() == package.replace("_", ""): + methods.append( + _GlobalField( + client_ctx, + name, + f, + parent, + f"{alias}.{function}", + owner=owner, + entry=True, + ) + ) + elif parent.name == "Query": + methods.append( + _GlobalField( + client_ctx, + name, + f, + parent, + f"{alias}.{function}", + owner=owner, + receiver=core_root, + ) + ) + else: + receiver = ( + f"_{parent.name}Client" + if is_interface_type(parent) + else parent.name + ) + patches.append((receiver, function, f"{alias}.{function}")) + + receivers = sorted({r for r, _, _ in patches if r.startswith("_")}) + yield _GLOBAL_HEADER + yield f"import {NAMESPACE}.{partition.CORE} as {core_alias}" + yield from (f"import {NAMESPACE}.{p} as _{p}" for p in sorted(clients)) + yield f"from {NAMESPACE}.{partition.CORE} import * # noqa: F403" + if receivers: + yield f"from {NAMESPACE}.{partition.CORE} import ({_lines(receivers)})" + yield from imports + yield "" + members = [ + '"""The API as methods on the default session, for code not migrated."""', + *(m.render() for m in sorted(methods, key=lambda m: m.name)), + ] + yield _block("class Client(_Session):", indent("\n\n".join(members))) + yield _block("dag = Client()", '"""The global client instance."""') + if patches: + yield "" + yield "# On the core classes at run time only: type checkers do not see" + yield "# these, which points to the migration." + yield from ( + f"{receiver}.{name} = {function} # type: ignore[attr-defined]" + for receiver, name, function in sorted(set(patches)) + ) + yield _all([*exported, "Client", "dag"]) + + +def _lines(names: Sequence[str]) -> str: + return "\n" + "".join(indent(f"{name},") + "\n" for name in names) + + +def global_package(schemas: Sequence[GraphQLSchema], schema_version: str = "") -> Files: + """Files of `dagger_global`, for core and every client the schemas hold. + + Each client is generated from a schema of core and that module, so the + global client, which names them all, takes one schema per client. + """ + if not schemas: + msg = "the global client needs at least one schema" + raise partition.ClientError(msg) + legacy = legacy_sdk_compat(schema_version) + digest = partition.core_digest(schemas[0], legacy_sdk_compat=legacy) + for schema in schemas: + partition.check_attribution(schema) + other = partition.core_digest(schema, legacy_sdk_compat=legacy) + if other != digest: + msg = f"the schemas hold two cores, {digest} and {other}" + raise partition.ClientError(msg) + return { + "__init__.py": _global_init(schemas, schema_version), + "py.typed": "", + } + + +def write_global(output: pathlib.Path, files: Files) -> pathlib.Path: + """Write the global client package, which is no member of the namespace.""" + root = output / GLOBAL + root.mkdir(parents=True, exist_ok=True) + for name, content in files.items(): + (root / name).write_text(content) + return root + + +def write_package(output: pathlib.Path, package: str, files: Files) -> pathlib.Path: + """Write a package under the namespace, which itself gets no `__init__.py`.""" + root = output / NAMESPACE / package + root.mkdir(parents=True, exist_ok=True) + for name, content in files.items(): + (root / name).write_text(content) + return root diff --git a/sdk/codegen/src/codegen/partition.py b/sdk/codegen/src/codegen/partition.py new file mode 100644 index 0000000..82fae56 --- /dev/null +++ b/sdk/codegen/src/codegen/partition.py @@ -0,0 +1,202 @@ +"""Attribution of a schema's types and fields to core or to a client.""" + +import hashlib +from collections.abc import Iterable, Iterator +from keyword import iskeyword + +import graphql +from graphql import ( + GraphQLEnumType, + GraphQLField, + GraphQLInputObjectType, + GraphQLInterfaceType, + GraphQLNamedType, + GraphQLObjectType, + GraphQLSchema, +) + +CORE = "core" +"""Package name of the core bindings, so no client can take it.""" + +_FieldOwner = GraphQLObjectType | GraphQLInterfaceType + + +class ClientError(ValueError): + """A client that can't be generated from the schema.""" + + +class ClientNameError(ClientError): + """A client name that can't become a package in `dagger_clients`.""" + + +def source_module(node: graphql.Node | None) -> str | None: + """Module that `@sourceMap(module:)` attributes a type or a field to.""" + for directive in getattr(node, "directives", None) or (): + if directive.name.value != "sourceMap": + continue + for arg in directive.arguments or (): + if arg.name.value == "module": + return graphql.value_from_ast_untyped(arg.value) or None + return None + + +def own_fields(t: _FieldOwner) -> dict[str, GraphQLField]: + """Fields of a type, without those a client contributes to a core type.""" + if source_module(t.ast_node) is not None: + return t.fields + return {n: f for n, f in t.fields.items() if source_module(f.ast_node) is None} + + +def contributed_fields( + schema: GraphQLSchema, module: str +) -> list[tuple[_FieldOwner, str, GraphQLField]]: + """Fields that a module contributes to core types.""" + return [ + (t, name, f) + for _, t in sorted(schema.type_map.items()) + if isinstance(t, _FieldOwner) and source_module(t.ast_node) is None + for name, f in sorted(t.fields.items()) + if source_module(f.ast_node) == module + ] + + +def check_attribution(schema: GraphQLSchema) -> None: + """Refuse a member attributed to a module that can't get it. + + A client contributes fields to core object and interface types, and + nothing else. Anything else with `@sourceMap` on it would silently land + in the package of its type, so it's an error at generation. + """ + for type_name, t in sorted(schema.type_map.items()): + if type_name.startswith("__"): + continue + owner = source_module(t.ast_node) + if isinstance(t, _FieldOwner | GraphQLInputObjectType): + members = {n: f.ast_node for n, f in t.fields.items()} + elif isinstance(t, GraphQLEnumType): + members = {n: v.ast_node for n, v in t.values.items()} + else: + continue + for name, node in sorted(members.items()): + module = source_module(node) + if module is None or module == owner: + continue + if owner is None and isinstance(t, _FieldOwner): + continue + place = f'"{type_name}.{name}" is attributed to the client "{module}"' + if owner is not None: + msg = ( + f'{place}, but its type "{type_name}" ' + f'is attributed to the client "{owner}"' + ) + else: + msg = ( + f"{place}, but only a field of a core object or interface " + "type can be contributed" + ) + raise ClientError(msg) + + +def modules(schema: GraphQLSchema) -> list[str]: + """Every module the schema attributes something to.""" + + def _attributions() -> Iterator[str | None]: + for t in schema.type_map.values(): + yield source_module(t.ast_node) + if isinstance(t, _FieldOwner): + yield from (source_module(f.ast_node) for f in t.fields.values()) + + return sorted({m for m in _attributions() if m is not None}) + + +def package_name(name: str) -> str: + """Package that a client name becomes in `dagger_clients`.""" + result = name.lower().replace("-", "_").replace(".", "_") + if not result.isidentifier(): + msg = f'client name "{name}" is not a Python identifier: "{result}"' + raise ClientNameError(msg) + if iskeyword(result): + msg = f'client name "{name}" is a Python keyword' + raise ClientNameError(msg) + if result.startswith("_"): + msg = f'client name "{name}" starts with "_"' + raise ClientNameError(msg) + if result == CORE: + msg = f'client name "{name}" is taken by the core bindings' + raise ClientNameError(msg) + return result + + +def package_names(names: Iterable[str]) -> dict[str, str]: + """Package of each client name, refusing two clients with one package.""" + taken: dict[str, str] = {} + for name in names: + package = package_name(name) + if taken.setdefault(package, name) != name: + msg = ( + f'clients "{taken[package]}" and "{name}" ' + f'both become the package "{package}"' + ) + raise ClientNameError(msg) + return {name: package for package, name in taken.items()} + + +def core_digest(schema: GraphQLSchema, *, legacy_sdk_compat: bool = False) -> str: + """Digest of what the schema holds for core, whatever its clients are.""" + # The version itself is not hashed: two versions that generate one core + # must give one digest, and only the compatibility mode changes the code. + digest = hashlib.sha256(b"legacy" if legacy_sdk_compat else b"") + for name, t in sorted(schema.type_map.items()): + if name.startswith("__") or source_module(t.ast_node) is not None: + continue + # A schema lists a built-in scalar only when something uses it, and + # that may be a client. + if graphql.is_specified_scalar_type(t): + continue + for line in _describe(t): + digest.update(line.encode()) + digest.update(b"\n") + return f"sha256:{digest.hexdigest()}" + + +def _describe(t: GraphQLNamedType) -> Iterator[str]: + # Members go by name: the order the engine lists them in is not part of + # the API, and must not make a client look stale. + # + # The printer drops applied directives, and the generated code depends on + # them, for example on `@expectedType`. So they are listed apart. + yield from _directives(t.name, t.ast_node) + + if isinstance(t, _FieldOwner): + fields = dict(sorted(own_fields(t).items())) + copy = type(t)(t.name, fields, t.interfaces, description=t.description) + yield graphql.print_type(copy) + for field_name, field in fields.items(): + path = f"{t.name}.{field_name}" + yield from _directives(path, field.ast_node) + for arg_name, arg in field.args.items(): + yield from _directives(f"{path}.{arg_name}", arg.ast_node) + + elif isinstance(t, GraphQLInputObjectType): + inputs = dict(sorted(t.fields.items())) + yield graphql.print_type( + GraphQLInputObjectType(t.name, inputs, description=t.description) + ) + for input_name, input_field in inputs.items(): + yield from _directives(f"{t.name}.{input_name}", input_field.ast_node) + + elif isinstance(t, GraphQLEnumType): + values = dict(sorted(t.values.items())) + yield graphql.print_type( + GraphQLEnumType(t.name, values, description=t.description) + ) + for value_name, value in values.items(): + yield from _directives(f"{t.name}.{value_name}", value.ast_node) + + else: + yield graphql.print_type(t) + + +def _directives(path: str, node: graphql.Node | None) -> Iterator[str]: + for directive in getattr(node, "directives", None) or (): + yield f"{path} {graphql.print_ast(directive)}" diff --git a/sdk/pyproject.toml b/sdk/pyproject.toml index 760b98f..485fdf8 100644 --- a/sdk/pyproject.toml +++ b/sdk/pyproject.toml @@ -95,3 +95,8 @@ pretty = true show_column_numbers = true warn_no_return = false warn_unused_ignores = true + +[[tool.mypy.overrides]] +# The temporary global client, generated into a scope, not into this tree. +module = ["dagger_global"] +ignore_missing_imports = true diff --git a/sdk/src/dagger/__init__.py b/sdk/src/dagger/__init__.py index c82edfe..693be3a 100644 --- a/sdk/src/dagger/__init__.py +++ b/sdk/src/dagger/__init__.py @@ -1,4 +1,10 @@ import contextlib +import functools as _functools +import importlib as _importlib +import importlib.util as _importlib_util +import types as _types +import typing as _typing +import warnings as _warnings # Make sure to place exceptions first as they're dependencies of other imports. from dagger._exceptions import * @@ -7,24 +13,95 @@ with contextlib.suppress(ModuleNotFoundError): from dagger.provisioning import * -# Client bindings -try: - # Custom extended API bindings can be placed in user's src/dagger_gen.py - from dagger_gen import * -except ModuleNotFoundError: - # Only core API bindings - from dagger.client.gen import * - # Client connection +from dagger.client import Session as Session +from dagger.client import _session as _sessions from dagger.client._config import Retry as Retry from dagger.client._config import Timeout as Timeout from dagger.client._connection import connect as connect from dagger.client._connection import close as close +# The temporary global client is the only generated code named here: it +# keeps dag.container() and dagger.Container working while a module migrates. +if _typing.TYPE_CHECKING: + try: + from dagger_global import * + except ModuleNotFoundError: + # With the global client, a type checker sees dag as its Client. + dag = _sessions.default_session() # type: ignore[assignment, unused-ignore] + + +@_functools.cache +def _global_client() -> _types.ModuleType | None: + # On first use, never on import: it imports core and every client, and + # each of them imports dagger first, so importing it here would fail + # whenever a generated package is the first import of the process. + try: + return _importlib.import_module("dagger_global") + except ModuleNotFoundError as e: + # Only its absence means no flag: a global client that cannot import + # one of its clients is broken, not off. + if e.name != "dagger_global": + raise + return None + + +def _global_dag() -> Session | None: + global_ = _global_client() + return None if global_ is None else global_.dag + + +_sessions.set_default_finder(_global_dag) + +# An earlier version star-imported its one-file bindings from here. Found +# only, never loaded: without this, dag just loses its API with no word why. +if (_legacy := _importlib_util.find_spec("dagger_gen")) is not None: + _warnings.warn( + f"{_legacy.origin} is no longer loaded. If an earlier SDK generated " + "it, `dagger generate` removes it; if you wrote it, delete or rename it.", + stacklevel=2, + ) +del _legacy + # Module support (only makes sense in a module runtime container) with contextlib.suppress(ModuleNotFoundError): from dagger.mod import * + +def _lazy_names() -> list[str]: + global_ = _global_client() + return ["dag", *(() if global_ is None else global_.__all__)] + + +def __dir__() -> list[str]: + return sorted({*globals(), *_lazy_names()}) + + +def __getattr__(name: str) -> _typing.Any: + """Names of the global client, or where a name of the legacy bindings went.""" + if name == "dag": + return _sessions.default_session() + if name == "__all__": + # What a star import took when these names were globals. + public = (n for n in globals() if not n.startswith("_")) + return sorted({*public, *_lazy_names()}) + global_ = None if name.startswith("_") else _global_client() + if global_ is not None and name in global_.__all__: + return getattr(global_, name) + msg = f"module {__name__!r} has no attribute {name!r}" + if name == "Client": + msg += ( + ". dagger.Connection now yields a dagger.Session, " + "and the API is on core() from dagger_clients.core." + ) + elif name[:1].isupper(): + msg += ( + f". Core types moved to dagger_clients.core: " + f"from dagger_clients.core import {name}" + ) + raise AttributeError(msg) + + # Re-export imports so they look like they live directly in this package. for _value in list(locals().values()): if getattr(_value, "__module__", "").startswith("dagger."): diff --git a/sdk/src/dagger/_engine/_version.py b/sdk/src/dagger/_engine/_version.py index 9187ccd..e2e4b85 100644 --- a/sdk/src/dagger/_engine/_version.py +++ b/sdk/src/dagger/_engine/_version.py @@ -1,3 +1,3 @@ # Code generated by dagger. DO NOT EDIT. -CLI_VERSION = "1.0.0-beta.10" +CLI_VERSION = "1.0.0-beta.14" diff --git a/sdk/src/dagger/_exceptions.py b/sdk/src/dagger/_exceptions.py index f70532d..22b577b 100644 --- a/sdk/src/dagger/_exceptions.py +++ b/sdk/src/dagger/_exceptions.py @@ -1,8 +1,11 @@ import dataclasses -from typing import Any +from typing import TYPE_CHECKING, Any import cattrs +if TYPE_CHECKING: + from dagger.client._descriptor import Target + class VersionMismatch(Warning): """Dagger CLI version doesn't match required version.""" @@ -34,6 +37,21 @@ class InvalidQueryError(ClientError): """Misuse of the query builder.""" +class ClientLoadError(ClientError): + """The module a generated client targets could not be loaded.""" + + def __init__(self, *args, target: "Target | None" = None): + super().__init__(*args) + self.target = target + + +class StaleClientError(ClientError, ImportError): + """A generated client no longer matches its core or its module. + + An ImportError, because the check runs when the client is imported. + """ + + @dataclasses.dataclass(slots=True) class QueryErrorLocation: """Error location returned by the API.""" @@ -164,10 +182,12 @@ def __str__(self): __all__ = [ "ClientConnectionError", "ClientError", + "ClientLoadError", "DaggerError", "ExecError", "InvalidQueryError", "QueryError", + "StaleClientError", "TransportError", "VersionMismatch", ] diff --git a/sdk/src/dagger/client/__init__.py b/sdk/src/dagger/client/__init__.py index e69de29..f9aee7f 100644 --- a/sdk/src/dagger/client/__init__.py +++ b/sdk/src/dagger/client/__init__.py @@ -0,0 +1,17 @@ +from dagger.client._clients import client_root as client_root +from dagger.client._clients import client_select as client_select +from dagger.client._session import Session as Session +from dagger.client._session import default_session as default_session +from dagger.client._descriptor import Target as Target +from dagger.client._descriptor import check_core as check_core +from dagger.client._descriptor import registering_types as registering_types + +__all__ = [ + "Session", + "Target", + "check_core", + "client_root", + "client_select", + "default_session", + "registering_types", +] diff --git a/sdk/src/dagger/client/_clients.py b/sdk/src/dagger/client/_clients.py new file mode 100644 index 0000000..5bb7b91 --- /dev/null +++ b/sdk/src/dagger/client/_clients.py @@ -0,0 +1,44 @@ +"""The calls generated client code makes into the SDK.""" + +import dataclasses +from typing import TypeVar + +from dagger.client._core import Arg, Context +from dagger.client._descriptor import Target +from dagger.client._session import Session, default_session +from dagger.client.base import Type + +T = TypeVar("T", bound=Type) + + +def client_root( + cls: type[T], + target: Target | None, + field: str | None, + args: list[Arg], + *, + session: Session | None = None, +) -> T: + """The root of a client: what to load, and the entry field if any. + + Core has neither, so ``core()`` passes ``None`` for both. + """ + ctx = Context( + session or default_session(), + targets=frozenset([target]) if target else frozenset(), + ) + if field is not None: + ctx = ctx.root_select(field, args) + return cls(ctx) + + +def client_select( + receiver: Type, + target: Target, + field: str, + args: list[Arg], +) -> Context: + """Select a field a client adds to the receiver's type.""" + ctx = receiver._ctx # noqa: SLF001 + ctx = dataclasses.replace(ctx, targets=ctx.targets | {target}) + return ctx.select(receiver._graphql_name(), field, args) # noqa: SLF001 diff --git a/sdk/src/dagger/client/_connection.py b/sdk/src/dagger/client/_connection.py index 6640389..d7bb6b3 100644 --- a/sdk/src/dagger/client/_connection.py +++ b/sdk/src/dagger/client/_connection.py @@ -1,5 +1,11 @@ -from dagger.client._session import SharedConnection +from dagger.client._session import Session, default_session -_shared = SharedConnection() -connect = _shared.connect -close = _shared.close + +async def connect() -> Session: + """Open the default session's connection.""" + return await default_session().connect() + + +async def close() -> None: + """Close the default session's connection.""" + await default_session().close() diff --git a/sdk/src/dagger/client/_core.py b/sdk/src/dagger/client/_core.py index 390d383..20eb055 100644 --- a/sdk/src/dagger/client/_core.py +++ b/sdk/src/dagger/client/_core.py @@ -21,8 +21,14 @@ from cattrs.preconf.json import make_converter as make_json_converter from typing_extensions import TypeForm -from dagger import DaggerError, InvalidQueryError -from dagger.client._session import BaseConnection, SharedConnection +from dagger._exceptions import DaggerError, InvalidQueryError, QueryError +from dagger.client._descriptor import Target, stale_client_error +from dagger.client._session import ( + BaseConnection, + Session, + as_session, + default_session, +) from dagger.client.base import Input, Scalar, Type from ._guards import ( @@ -39,6 +45,7 @@ INDENT = " " _SNAKE_TO_CAMEL_RE = re.compile(r"(_)([a-z\d])") +_ENUM_NAME_RE = re.compile(r"[_A-Za-z][_0-9A-Za-z]*") def snake_to_camel(s: str, upper: bool = True) -> str: @@ -53,6 +60,12 @@ def snake_to_camel(s: str, upper: bool = True) -> str: return s +class EnumName(str): + """A schema enum value, for callers that have no generated enum to send.""" + + __slots__ = () + + class Arg(typing.NamedTuple): name: str # GraphQL name value: Any @@ -115,8 +128,8 @@ def _scalar_literal(value: Any) -> str: if isinstance(value, bool): return "true" if value else "false" # Before str: an enum may subclass str, and goes by name. - if isinstance(value, enum.Enum): - return value.name + if isinstance(value, (enum.Enum, EnumName)): + return _enum_literal(value) if isinstance(value, str): # GraphQL string escapes are a subset of JSON's. return json.dumps(value) @@ -128,6 +141,16 @@ def _scalar_literal(value: Any) -> str: raise InvalidQueryError(msg) +def _enum_literal(value: enum.Enum | EnumName) -> str: + if isinstance(value, enum.Enum): + return value.name + # Rendered bare, so anything but a name would inject query syntax. + if not _ENUM_NAME_RE.fullmatch(value): + msg = f"Invalid enum value name: {str(value)!r}" + raise InvalidQueryError(msg) + return str(value) + + def _input_literal(obj: Input) -> str: # The generator records only the GraphQL names snake_to_camel can't derive. names = dict(getattr(type(obj), "_graphql_names", ())) @@ -161,7 +184,7 @@ def _snapshot(value: Any) -> Any: @dataclasses.dataclass(slots=True) class Context: conn: BaseConnection = dataclasses.field( - default_factory=SharedConnection, + default_factory=default_session, compare=False, ) selections: collections.deque[Field] = dataclasses.field( @@ -171,6 +194,9 @@ class Context: init=False, compare=False, ) + # On the context because it is what survives chained selections and + # ID resolution, so it is the only thing that always reaches execute. + targets: frozenset[Target] = frozenset() def __post_init__(self): self.converter = make_converter(self) @@ -244,10 +270,22 @@ async def execute(self, return_type: TypeForm[T] | type[T]) -> T: ... async def execute( self, return_type: TypeForm[T] | type[T] | None = None ) -> T | None: + session = as_session(self.conn) + await self.load_targets(session) await self.resolve_ids() - result = await self.conn.session.execute(self.build()) + try: + result = await session.execute(self.build()) + except QueryError as e: + if self.targets and (stale := stale_client_error(e, self.targets)): + raise stale from e + raise return self.get_value(result, return_type) if return_type else None + async def load_targets(self, session: Session) -> None: + """Serve every module the query needs.""" + for target in sorted(self.targets, key=lambda t: t.name): + await session.load(target) + async def execute_object_list( self, element_type: type[Obj_T], diff --git a/sdk/src/dagger/client/_descriptor.py b/sdk/src/dagger/client/_descriptor.py new file mode 100644 index 0000000..52da6ff --- /dev/null +++ b/sdk/src/dagger/client/_descriptor.py @@ -0,0 +1,108 @@ +"""What a generated client hands the SDK: its target and its core digest.""" + +import contextlib +import dataclasses +import logging +import re +import threading +from collections.abc import Iterable, Iterator + +from dagger._exceptions import QueryError, StaleClientError + +logger = logging.getLogger(__name__) + +GENERATE_HINT = "Run `dagger generate`." + +# What the engine's validator answers when the schema lacks a field the +# bindings have. Anchored, because a resolver may quote the same words. +MISSING_FIELD = re.compile( + r'^Cannot query field "([^"]*)" on type "([^"]*)"', re.IGNORECASE +) + + +@dataclasses.dataclass(frozen=True, slots=True) +class Target: + """The module a client was generated for.""" + + name: str + ref: str + pin: str | None = None + + +# Registration is a phase of the process, not of one task: a context +# variable would follow a child task out of the window and never reach a +# thread. The cost is that a concurrent session in this process, while a +# module registers, also sees a genuine mismatch downgraded to a warning. +_registering = 0 +_registering_lock = threading.Lock() + + +@contextlib.contextmanager +def registering_types() -> Iterator[None]: + """Mark the window in which the SDK registers a module's types. + + A stale client only warns here: a module with a client to itself has + to run before that client can be regenerated. + """ + global _registering # noqa: PLW0603 + with _registering_lock: + _registering += 1 + try: + yield + finally: + with _registering_lock: + _registering -= 1 + + +def check_core(client: str, expected: str, installed: str) -> None: + """Refuse a client generated against another core.""" + if expected == installed: + return + msg = ( + f"Client {client!r} was generated for core {expected}, " + f"but the installed core is {installed}. {GENERATE_HINT}" + ) + if _registering: + logger.warning(msg) + return + raise StaleClientError(msg) + + +def missing_field(error: QueryError) -> "re.Match[str] | None": + """The validation error for a field the schema lacks, with its two names.""" + # Validation fails before anything runs, so it carries no path, and says + # it is validation in its code; any other error comes from something + # that ran, whatever its message says. + for e in error.errors: + if ( + e.path is None + and e.extensions.get("code") == "GRAPHQL_VALIDATION_FAILED" + and (match := MISSING_FIELD.match(e.message)) + ): + return match + return None + + +def stale_client_error( + error: QueryError, targets: Iterable[Target] +) -> StaleClientError | None: + """The error a missing field means once the module was loaded.""" + missing = missing_field(error) + if missing is None: + return None + # The name and the address both: a module served under another name + # than the one generated against lands here too, and the address is + # what the user has to look at. + clients = [_describe(t) for t in sorted(targets, key=lambda t: t.name)] + which = ( + f"The client {clients[0]} is" + if len(clients) == 1 + else f"The clients {', '.join(clients)} are" + ) + msg = f"{missing.string} {which} out of date. {GENERATE_HINT}" + return StaleClientError(msg) + + +def _describe(target: Target) -> str: + where = f"{target.ref} at {target.pin}" if target.pin else target.ref + return f"{target.name!r} from {where}" diff --git a/sdk/src/dagger/client/_guards.py b/sdk/src/dagger/client/_guards.py index 674599e..cc81f14 100644 --- a/sdk/src/dagger/client/_guards.py +++ b/sdk/src/dagger/client/_guards.py @@ -1,3 +1,4 @@ +import sys import typing from collections.abc import Sequence from typing import TypeGuard @@ -40,7 +41,11 @@ def type_error(method: str, param: str, value: object, expected: str) -> TypeErr shown = repr(value) if len(shown) > 60: # noqa: PLR2004 shown = shown[:57] + "..." + # Bindings can be generated into any package, so the caller says where + # it lives. Generated code only passes the class and method. + module = sys._getframe(1).f_globals.get("__name__", "") # noqa: SLF001 + qualname = f"{module}.{method}" if module else method return TypeError( - f"Method dagger.client.gen.{method}() parameter {param}={shown} " + f"Method {qualname}() parameter {param}={shown} " f"expected to be of type {expected}." ) diff --git a/sdk/src/dagger/client/_load.py b/sdk/src/dagger/client/_load.py new file mode 100644 index 0000000..86aa9ba --- /dev/null +++ b/sdk/src/dagger/client/_load.py @@ -0,0 +1,138 @@ +"""How the module a target names gets served into a session.""" + +import json +from collections.abc import Mapping +from typing import Any + +from dagger._exceptions import ClientLoadError +from dagger.client._core import Arg, Context +from dagger.client._descriptor import GENERATE_HINT, Target +from dagger.client._session import Session + +# A descriptor has one shape for a git ref and for a workspace path: `ref` is +# the address, `pin` the refPin. There is no name to pin: the engine derives +# it from the module's own config, as it did for the schema this client was +# generated from, so the two agree unless the module renamed itself since, +# and then the client is stale and its first selection says so (see +# stale_client_error). +# +# Two queries serve a target, and one condition picks between them: +# +# a local target, in a process a module entrypoint runs +# node(id: ) { +# ... on ModuleSource { asModule { serve } } } +# +# any other target +# serveModule(address: , refPin: ) +# +# serveModule resolves a git address itself, and a path in the caller's +# current workspace. That is the right workspace for a plain program. Under +# a Dang entrypoint it is not: the module's code runs in an exec the +# entrypoint starts, the engine gives that exec no module context, and the +# process is a plain nested client whose current workspace is the one the +# engine finds in its own container. The entrypoint resolves the clients the +# caller declared on the module's scope and hands each over by name, as a +# source over that client's own files (entrypoint/handover.dang). The +# workspace never reaches this process: in Dagger an ID is a capability, and +# this is third-party code. A git address depends on no workspace, so it +# keeps serveModule either way. + + +class _Handover: + """What a module entrypoint handed this process with its call.""" + + def __init__(self, clients: Mapping[str, str] | None): + # None when the entrypoint sent nothing: one from before the + # handover, or not one this SDK wrote. + self.clients = clients + + def source(self, target: Target) -> str: + if self.clients is None: + msg = ( + f"The module's entrypoint handed over no clients, so the local " + f"client {target.name!r} cannot load: the entrypoint is not the " + f"one this SDK writes, or predates it. {GENERATE_HINT}" + ) + raise ClientLoadError(msg, target=target) + if (source := self.clients.get(target.name)) is None: + msg = ( + f"The local client {target.name!r} is not declared on this " + f"module's scope in the caller's workspace, so the module's " + f"entrypoint did not hand it over. Declare it with `dagger " + f"module client add`. {GENERATE_HINT}" + ) + raise ClientLoadError(msg, target=target) + return source + + +# One per process: the entrypoint runs one call per process. +_handover: _Handover | None = None + + +def use_entrypoint_clients(clients: Mapping[str, str] | None) -> None: + """Resolve local targets through what a module entrypoint handed over. + + Only ``python -m dagger.mod call`` sets it, from the request, and always: + a process that an entrypoint runs never falls back to serveModule for a + local target, which would resolve it in the process's own container. + None means the entrypoint sent no clients at all. + """ + global _handover # noqa: PLW0603 + _handover = _Handover(None if clients is None else dict(clients)) + + +def leave_entrypoint() -> None: + """Forget the handover, as in a process no entrypoint runs.""" + global _handover # noqa: PLW0603 + _handover = None + + +def parse_handed_clients(value: Any) -> dict[str, str] | None: + """The clients of a call request: name to module source ID. + + The entrypoint sends a list of ``{"name", "source"}``, as JSON or as its + encoding; a missing field is None, an entrypoint that sends none. + """ + if value is None: + return None + if isinstance(value, str): + value = json.loads(value) + if not isinstance(value, list): + msg = f"expected a list of handed clients, got {type(value).__name__}" + raise TypeError(msg) + clients: dict[str, str] = {} + for entry in value: + name, source = entry["name"], entry["source"] + if not isinstance(name, str) or not isinstance(source, str): + msg = f"a handed client needs a string name and source: {entry!r}" + raise TypeError(msg) + clients[name] = source + return clients + + +async def load_target(session: Session, target: Target) -> None: + """Serve the module a target names.""" + ctx = Context(session) + if _is_local(target) and _handover is not None: + await _serve_handed(ctx, _handover.source(target)) + return + # An engine without the field, below this SDK's floor, fails the load + # here: a ClientLoadError, not a stale client, since regenerating the + # client cannot give the engine a field. + args = [Arg("address", target.ref), Arg("refPin", target.pin, None)] + await ctx.root_select("serveModule", args).execute() + + +def _is_local(target: Target) -> bool: + # The engine's own rule for a workspace path: an explicit one. A bare name + # is refused there, and never written into a descriptor. + return target.ref.startswith((".", "/")) + + +async def _serve_handed(ctx: Context, source_id: str) -> None: + await ( + ctx.select_id("ModuleSource", source_id) + .select("ModuleSource", "asModule", []) + .select("Module", "serve", []) + .execute() + ) diff --git a/sdk/src/dagger/client/_session.py b/sdk/src/dagger/client/_session.py index 4492eb5..834ed74 100644 --- a/sdk/src/dagger/client/_session.py +++ b/sdk/src/dagger/client/_session.py @@ -1,18 +1,30 @@ +import atexit import logging import os +import threading +from collections.abc import Callable from dataclasses import dataclass, field -from typing import Any +from typing import TYPE_CHECKING, Any import anyio import httpx from opentelemetry import propagate from typing_extensions import Self -from dagger import ClientConnectionError, TransportError, telemetry -from dagger._exceptions import _query_error_from_response +from dagger import telemetry +from dagger._exceptions import ( + ClientConnectionError, + ClientLoadError, + DaggerError, + TransportError, + _query_error_from_response, +) from dagger._managers import ResourceManager from dagger.client._config import ConnectConfig +if TYPE_CHECKING: + from dagger.client._descriptor import Target + logger = logging.getLogger(__name__) # Safe to retry: every API call is cached on its inputs. @@ -176,12 +188,22 @@ def _unexpected(response: httpx.Response) -> str: class BaseConnection: session: ClientSession + # Kept on the connection so the session dies with it, and so every + # execution over one connection shares one load memo and name guard. + _as_session: "Session | None" = None + + async def _ready(self) -> ClientSession: + """The session, once there is an engine behind it.""" + return self.session async def connect(self) -> Self: - await self.session.start() + await (await self._ready()).start() return self async def close(self) -> None: + # The transport forgets its served modules; so must the session. + if self._as_session is not None: + self._as_session.forget() await self.session.close() async def aclose(self) -> None: @@ -215,6 +237,9 @@ class SharedConnection(BaseConnection): _session: ClientSession | None = None _params: ConnectParams | None = None _cfg: ConnectConfig + # Set while this process runs an engine it provisioned itself. + _stop_engine: Callable[[], None] | None = None + _provisioning: anyio.Lock | None = None def __new__(cls): if not cls._instance: @@ -262,6 +287,55 @@ def session(self) -> ClientSession: self._session = ClientSession(self._params, self._cfg) return self._session + async def _ready(self) -> ClientSession: + """The session, provisioning an engine if nothing gave one. + + A module and `dagger run` put a session in the environment, and + `dagger.connection()` passes its own, so both come first: a module + must never try to provision. Only a plain program with no connection + handling gets here with neither. + """ + if not self._session and not self._params: + self._params = ConnectParams.from_env() + if not self._params: + await self._provision() + return self.session + + async def _provision(self) -> None: + if self._provisioning is None: + self._provisioning = anyio.Lock() + async with self._provisioning: + if self._params: + return + if _module_runtime: + # The engine gives a module its session; one that lacks it + # must say so, not download a CLI in the module's container. + msg = "No active engine session to connect to" + raise ClientConnectionError(msg) + try: + # Not at import: an older module runtime has no provisioning. + from dagger.provisioning._config import Config + from dagger.provisioning._engine import provision_default_session + except ModuleNotFoundError as e: + if not (e.name or "").startswith("dagger.provisioning"): + raise + msg = "No active engine session to connect to" + raise ClientConnectionError(msg) from e + + cfg = Config(timeout=self._cfg.timeout, retry=self._cfg.retry) + self._params, self._stop_engine = await provision_default_session(cfg) + # The engine is a subprocess, and closing it is sync, so it can + # run at exit, after the program's event loop is gone. + atexit.register(self.stop_engine) + + def stop_engine(self) -> None: + """End the engine this process provisioned, if it did.""" + stop, self._stop_engine = self._stop_engine, None + if stop is not None: + atexit.unregister(self.stop_engine) + self._params = None + stop() + def is_connected(self) -> bool: return self._session is not None and self._session.has_session() @@ -269,3 +343,151 @@ async def close(self) -> None: if self._session: await super().close() self._session = None + if self._stop_engine is not None: + await anyio.to_thread.run_sync(self.stop_engine) + + +@dataclass(slots=True) +class _Load: + target: "Target" + lock: anyio.Lock = field(default_factory=anyio.Lock) + done: bool = False + + +class Session(BaseConnection): + """A connection to one engine, and what has been loaded into it. + + Owns the connection, the query transport and the load memo. It has no + API field: a client is the way in. + """ + + def __init__(self, connection: BaseConnection | None = None) -> None: + # No connection is the shared one, which a module and `dagger run` + # set up: the session the generated global client builds is over it. + self.connection = SharedConnection() if connection is None else connection + self._loads: dict[str, _Load] = {} + + @property + def session(self) -> ClientSession: # type: ignore[override] + return self.connection.session + + async def _ready(self) -> ClientSession: + return await self.connection._ready() # noqa: SLF001 + + async def connect(self) -> Self: + await self.connection.connect() + return self + + async def close(self) -> None: + self.forget() + await self.connection.close() + + def forget(self) -> None: + """Drop the load memo: a new engine behind the connection has nothing.""" + self._loads.clear() + + async def execute(self, query: str) -> Any: + return await (await self._ready()).execute(query) + + def __getattr__(self, name: str) -> Any: + # Only reached for a name the session lacks, and the likely ask is + # an API field of the global client that dag used to be. Telling a + # core field from a client would take importing core, so the message + # names both. + msg = f"{type(self).__name__!r} object has no attribute {name!r}" + if not name.startswith("_"): + msg += ( + f". The API is on the clients now: core().{name}() for a core " + "field (from dagger_clients.core import core), " + f"or {name}() from the client's package for a client. " + f"To keep dag.{name}() while migrating, set " + "global-client = true under [tool.dagger] and run dagger generate." + ) + raise AttributeError(msg, name=name, obj=self) + + async def load(self, target: "Target") -> None: + """Serve the module a target names, once per session.""" + entry = self._loads.setdefault(target.name, _Load(target)) + if entry.target != target: + # Whole descriptors: the two may differ only by pin, and the + # first is held from its first attempt, loaded or not. + msg = ( + f"This session already holds {entry.target!r} " + f"and cannot also take {target!r}" + ) + raise ClientLoadError(msg, target=target) + async with entry.lock: + if entry.done: + return + # The loader builds its query with Context, which imports this + # module, so it can only be reached from inside a function. + from dagger.client._load import load_target + + try: + await load_target(self, target) + except DaggerError as e: + msg = f"Failed to load client {target.name!r} from {target.ref!r}: {e}" + raise ClientLoadError(msg, target=target) from e + entry.done = True + + +_module_runtime = False + + +def mark_module_runtime() -> None: + """Say this process serves a module, so it never provisions an engine. + + The module entrypoints call it first. A module's session comes from the + engine that runs it; without this, only that session being there would + keep the default session from provisioning one. + """ + global _module_runtime # noqa: PLW0603 + _module_runtime = True + + +_default: Session | None = None +# Sessions are looked up from threads too, before any event loop exists. +_sessions_lock = threading.Lock() + + +def _no_default() -> Session | None: + return None + + +_find_default: Callable[[], Session | None] = _no_default + + +def set_default_finder(find: Callable[[], Session | None]) -> None: + """Say where to find a default session before making one. + + Only the temporary global client needs this: its ``dag`` is a Session, + and it has to be the default one, so that a client called without + ``session=`` and ``dagger.connection()`` share its loads. + """ + global _find_default # noqa: PLW0603 + _find_default = find + + +def default_session() -> Session: + """The one session per process, over the shared connection.""" + global _default # noqa: PLW0603 + if _default is None: + # Outside the lock: finding it can import generated code, which must + # not run while other threads wait on the lock. + found = _find_default() + with _sessions_lock: + if _default is None: + _default = Session() if found is None else found + return _default + + +def as_session(conn: BaseConnection) -> Session: + """The session a connection belongs to.""" + if isinstance(conn, Session): + return conn + if isinstance(conn, SharedConnection): + return default_session() + with _sessions_lock: + if conn._as_session is None: # noqa: SLF001 + conn._as_session = Session(conn) # noqa: SLF001 + return conn._as_session # noqa: SLF001 diff --git a/sdk/src/dagger/client/base.py b/sdk/src/dagger/client/base.py index 2c6daaa..51aa662 100644 --- a/sdk/src/dagger/client/base.py +++ b/sdk/src/dagger/client/base.py @@ -7,7 +7,6 @@ if typing.TYPE_CHECKING: from dagger.client._core import Context - from dagger.client._session import BaseConnection class Scalar(str): @@ -91,13 +90,6 @@ def __init__(self, ctx: Context | None = None): super().__init__(ctx) - @classmethod - def from_connection(cls, conn: BaseConnection): - """Create a new instance of the root type, using the given connection.""" - from ._core import Context - - return cls(Context(conn)) - @classmethod def _graphql_name(cls) -> str: return "Query" diff --git a/sdk/src/dagger/mod/__main__.py b/sdk/src/dagger/mod/__main__.py index e8edf15..2de1938 100644 --- a/sdk/src/dagger/mod/__main__.py +++ b/sdk/src/dagger/mod/__main__.py @@ -9,8 +9,9 @@ import anyio -import dagger from dagger import telemetry +from dagger._exceptions import QueryError +from dagger.client._session import mark_module_runtime from dagger.mod._exceptions import ModuleError logger = logging.getLogger(__package__) @@ -18,6 +19,7 @@ def main(argv: list[str] | None = None) -> int: """Run one command and return the exit status.""" + mark_module_runtime() parser = argparse.ArgumentParser(prog="python -m dagger.mod") commands = parser.add_subparsers(required=True) @@ -26,9 +28,6 @@ def main(argv: list[str] | None = None) -> int: help="render the static entrypoint of the module in the current directory", ) entrypoint.add_argument("--name", required=True, help="module name") - entrypoint.add_argument( - "--path", required=True, help="module directory, relative to the workspace" - ) entrypoint.add_argument("--output", required=True, type=pathlib.Path) entrypoint.set_defaults(run=_entrypoint) @@ -49,7 +48,7 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) try: args.run(args) - except (ModuleError, dagger.QueryError) as e: + except (ModuleError, QueryError) as e: logger.error(str(e)) # noqa: TRY400 - the message is the whole story return 2 except Exception: @@ -59,24 +58,29 @@ def main(argv: list[str] | None = None) -> int: def _entrypoint(args: argparse.Namespace) -> None: + from dagger.client._descriptor import registering_types from dagger.mod._entrypoint import write_entrypoint from dagger.mod.cli import load_module + with registering_types(): + mod = load_module() write_entrypoint( - load_module().describe(), + mod.describe(), name=args.name, - path=args.path, root=pathlib.Path.cwd(), output=args.output, ) def _describe(args: argparse.Namespace) -> None: + from dagger.client._descriptor import registering_types from dagger.mod._describe import describe_json from dagger.mod.cli import load_module + with registering_types(): + mod = load_module() args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(describe_json(load_module().describe())) + args.output.write_text(describe_json(mod.describe())) def _call(args: argparse.Namespace) -> None: @@ -91,8 +95,19 @@ def _call(args: argparse.Namespace) -> None: async def _dispatch(request: dict[str, Any]) -> Any: + from dagger.client._load import parse_handed_clients, use_entrypoint_clients + from dagger.mod._exceptions import InvalidInputError from dagger.mod.cli import load_module + # The module's declared local clients, which the entrypoint resolved and + # this process cannot. Before the user's code is imported, so nothing it + # starts can load without them. + try: + clients = parse_handed_clients(request.get("clients")) + except (TypeError, KeyError, ValueError) as e: + msg = f"Failed to read the clients the entrypoint handed over: {e}" + raise InvalidInputError(msg) from e + use_entrypoint_clients(clients) return await load_module().dispatch(request) diff --git a/sdk/src/dagger/mod/_api.py b/sdk/src/dagger/mod/_api.py new file mode 100644 index 0000000..152763a --- /dev/null +++ b/sdk/src/dagger/mod/_api.py @@ -0,0 +1,237 @@ +"""The engine API that module support calls, on the raw query builder. + +The SDK files can't import generated bindings, so the few fields needed to +register types and to answer a function call are selected by name here. +Arguments and their defaults follow the generated bindings, so the queries +are the same. +""" + +from __future__ import annotations + +import dataclasses +from collections.abc import Sequence + +from typing_extensions import Self + +from dagger.client._core import Arg, Context, EnumName +from dagger.client.base import Type + + +class _Object(Type): + async def id(self) -> str: + return await self._select("id", []).execute(str) + + def _with(self, field: str, *args: Arg) -> Self: + return type(self)(self._select(field, args)) + + +class TypeDef(_Object): + def with_optional(self, optional: bool) -> Self: + return self._with("withOptional", Arg("optional", optional)) + + def with_kind(self, kind: str) -> Self: + return self._with("withKind", Arg("kind", EnumName(kind))) + + def with_list_of(self, element_type: TypeDef) -> Self: + return self._with("withListOf", Arg("elementType", element_type)) + + def with_scalar(self, name: str, *, description: str | None = "") -> Self: + return self._with( + "withScalar", + Arg("name", name), + Arg("description", description, ""), + ) + + def with_enum(self, name: str, *, description: str | None = "") -> Self: + return self._with( + "withEnum", + Arg("name", name), + Arg("description", description, ""), + ) + + def with_enum_member( + self, + name: str, + *, + value: str | None = "", + description: str | None = "", + deprecated: str | None = None, + ) -> Self: + return self._with( + "withEnumMember", + Arg("name", name), + Arg("value", value, ""), + Arg("description", description, ""), + Arg("deprecated", deprecated, None), + ) + + def with_interface(self, name: str, *, description: str | None = "") -> Self: + return self._with( + "withInterface", + Arg("name", name), + Arg("description", description, ""), + ) + + def with_object( + self, + name: str, + *, + description: str | None = "", + deprecated: str | None = None, + ) -> Self: + return self._with( + "withObject", + Arg("name", name), + Arg("description", description, ""), + Arg("deprecated", deprecated, None), + ) + + def with_field( + self, + name: str, + type_def: TypeDef, + *, + description: str | None = "", + deprecated: str | None = None, + ) -> Self: + return self._with( + "withField", + Arg("name", name), + Arg("typeDef", type_def), + Arg("description", description, ""), + Arg("deprecated", deprecated, None), + ) + + def with_function(self, function: Function) -> Self: + return self._with("withFunction", Arg("function", function)) + + def with_constructor(self, function: Function) -> Self: + return self._with("withConstructor", Arg("function", function)) + + +class Function(_Object): + def with_description(self, description: str) -> Self: + return self._with("withDescription", Arg("description", description)) + + def with_cache_policy( + self, policy: str, *, time_to_live: str | None = None + ) -> Self: + return self._with( + "withCachePolicy", + Arg("policy", EnumName(policy)), + Arg("timeToLive", time_to_live, None), + ) + + def with_deprecated(self, *, reason: str | None = None) -> Self: + return self._with("withDeprecated", Arg("reason", reason, None)) + + def with_check(self) -> Self: + return self._with("withCheck") + + def with_generator(self) -> Self: + return self._with("withGenerator") + + def with_up(self) -> Self: + return self._with("withUp") + + def with_agent(self) -> Self: + return self._with("withAgent") + + def with_arg( # noqa: PLR0913 + self, + name: str, + type_def: TypeDef, + *, + description: str | None = "", + default_value: str | None = None, + default_path: str | None = "", + default_address: str | None = "", + ignore: Sequence[str] | None = None, + deprecated: str | None = None, + ) -> Self: + return self._with( + "withArg", + Arg("name", name), + Arg("typeDef", type_def), + Arg("description", description, ""), + Arg("defaultValue", default_value, None), + Arg("defaultPath", default_path, ""), + Arg("ignore", [] if ignore is None else list(ignore), []), + Arg("deprecated", deprecated, None), + Arg("defaultAddress", default_address, ""), + ) + + +class Module(_Object): + def with_description(self, description: str) -> Self: + return self._with("withDescription", Arg("description", description)) + + def with_object(self, object_: TypeDef) -> Self: + return self._with("withObject", Arg("object", object_)) + + def with_interface(self, iface: TypeDef) -> Self: + return self._with("withInterface", Arg("iface", iface)) + + def with_enum(self, enum: TypeDef) -> Self: + return self._with("withEnum", Arg("enum", enum)) + + +class Error(_Object): + def with_value(self, name: str, value: str) -> Self: + """Attach a value, as JSON text.""" + return self._with("withValue", Arg("name", name), Arg("value", value)) + + +@dataclasses.dataclass(slots=True) +class ArgValue: + name: str + # JSON text. + value: str + + +class FunctionCall(Type): + async def parent_name(self) -> str: + return await self._select("parentName", []).execute(str) + + async def name(self) -> str: + return await self._select("name", []).execute(str) + + async def parent(self) -> str: + """The parent object's state, as JSON text.""" + return await self._select("parent", []).execute(str) + + async def input_args(self) -> list[ArgValue]: + ctx = self._select("inputArgs", []).select_multiple( + "FunctionCallArgValue", + name="name", + value="value", + ) + return await ctx.execute(list[ArgValue]) + + async def return_value(self, value: str) -> None: + """Set the function's result, as JSON text.""" + await self._select("returnValue", [Arg("value", value)]).execute() + + async def return_error(self, error: Error) -> None: + await self._select("returnError", [Arg("error", error)]).execute() + + +def type_def() -> TypeDef: + return TypeDef(Context().root_select("typeDef", [])) + + +def function(name: str, return_type: TypeDef) -> Function: + args = [Arg("name", name), Arg("returnType", return_type)] + return Function(Context().root_select("function", args)) + + +def module() -> Module: + return Module(Context().root_select("module", [])) + + +def error(message: str) -> Error: + return Error(Context().root_select("error", [Arg("message", message)])) + + +def current_function_call() -> FunctionCall: + return FunctionCall(Context().root_select("currentFunctionCall", [])) diff --git a/sdk/src/dagger/mod/_arguments.py b/sdk/src/dagger/mod/_arguments.py index ccf8c05..f35ebe7 100644 --- a/sdk/src/dagger/mod/_arguments.py +++ b/sdk/src/dagger/mod/_arguments.py @@ -4,7 +4,6 @@ from cattrs.preconf.json import JsonConverter -import dagger from dagger.mod._exceptions import BadUsageError from dagger.mod._types import APIName, ContextPath @@ -138,7 +137,8 @@ class Parameter: ignore: list[str] | None = None default_path: ContextPath | None = None default_address: str | None = None - default_value: dagger.JSON | None = None + # JSON text. + default_value: str | None = None deprecated: str | None = None conv: dataclasses.InitVar[JsonConverter] @@ -149,7 +149,7 @@ def __post_init__(self, conv: JsonConverter): if not self.has_default: return try: - self.default_value = dagger.JSON(conv.dumps(self.signature.default)) + self.default_value = conv.dumps(self.signature.default) except TypeError as e: # Rather than failing on a default value that's not JSON # serializable and going through hoops to support more and more diff --git a/sdk/src/dagger/mod/_converter.py b/sdk/src/dagger/mod/_converter.py index 24ad207..470fa53 100644 --- a/sdk/src/dagger/mod/_converter.py +++ b/sdk/src/dagger/mod/_converter.py @@ -4,11 +4,10 @@ from cattrs.preconf.json import make_converter as make_json_converter -import dagger -from dagger import dag -from dagger.client._core import Arg, configure_converter_enum +from dagger.client._core import Arg, Context, configure_converter_enum from dagger.client._guards import is_id_type, is_id_type_subclass from dagger.client.base import Interface, Scalar, Type +from dagger.mod import _api from dagger.mod._describe import TypeRef, describe_type from dagger.mod._resolver import Function from dagger.mod._utils import ( @@ -24,9 +23,6 @@ logger = logging.getLogger(__name__) -if typing.TYPE_CHECKING: - from dagger import TypeDef - def make_converter(): conv = make_json_converter() @@ -58,9 +54,7 @@ def dagger_type_structure(id_: str | Scalar, cls: type[Type]): msg = f"Unsupported type '{cls.__name__}'" raise TypeError(msg) - return cls( - dag._ctx.select_id(cls._graphql_name(), id_) # noqa: SLF001 - ) + return cls(Context().select_id(cls._graphql_name(), id_)) def dagger_interface_structure(id_, cls: type[Interface]): @@ -155,29 +149,29 @@ async def exec_method(self, *args, **kwargs): @functools.cache -def to_typedef(annotation: typing.Any, context: str = "type") -> "TypeDef": +def to_typedef(annotation: typing.Any, context: str = "type") -> _api.TypeDef: """Convert Python object to API type.""" return typedef_from(describe_type(annotation, context)) -def typedef_from(ref: TypeRef) -> "TypeDef": +def typedef_from(ref: TypeRef) -> _api.TypeDef: """Build the API type from its description.""" - td = dag.type_def() + td = _api.type_def() if ref.optional: td = td.with_optional(True) match ref.kind: - case dagger.TypeDefKind.LIST_KIND: + case "LIST_KIND": assert ref.elem is not None return td.with_list_of(typedef_from(ref.elem)) - case dagger.TypeDefKind.ENUM_KIND: + case "ENUM_KIND": return td.with_enum(ref.name, description=ref.description) - case dagger.TypeDefKind.SCALAR_KIND: + case "SCALAR_KIND": return td.with_scalar(ref.name, description=ref.description) - case dagger.TypeDefKind.INTERFACE_KIND: + case "INTERFACE_KIND": return td.with_interface(ref.name) - case dagger.TypeDefKind.OBJECT_KIND: + case "OBJECT_KIND": return td.with_object(ref.name) case _: return td.with_kind(ref.kind) diff --git a/sdk/src/dagger/mod/_describe.py b/sdk/src/dagger/mod/_describe.py index ec5a488..47fa573 100644 --- a/sdk/src/dagger/mod/_describe.py +++ b/sdk/src/dagger/mod/_describe.py @@ -12,9 +12,8 @@ import enum import inspect import json -from typing import Any +from typing import Any, Literal -import dagger from dagger.client._guards import is_id_type_subclass from dagger.client.base import Scalar from dagger.mod._utils import ( @@ -30,14 +29,26 @@ strip_annotations, ) -Kind = dagger.TypeDefKind - -_BUILTINS: dict[Any, dagger.TypeDefKind] = { - str: Kind.STRING_KIND, - int: Kind.INTEGER_KIND, - float: Kind.FLOAT_KIND, - bool: Kind.BOOLEAN_KIND, - type(None): Kind.VOID_KIND, +# TypeDefKind values by schema name: the SDK files can't import the enum. +Kind = Literal[ + "STRING_KIND", + "INTEGER_KIND", + "FLOAT_KIND", + "BOOLEAN_KIND", + "VOID_KIND", + "LIST_KIND", + "ENUM_KIND", + "SCALAR_KIND", + "INTERFACE_KIND", + "OBJECT_KIND", +] + +_BUILTINS: dict[Any, Kind] = { + str: "STRING_KIND", + int: "INTEGER_KIND", + float: "FLOAT_KIND", + bool: "BOOLEAN_KIND", + type(None): "VOID_KIND", } @@ -45,7 +56,7 @@ class TypeRef: """A reference to an API type.""" - kind: dagger.TypeDefKind + kind: Kind name: str = "" description: str | None = None optional: bool = False @@ -129,9 +140,9 @@ def describe_json(desc: ModuleDescription) -> str: The module-kind entrypoint reads this in its own session and replays the same builder calls the API build makes, so the definitions it returns belong to that session instead of the module's nested one. A TypeDefKind - becomes its schema name; a tuple becomes a list; nothing else is special. + is its schema name; a tuple becomes a list; nothing else is special. """ - return json.dumps(dataclasses.asdict(desc), default=lambda value: value.value) + return json.dumps(dataclasses.asdict(desc)) def describe_type( # noqa: C901, PLR0911 @@ -159,24 +170,24 @@ def describe_type( # noqa: C901, PLR0911 return TypeRef(_BUILTINS[typ], optional=optional) if el := list_of(typ): - return TypeRef(Kind.LIST_KIND, optional=optional, elem=describe_type(el)) + return TypeRef("LIST_KIND", optional=optional, elem=describe_type(el)) if inspect.isclass(cls := typ): name = cls.__name__ if is_subclass(cls, enum.Enum): - return TypeRef(Kind.ENUM_KIND, name, get_doc(cls), optional) + return TypeRef("ENUM_KIND", name, get_doc(cls), optional) if is_subclass(cls, Scalar): - return TypeRef(Kind.SCALAR_KIND, name, get_doc(cls), optional) + return TypeRef("SCALAR_KIND", name, get_doc(cls), optional) # object defined in this module if obj_type := get_object_type(cls): - kind = Kind.INTERFACE_KIND if obj_type.interface else Kind.OBJECT_KIND + kind: Kind = "INTERFACE_KIND" if obj_type.interface else "OBJECT_KIND" return TypeRef(kind, name, optional=optional) # object type from API (codegen) if is_id_type_subclass(cls): - return TypeRef(Kind.OBJECT_KIND, name, optional=optional) + return TypeRef("OBJECT_KIND", name, optional=optional) raise TypeError(error_msg) diff --git a/sdk/src/dagger/mod/_entrypoint.py b/sdk/src/dagger/mod/_entrypoint.py index 72035e4..bac172d 100644 --- a/sdk/src/dagger/mod/_entrypoint.py +++ b/sdk/src/dagger/mod/_entrypoint.py @@ -8,7 +8,6 @@ import pathlib from collections.abc import Iterator -import dagger from dagger.mod._describe import ( ArgumentDescription, EnumDescription, @@ -31,8 +30,6 @@ The rendered guard skips the same names, so both sides scan one set of files. """ -Kind = dagger.TypeDefKind - @dataclasses.dataclass(frozen=True, slots=True) class SourceFile: @@ -44,14 +41,13 @@ def write_entrypoint( desc: ModuleDescription, *, name: str, - path: str, root: pathlib.Path, output: pathlib.Path, ) -> None: """Write types.dang and main.dang for the module at root.""" output.mkdir(parents=True, exist_ok=True) (output / "types.dang").write_text(render_types(desc)) - (output / "main.dang").write_text(render_main(name, path, source_files(root))) + (output / "main.dang").write_text(render_main(name, source_files(root))) def source_files(root: pathlib.Path) -> list[SourceFile]: @@ -93,8 +89,7 @@ def render_types(desc: ModuleDescription) -> str: return "\n".join(lines) -def render_main(name: str, path: str, files: list[SourceFile]) -> str: - _check_path(path) +def render_main(name: str, files: list[SourceFile]) -> str: file_lines = "".join( f" SourceFile(path: {_quote(f.path)}, digest: {_quote(f.digest)}),\n" for f in files @@ -102,19 +97,11 @@ def render_main(name: str, path: str, files: list[SourceFile]) -> str: return MAIN_TEMPLATE.format( header=HEADER, name=_quote(name), - path=_quote(path), files=file_lines, skipped=", ".join(_quote(d) for d in sorted(SKIPPED_DIRS)), ) -def _check_path(path: str) -> None: - parts = pathlib.PurePosixPath(path).parts - if pathlib.PurePosixPath(path).is_absolute() or ".." in parts: - msg = f"module path must be relative to the workspace and not escape it: {path}" - raise BadUsageError(msg) - - def _object(obj: ObjectDescription) -> list[str]: description = _opt("description", obj.description) if obj.interface: @@ -197,21 +184,21 @@ def _type(ref: TypeRef) -> str: if ref.optional: expr += ".withOptional(true)" match ref.kind: - case Kind.LIST_KIND: + case "LIST_KIND": assert ref.elem is not None return f"{expr}.withListOf({_type(ref.elem)})" - case Kind.ENUM_KIND: + case "ENUM_KIND": described = f"{_quote(ref.name)}{_opt('description', ref.description)}" return f"{expr}.withEnum({described})" - case Kind.SCALAR_KIND: + case "SCALAR_KIND": described = f"{_quote(ref.name)}{_opt('description', ref.description)}" return f"{expr}.withScalar({described})" - case Kind.INTERFACE_KIND: + case "INTERFACE_KIND": return f"{expr}.withInterface({_quote(ref.name)})" - case Kind.OBJECT_KIND: + case "OBJECT_KIND": return f"{expr}.withObject({_quote(ref.name)})" case _: - return f"{expr}.withKind(TypeDefKind.{ref.kind.value})" + return f"{expr}.withKind(TypeDefKind.{ref.kind})" def _opt(arg: str, value: str | None) -> str: @@ -237,7 +224,6 @@ def _indent(lines: list[str], width: int) -> list[str]: MAIN_TEMPLATE = """{header} type Entrypoint implements ModuleEntrypoint {{ let moduleName: String! = {name} - let modulePath: String! = {path} let skippedDirs: [String!]! = [{skipped}] let sourceFiles: [SourceFile!]! = [ {files} ] @@ -253,43 +239,46 @@ def _indent(lines: list[str], width: int) -> list[str]: fnName: String!, fnArgs: JSON!, ): JSON! {{ + # The module's declared local clients go with the call, never the + # workspace (see handover.dang). let request = JSON.encode({{{{ receiverType: receiverType, receiverValue: receiverValue, fnName: fnName, fnArgs: fnArgs, + clients: ClientHandover(workspace: workspace).clients.map {{ client => + {{{{name: client.name, source: client.source}}}} + }}, }}}}) - let result = runtime(workspace) + let result = runtime .withExec(["python", "-m", "dagger.mod", "call", "--output", "/dagger/result.json"], stdin: request, experimentalPrivilegedNesting: true) .file("/dagger/result.json") .contents (result :: JSON!) }} - let runtime(workspace: Workspace!): Container! {{ - let module = workspace.directory(if (modulePath == ".") {{ "/" }} else {{ "/" + modulePath }}) - if (module.exists("pyproject.toml") == false) {{ - raise "module \\"" + moduleName + "\\" was generated at \\"" + modulePath + "\\" and is not there; run `dagger generate` after moving it" - }} else {{ - let changed = sourceFiles.filter {{ f => - if (f.digest == "") {{ - module.exists(f.path) - }} else {{ - module.exists(f.path) == false or module.file(f.path).digest(excludeMetadata: true) != f.digest - }} - }}.map {{ f => f.path }} - let added = module.glob("**/*.py").filter {{ p => - isSource(p) and sourceFiles.filter {{ f => f.path == p }}.length == 0 - }} - if ((changed + added).length > 0) {{ - raise "module \\"" + moduleName + "\\" changed since its entrypoint was generated (" + (changed + added).join(", ") + "); run `dagger generate`" + # The module's own source, not the workspace the engine hands over: that is + # the caller's, which holds the module only when the module sits in it. + let runtime: Container! {{ + let module = currentModule.source + let changed = sourceFiles.filter {{ f => + if (f.digest == "") {{ + module.exists(f.path) }} else {{ - PythonModuleBuild( - contextDir: workspace.directory("/", include: [if (modulePath == ".") {{ "**" }} else {{ modulePath + "/**" }}], exclude: ["**/.venv", "**/__pycache__"]), - subPath: modulePath, - moduleName: moduleName, - ).installed + module.exists(f.path) == false or module.file(f.path).digest(excludeMetadata: true) != f.digest }} + }}.map {{ f => f.path }} + let added = module.glob("**/*.py").filter {{ p => + isSource(p) and sourceFiles.filter {{ f => f.path == p }}.length == 0 + }} + if ((changed + added).length > 0) {{ + raise "module \\"" + moduleName + "\\" changed since its entrypoint was generated (" + (changed + added).join(", ") + "); run `dagger generate`" + }} else {{ + PythonModuleBuild( + contextDir: directory.withDirectory(".", module, exclude: ["**/.venv", "**/__pycache__"]), + subPath: ".", + moduleName: moduleName, + ).installed }} }} diff --git a/sdk/src/dagger/mod/_exceptions.py b/sdk/src/dagger/mod/_exceptions.py index a55b09b..e1a0371 100644 --- a/sdk/src/dagger/mod/_exceptions.py +++ b/sdk/src/dagger/mod/_exceptions.py @@ -11,8 +11,9 @@ EXCEPTION_TYPE, ) -import dagger -from dagger import DaggerError, dag, telemetry +from dagger import telemetry +from dagger._exceptions import DaggerError, QueryError +from dagger.mod import _api logger = logging.getLogger(__package__) @@ -136,15 +137,15 @@ async def record_exception(exc: Exception): attrs = {**extra, **attrs} # Preserve original API error so it's properly propagated. - if isinstance(exc, dagger.QueryError): + if isinstance(exc, QueryError): msg = str(exc) attrs.update(exc.error.extensions) - dag_err = dag.error(msg) + dag_err = _api.error(msg) for key, value in attrs.items(): - dag_err = dag_err.with_value(key, dagger.JSON(_safe_json_dumps(value))) + dag_err = dag_err.with_value(key, _safe_json_dumps(value)) - await dag.current_function_call().return_error(dag_err) + await _api.current_function_call().return_error(dag_err) # When an error occurs within a started span context the OTel SDK # automatically sends an event with details about the exception. diff --git a/sdk/src/dagger/mod/_module.py b/sdk/src/dagger/mod/_module.py index c0c515a..77094b1 100644 --- a/sdk/src/dagger/mod/_module.py +++ b/sdk/src/dagger/mod/_module.py @@ -16,9 +16,9 @@ from cattrs.preconf.json import JsonConverter from typing_extensions import dataclass_transform, overload -import dagger -from dagger import dag +from dagger._exceptions import QueryError from dagger.client._core import configure_converter_enum +from dagger.mod import _api from dagger.mod._converter import make_converter, typedef_from from dagger.mod._describe import ( ArgumentDescription, @@ -98,7 +98,7 @@ def is_main(self, other: ObjectType) -> bool: return self.main_cls is other.cls async def serve(self): - if await dag.current_function_call().parent_name(): + if await _api.current_function_call().parent_name(): result = await self.invoke() else: try: @@ -122,7 +122,7 @@ async def serve(self): textwrap.shorten(repr(output), 144), ) - await dag.current_function_call().return_value(dagger.JSON(output)) + await _api.current_function_call().return_value(output) async def register(self): """Register the module and its types with the Dagger API.""" @@ -174,7 +174,7 @@ async def invoke(self) -> str: This includes getting the call context from the API and deserializing data. """ - fn_call = dag.current_function_call() + fn_call = _api.current_function_call() parent_name = await fn_call.parent_name() if not parent_name: @@ -202,19 +202,15 @@ async def invoke(self) -> str: inputs = {} for arg in input_args: - # NB: These are already loaded by `input_args`, - # the await just returns the cached value. - arg_name = await arg.name() - arg_value = await arg.value() try: # Cattrs can decode JSON strings but use `json` directly # for more granular control over the error. - inputs[arg_name] = json.loads(arg_value) - except ValueError as e: + inputs[arg.name] = json.loads(arg.value) + except ValueError as e: # noqa: PERF203 logger.exception("Failed to decode JSON input value") - msg = f"Unable to decode input argument '{arg_name}'" + msg = f"Unable to decode input argument '{arg.name}'" extra = { - "json_value": arg_value, + "json_value": arg.value, } raise InvalidInputError(msg, extra=extra) from e @@ -359,7 +355,7 @@ async def call(self, func: Func[P, R], *args: P.args, **kwargs: P.kwargs) -> R: except FunctionError: # Escape hatch to fully control logging from user code. raise - except dagger.QueryError as e: + except QueryError as e: tb = e.__traceback__ # Exclude the line in "try" above if tb: @@ -1051,8 +1047,8 @@ def _describe_enum(name: str, cls: type[enum.Enum]) -> EnumDescription: return EnumDescription(name, get_doc(cls), tuple(members)) -def _module_from(desc: ModuleDescription) -> dagger.Module: - mod = dag.module() +def _module_from(desc: ModuleDescription) -> _api.Module: + mod = _api.module() for obj in desc.objects: if obj.name == desc.main_object and desc.description: mod = mod.with_description(desc.description) @@ -1065,8 +1061,8 @@ def _module_from(desc: ModuleDescription) -> dagger.Module: return mod -def _object_from(obj: ObjectDescription) -> dagger.TypeDef: - type_def = dag.type_def() +def _object_from(obj: ObjectDescription) -> _api.TypeDef: + type_def = _api.type_def() if obj.interface: type_def = type_def.with_interface(obj.name, description=obj.description) else: @@ -1089,19 +1085,16 @@ def _object_from(obj: ObjectDescription) -> dagger.TypeDef: return type_def -def _function_from(func: FunctionDescription) -> dagger.Function: # noqa: C901 - func_def = dag.function(func.name, typedef_from(func.returns)) +def _function_from(func: FunctionDescription) -> _api.Function: # noqa: C901 + func_def = _api.function(func.name, typedef_from(func.returns)) if func.description: func_def = func_def.with_description(func.description) if func.cache == "never": - func_def = func_def.with_cache_policy(dagger.FunctionCachePolicy.Never) + func_def = func_def.with_cache_policy("Never") elif func.cache == "session": - func_def = func_def.with_cache_policy(dagger.FunctionCachePolicy.PerSession) + func_def = func_def.with_cache_policy("PerSession") elif func.cache: - func_def = func_def.with_cache_policy( - dagger.FunctionCachePolicy.Default, - time_to_live=func.cache, - ) + func_def = func_def.with_cache_policy("Default", time_to_live=func.cache) if func.deprecated: func_def = func_def.with_deprecated(reason=func.deprecated) if func.check: @@ -1120,11 +1113,7 @@ def _function_from(func: FunctionDescription) -> dagger.Function: # noqa: C901 arg.name, arg_def, description=arg.description, - default_value=( - dagger.JSON(arg.default_value) - if arg.default_value is not None - else None - ), + default_value=arg.default_value, default_path=arg.default_path, default_address=arg.default_address, ignore=list(arg.ignore) if arg.ignore is not None else None, @@ -1133,8 +1122,8 @@ def _function_from(func: FunctionDescription) -> dagger.Function: # noqa: C901 return func_def -def _enum_from(enum_desc: EnumDescription) -> dagger.TypeDef: - enum_def = dag.type_def().with_enum( +def _enum_from(enum_desc: EnumDescription) -> _api.TypeDef: + enum_def = _api.type_def().with_enum( enum_desc.name, description=enum_desc.description ) for member in enum_desc.members: diff --git a/sdk/src/dagger/mod/cli.py b/sdk/src/dagger/mod/cli.py index 86d0866..21a11e2 100644 --- a/sdk/src/dagger/mod/cli.py +++ b/sdk/src/dagger/mod/cli.py @@ -1,5 +1,6 @@ """Command line interface for the dagger extension runtime.""" +import contextlib import importlib import importlib.metadata import importlib.util @@ -9,8 +10,11 @@ import anyio -import dagger from dagger import telemetry +from dagger._exceptions import QueryError +from dagger.client._connection import connect +from dagger.client._descriptor import registering_types +from dagger.client._session import mark_module_runtime from dagger.mod._exceptions import ModuleError, ModuleLoadError, record_exception from dagger.mod._module import MAIN_OBJECT, Module @@ -23,6 +27,7 @@ def app(mod: Module | None = None, register: bool = False) -> int | None: """Entrypoint for a Python Dagger module.""" + mark_module_runtime() telemetry.initialize() try: return anyio.run(main, mod, register) @@ -35,14 +40,17 @@ async def main(mod: Module | None = None, register: bool = False) -> int | None: # Establishing connection early on to allow returning dag.error(). # Note: if there's a connection error dag.error() won't be sent but # should be logged and the traceback shown on the function's stderr output. - async with await dagger.connect(): + async with await connect(): try: if mod is None: - mod = load_module() + # Only an explicit registration is known before the user's + # code is imported; serve() decides after. + with registering_types() if register else contextlib.nullcontext(): + mod = load_module() if register: return await mod.register() return await mod.serve() - except (ModuleError, dagger.QueryError) as e: + except (ModuleError, QueryError) as e: await record_exception(e) return 2 except Exception as e: diff --git a/sdk/src/dagger/provisioning/_connection.py b/sdk/src/dagger/provisioning/_connection.py index 630ed74..326565c 100644 --- a/sdk/src/dagger/provisioning/_connection.py +++ b/sdk/src/dagger/provisioning/_connection.py @@ -1,34 +1,31 @@ import contextlib import logging -from typing import TYPE_CHECKING from dagger import telemetry from dagger._managers import ResourceManager +from dagger.client._session import Session, as_session from ._config import Config from ._engine import Engine, provision_engine -if TYPE_CHECKING: - from dagger import Client - logger = logging.getLogger(__name__) class Connection(ResourceManager): - """Connect to a Dagger Engine with an isolated client (legacy). + """Connect to a Dagger Engine with an isolated session (legacy). - This is an older version of :py:func:`dagger.connection` that uses an isolated - client instance. Should no longer be used in newer projects unless there's - a specific reason to do so. + This is an older version of :py:func:`dagger.connection` that uses an + isolated session instead of the default one. Should no longer be used in + newer projects unless there's a specific reason to do so. - Example:: + Example, with ``core`` the entry function of the generated core client:: import dagger async def main(): - async with dagger.Connection() as client: - ctr = client.container().from_("alpine") + async with dagger.Connection() as session: + ctr = core(session=session).container().from_("alpine") You can stream the logs from the engine to see progress:: @@ -41,8 +38,8 @@ async def main(): async def main(): cfg = dagger.Config(log_output=sys.stderr) - async with dagger.Connection(cfg) as client: - ctr = client.container().from_("python:3.11.1-alpine") + async with dagger.Connection(cfg) as session: + ctr = core(session=session).container().from_("python:3.11.1-alpine") version = await ctr.with_exec(["python", "-V"]).stdout() print(version) @@ -56,36 +53,36 @@ def __init__(self, config: Config | None = None) -> None: super().__init__() self.cfg = config or Config() - async def __aenter__(self) -> "Client": + async def __aenter__(self) -> Session: telemetry.initialize() - logger.debug("Establishing connection with isolated client") + logger.debug("Establishing connection with isolated session") async with self.get_stack() as stack: engine = await Engine(self.cfg, stack).provision() - conn = engine.get_client_connection() - return await engine.setup_client(conn) + session = as_session(engine.get_client_connection()) + await engine.setup_client(session) + return session async def close(self): - logger.debug("Closing connection with isolated client") + logger.debug("Closing connection with isolated session") await super().close() @contextlib.asynccontextmanager async def connection(config: Config | None = None): - """Connect to a Dagger Engine using the global client. + """Connect to a Dagger Engine using the default session. - This is similar to :py:class:`dagger.Connection` but uses a global client - (:py:attr:`dagger.dag`) so there's no need to pass around a client instance - with this. + This is similar to :py:class:`dagger.Connection` but uses the default + session (:py:attr:`dagger.dag`), the one a client called without + ``session=`` uses, so there's no need to pass a session around. - Example:: + Example, with ``core`` the entry function of the generated core client:: import dagger - from dagger import dag async def main(): async with dagger.connection(): - ctr = dag.container().from_("alpine") + ctr = core().container().from_("alpine") # Connection is closed when leaving the context manager's scope. @@ -95,14 +92,13 @@ async def main(): import sys import anyio import dagger - from dagger import dag async def main(): cfg = dagger.Config(log_output=sys.stderr) async with dagger.connection(cfg): - ctr = dag.container().from_("python:3.11.1-alpine") + ctr = core().container().from_("python:3.11.1-alpine") version = await ctr.with_exec(["python", "-V"]).stdout() print(version) @@ -112,9 +108,9 @@ async def main(): anyio.run(main) """ telemetry.initialize() - logger.debug("Establishing connection with shared client") + logger.debug("Establishing connection with the default session") async with provision_engine(config or Config()) as engine: - conn = engine.get_shared_client_connection() - await engine.setup_client(conn) - yield conn - logger.debug("Closing connection with shared client") + session = as_session(engine.get_shared_client_connection()) + await engine.setup_client(session) + yield session + logger.debug("Closing connection with the default session") diff --git a/sdk/src/dagger/provisioning/_engine.py b/sdk/src/dagger/provisioning/_engine.py index d81980f..9f30813 100644 --- a/sdk/src/dagger/provisioning/_engine.py +++ b/sdk/src/dagger/provisioning/_engine.py @@ -3,14 +3,16 @@ import os import shutil import sys -import typing +from collections.abc import Callable from typing import TextIO from exceptiongroup import ExceptionGroup from typing_extensions import Self -import dagger from dagger._engine._version import CLI_VERSION +from dagger._exceptions import QueryError +from dagger._managers import SyncResource, asyncify +from dagger.client._core import Context from dagger.client._session import ( BaseConnection, ConnectConfig, @@ -23,13 +25,10 @@ from ._download import Downloader from ._exceptions import CLIReleaseUnavailableError, ProvisionError from ._progress import Progress -from ._session import start_cli_session +from ._session import start_cli_session_sync logger = logging.getLogger(__name__) -if typing.TYPE_CHECKING: - from dagger import Client - @contextlib.asynccontextmanager async def provision_engine(cfg: Config): @@ -99,8 +98,8 @@ async def provision(self) -> Self: await self.progress.update("Creating new Engine session") try: - connect_params = await self.stack.enter_async_context( - start_cli_session(self.cfg, cli_bin) + connect_params = await self.enter_session( + start_cli_session_sync(self.cfg, cli_bin) ) except Exception as e: if download_error is not None: @@ -119,6 +118,12 @@ async def provision(self) -> Self: return self + async def enter_session( + self, session: contextlib.AbstractContextManager[ConnectParams] + ) -> ConnectParams: + """Start the CLI session, closed with this engine's stack.""" + return await self.stack.enter_async_context(SyncResource(session)) + async def get_cli(self) -> str: """Get path to CLI.""" if cli_bin := os.getenv("_EXPERIMENTAL_DAGGER_CLI_BIN"): @@ -127,15 +132,14 @@ async def get_cli(self) -> str: # Get from cache or download. return await Downloader(progress=self.progress) - async def setup_client(self, conn: BaseConnection) -> "Client": - """Setup client instance from connection.""" + async def setup_client(self, conn: BaseConnection) -> BaseConnection: + """Open the connection and check the engine behind it.""" await self.progress.update("Establishing connection to the API server") conn = await self.stack.enter_async_context(conn) - client = dagger.Client.from_connection(conn) self.stack.push_async_callback(self.progress.stop) - return await self.verify(client) + return await self.verify(conn) def get_shared_client_connection(self) -> SharedConnection: """Global client connection to the GraphQL server.""" @@ -156,15 +160,53 @@ def get_client_connection(self) -> SingleConnection: self.connect_config, ) - async def verify(self, client: "Client") -> "Client": + async def verify(self, conn: BaseConnection) -> BaseConnection: """Check if the Dagger CLI version is compatible with the engine.""" await self.progress.update("Checking version compatibility") try: - await client.version() - except dagger.QueryError as e: + await Context(conn).root_select("version", []).execute(str) + except QueryError as e: logger.warning("Failed to check Dagger engine version compatibility: %s", e) await self.progress.update("Running pipelines") await self.progress.stop() - return client + return conn + + +class _UnmanagedEngine(Engine): + """An engine whose CLI session outlives any event loop.""" + + def __init__(self, cfg: Config, stack: contextlib.AsyncExitStack) -> None: + super().__init__(cfg, stack) + # Nothing to close until a CLI session starts. + self.close_session: Callable[[], None] = lambda: None + + async def enter_session( + self, session: contextlib.AbstractContextManager[ConnectParams] + ) -> ConnectParams: + params = await asyncify(session.__enter__) + + def close() -> None: + # Closing stdin ends the CLI; this waits for it to drain its logs. + session.__exit__(None, None, None) + + self.close_session = close + return params + + +async def provision_default_session( + cfg: Config, +) -> tuple[ConnectParams, Callable[[], None]]: + """Provision an engine for the default session of a plain program. + + The default session has no ``async with`` to close it, so the caller gets + the close instead. It is sync: it may run at exit, when no event loop does. + """ + engine = _UnmanagedEngine(cfg, contextlib.AsyncExitStack()) + try: + await engine.provision() + finally: + await engine.progress.stop() + assert engine.connect_params + return engine.connect_params, engine.close_session diff --git a/sdk/tests/client/test_clients.py b/sdk/tests/client/test_clients.py new file mode 100644 index 0000000..652a7ef --- /dev/null +++ b/sdk/tests/client/test_clients.py @@ -0,0 +1,740 @@ +"""The SDK side of a generated client: its session, its target, its load.""" + +import gc +import json +import logging +import threading +import time +import weakref + +import anyio +import anyio.lowlevel +import pytest + +import dagger +from dagger._exceptions import ( + ClientLoadError, + QueryError, + QueryErrorValue, + StaleClientError, + TransportError, +) +from dagger.client import ( + Session, + Target, + check_core, + client_root, + client_select, + default_session, + registering_types, +) +from dagger.client._core import Arg, Context +from dagger.client._load import ( + leave_entrypoint, + parse_handed_clients, + use_entrypoint_clients, +) +from dagger.client._session import BaseConnection, as_session +from dagger.client.base import Type + +pytestmark = pytest.mark.anyio + +GLOW = Target(name="glow", ref="github.com/eunomie/glow", pin="4f1c9e") +# The module's own path in the workspace: that is what the engine serves. +LINTER = Target(name="linter", ref="./.dagger/modules/linter") + +MISSING_FIELD = 'Cannot query field "glow" on type "Query".' +# What beta.13 answers: a validation error, before anything runs. +VALIDATION = {"code": "GRAPHQL_VALIDATION_FAILED"} + + +class Answer(dict): + """A response with every field, at every depth.""" + + def __missing__(self, key): + return Answer() + + +class FakeSession: + """Answers every query, and keeps what it was asked. + + A query that contains a key of ``fail`` raises that value instead. + """ + + def __init__(self): + self.data: dict = Answer() + self.queries: list[str] = [] + self.fail: dict[str, Exception] = {} + self.checkpoints = 0 + + async def execute(self, query: str): + self.queries.append(query) + for needle, error in self.fail.items(): + if needle in query: + raise error + if is_load(query): + # Let another task in, so a missing lock shows as a second load. + for _ in range(self.checkpoints): + await anyio.lowlevel.checkpoint() + return None + return self.data + + async def close(self): + pass + + @property + def loads(self) -> list[str]: + return [q for q in self.queries if is_load(q)] + + +def is_load(query: str) -> bool: + return "serve" in query + + +class FakeConnection(BaseConnection): + def __init__(self): + self.session = FakeSession() + + +def session() -> Session: + return Session(FakeConnection()) + + +class Query(Type): + pass + + +class Glow(Type): + def again(self) -> "Glow": + return Glow(self._select("again", [])) + + async def output(self) -> str: + return await self._select("output", []).execute(str) + + +def glow(*, session: Session | None = None) -> Glow: + return client_root(Glow, GLOW, "glow", [], session=session) + + +async def test_load_once_per_session(): + s = session() + s.session.data = {"glow": {"output": "hi", "again": {"output": "again"}}} + + assert await glow(session=s).output() == "hi" + assert await glow(session=s).again().output() == "again" + + assert len(s.session.loads) == 1 + assert s.session.queries.index(s.session.loads[0]) == 0 + + +async def test_load_once_per_target_in_each_session(): + first, second = session(), session() + + await glow(session=first).output() + await glow(session=second).output() + await glow(session=first).output() + + assert len(first.session.loads) == 1 + assert len(second.session.loads) == 1 + + +async def test_concurrent_queries_load_once(): + s = session() + s.session.checkpoints = 3 + + async with anyio.create_task_group() as tg: + tg.start_soon(glow(session=s).output) + tg.start_soon(glow(session=s).output) + + assert len(s.session.loads) == 1 + + +async def test_git_target_loads_before_its_query(): + s = session() + + await glow(session=s).output() + + load, query = s.session.queries + assert is_load(load) + assert GLOW.ref in load + assert GLOW.pin in load + assert query == "query {\n glow {\n output\n }\n}" + + +async def test_local_target_loads_before_its_query(): + s = session() + + await client_root(Glow, LINTER, "linter", [], session=s).output() + + load, query = s.session.queries + assert is_load(load) + assert LINTER.ref in load + assert query == "query {\n linter {\n output\n }\n}" + + +NO_SERVE_MODULE = QueryError( + [ + QueryErrorValue( + 'Cannot query field "serveModule" on type "Query".', extensions=VALIDATION + ) + ], + "query", +) + + +async def test_engine_without_serve_module_fails_the_load_not_as_stale(): + # An engine below the floor lacks the field. That is the engine's age, + # not the client's, so it is no stale client: regenerating cannot help. + s = session() + s.session.fail["serveModule"] = NO_SERVE_MODULE + + with pytest.raises(ClientLoadError) as info: + await glow(session=s).output() + + assert not isinstance(info.value, StaleClientError) + assert info.value.__cause__ is NO_SERVE_MODULE + assert len(s.session.loads) == 1 + + +HANDED = "bW9kdWxlU291cmNl" + + +@pytest.fixture +def handed_clients(): + use_entrypoint_clients({"linter": HANDED}) + yield HANDED + leave_entrypoint() + + +async def test_handed_client_serves_a_local_target_by_name(handed_clients): + # Under a module entrypoint this process is not the module, and its own + # current workspace is its container: the entrypoint hands over each + # declared client as a source, by name, and serveModule is never asked. + s = session() + + await client_root(Glow, LINTER, "linter", [], session=s).output() + + load, query = s.session.queries + assert load == ( + "query {\n" + f' node(id: "{handed_clients}") {{\n' + " ... on ModuleSource {\n" + " asModule {\n" + " serve\n" + " }\n" + " }\n" + " }\n" + "}" + ) + assert query == "query {\n linter {\n output\n }\n}" + + +async def test_handed_clients_leave_a_git_target_to_serve_module(handed_clients): + s = session() + + await glow(session=s).output() + + (load,) = s.session.loads + assert "serveModule" in load + assert handed_clients not in load + + +@pytest.mark.usefixtures("handed_clients") +async def test_undeclared_local_client_fails_naming_it(): + # A local target the caller did not declare on the scope was not handed + # over. It never falls back to serveModule, which would resolve the path + # in this process's container. + s = session() + other = Target(name="other", ref="/.dagger/modules/other") + + with pytest.raises(ClientLoadError) as info: + await client_root(Glow, other, "other", [], session=s).output() + + assert "'other' is not declared" in str(info.value) + assert "dagger generate" in str(info.value) + assert not s.session.queries + + +async def test_entrypoint_without_a_handover_fails_a_local_target(): + # An entrypoint that sends no clients, one from before the handover or + # not this SDK's, is still an entrypoint: the path cannot resolve here. + use_entrypoint_clients(None) + try: + s = session() + with pytest.raises(ClientLoadError) as info: + await client_root(Glow, LINTER, "linter", [], session=s).output() + await glow(session=s).output() + finally: + leave_entrypoint() + + assert "handed over no clients" in str(info.value) + assert "dagger generate" in str(info.value) + (load,) = s.session.loads + assert "serveModule" in load + assert GLOW.ref in load + + +@pytest.mark.usefixtures("handed_clients") +async def test_handed_client_failure_does_not_fall_back(): + s = session() + s.session.fail["asModule"] = QueryError([QueryErrorValue("boom")], "query") + + with pytest.raises(ClientLoadError): + await client_root(Glow, LINTER, "linter", [], session=s).output() + + (load,) = s.session.loads + assert "serveModule" not in load + + +async def test_outside_an_entrypoint_a_local_target_uses_serve_module(): + s = session() + + await client_root(Glow, LINTER, "linter", [], session=s).output() + + (load,) = s.session.loads + assert "serveModule" in load + assert "node(" not in load + + +def test_handed_clients_parse_from_the_request(): + handed = [{"name": "linter", "source": HANDED}] + + assert parse_handed_clients(handed) == {"linter": HANDED} + assert parse_handed_clients(json.dumps(handed)) == {"linter": HANDED} + assert parse_handed_clients([]) == {} + assert parse_handed_clients(None) is None + with pytest.raises(TypeError): + parse_handed_clients({"linter": HANDED}) + with pytest.raises(TypeError): + parse_handed_clients([{"name": "linter", "source": 1}]) + + +async def test_failed_load_names_the_target_and_the_cause(): + s = session() + cause = TransportError("engine went away") + s.session.fail["serve"] = cause + + with pytest.raises(ClientLoadError, match="glow") as info: + await glow(session=s).output() + + assert info.value.target == GLOW + assert "engine went away" in str(info.value) + assert info.value.__cause__ is cause + assert isinstance(info.value, dagger.ClientLoadError) + + +async def test_failed_load_is_retried(): + s = session() + s.session.fail["serve"] = TransportError("not yet") + + with pytest.raises(ClientLoadError): + await glow(session=s).output() + s.session.fail.clear() + await glow(session=s).output() + + assert len(s.session.loads) == 2 + + +async def test_conflict_names_both_descriptors_pins_included(): + s = session() + repinned = Target(name=GLOW.name, ref=GLOW.ref, pin="9b2d7a") + + await glow(session=s).output() + with pytest.raises(ClientLoadError) as info: + await client_root(Glow, repinned, "glow", [], session=s).output() + + assert repr(GLOW) in str(info.value) + assert repr(repinned) in str(info.value) + assert info.value.target == repinned + + +async def test_conflict_after_a_failed_load_does_not_say_loaded(): + s = session() + s.session.fail["serve"] = TransportError("engine went away") + other = Target(name="glow", ref="github.com/eunomie/glow-fork") + + with pytest.raises(ClientLoadError): + await glow(session=s).output() + with pytest.raises(ClientLoadError) as info: + await client_root(Glow, other, "glow", [], session=s).output() + + assert "already holds" in str(info.value) + assert "loaded" not in str(info.value) + + +async def test_no_target_attaches_nothing(): + s = session() + s.session.data = {"version": "v1"} + root = client_root(Query, None, None, [], session=s) + + assert root._ctx.targets == frozenset() + assert not root._ctx.selections + assert await root._select("version", []).execute(str) == "v1" + assert s.session.loads == [] + + +def test_root_field_with_args(): + s = session() + obj = client_root(Glow, GLOW, "glow", [Arg("name", "x")], session=s) + + assert obj._ctx.targets == {GLOW} + assert obj._ctx.build() == 'query {\n glow(name: "x")\n}' + + +def test_selections_keep_the_target(): + obj = glow(session=session()) + + assert obj.again()._ctx.targets == {GLOW} + assert obj._select("output", []).targets == {GLOW} + + +def test_default_session_when_none_given(): + assert glow()._ctx.conn is default_session() + assert default_session() is default_session() + + +def test_default_session_is_created_once_under_contention(monkeypatch): + from dagger.client import _session + + in_init = threading.Event() + + class SlowSession(Session): + def __init__(self, connection=None): + in_init.set() + # Long enough for the other thread to read the default meanwhile. + time.sleep(0.1) + super().__init__(connection) + + monkeypatch.setattr(_session, "_default", None) + monkeypatch.setattr(_session, "Session", SlowSession) + seen: list[Session] = [] + + def second(): + in_init.wait() + seen.append(default_session()) + + thread = threading.Thread(target=second) + thread.start() + seen.append(default_session()) + thread.join() + + assert seen[0] is seen[1] + + +def test_one_session_per_connection(): + conn = FakeConnection() + + assert as_session(conn) is as_session(conn) + + +def over(conn: BaseConnection, target: Target) -> Glow: + """A client built on a bare connection, the way a root type does.""" + ctx = Context(conn, targets=frozenset([target])).root_select("glow", []) + return Glow(ctx) + + +async def test_executions_through_one_connection_load_once(): + conn = FakeConnection() + + await over(conn, GLOW).output() + await over(conn, GLOW).output() + + assert len(conn.session.loads) == 1 + + +async def test_name_guard_holds_across_wrappers_of_one_connection(): + conn = FakeConnection() + other = Target(name="glow", ref="github.com/eunomie/glow-fork") + + await over(conn, GLOW).output() + with pytest.raises(ClientLoadError): + await over(conn, other).output() + + +async def test_closing_a_bare_connection_forgets_what_was_loaded(): + conn = FakeConnection() + + await over(conn, GLOW).output() + await conn.close() + await over(conn, GLOW).output() + + assert len(conn.session.loads) == 2 + + +def test_session_dies_with_its_connection(): + conn = FakeConnection() + gone = weakref.ref(conn) + as_session(conn) + + del conn + gc.collect() + + assert gone() is None + + +def test_client_select_keeps_the_receiver_session(): + s = session() + receiver = client_root(Query, None, None, [], session=s) + + ctx = client_select(receiver, GLOW, "asGlow", [Arg("strict", True)]) + + assert ctx.conn is s + assert ctx.targets == {GLOW} + assert [(f.type_name, f.name) for f in ctx.selections] == [("Query", "asGlow")] + assert ctx.build() == "query {\n asGlow(strict: true)\n}" + + +def test_client_select_adds_to_the_receiver_targets(): + receiver = client_root(Glow, LINTER, "linter", [], session=session()) + + ctx = client_select(receiver, GLOW, "asGlow", []) + + assert ctx.targets == {LINTER, GLOW} + assert [f.name for f in ctx.selections] == ["linter", "asGlow"] + + +async def test_client_select_loads_in_the_receiver_session(): + s = session() + receiver = client_root(Query, None, None, [], session=s) + + await Glow(client_select(receiver, GLOW, "asGlow", [])).output() + + assert len(s.session.loads) == 1 + + +async def test_missing_field_becomes_stale_client_error(): + s = session() + error = QueryError([QueryErrorValue(MISSING_FIELD, extensions=VALIDATION)], "query") + s.session.fail["output"] = error + + with pytest.raises(StaleClientError, match="dagger generate") as info: + await glow(session=s).output() + + assert str(info.value) == ( + f"{MISSING_FIELD} The client 'glow' from github.com/eunomie/glow at 4f1c9e " + "is out of date. Run `dagger generate`." + ) + assert info.value.__cause__ is error + + +async def test_stale_message_names_each_client_and_its_address(): + s = session() + s.session.fail["output"] = QueryError( + [QueryErrorValue(MISSING_FIELD, extensions=VALIDATION)], "query" + ) + receiver = client_root(Glow, LINTER, "linter", [], session=s) + + with pytest.raises(StaleClientError) as info: + await Glow(client_select(receiver, GLOW, "asGlow", [])).output() + + assert str(info.value).endswith( + "The clients 'glow' from github.com/eunomie/glow at 4f1c9e, " + "'linter' from ./.dagger/modules/linter are out of date. Run `dagger generate`." + ) + + +async def test_missing_field_without_target_stays_a_query_error(): + s = session() + s.session.fail["version"] = QueryError( + [QueryErrorValue(MISSING_FIELD, extensions=VALIDATION)], "query" + ) + root = client_root(Query, None, None, [], session=s) + + with pytest.raises(QueryError) as info: + await root._select("version", []).execute(str) + + assert not isinstance(info.value, StaleClientError) + + +async def test_other_query_errors_pass_through(): + s = session() + error = QueryError([QueryErrorValue("boom")], "query") + s.session.fail["output"] = error + + with pytest.raises(QueryError) as info: + await glow(session=s).output() + + assert info.value is error + + +async def test_missing_field_in_a_later_error_is_stale(): + s = session() + error = QueryError( + [ + QueryErrorValue("boom"), + QueryErrorValue(MISSING_FIELD, extensions=VALIDATION), + ], + "query", + ) + s.session.fail["output"] = error + + with pytest.raises(StaleClientError) as info: + await glow(session=s).output() + + assert MISSING_FIELD in str(info.value) + + +async def test_resolver_error_with_the_phrase_stays_a_query_error(): + s = session() + error = QueryError( + [QueryErrorValue(MISSING_FIELD, path=["glow"], extensions=VALIDATION)], + "query", + ) + s.session.fail["output"] = error + + with pytest.raises(QueryError) as info: + await glow(session=s).output() + + assert info.value is error + + +async def test_internal_error_with_the_phrase_stays_a_query_error(): + s = session() + internal = {"code": "INTERNAL_SERVER_ERROR"} + error = QueryError([QueryErrorValue(MISSING_FIELD, extensions=internal)], "query") + s.session.fail["output"] = error + + with pytest.raises(QueryError) as info: + await glow(session=s).output() + + assert not isinstance(info.value, StaleClientError) + + +async def test_phrase_inside_a_message_stays_a_query_error(): + s = session() + error = QueryError([QueryErrorValue(f"stdout: {MISSING_FIELD}")], "query") + s.session.fail["output"] = error + + with pytest.raises(QueryError) as info: + await glow(session=s).output() + + assert info.value is error + + +def test_check_core_accepts_a_matching_digest(): + check_core("glow", "sha256:aa", "sha256:aa") + + +def test_check_core_refuses_a_stale_client(): + with pytest.raises(StaleClientError, match="dagger generate") as info: + check_core("glow", "sha256:aa", "sha256:bb") + + assert isinstance(info.value, ImportError) + assert isinstance(info.value, dagger.StaleClientError) + assert "glow" in str(info.value) + assert "sha256:aa" in str(info.value) + assert "sha256:bb" in str(info.value) + + +def test_check_core_warns_while_registering(caplog: pytest.LogCaptureFixture): + with caplog.at_level(logging.WARNING), registering_types(): + check_core("glow", "sha256:aa", "sha256:bb") + + assert len(caplog.records) == 1 + assert "sha256:aa" in caplog.text + assert "sha256:bb" in caplog.text + assert "dagger generate" in caplog.text + + +def test_registering_ends_with_its_block(): + with registering_types(): + pass + + with pytest.raises(StaleClientError): + check_core("glow", "sha256:aa", "sha256:bb") + + +async def test_registering_ends_for_tasks_started_inside_it(): + started, left = anyio.Event(), anyio.Event() + + async def child(): + started.set() + await left.wait() + with pytest.raises(StaleClientError): + check_core("glow", "sha256:aa", "sha256:bb") + + async with anyio.create_task_group() as tg: + with registering_types(): + tg.start_soon(child) + await started.wait() + left.set() + + +def test_registering_covers_threads_started_inside_it( + caplog: pytest.LogCaptureFixture, +): + with caplog.at_level(logging.WARNING), registering_types(): + thread = threading.Thread( + target=check_core, args=("glow", "sha256:aa", "sha256:bb") + ) + thread.start() + thread.join() + + assert "sha256:bb" in caplog.text + + +async def test_session_close_forgets_what_was_loaded(): + s = session() + + await glow(session=s).output() + await s.close() + await glow(session=s).output() + + assert len(s.session.loads) == 2 + + +async def test_connection_yields_the_default_session(monkeypatch): + from dagger.provisioning import _connection + + class Engine: + def get_shared_client_connection(self): + return default_session().connection + + async def setup_client(self, conn): + return conn + + class provision_engine: # noqa: N801 + def __init__(self, cfg): + pass + + async def __aenter__(self): + return Engine() + + async def __aexit__(self, *_): + pass + + monkeypatch.setattr(_connection, "provision_engine", provision_engine) + + async with dagger.connection() as s: + assert s is default_session() + + +def test_session_with_no_connection_is_over_the_shared_one(): + from dagger.client._session import SharedConnection + + assert Session().connection is SharedConnection() + assert as_session(SharedConnection()) is default_session() + + +async def test_legacy_connection_yields_an_isolated_session(monkeypatch): + from dagger.provisioning import _connection + + class Engine: + def __init__(self, cfg, stack): + pass + + async def provision(self): + return self + + def get_client_connection(self): + return FakeConnection() + + async def setup_client(self, conn): + return conn + + monkeypatch.setattr(_connection, "Engine", Engine) + + async with dagger.Connection() as s: + assert isinstance(s, Session) + assert s is not default_session() + assert isinstance(s.connection, FakeConnection) diff --git a/sdk/tests/client/test_default_engine.py b/sdk/tests/client/test_default_engine.py new file mode 100644 index 0000000..6dc450c --- /dev/null +++ b/sdk/tests/client/test_default_engine.py @@ -0,0 +1,184 @@ +"""The default session provisions an engine in a plain program. + +Each program runs in its own interpreter: the shared connection is a +singleton, and closing at exit only happens when an interpreter exits. A fake +CLI stands in for `dagger session`: it prints connection params, and when its +stdin closes it waits a little before writing that it ended, so a program +that did not wait for it exits before the mark is there. +""" + +import os +import pathlib +import subprocess +import sys +import textwrap + +import pytest + +FAKE_CLI = """\ +#!{python} +import pathlib, sys, time +marks = pathlib.Path({marks!r}) +with (marks / "started").open("a") as f: + f.write("session\\n") +print('{{"port": 4242, "session_token": "fake"}}', flush=True) +sys.stdin.read() +time.sleep(0.5) +(marks / "ended").write_text("ended") +""" + + +@pytest.fixture +def marks(tmp_path: pathlib.Path) -> pathlib.Path: + cli = tmp_path / "dagger" + cli.write_text(FAKE_CLI.format(python=sys.executable, marks=str(tmp_path))) + cli.chmod(0o755) + return tmp_path + + +def run(program: str, marks: pathlib.Path, **env: str) -> subprocess.CompletedProcess: + environ = { + k: v + for k, v in os.environ.items() + if k not in ("DAGGER_SESSION_PORT", "DAGGER_SESSION_TOKEN") + } + environ["_EXPERIMENTAL_DAGGER_CLI_BIN"] = str(marks / "dagger") + environ.update(env) + return subprocess.run( + [sys.executable, "-c", textwrap.dedent(program)], + capture_output=True, + text=True, + env=environ, + timeout=60, + check=False, + ) + + +READY = """ +import anyio +from dagger.client._session import SharedConnection + +async def main(): + session = await SharedConnection()._ready() + print(session.conn.port) + +anyio.run(main) +""" + + +def test_a_plain_program_provisions_and_ends_the_engine_at_exit(marks): + proc = run(READY, marks) + + assert proc.returncode == 0, proc.stderr + assert proc.stdout.strip() == "4242" + assert (marks / "started").read_text() == "session\n" + # Written after the CLI's stdin closed and a pause: the program waited. + assert (marks / "ended").exists() + + +def test_a_session_in_the_environment_is_never_provisioned(marks): + proc = run(READY, marks, DAGGER_SESSION_PORT="5151", DAGGER_SESSION_TOKEN="t") + + assert proc.returncode == 0, proc.stderr + assert proc.stdout.strip() == "5151" + assert not (marks / "started").exists() + + +def test_without_provisioning_there_is_no_session(marks): + # A module's runtime has no dagger.provisioning. + proc = run( + """ + import importlib.abc, sys + + class NoProvisioning(importlib.abc.MetaPathFinder): + def find_spec(self, name, path=None, target=None): + if name.startswith("dagger.provisioning"): + raise ModuleNotFoundError(name, name=name) + + sys.meta_path.insert(0, NoProvisioning()) + """ + + textwrap.indent(READY, " "), + marks, + ) + + assert proc.returncode == 1 + assert "No active engine session to connect to" in proc.stderr + assert not (marks / "started").exists() + + +def test_concurrent_first_queries_provision_once(marks): + proc = run( + """ + import anyio + from dagger.client._session import SharedConnection + + async def main(): + async with anyio.create_task_group() as tg: + for _ in range(3): + tg.start_soon(SharedConnection()._ready) + + anyio.run(main) + """, + marks, + ) + + assert proc.returncode == 0, proc.stderr + assert (marks / "started").read_text().count("session") == 1 + + +def test_close_ends_the_engine_before_it_returns(marks): + proc = run( + """ + import os, pathlib, anyio, dagger + from dagger.client._session import SharedConnection + + marks = pathlib.Path(os.environ["_EXPERIMENTAL_DAGGER_CLI_BIN"]).parent + + async def main(): + await SharedConnection()._ready() + await dagger.close() + print((marks / "ended").exists()) + + anyio.run(main) + """, + marks, + ) + + assert proc.returncode == 0, proc.stderr + assert proc.stdout.strip() == "True" + + +MODULE_ENTRYPOINTS = { + # The runtime executable's entrypoint. + "cli": """ + import dagger.mod.cli as cli + from dagger.client._session import SharedConnection + + async def main(mod=None, register=False): + await SharedConnection()._ready() + + cli.main = main + cli.app() + """, + # The generated entrypoint's commands. + "python -m dagger.mod": """ + import anyio + import dagger.mod.__main__ as entry + from dagger.client._session import SharedConnection + + def call(args): + anyio.run(SharedConnection()._ready) + + entry._call = call + entry.main(["call", "--output", "/dev/null"]) + """, +} + + +@pytest.mark.parametrize("entry", MODULE_ENTRYPOINTS) +def test_a_module_never_provisions(marks, entry): + # Not even when its session is missing from the environment. + proc = run(MODULE_ENTRYPOINTS[entry], marks) + + assert "No active engine session to connect to" in proc.stderr + assert not (marks / "started").exists() diff --git a/sdk/tests/client/test_guards.py b/sdk/tests/client/test_guards.py new file mode 100644 index 0000000..9a08a1e --- /dev/null +++ b/sdk/tests/client/test_guards.py @@ -0,0 +1,19 @@ +import pytest + +from dagger.client._guards import type_error + + +def test_type_error_names_the_calling_module(): + err = type_error("Container.with_exec", "args", "nope", "list[str]") + + assert str(err) == ( + f"Method {__name__}.Container.with_exec() parameter args='nope' " + "expected to be of type list[str]." + ) + + +def test_generated_bindings_keep_their_message(): + from dagger.client.gen import dag + + with pytest.raises(TypeError, match=r"Method dagger\.client\.gen\.Container\."): + dag.container().with_exec("nope") diff --git a/sdk/tests/client/test_query_builder.py b/sdk/tests/client/test_query_builder.py index 88ed267..5938d00 100644 --- a/sdk/tests/client/test_query_builder.py +++ b/sdk/tests/client/test_query_builder.py @@ -3,7 +3,7 @@ import pytest -from dagger.client._core import Arg, Context, snake_to_camel, to_literal +from dagger.client._core import Arg, Context, EnumName, snake_to_camel, to_literal from dagger.client.base import Enum, Input, Scalar, Type @@ -190,3 +190,13 @@ class StrColor(str, enum.Enum): BLUE = "blue" assert to_literal(StrColor.BLUE) == "BLUE" + + +def test_enum_name_literal_is_bare(): + assert to_literal(EnumName("OBJECT_KIND")) == "OBJECT_KIND" + assert to_literal("OBJECT_KIND") == '"OBJECT_KIND"' + + +def test_enum_name_literal_rejects_query_syntax(): + with pytest.raises(Exception, match="Invalid enum value name"): + to_literal(EnumName("A) { id } #")) diff --git a/sdk/tests/client/test_sdk_isolation.py b/sdk/tests/client/test_sdk_isolation.py new file mode 100644 index 0000000..f6cd8ee --- /dev/null +++ b/sdk/tests/client/test_sdk_isolation.py @@ -0,0 +1,311 @@ +"""The hand-written SDK files depend on nothing generated. + +Only ``dagger/__init__.py`` may name generated code: the optional import of +the temporary global client. +""" + +import ast +import importlib.util +import pathlib +import re +import subprocess +import sys + +import pytest + +SRC = pathlib.Path( + next(iter(importlib.util.find_spec("dagger").submodule_search_locations)) +) +PACKAGE_INIT = SRC / "__init__.py" + +GENERATED_NAME_RE = re.compile( + r"(? list[pathlib.Path]: + return sorted(p for p in SRC.rglob("*.py") if p != PACKAGE_INIT) + + +def test_sdk_files_import_without_generated_code(): + proc = subprocess.run( + [sys.executable, "-c", IMPORT_ALL, str(SRC)], + capture_output=True, + text=True, + check=False, + ) + + assert proc.returncode == 0, proc.stderr + imported = proc.stdout.split() + assert "dagger.mod._module" in imported + assert "dagger.provisioning._engine" in imported + assert len(imported) == len(sdk_files()) - 1 + + +# The rule is that the SDK imports nothing generated, and the AST scan below +# enforces it. A help message naming the package a user has to import is not +# a dependency, so this one string is let through; every other name in the +# text still fails. +HELP_TEXT = { + pathlib.Path("client/_session.py"): "(from dagger_clients.core import core)", +} + + +@pytest.mark.parametrize("path", sdk_files(), ids=lambda p: str(p.relative_to(SRC))) +def test_sdk_file_text_names_no_generated_package(path: pathlib.Path): + """Catches a name in a string, which no import resolution sees.""" + allowed = HELP_TEXT.get(path.relative_to(SRC)) + found = [ + f"{path.relative_to(SRC)}:{number}: {line.strip()}" + for number, line in enumerate(path.read_text().splitlines(), 1) + if GENERATED_NAME_RE.search(line.replace(allowed, "") if allowed else line) + ] + + assert not found, "\n".join(found) + + +def _is_submodule(name: str) -> bool: + # Listed, not stat'ed: a case-insensitive filesystem matches Client to client/. + return name in {p.stem for p in SRC.iterdir()} + + +GENERATED = ("dagger.client.gen", "dagger_gen", "dagger_clients", "dagger_global") + + +def _is_generated(name: str) -> bool: + return any(name == g or name.startswith(g + ".") for g in GENERATED) + + +def _generated_imports(source: str, module: str) -> list[tuple[int, str]]: + """Every import that lands on generated code, wherever it sits in the file. + + Names are resolved to absolute before judging, so ``from .gen import X`` + counts the same as ``from dagger.client.gen import X``. + """ + package = module if _is_package(module) else module.rpartition(".")[0] + found = [] + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Import): + bad = [ + f"import {a.name}" + for a in node.names + if _is_generated(a.name) or a.name == "dagger" + ] + elif isinstance(node, ast.ImportFrom): + base = importlib.util.resolve_name( + "." * node.level + (node.module or ""), package + ) + bad = [ + f"from {base} import {a.name}" + for a in node.names + if _is_generated(base) + or _is_generated(f"{base}.{a.name}") + # A name the init provides may be generated; a submodule never is. + or (base == "dagger" and not _is_submodule(a.name)) + ] + elif isinstance(node, ast.Call) and (name := _literal_import(node, package)): + bad = [f"{_call_name(node)}({name!r})"] if _is_generated(name) else [] + else: + continue + found += [(node.lineno, stmt) for stmt in bad] + return found + + +def _call_name(call: ast.Call) -> str: + func = call.func + return func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", "") + + +def _literal_import(call: ast.Call, package: str) -> str | None: + """The absolute target of ``import_module``/``__import__`` on a literal name. + + A computed name can't be judged statically, so the rule stops at "no + static or literal-dynamic import of generated code"; the subprocess + import test is what catches the rest. + """ + if _call_name(call) not in ("import_module", "__import__") or not call.args: + return None + name = call.args[0] + if not isinstance(name, ast.Constant) or not isinstance(name.value, str): + return None + anchor = next( + (k.value for k in call.keywords if k.arg == "package"), + call.args[1] if len(call.args) > 1 else None, + ) + if isinstance(anchor, ast.Constant) and isinstance(anchor.value, str): + package = anchor.value + return importlib.util.resolve_name(name.value, package) + + +def _is_package(module: str) -> bool: + return (SRC.parent / module.replace(".", "/")).is_dir() + + +def _module_name(path: pathlib.Path) -> str: + parts = path.relative_to(SRC).with_suffix("").parts + if parts[-1] == "__init__": + parts = parts[:-1] + return ".".join(("dagger", *parts)) + + +@pytest.mark.parametrize("path", sdk_files(), ids=lambda p: str(p.relative_to(SRC))) +def test_sdk_file_imports_no_generated_package(path: pathlib.Path): + found = _generated_imports(path.read_text(), _module_name(path)) + + assert not found, "\n".join( + f"{path.relative_to(SRC)}:{line}: {stmt}" for line, stmt in found + ) + + +LAZY_MUTATION = """ +def root_type(): + from .gen import Client + + return Client +""" + +DYNAMIC_MUTATION = """ +def f(): + from importlib import import_module + return import_module(".gen", __package__) +""" + + +@pytest.mark.parametrize( + ("source", "module"), + [ + pytest.param(LAZY_MUTATION, "dagger.client.base", id="lazy-relative"), + pytest.param(DYNAMIC_MUTATION, "dagger.client.base", id="dynamic-relative"), + pytest.param( + "import importlib\nimportlib.import_module('dagger_gen')\n", + "dagger.log", + id="dynamic-attribute", + ), + pytest.param( + "import_module('.gen', package='dagger.client')\n", + "dagger.log", + id="dynamic-package-kwarg", + ), + pytest.param( + "__import__('dagger.client.gen')\n", "dagger.log", id="dunder-import" + ), + pytest.param("from . import gen\n", "dagger.client.base", id="from-dot"), + pytest.param("from .. import gen\n", "dagger.client.sub.x", id="from-dotdot"), + pytest.param( + "from ..client import gen\n", + "dagger.provisioning._engine", + id="sibling-package", + ), + pytest.param( + "from dagger.client import gen\n", "dagger.mod._module", id="from-parent" + ), + pytest.param( + "from typing import TYPE_CHECKING\n" + "if TYPE_CHECKING:\n" + " from dagger.client.gen import Client\n", + "dagger.mod._module", + id="type-checking", + ), + pytest.param( + "class A:\n def m(self):\n import dagger.client.gen\n", + "dagger.mod._module", + id="method-body", + ), + pytest.param("import dagger_gen\n", "dagger.log", id="dagger_gen"), + pytest.param("import dagger_global\n", "dagger.log", id="dagger_global"), + pytest.param( + "from dagger_clients.core import core\n", "dagger.log", id="dagger_clients" + ), + pytest.param("def f():\n import dagger\n", "dagger.log", id="lazy-package"), + pytest.param("from . import Client\n", "dagger.log", id="dot-is-the-init"), + pytest.param( + "from ... import Client\n", "dagger.client.sub.x", id="dots-to-the-init" + ), + ], +) +def test_guard_rejects_generated_import(source: str, module: str): + assert _generated_imports(source, module) + + +@pytest.mark.parametrize( + ("source", "module"), + [ + pytest.param( + "from ._core import Context\n", "dagger.client.base", id="sibling" + ), + pytest.param("from . import _core\n", "dagger.client.base", id="from-dot"), + pytest.param("from dagger import mod\n", "dagger.log", id="submodule"), + pytest.param( + "from dagger.client.base import Root\n", "dagger.mod._module", id="absolute" + ), + pytest.param("import dagger_gen_tools\n", "dagger.log", id="prefix-only"), + pytest.param("import gen\n", "dagger.client.base", id="third-party-gen"), + pytest.param( + "import_module(name)\n", "dagger.client.base", id="computed-dynamic" + ), + pytest.param( + "import_module('.' + name, __package__)\n", + "dagger.client.base", + id="computed-relative", + ), + pytest.param( + "import_module('._core', __package__)\n", + "dagger.client.base", + id="dynamic-sibling", + ), + ], +) +def test_guard_accepts_sdk_import(source: str, module: str): + assert not _generated_imports(source, module) + + +def test_package_init_names_only_the_global_client(): + """The init's generated imports are the optional global client's. + + One for type checkers, and the lazy one at run time. + """ + found = [stmt for _, stmt in _generated_imports(PACKAGE_INIT.read_text(), "dagger")] + + assert found == ["from dagger_global import *", "import_module('dagger_global')"] diff --git a/sdk/tests/codegen/test_packages.py b/sdk/tests/codegen/test_packages.py new file mode 100644 index 0000000..3645ca1 --- /dev/null +++ b/sdk/tests/codegen/test_packages.py @@ -0,0 +1,1122 @@ +import ast +import json +import sys +import types +from textwrap import dedent, indent + +import graphql +import pytest +from graphql import build_schema + +import dagger.client +from codegen import cli, partition +from codegen.packages import ( + SESSION_NAMES, + client_package, + core_package, + global_package, + write_global, + write_package, +) +from codegen.partition import ClientError, ClientNameError, core_digest +from dagger.client import Session, Target +from dagger.client._core import Context +from dagger.client._session import SharedConnection + +_CORE = """ + directive @sourceMap(module: String, filename: String) + on OBJECT | FIELD_DEFINITION | ENUM | ENUM_VALUE | INPUT_OBJECT + directive @expectedType(name: String!) on FIELD_DEFINITION | ARGUMENT_DEFINITION + + enum Severity { LOW HIGH } + type Directory { + id: ID! @expectedType(name: "Directory") + entries: [String!]! + } + type File { id: ID! @expectedType(name: "File") } +""" + +_LINTER = """ + type Linter @sourceMap(module: "linter") { + id: ID! @expectedType(name: "Linter") + lint(src: ID! @expectedType(name: "Directory"), level: Severity): String! + report: LinterReport! + } + type LinterReport @sourceMap(module: "linter") { errors: Int! } +""" + +_GLOW = """ + type Glow @sourceMap(module: "glow") { render(text: String!): String! } +""" + +_MEMBERS = { + _LINTER: { + "Binding": 'asLinter: Linter! @sourceMap(module: "linter")', + "Query": """ + linter( + source: ID! @expectedType(name: "Directory"), + config: String + ): Linter! @sourceMap(module: "linter") + """, + }, + _GLOW: { + "Binding": 'asGlow: Glow! @sourceMap(module: "glow")', + "Query": 'glow: Glow! @sourceMap(module: "glow")', + }, +} + + +def _sdl(*clients: str, **members: str) -> str: + """Core and the given clients, with the fields they contribute to core types. + + A keyword replaces what the clients contribute to that type. + """ + fields = { + "Binding": ["name: String!"], + "Env": ["name: String!"], + "Query": ["directory: Directory!"], + } + for contributed in (*(_MEMBERS[c] for c in clients), members): + for type_name, field in contributed.items(): + if contributed is members or type_name not in members: + fields[type_name].append(field) + return ( + _CORE + + "".join(clients) + + "".join(f"type {name} {{ {' '.join(f)} }}" for name, f in fields.items()) + ) + + +def _schema(*clients: str, **members: str) -> graphql.GraphQLSchema: + return build_schema(_sdl(*clients, **members)) + + +def _linter(*clients: str, **members: str) -> str: + _, files = client_package(_schema(*clients, **members), "linter", "./linter") + return files["__init__.py"] + + +def test_core_holds_no_client_type(): + code = core_package(_schema(_LINTER, _GLOW))["__init__.py"] + + assert "class Directory(_Type):" in code + assert "class Binding(_Type):" in code + assert "Linter" not in code + assert "Glow" not in code + assert "def linter(" not in code + assert "def glow(" not in code + + +def test_core_entry_point(): + schema = _schema(_LINTER) + files = core_package(schema) + code = files["__init__.py"] + + assert files["py.typed"] == "" + assert "def core(*, session: _Session | None = None) -> Query:" in code + assert "return _client_root(Query, None, None, [], session=session)" in code + assert "check_core" not in code.replace("check_core as _check_core", "") + assert f'CORE_DIGEST = "{core_digest(schema)}"' in code + assert '"CORE_DIGEST",' in code + assert '"core",' in code + # Only the temporary global client has them. + assert "class Client(" not in code + assert "dag = " not in code + + +def test_core_is_the_same_whatever_the_clients(): + core = core_package(_schema()) + + assert core_package(_schema(_LINTER)) == core + assert core_package(_schema(_LINTER, _GLOW)) == core + + +def test_legacy_core_is_the_same_whatever_the_clients(): + core = core_package(_schema(), "v0.20.0") + # A client type named like a legacy ID must not take the class out of core. + taken = 'type DirectoryID @sourceMap(module: "linter") { size: Int! }' + schema = build_schema(_sdl(_LINTER) + taken) + + assert "class DirectoryID(_Scalar):" in core["__init__.py"] + assert core_package(schema, "v0.20.0") == core + + +def _digest(files: dict[str, str]) -> str: + code = files["__init__.py"] + return code[code.index("CORE_DIGEST = ") :].splitlines()[0] + + +def test_core_digest_follows_the_compatibility_mode_only(): + schema = _schema() + + assert _digest(core_package(schema, "v0.21.0")) == _digest( + core_package(schema, "v0.22.0") + ) + assert _digest(core_package(schema, "v0.20.0")) != _digest( + core_package(schema, "v0.21.0") + ) + + +def test_client_holds_its_own_types(): + code = _linter(_LINTER, _GLOW) + + assert "class Linter(_Type):" in code + assert "class LinterReport(_Type):" in code + assert "async def lint(self, src: Directory, *, level: Severity" in code + assert "class Glow(" not in code + assert "as_glow" not in code + assert "class Directory(" not in code + assert "class Query(" not in code + + +def _imports(code: str, module: str) -> dict[str, str]: + """Names imported from a module, by the name they get.""" + return { + alias.asname or alias.name: alias.name + for node in ast.parse(code).body + if isinstance(node, ast.ImportFrom) + and f"{'.' * node.level}{node.module}" == module + for alias in node.names + } + + +def _defs(code: str, name: str) -> list[ast.FunctionDef | ast.AsyncFunctionDef]: + return [ + node + for node in ast.parse(code).body + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) + and node.name == name + ] + + +def test_client_imports_the_core_types_it_names(): + code = _linter(_LINTER, _GLOW) + + imports = _imports(code, "dagger_clients.core") + assert {imports[n] for n in ("Binding", "Directory", "Severity")} == { + "Binding", + "Directory", + "Severity", + } + assert "File" not in imports.values() + assert "Glow" not in imports.values() + assert _imports(code, "._target") == { + "CORE_DIGEST": "CORE_DIGEST", + "NAME": "NAME", + "PIN": "PIN", + "REF": "REF", + } + assert "\n_TARGET = _Target(name=NAME, ref=REF, pin=PIN)\n" in code + + +def test_client_checks_core_before_importing_its_symbols(): + # A stale core that dropped a symbol must fail on check_core, with its + # "run dagger generate" message, not on a plain ImportError before it. + code = _linter(_LINTER, _GLOW) + + statements = [ast.unparse(node) for node in ast.parse(code).body] + digest = statements.index( + "from dagger_clients.core import CORE_DIGEST as _installed_core" + ) + check = statements.index("_check_core(NAME, CORE_DIGEST, _installed_core)") + symbols = [ + i + for i, statement in enumerate(statements) + if statement.startswith("from dagger_clients.core import") + and "CORE_DIGEST" not in statement + ] + assert digest < check + assert symbols + assert all(check < i for i in symbols) + + +def test_client_entry_function(): + code = _linter(_LINTER) + + assert ( + "def linter(source: Directory, *, config: str | None = None, " + "session: _Session | None = None,) -> Linter:" + ) in code + assert 'raise _type_error("linter", "source", source, "Directory")' in code + assert ( + indent( + dedent( + """\ + _args = [ + _Arg("source", source), + _Arg("config", config, None), + ] + return _client_root(Linter, _TARGET, "linter", _args, session=session) + """ + ), + " ", + ) + in code + ) + + +def test_client_entry_function_renames_a_session_argument(): + query = 'linter(session: String): Linter! @sourceMap(module: "linter")' + _, files = client_package(_schema(_LINTER, Query=query), "linter", "./linter") + code = files["__init__.py"] + + assert ( + "def linter(*, session_: str | None = None, " + "session: _Session | None = None,) -> Linter:" + ) in code + assert '_Arg("session", session_, None),' in code + + +def test_client_contributed_field(): + code = _linter(_LINTER, _GLOW) + + assert ( + dedent( + """ + def as_linter(binding: Binding, /) -> Linter: + _args: list[_Arg] = [] + _ctx = _client_select(binding, _TARGET, "asLinter", _args) + return Linter(_ctx) + """ + ) + in code + ) + assert '"as_linter",' in code + assert "@overload" not in code + + +def test_client_contributed_field_executes_a_leaf(): + env = 'linterCount: Int! @sourceMap(module: "linter")' + code = _linter(_LINTER, Env=env) + + assert "async def linter_count(env: Env, /) -> int:" in code + assert '_ctx = _client_select(env, _TARGET, "linterCount", _args)' in code + assert "return await _ctx.execute(int)" in code + + +def test_client_contributed_field_that_returns_an_id_of_its_receiver(): + binding = 'linted: ID! @expectedType(name: "Binding") @sourceMap(module: "linter")' + code = _linter(_LINTER, Binding=binding) + + assert "async def linted(binding: Binding, /) -> Binding:" in code + # Through client_select like any contributed field, so that the module is + # loaded before the query, never straight through the receiver's context. + assert '_ctx = _client_select(binding, _TARGET, "linted", _args)' in code + assert "binding._ctx" not in code + assert "return Binding(" in code + + +def test_client_contributed_field_receiver_avoids_an_argument_name(): + env = """ + withLinter(env: ID @expectedType(name: "Env")): Env! + @sourceMap(module: "linter") + """ + code = _linter(_LINTER, Env=env) + + assert "def with_linter(env_: Env, /, *, env: Env | None = None) -> Env:" in code + assert '_ctx = _client_select(env_, _TARGET, "withLinter", _args)' in code + + +def test_client_overloads_one_name_on_two_receivers(): + env = 'asLinter(strict: Boolean): Linter! @sourceMap(module: "linter")' + code = _linter(_LINTER, Env=env) + + *overloads, dispatcher = _defs(code, "as_linter") + assert [ast.unparse(d.args) for d in overloads] == [ + "binding: Binding, /", + "env: Env, /, *, strict: bool | None=None", + ] + assert all(ast.unparse(d.decorator_list) == "_overload" for d in overloads) + assert not dispatcher.decorator_list + # One selection per receiver, each with its own field arguments. + assert '_client_select(binding, _TARGET, "asLinter", _args)' in code + assert '_client_select(env, _TARGET, "asLinter", _args)' in code + assert '_Arg("strict", strict, None)' in code + exported = code[code.index("__all__") :] + assert exported.count('"as_linter",') == 1 + assert exported.count('"') == 2 * len( + ("Linter", "LinterReport", "as_linter", "linter") + ) + compile(code, "linter", "exec") + + +def _names(code: str) -> tuple[set[str], set[str]]: + """Names a module imports, and names it defines at the top level.""" + tree = ast.parse(code) + imported = { + alias.asname or alias.name + for node in tree.body + if isinstance(node, ast.Import | ast.ImportFrom) + for alias in node.names + } + defined = { + node.name + for node in tree.body + if isinstance(node, ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef) + } + return imported, defined + + +@pytest.mark.parametrize("name", ["Arg", "Type", "Session", "Callable"]) +def test_client_type_named_like_a_helper_does_not_shadow_it(name: str): + own = f'type {name} @sourceMap(module: "linter") {{ value: Int! }}' + _, files = client_package(build_schema(_sdl(_LINTER) + own), "linter", ".") + code = files["__init__.py"] + + assert f"class {name}(" in code + imported, defined = _names(code) + assert not imported & defined + # The entry function still builds its arguments with the query builder. + assert '_Arg("source", source)' in code + + +def test_core_type_named_like_a_helper_does_not_shadow_it(): + own = "type Type { value: Int! }" + code = core_package(build_schema(_sdl() + own))["__init__.py"] + + assert "class Type(" in code + imported, defined = _names(code) + assert not imported & defined + + +def test_client_is_the_same_whatever_the_other_clients(): + alone = client_package(_schema(_LINTER), "linter", "./linter") + + assert client_package(_schema(_LINTER, _GLOW), "linter", "./linter") == alone + assert client_package(_schema(_GLOW, _LINTER), "linter", "./linter") == alone + + +@pytest.mark.parametrize("schema_version", ["v0.20.0", "v0.21.0"]) +def test_packages_compile(schema_version: str): + env = 'asLinter(strict: Boolean): Linter! @sourceMap(module: "linter")' + schema = _schema(_LINTER, _GLOW, Env=env) + _, client = client_package(schema, "linter", ".", schema_version=schema_version) + + for files in (core_package(schema, schema_version), client): + for name, content in files.items(): + compile(content, name, "exec") + + +def test_core_type_whose_fields_are_all_contributed_compiles(): + # Partitioning can leave a class with no body of its own. + query = 'type Query { linter: Linter! @sourceMap(module: "linter") }' + binding = "type Binding { name: String! }" + schema = build_schema(_CORE + _LINTER + query + binding) + + code = core_package(schema)["__init__.py"] + + assert "class Query(_Root):\n ...\n" in code + compile(code, "core", "exec") + + +def test_interface_with_no_own_member_compiles(): + # id is on every Type, so the protocol of this interface has no member. + bare = 'interface Bare { id: ID! @expectedType(name: "Bare") }' + schema = build_schema(_sdl() + bare) + + code = core_package(schema)["__init__.py"] + + assert "class Bare(_Protocol):\n ...\n" in code + compile(code, "core", "exec") + + +def test_target_is_plain_data(): + schema = _schema(_LINTER) + package, files = client_package(schema, "linter", "./modules/linter") + + assert package == "linter" + assert files["py.typed"] == "" + assert files["_target.py"] == dedent( + f"""\ + # Code generated by dagger. DO NOT EDIT. + + NAME = "linter" + REF = "./modules/linter" + PIN = None + CORE_DIGEST = "{core_digest(schema)}" + """ + ) + + +def test_target_with_a_pin_and_a_given_core_digest(): + schema = _schema(_GLOW) + _, files = client_package( + schema, + "glow", + "github.com/eunomie/glow", + "4f1c9e", + core_digest=core_digest(schema), + ) + + assert 'NAME = "glow"' in files["_target.py"] + assert 'REF = "github.com/eunomie/glow"' in files["_target.py"] + assert 'PIN = "4f1c9e"' in files["_target.py"] + assert f'CORE_DIGEST = "{core_digest(schema)}"' in files["_target.py"] + assert "import" not in files["_target.py"] + + +@pytest.mark.parametrize("given", ["", "sha256:other"]) +def test_client_refuses_a_core_digest_that_is_not_its_schema_core(given: str): + # Otherwise check_core would bless a client generated against other core + # types, which is the version skew it is there to catch. + schema = _schema(_GLOW) + + with pytest.raises(ClientError, match=f'"{given}".*{core_digest(schema)}'): + client_package(schema, "glow", "./glow", core_digest=given) + + +class _Runtime: + """Fake of what the generated code needs from dagger.client, recording calls.""" + + def __init__(self) -> None: + self.selected: list[tuple] = [] + self.rooted: list[tuple] = [] + + class Session: ... + + class Target: + def __init__(self, **kwargs) -> None: + self.kwargs = kwargs + + def check_core(self, name: str, wanted: str, installed: str) -> None: + assert wanted == installed, name + + def client_root(self, cls, target, name, args, *, session=None): + self.rooted.append((cls, target, name, args)) + return cls(Context()) + + def client_select(self, receiver, target, name, args): + self.selected.append((receiver, target, name, args)) + return Context() + + +@pytest.fixture +def runtime(monkeypatch): + """Load generated packages against a fake runtime; returns the loader.""" + fake = _Runtime() + for name in ("Session", "Target", "check_core", "client_root", "client_select"): + monkeypatch.setattr(dagger.client, name, getattr(fake, name), raising=False) + namespace = types.ModuleType("dagger_clients") + monkeypatch.setitem(sys.modules, "dagger_clients", namespace) + + def _module(name: str, code: str) -> types.ModuleType: + module = types.ModuleType(name) + module.__package__ = name.rpartition(".")[0] or name + monkeypatch.setitem(sys.modules, name, module) + exec(compile(code, name, "exec"), module.__dict__) + return module + + def load(schema: graphql.GraphQLSchema, name: str) -> types.ModuleType: + _module("dagger_clients.core", core_package(schema)["__init__.py"]) + package, files = client_package(schema, name, ".") + client = f"dagger_clients.{package}" + module = types.ModuleType(client) + module.__package__ = client + monkeypatch.setitem(sys.modules, client, module) + _module(f"{client}._target", files["_target.py"]) + exec(compile(files["__init__.py"], client, "exec"), module.__dict__) + return module + + load.selected = fake.selected # type: ignore[attr-defined] + load.rooted = fake.rooted # type: ignore[attr-defined] + load.Target = fake.Target # type: ignore[attr-defined] + return load + + +_ANIMALS = """ + interface Animal { + id: ID! @expectedType(name: "Animal") + sound: String! + asLinter: Linter! @sourceMap(module: "linter") + } + type Zebra implements Animal { + id: ID! @expectedType(name: "Zebra") + sound: String! + asLinter(strict: Boolean!): Linter! @sourceMap(module: "linter") + } +""" + + +def test_overload_dispatches_on_the_exact_type_of_the_receiver(runtime): + # A Zebra is an Animal structurally, so an isinstance chain would pick the + # Animal hook first and reject Zebra's own argument. + schema = build_schema(_sdl(_LINTER) + _ANIMALS) + linter = runtime(schema, "linter") + core = sys.modules["dagger_clients.core"] + zebra = core.Zebra(Context()) + + result = linter.as_linter(zebra, strict=True) + + assert isinstance(result, linter.Linter) + receiver, _, name, args = runtime.selected[-1] + assert receiver is zebra + assert name == "asLinter" + assert [(a.name, a.value) for a in args] == [("strict", True)] + + +def test_overload_dispatches_an_implementation_to_its_interface_hook(runtime): + # With no hook of its own, a Zebra is still an Animal. + animals = _ANIMALS.replace( + 'asLinter(strict: Boolean!): Linter! @sourceMap(module: "linter")', "" + ) + schema = build_schema(_sdl(_LINTER) + animals) + linter = runtime(schema, "linter") + core = sys.modules["dagger_clients.core"] + zebra = core.Zebra(Context()) + + assert isinstance(linter.as_linter(zebra), linter.Linter) + assert runtime.selected[-1][0] is zebra + with pytest.raises(TypeError, match="as_linter"): + linter.as_linter("zebra") + + +def test_client_that_names_no_core_symbol_compiles(runtime): + # Nothing to import from core but the digest: no empty import block. + schema = build_schema(_sdl() + _named("solo", "Solo", "solo")) + + solo = runtime(schema, "solo") + + assert isinstance(solo.solo(), solo.Solo) + assert "import (" not in client_package(schema, "solo", ".")[1]["__init__.py"] + + +def test_client_accepts_any_object_for_a_generic_id(runtime): + # No @expectedType: any Type is accepted, under the alias the package has. + query = 'linter(source: ID!): Linter! @sourceMap(module: "linter")' + schema = _schema(_LINTER, Query=query) + code = client_package(schema, "linter", ".")[1]["__init__.py"] + + assert "def linter(source: _Type, *, session: _Session | None = None,)" in code + linter = runtime(schema, "linter") + directory = sys.modules["dagger_clients.core"].Directory(Context()) + assert isinstance(linter.linter(directory), linter.Linter) + with pytest.raises(TypeError, match="linter"): + linter.linter("source") + + +def test_client_type_named_target_does_not_shadow_the_descriptor(runtime): + own = 'type TARGET @sourceMap(module: "linter") { value: Int! }' + schema = build_schema(_sdl(_LINTER) + own) + + linter = runtime(schema, "linter") + directory = sys.modules["dagger_clients.core"].Directory(Context()) + linter.linter(directory) + + _, target, _, _ = runtime.rooted[-1] + assert isinstance(target, runtime.Target) + assert target.kwargs == {"name": "linter", "ref": ".", "pin": None} + assert ( + "class TARGET(_Type):" + in client_package(schema, "linter", ".")[1]["__init__.py"] + ) + + +def _named(module: str, root: str, constructor: str) -> str: + """A client whose name the engine turned into a type and a constructor.""" + return f""" + type {root} @sourceMap(module: "{module}") {{ run: String! }} + extend type Query {{ {constructor}: {root}! @sourceMap(module: "{module}") }} + """ + + +def test_client_name_becomes_a_package(): + schema = build_schema( + _sdl() + _named("My-Project.dev", "MyProjectDev", "myProjectDev") + ) + package, files = client_package(schema, "my-project.dev", ".") + + assert package == "my_project_dev" + assert ( + "def my_project_dev(*, session: _Session | None = None) -> MyProjectDev:" + in (files["__init__.py"]) + ) + # The field name is the engine's, so the SDK never derives it. + assert ( + '_client_root(MyProjectDev, _TARGET, "myProjectDev", _args, session=session)' + in files["__init__.py"] + ) + # The descriptor pins the name as given, which is the one the engine knows. + assert 'NAME = "my-project.dev"' in files["_target.py"] + + +@pytest.mark.parametrize( + ("module", "reason"), + [ + ("class", "keyword"), + ("core", "core bindings"), + ("_hidden", 'starts with "_"'), + ("my linter", "not a Python identifier"), + ], +) +def test_client_name_refused(module: str, reason: str): + schema = build_schema(_sdl() + _named(module, "Thing", "thing")) + + with pytest.raises(ClientNameError, match=reason): + client_package(schema, module, ".") + + +def test_client_name_refused_when_two_clients_become_one_package(): + schema = build_schema( + _sdl() + + _named("my-linter", "MyLinter", "myLinter") + + _named("my.linter", "MyLinter2", "myLinter2") + ) + + with pytest.raises(ClientNameError, match='both become the package "my_linter"'): + client_package(schema, "my-linter", ".") + + +def test_client_missing_from_the_schema(): + with pytest.raises( + ClientError, match='nothing to the client "glow"; it has: linter' + ): + client_package(_schema(_LINTER), "glow", ".") + + +def test_packages_refuse_a_wrong_attribution(): + # Otherwise the field lands in the linter package, and glow gets nothing. + stray = """ + type Glow @sourceMap(module: "glow") { + lint: Int! @sourceMap(module: "linter") + } + """ + schema = build_schema(_sdl(_LINTER) + stray) + + with pytest.raises(ClientError, match=r'"Glow\.lint"'): + client_package(schema, "linter", ".") + with pytest.raises(ClientError, match=r'"Glow\.lint"'): + core_package(schema) + + +def test_client_refuses_a_type_of_another_client(): + env = 'glowLinter: Glow! @sourceMap(module: "linter")' + + with pytest.raises(ClientError, match=r'"Env\.glowLinter" .* names "Glow"'): + _linter(_LINTER, _GLOW, Env=env) + + +def _introspection(sdl: str) -> dict: + """Introspection result, with directives the way the engine adds them.""" + schema = build_schema(sdl) + + def directives(node) -> list[dict]: + return [ + { + "name": d.name.value, + "args": [ + {"name": a.name.value, "value": graphql.print_ast(a.value)} + for a in d.arguments + ], + } + for d in (node.directives if node else ()) + ] + + result = graphql.introspection_from_schema(schema) + for tp in result["__schema"]["types"]: + named = schema.type_map[tp["name"]] + tp["directives"] = directives(named.ast_node) + for field in tp["fields"] or (): + member = named.fields[field["name"]] + field["directives"] = directives(member.ast_node) + for arg in field["args"]: + arg["directives"] = directives(member.args[arg["name"]].ast_node) + for field in tp["inputFields"] or (): + field["directives"] = directives(named.fields[field["name"]].ast_node) + for value in tp["enumValues"] or (): + value["directives"] = directives(named.values[value["name"]].ast_node) + return {**result, "__schemaVersion": "v0.21.0"} + + +@pytest.fixture +def introspection(tmp_path): + path = tmp_path / "schema.json" + path.write_text(json.dumps(_introspection(_sdl(_LINTER, _GLOW)))) + return path + + +def test_cli_generates_packages(tmp_path, introspection): + out = tmp_path / "src" + + cli.main(["generate-core", "-i", str(introspection), "-o", str(out)]) + core = (out / "dagger_clients/core/__init__.py").read_text() + digest = core[core.index("CORE_DIGEST = ") :].splitlines()[0] + + # In real use, the caller hands the client the digest of the core it + # generated, because the two must match for the client to import. + cli.main( + [ + "generate-client", + *("-i", str(introspection)), + *("-o", str(out)), + *("--name", "linter"), + *("--ref", "github.com/acme/linter"), + *("--pin", "4f1c9e"), + *("--core-digest", digest.removeprefix("CORE_DIGEST = ").strip('"')), + ] + ) + cli.main( + [ + "generate-client", + *("-i", str(introspection)), + *("-o", str(out)), + *("--name", "glow"), + *("--ref", "./glow"), + ] + ) + + assert sorted(str(p.relative_to(out)) for p in out.rglob("*") if p.is_file()) == [ + "dagger_clients/core/__init__.py", + "dagger_clients/core/py.typed", + "dagger_clients/glow/__init__.py", + "dagger_clients/glow/_target.py", + "dagger_clients/glow/py.typed", + "dagger_clients/linter/__init__.py", + "dagger_clients/linter/_target.py", + "dagger_clients/linter/py.typed", + ] + client = (out / "dagger_clients/linter/__init__.py").read_text() + target = (out / "dagger_clients/linter/_target.py").read_text() + assert "Linter" not in core + assert "def as_linter(binding: Binding, /) -> Linter:" in client + assert 'PIN = "4f1c9e"' in target + assert digest in target + # Without the flag, the digest is the one of the same schema's core. + assert digest in (out / "dagger_clients/glow/_target.py").read_text() + + +def test_cli_refuses_a_core_digest_of_another_core(tmp_path, introspection, capsys): + args = ["generate-client", "-i", str(introspection), "-o", str(tmp_path)] + + with pytest.raises(SystemExit): + cli.main([*args, "--name", "glow", "--ref", ".", "--core-digest", "sha256:x"]) + + assert '"sha256:x"' in capsys.readouterr().err + assert not (tmp_path / "dagger_clients").exists() + + +@pytest.mark.parametrize("flag", ["--name", "--ref"]) +def test_cli_refuses_an_empty_name_or_ref(tmp_path, introspection, capsys, flag): + args = ["generate-client", "-i", str(introspection), "-o", str(tmp_path)] + given = {"--name": "glow", "--ref": ".", flag: ""} + + with pytest.raises(SystemExit): + cli.main([*args, *(a for f, v in given.items() for a in (f, v))]) + + assert f"argument {flag}" in capsys.readouterr().err + assert not (tmp_path / "dagger_clients").exists() + + +def test_cli_reads_what_write_package_writes(tmp_path): + schema = _schema(_LINTER) + package, files = client_package(schema, "linter", ".") + + root = write_package(tmp_path, package, files) + + assert root == tmp_path / "dagger_clients" / "linter" + assert {p.name: p.read_text() for p in root.iterdir()} == files + + +def test_cli_refuses_a_client_name(tmp_path, introspection, capsys): + args = ["generate-client", "-i", str(introspection), "-o", str(tmp_path)] + + with pytest.raises(SystemExit): + cli.main([*args, "--name", "core", "--ref", "."]) + + assert 'client name "core" is taken by the core bindings' in capsys.readouterr().err + assert not (tmp_path / "dagger_clients").exists() + + +def test_cli_still_generates_one_file(tmp_path, introspection): + output = tmp_path / "gen.py" + + cli.main(["generate", "-i", str(introspection), "-o", str(output)]) + + code = output.read_text() + assert "class Linter(Type):" in code + assert "class Client(Query):" in code + assert "dag = Client()" in code + + +# The temporary global client, generated only with the flag. + + +def _global(*clients: str, **members: str) -> str: + return global_package([_schema(*clients, **members)])["__init__.py"] + + +def _exported(code: str) -> set[str]: + node = next( + n + for n in ast.parse(code).body + if isinstance(n, ast.Assign) and ast.unparse(n.targets[0]) == "__all__" + ) + return set(ast.literal_eval(node.value)) + + +def test_global_client_is_temporary_and_says_so(): + code = _global(_LINTER) + + assert "Temporary" in code + assert "global-client = true" in code + assert "dagger generate" in code + + +def test_global_client_delegates_root_fields_to_core(): + code = _global(_LINTER) + + assert "from dagger.client import Session as _Session" in code + assert "class Client(_Session):" in code + assert ( + " def directory(self) -> Directory:\n" + " return _core.core(session=self).directory()\n" + ) in code + assert "\ndag = Client()\n" in code + assert "from dagger_clients.core import *" in code + + +def test_global_client_has_one_method_per_client(): + code = _global(_LINTER, _GLOW) + + assert ( + " def linter(self, source: Directory, *, config: str | None = None,) " + "-> Linter:\n" + " return _linter.linter(source, config=config, session=self)\n" + ) in code + assert ( + " def glow(self) -> Glow:\n return _glow.glow(session=self)\n" in code + ) + assert "import dagger_clients.linter as _linter" in code + assert "import dagger_clients.glow as _glow" in code + + +def test_global_client_keeps_the_legacy_name_of_a_session_argument(): + query = 'linter(session: String): Linter! @sourceMap(module: "linter")' + code = _global(_LINTER, Query=query) + + assert "def linter(self, *, session: str | None = None) -> Linter:" in code + assert "return _linter.linter(session_=session, session=self)" in code + + +def test_global_client_has_a_method_for_a_contributed_root_field(): + query = 'lintAll(strict: Boolean!): String! @sourceMap(module: "linter")' + code = _global(_LINTER, Query=query) + + # No entry: the schema has no constructor, but the global client still + # carries the field the way the legacy dag did. + assert ( + " async def lint_all(self, strict: bool) -> str:\n" + " return await _linter.lint_all(_core.core(session=self), strict)\n" + ) in code + + +def test_global_client_puts_contributed_fields_on_the_core_classes(): + env = 'asLinter(strict: Boolean): Linter! @sourceMap(module: "linter")' + schema = build_schema(_sdl(_LINTER, _GLOW, Env=env) + _ANIMALS) + code = global_package([schema])["__init__.py"] + + assert "Binding.as_linter = _linter.as_linter # type: ignore[attr-defined]" in code + assert "Env.as_linter = _linter.as_linter # type: ignore[attr-defined]" in code + assert "Binding.as_glow = _glow.as_glow # type: ignore[attr-defined]" in code + assert "Zebra.as_linter = _linter.as_linter" in code + assert "_AnimalClient.as_linter = _linter.as_linter" in code + assert code.count("_AnimalClient.as_linter =") == 1 + assert "_AnimalClient,\n" in code + + +def test_global_client_exports_the_types_and_itself_only(): + exported = _exported(_global(_LINTER, _GLOW)) + + assert {"Directory", "Binding", "Query", "Severity", "File"} <= exported + assert {"Linter", "LinterReport", "Glow"} <= exported + assert {"Client", "dag"} <= exported + assert not exported & {"core", "CORE_DIGEST", "linter", "glow", "as_linter"} + assert not {n for n in exported if n.startswith("_")} + + +def test_global_client_takes_one_schema_per_client(): + together = global_package([_schema(_LINTER, _GLOW)]) + + assert global_package([_schema(_LINTER), _schema(_GLOW)]) == together + assert global_package([_schema(_GLOW), _schema(_LINTER)]) == together + + +def test_global_client_refuses_schemas_of_two_cores(): + other = build_schema(_sdl(_GLOW) + "type Extra { n: Int! }") + + with pytest.raises(ClientError, match="two cores"): + global_package([_schema(_LINTER), other]) + + +def test_global_client_refuses_two_clients_that_become_one_package(): + schemas = [ + build_schema(_sdl() + _named("my-linter", "MyLinter", "myLinter")), + build_schema(_sdl() + _named("my.linter", "MyLinter2", "myLinter2")), + ] + + with pytest.raises(ClientNameError, match='both become the package "my_linter"'): + global_package(schemas) + + +@pytest.mark.parametrize("field", ["close", "load", "execute"]) +def test_global_client_refuses_a_core_field_that_hides_the_session(field: str): + schema = _schema(Query=f"{field}: String!") + + with pytest.raises(ClientError) as info: + global_package([schema]) + + assert str(info.value) == ( + f'the global client cannot have a method for "Query.{field}" of core: ' + f"it would hide Session.{field}" + ) + + +def test_global_client_refuses_a_client_that_hides_the_session(): + schema = build_schema(_sdl() + _named("connect", "Connect", "connect")) + + with pytest.raises(ClientError) as info: + global_package([schema]) + + assert str(info.value) == ( + 'the global client cannot have a method for "Query.connect" of the ' + 'client "connect": it would hide Session.connect' + ) + + +def test_global_client_knows_every_name_of_a_session(): + # The generator cannot import the SDK, so it keeps its own list. + instance = Session(SharedConnection()) + + assert {n for n in dir(instance) if not n.startswith("__")} == SESSION_NAMES + + +def test_global_client_with_no_client(): + code = _global() + + assert "def directory(self) -> Directory:" in code + assert "import dagger_clients.core as _core" in code + assert code.count("dagger_clients.") == 2 + compile(code, "dagger_global", "exec") + + +@pytest.mark.parametrize("schema_version", ["v0.20.0", "v0.21.0"]) +def test_global_package_compiles(schema_version: str): + env = 'asLinter(strict: Boolean): Linter! @sourceMap(module: "linter")' + schema = build_schema(_sdl(_LINTER, _GLOW, Env=env) + _ANIMALS) + + files = global_package([schema], schema_version) + + assert files["py.typed"] == "" + compile(files["__init__.py"], "dagger_global", "exec") + + +@pytest.fixture +def installed(monkeypatch): + """Load core, every client and the global client on the real runtime.""" + namespace = types.ModuleType("dagger_clients") + monkeypatch.setitem(sys.modules, "dagger_clients", namespace) + + def _module(name: str, code: str) -> types.ModuleType: + module = types.ModuleType(name) + module.__package__ = name.rpartition(".")[0] or name + monkeypatch.setitem(sys.modules, name, module) + exec(compile(code, name, "exec"), module.__dict__) + return module + + def load(schema: graphql.GraphQLSchema) -> types.ModuleType: + _module("dagger_clients.core", core_package(schema)["__init__.py"]) + for name in partition.modules(schema): + package, files = client_package(schema, name, ".") + client = f"dagger_clients.{package}" + module = types.ModuleType(client) + module.__package__ = client + monkeypatch.setitem(sys.modules, client, module) + _module(f"{client}._target", files["_target.py"]) + exec(compile(files["__init__.py"], client, "exec"), module.__dict__) + return _module("dagger_global", global_package([schema])["__init__.py"]) + + return load + + +def test_global_dag_is_a_session_over_the_shared_connection(installed): + global_ = installed(_schema(_LINTER)) + + assert isinstance(global_.dag, Session) + assert type(global_.dag) is global_.Client + assert global_.dag.connection is SharedConnection() + + +def test_global_dag_returns_the_core_classes(installed): + global_ = installed(_schema(_LINTER)) + core = sys.modules["dagger_clients.core"] + + directory = global_.dag.directory() + + assert type(directory) is core.Directory + assert directory._ctx.conn is global_.dag + assert global_.Directory is core.Directory + + +def test_old_and_new_calls_mix(installed): + global_ = installed(_schema(_LINTER)) + core = sys.modules["dagger_clients.core"] + linter = sys.modules["dagger_clients.linter"] + target = Target(name="linter", ref=".") + + new_with_old = linter.linter(global_.dag.directory()) + old_with_new = global_.dag.linter(core.core().directory(), config="x") + + assert type(new_with_old) is type(old_with_new) is linter.Linter + assert new_with_old._ctx.targets == old_with_new._ctx.targets == {target} + assert old_with_new._ctx.conn is global_.dag + assert [f.name for f in old_with_new._ctx.selections] == ["linter"] + assert old_with_new._ctx.selections[0].args["config"] == "x" + + +def test_contributed_field_is_a_method_at_run_time(installed): + global_ = installed(_schema(_LINTER)) + core = sys.modules["dagger_clients.core"] + linter = sys.modules["dagger_clients.linter"] + binding = core.Binding(Context(global_.dag)) + + result = binding.as_linter() + + assert type(result) is linter.Linter + assert result._ctx.targets == {Target(name="linter", ref=".")} + assert result._ctx.conn is global_.dag + + +def test_cli_generates_the_global_client_from_one_schema_per_client(tmp_path): + linter = tmp_path / "linter.json" + glow = tmp_path / "glow.json" + linter.write_text(json.dumps(_introspection(_sdl(_LINTER)))) + glow.write_text(json.dumps(_introspection(_sdl(_GLOW)))) + out = tmp_path / "src" + + cli.main(["generate-global", "-i", str(linter), "-i", str(glow), "-o", str(out)]) + + code = (out / "dagger_global/__init__.py").read_text() + assert (out / "dagger_global/py.typed").read_text() == "" + assert "def linter(self, source: Directory" in code + assert "def glow(self) -> Glow:" in code + assert not (out / "dagger_clients").exists() + + +def test_cli_refuses_global_schemas_of_two_cores(tmp_path, introspection, capsys): + other = tmp_path / "other.json" + other.write_text(json.dumps(_introspection(_sdl() + "type Extra { n: Int! }"))) + + with pytest.raises(SystemExit): + cli.main( + ["generate-global", "-i", str(introspection), "-i", str(other), "-o", "x"] + ) + + assert "two cores" in capsys.readouterr().err + + +def test_write_global_writes_next_to_the_namespace(tmp_path): + files = global_package([_schema(_LINTER)]) + + root = write_global(tmp_path, files) + + assert root == tmp_path / "dagger_global" + assert {p.name: p.read_text() for p in root.iterdir()} == files diff --git a/sdk/tests/codegen/test_partition.py b/sdk/tests/codegen/test_partition.py new file mode 100644 index 0000000..4836c44 --- /dev/null +++ b/sdk/tests/codegen/test_partition.py @@ -0,0 +1,198 @@ +import pytest +from graphql import build_schema + +from codegen.partition import ( + ClientError, + ClientNameError, + check_attribution, + contributed_fields, + core_digest, + modules, + own_fields, + package_name, + package_names, + source_module, +) + +_DIRECTIVES = """ + directive @sourceMap(module: String, filename: String) + on OBJECT | FIELD_DEFINITION | ENUM | ENUM_VALUE | INPUT_FIELD_DEFINITION + directive @expectedType(name: String!) on FIELD_DEFINITION | ARGUMENT_DEFINITION +""" + +_CORE = """ + type Directory { id: ID! @expectedType(name: "Directory") } +""" + +_LINTER = """ + type Linter @sourceMap(module: "linter", filename: "main.py") { + lint: String! @sourceMap(module: "linter") + } + enum LinterLevel @sourceMap(module: "linter") { LOW HIGH } +""" + +_GLOW = """ + type Glow @sourceMap(module: "glow") { render: String! size: Int! } +""" + + +def _schema(*clients: str, core: str = _CORE): + binding = ["name: String!"] + query = ["directory: Directory!"] + if _LINTER in clients: + binding.append('asLinter: Linter! @sourceMap(module: "linter")') + query.append('linter: Linter! @sourceMap(module: "linter")') + if _GLOW in clients: + binding.append('asGlow: Glow! @sourceMap(module: "glow")') + query.append('glow: Glow! @sourceMap(module: "glow")') + return build_schema( + _DIRECTIVES + + core + + "".join(clients) + + f"type Binding {{ {' '.join(binding)} }}" + + f"type Query {{ {' '.join(query)} }}" + ) + + +def test_source_module(): + schema = _schema(_LINTER) + + assert source_module(schema.type_map["Linter"].ast_node) == "linter" + assert source_module(schema.type_map["LinterLevel"].ast_node) == "linter" + assert source_module(schema.type_map["Directory"].ast_node) is None + # A type built without a definition is core, like a type with no directive. + assert source_module(None) is None + + +def test_modules(): + assert modules(_schema()) == [] + assert modules(_schema(_LINTER, _GLOW)) == ["glow", "linter"] + + +def test_own_fields_leave_out_contributed_fields(): + schema = _schema(_LINTER, _GLOW) + + assert list(own_fields(schema.type_map["Binding"])) == ["name"] + assert list(own_fields(schema.type_map["Query"])) == ["directory"] + # Every field of a client's type goes with the type. + assert list(own_fields(schema.type_map["Linter"])) == ["lint"] + + +def test_check_attribution_accepts_fields_contributed_to_core_types(): + assert check_attribution(_schema(_LINTER, _GLOW)) is None + + +def test_check_attribution_refuses_a_field_of_one_client_given_to_another(): + linter = """ + type Linter @sourceMap(module: "linter") { + lint: String! @sourceMap(module: "glow") + } + """ + + with pytest.raises( + ClientError, match=r'"Linter\.lint" .* "glow", but .* "Linter" .* "linter"' + ): + check_attribution(_schema(linter)) + + +@pytest.mark.parametrize( + ("member", "place"), + [ + ('enum Severity { LOW HIGH @sourceMap(module: "linter") }', "Severity.HIGH"), + ('input Options { level: Int @sourceMap(module: "linter") }', "Options.level"), + ], +) +def test_check_attribution_refuses_a_member_that_is_not_a_field(member, place): + with pytest.raises( + ClientError, match=f'"{place}" .* "linter", but only a field of a core' + ): + check_attribution(_schema(_LINTER, core=_CORE + member)) + + +def test_contributed_fields(): + schema = _schema(_LINTER, _GLOW) + + assert [(t.name, name) for t, name, _ in contributed_fields(schema, "linter")] == [ + ("Binding", "asLinter"), + ("Query", "linter"), + ] + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("linter", "linter"), + ("my-project-dev", "my_project_dev"), + ("My.Project", "my_project"), + ], +) +def test_package_name(name: str, expected: str): + assert package_name(name) == expected + + +@pytest.mark.parametrize( + ("name", "reason"), + [ + ("my linter", "not a Python identifier"), + ("2fast", "not a Python identifier"), + ("", "not a Python identifier"), + ("class", "keyword"), + ("Import", "keyword"), + ("_private", 'starts with "_"'), + ("-dash", 'starts with "_"'), + ("core", "core bindings"), + ("CORE", "core bindings"), + ], +) +def test_package_name_refused(name: str, reason: str): + with pytest.raises(ClientNameError, match=reason): + package_name(name) + + +def test_package_names_refuse_a_collision(): + assert package_names(["my-linter", "glow"]) == { + "my-linter": "my_linter", + "glow": "glow", + } + with pytest.raises(ClientNameError, match=r'"my-linter" and "my\.linter"'): + package_names(["my-linter", "glow", "my.linter"]) + + +def test_core_digest_ignores_clients(): + digest = core_digest(_schema()) + + assert digest.startswith("sha256:") + assert core_digest(_schema(_LINTER)) == digest + # Glow brings in `Int`, which the schema lists only because of it. + assert core_digest(_schema(_LINTER, _GLOW)) == digest + + +@pytest.mark.parametrize( + "core", + [ + # a new field + 'type Directory { id: ID! @expectedType(name: "Directory") name: String! }', + # a directive that the printed schema doesn't show + 'type Directory { id: ID! @expectedType(name: "File") }', + ], +) +def test_core_digest_follows_core(core: str): + assert core_digest(_schema(_LINTER, core=core)) != core_digest(_schema(_LINTER)) + + +def test_core_digest_follows_the_compatibility_mode_only(): + schema = _schema() + + assert core_digest(schema, legacy_sdk_compat=True) != core_digest(schema) + # Not the version: two modern versions render one core. + assert core_digest(schema, legacy_sdk_compat=False) == core_digest(schema) + + +def test_core_digest_ignores_member_order(): + reordered = """ + type Directory { name: String! id: ID! @expectedType(name: "Directory") } + """ + ordered = """ + type Directory { id: ID! @expectedType(name: "Directory") name: String! } + """ + assert core_digest(_schema(core=reordered)) == core_digest(_schema(core=ordered)) diff --git a/sdk/tests/invert.py b/sdk/tests/invert.py new file mode 100644 index 0000000..3aab75a --- /dev/null +++ b/sdk/tests/invert.py @@ -0,0 +1,415 @@ +"""Invert each new assertion once and confirm that its test fails. + +Run from ``sdk/``:: + + uv run --frozen python tests/invert.py + +Each entry names a test, the exact text of one of its assertions and the +inverted text. The test runs as written, then with the inversion applied, +and the file is restored either way. The exit code is non-zero if a test +fails as written, still passes when inverted, or if the text to invert is +not found exactly once. +""" + +# ruff: noqa: T201 + +import dataclasses +import pathlib +import subprocess +import sys + +HERE = pathlib.Path(__file__).parent + + +@dataclasses.dataclass(frozen=True) +class Inversion: + test: str + old: str + new: str + + @property + def path(self) -> pathlib.Path: + return HERE.parent / self.test.partition("::")[0] + + +PACKAGE = "tests/test_dagger_package.py::" +CLIENTS = "tests/client/test_clients.py::" +ISOLATION = "tests/client/test_sdk_isolation.py::" +PACKAGES = "tests/codegen/test_packages.py::" +DEFAULT_ENGINE = "tests/client/test_default_engine.py::" +DISPATCH = "tests/mod/test_dispatch.py::" + +INVERSIONS = [ + # The default session provisions an engine only in a plain program, once, + # and ends it at exit or on dagger.close(). + Inversion( + DEFAULT_ENGINE + "test_a_plain_program_provisions_and_ends_the_engine_at_exit", + 'assert (marks / "ended").exists()', + 'assert not (marks / "ended").exists()', + ), + Inversion( + DEFAULT_ENGINE + "test_a_session_in_the_environment_is_never_provisioned", + 'assert proc.stdout.strip() == "5151"', + 'assert proc.stdout.strip() == "4242"', + ), + Inversion( + DEFAULT_ENGINE + "test_without_provisioning_there_is_no_session", + 'assert "No active engine session to connect to" in proc.stderr', + 'assert "No active engine session to connect to" not in proc.stderr', + ), + Inversion( + DEFAULT_ENGINE + "test_concurrent_first_queries_provision_once", + 'assert (marks / "started").read_text().count("session") == 1', + 'assert (marks / "started").read_text().count("session") > 1', + ), + Inversion( + DEFAULT_ENGINE + "test_close_ends_the_engine_before_it_returns", + 'assert proc.stdout.strip() == "True"', + 'assert proc.stdout.strip() == "False"', + ), + # dagger.Container names its new home. + Inversion( + PACKAGE + "test_core_name_points_to_its_new_home", + 'dagger_clients.core: from dagger_clients.core import Container"', + 'dagger_clients.core: from dagger_clients.core import Directory"', + ), + Inversion( + PACKAGE + "test_legacy_client_type_points_to_session", + "and the API is on core() from dagger_clients.core.", + "and the API is on core() from dagger.client.gen.", + ), + # dag.container() names core().container(), where core comes from and + # the flag, and the same message holds for a client. + Inversion( + PACKAGE + "test_session_field_points_to_core", + '"clients now: core().container() for a core field "', + '"clients now: core().directory() for a core field "', + ), + Inversion( + PACKAGE + "test_session_field_points_to_core", + '"(from dagger_clients.core import core), or container() from "', + '"(from dagger_clients.glow import core), or container() from "', + ), + Inversion( + PACKAGE + "test_session_field_points_to_core", + '"migrating, set global-client = true under [tool.dagger] and run "', + '"migrating, set global-client = false under [tool.dagger] and run "', + ), + Inversion( + PACKAGE + "test_session_field_points_to_a_client_too", + """"or linter() from the client's package for a client." in""", + """"or linter() from the core package for a client." in""", + ), + # dag is the default Session. + Inversion( + PACKAGE + "test_dag_is_the_default_session", + "assert type(dagger.dag) is Session\n" + " assert dagger.dag is default_session()", + "assert type(dagger.dag) is Session\n" + " assert dagger.dag is not default_session()", + ), + # The package imports with no generated code present. + Inversion( + PACKAGE + "test_package_imports_with_no_generated_code", + 'assert _run(NO_GENERATED_CODE) == "ok\\n"', + 'assert _run(NO_GENERATED_CODE) == "ko\\n"', + ), + # With dagger_global installed, dag.container() is a core Container and + # old and new calls mix. + Inversion( + PACKAGE + "test_global_client_makes_dag_the_client", + "assert type(ctr) is core.Container, type(ctr)", + "assert type(ctr) is core.Directory, type(ctr)", + ), + Inversion( + PACKAGE + "test_global_client_makes_dag_the_client", + "assert type(linter.linter(old)) is linter.Linter", + "assert type(linter.linter(old)) is core.Directory", + ), + Inversion( + PACKAGE + "test_global_client_makes_dag_the_client", + "assert isinstance(dagger.dag, Session)\n" + " assert dagger.dag is default_session()", + "assert isinstance(dagger.dag, Session)\n" + " assert dagger.dag is not default_session()", + ), + # The global client loads on first use, whatever was imported first, and + # a client used before dag gets it too. + Inversion( + PACKAGE + "test_generated_code_imported_before_dagger", + " assert type(dagger.dag) is dagger_global.Client, type(dagger.dag)", + " assert type(dagger.dag) is not dagger_global.Client, type(dagger.dag)", + ), + Inversion( + PACKAGE + "test_client_called_before_dag_gets_the_global_session", + "assert root._ctx.conn is dagger_global.dag, type(root._ctx.conn)", + "assert root._ctx.conn is not dagger_global.dag, type(root._ctx.conn)", + ), + # dir() and a star import see the lazy names. + Inversion( + PACKAGE + "test_dir_lists_dag_with_the_sdk_names", + 'assert {"dag", "Session", "connection", "function"} <= set(names)', + 'assert {"dag", "Session", "connection", "function"} > set(names)', + ), + Inversion( + PACKAGE + "test_star_import_without_the_global_client", + 'assert namespace["dag"] is default_session()', + 'assert namespace["dag"] is not default_session()', + ), + Inversion( + PACKAGE + "test_global_client_names_are_listed_and_star_imported", + 'assert namespace["Container"] is core.Container', + 'assert namespace["Container"] is not core.Container', + ), + Inversion( + PACKAGE + "test_global_client_names_are_listed_and_star_imported", + 'assert {"dag", "Container", "Linter", "Client", "Session"} <= set(listed)', + 'assert {"dag", "Container", "Linter", "Client", "Session"} > set(listed)', + ), + # A global client that cannot import is an error, not an absent one. + Inversion( + PACKAGE + "test_global_client_that_cannot_import_is_not_skipped", + 'assert _run(BROKEN_GLOBAL_CLIENT, tmp_path) == "dagger_clients.gone\\n"', + 'assert _run(BROKEN_GLOBAL_CLIENT, tmp_path) == "Session\\n"', + ), + # Bindings an earlier version left in dagger_gen are named, not loaded. + Inversion( + PACKAGE + "test_legacy_bindings_are_named_not_loaded", + '"it, `dagger generate` removes it; if you wrote it, delete or rename it.\\n"', + '"it, `dagger generate` removes it.\\n"', + ), + Inversion( + PACKAGE + "test_no_legacy_bindings_no_warning", + 'assert _run(IMPORT_WARNINGS) == ""', + 'assert _run(IMPORT_WARNINGS) != ""', + ), + Inversion( + PACKAGE + "test_connection_yields_the_global_client", + "assert session is dagger.dag", + "assert session is not dagger.dag", + ), + Inversion( + PACKAGE + "test_mypy_types_dag_as_the_global_client", + "assert 'Revealed type is \"dagger_global.Client\"' in out, out", + "assert 'Revealed type is \"dagger.client._session.Session\"' in out, out", + ), + # dagger.Connection yields a Session. + Inversion( + CLIENTS + "test_legacy_connection_yields_an_isolated_session", + "assert isinstance(s, Session)\n assert s is not default_session()", + "assert not isinstance(s, Session)\n assert s is not default_session()", + ), + Inversion( + CLIENTS + "test_session_with_no_connection_is_over_the_shared_one", + "assert Session().connection is SharedConnection()", + "assert Session().connection is not SharedConnection()", + ), + # serveModule is the load, for a git and a local target alike. + Inversion( + CLIENTS + "test_git_target_loads_before_its_query", + "assert GLOW.pin in load", + "assert GLOW.pin not in load", + ), + Inversion( + CLIENTS + "test_local_target_loads_before_its_query", + "assert is_load(load)\n assert LINTER.ref in load", + "assert is_load(load)\n assert LINTER.ref not in load", + ), + # Under a module entrypoint, a local target loads through the source the + # entrypoint handed over for it, by name, and nothing else changes query. + Inversion( + CLIENTS + "test_handed_client_serves_a_local_target_by_name", + '" ... on ModuleSource {\\n"', + '" ... on Workspace {\\n"', + ), + Inversion( + CLIENTS + "test_handed_clients_leave_a_git_target_to_serve_module", + "assert handed_clients not in load", + "assert handed_clients in load", + ), + Inversion( + CLIENTS + "test_undeclared_local_client_fails_naming_it", + "assert not s.session.queries", + "assert s.session.queries", + ), + Inversion( + CLIENTS + "test_entrypoint_without_a_handover_fails_a_local_target", + 'assert "handed over no clients" in str(info.value)', + 'assert "handed over no clients" not in str(info.value)', + ), + Inversion( + CLIENTS + "test_handed_client_failure_does_not_fall_back", + 'assert "serveModule" not in load', + 'assert "serveModule" in load', + ), + Inversion( + CLIENTS + "test_outside_an_entrypoint_a_local_target_uses_serve_module", + 'assert "node(" not in load', + 'assert "node(" in load', + ), + Inversion( + DISPATCH + "test_command_hands_the_clients_to_the_load", + "assert got == \"{'linter': 'bW9kdWxlU291cmNl'}\"", + 'assert got == "None"', + ), + Inversion( + DISPATCH + "test_command_without_clients_is_still_under_an_entrypoint", + 'read_text()) == "None"', + 'read_text()) != "None"', + ), + Inversion( + DISPATCH + "test_command_refuses_malformed_clients", + 'assert "clients the entrypoint handed over" in proc.stderr', + 'assert "clients the entrypoint handed over" not in proc.stderr', + ), + # An engine below the floor fails the load; it is no stale client. + Inversion( + CLIENTS + "test_engine_without_serve_module_fails_the_load_not_as_stale", + "assert not isinstance(info.value, StaleClientError)\n" + " assert info.value.__cause__ is NO_SERVE_MODULE", + "assert isinstance(info.value, StaleClientError)\n" + " assert info.value.__cause__ is NO_SERVE_MODULE", + ), + # Only a validation error is a missing field, for the stale check too. + Inversion( + CLIENTS + "test_internal_error_with_the_phrase_stays_a_query_error", + 'internal = {"code": "INTERNAL_SERVER_ERROR"}', + 'internal = {"code": "GRAPHQL_VALIDATION_FAILED"}', + ), + # The staleness message names the client and its address. + Inversion( + CLIENTS + "test_missing_field_becomes_stale_client_error", + "The client 'glow' from github.com/eunomie/glow at 4f1c9e ", + "The client 'glow' from github.com/eunomie/glow at 9b2d7a ", + ), + Inversion( + CLIENTS + "test_stale_message_names_each_client_and_its_address", + "'linter' from ./.dagger/modules/linter are out of date.", + "'linter' from ./linter are out of date.", + ), + # The SDK files import nothing generated; the init names it once. + Inversion( + ISOLATION + "test_sdk_files_import_without_generated_code", + "assert proc.returncode == 0, proc.stderr", + "assert proc.returncode != 0, proc.stderr", + ), + # The text scan lets through the one help string, exactly. + Inversion( + ISOLATION + "test_sdk_file_text_names_no_generated_package[client/_session.py]", + '"(from dagger_clients.core import core)",', + '"(from dagger_clients.core import Container)",', + ), + Inversion( + ISOLATION + "test_package_init_names_only_the_global_client", + "\"import_module('dagger_global')\"]", + "\"import_module('dagger_gen')\"]", + ), + # The generated global client. + Inversion( + PACKAGES + "test_global_client_delegates_root_fields_to_core", + 'assert "class Client(_Session):" in code', + 'assert "class Client(_Root):" in code', + ), + Inversion( + PACKAGES + "test_global_client_has_one_method_per_client", + 'assert "import dagger_clients.glow as _glow" in code', + 'assert "import dagger_clients.glow as _glow" not in code', + ), + Inversion( + PACKAGES + "test_global_client_puts_contributed_fields_on_the_core_classes", + 'assert "Env.as_linter = _linter.as_linter # type: ignore', + 'assert "Env.as_glow = _linter.as_linter # type: ignore', + ), + # A method of the global client never hides the Session it is. + Inversion( + PACKAGES + "test_global_client_refuses_a_core_field_that_hides_the_session", + 'f"it would hide Session.{field}"', + 'f"it would hide Query.{field}"', + ), + Inversion( + PACKAGES + "test_global_client_refuses_a_client_that_hides_the_session", + """'client "connect": it would hide Session.connect'""", + """'client "connect": it would hide Session.close'""", + ), + Inversion( + PACKAGES + "test_global_client_knows_every_name_of_a_session", + 'if not n.startswith("__")} == SESSION_NAMES', + 'if not n.startswith("__")} != SESSION_NAMES', + ), + Inversion( + PACKAGES + "test_global_client_is_temporary_and_says_so", + 'assert "global-client = true" in code', + 'assert "global-client = true" not in code', + ), + Inversion( + PACKAGES + "test_global_dag_returns_the_core_classes", + "assert type(directory) is core.Directory", + "assert type(directory) is core.File", + ), + Inversion( + PACKAGES + "test_old_and_new_calls_mix", + "assert type(new_with_old) is type(old_with_new) is linter.Linter", + "assert type(new_with_old) is type(old_with_new) is linter.LinterReport", + ), + Inversion( + PACKAGES + "test_contributed_field_is_a_method_at_run_time", + "assert type(result) is linter.Linter\n", + "assert type(result) is linter.LinterReport\n", + ), + Inversion( + PACKAGES + "test_cli_generates_the_global_client_from_one_schema_per_client", + 'assert "def glow(self) -> Glow:" in code', + 'assert "def glow(self) -> Glow:" not in code', + ), + # The one-file path still works. + Inversion( + PACKAGES + "test_cli_still_generates_one_file", + 'assert "dag = Client()" in code', + 'assert "dag = Client()" not in code', + ), +] + + +def run(test: str) -> bool: + """Whether the test passes.""" + # No bytecode: pytest keys its rewritten test modules on mtime and size, + # so an inversion of the same length, restored within the same second, + # would otherwise run the inverted code again as the original. + proc = subprocess.run( + [sys.executable, "-B", "-m", "pytest", "-q", "-p", "no:cacheprovider", test], + capture_output=True, + text=True, + check=False, + cwd=HERE.parent, + ) + return proc.returncode == 0 + + +def check(inversion: Inversion) -> str | None: + """The problem with an inversion, or None when it behaves.""" + source = inversion.path.read_text() + if (count := source.count(inversion.old)) != 1: + return f"text to invert found {count} times, not once" + if not run(inversion.test): + return "fails as written" + inversion.path.write_text(source.replace(inversion.old, inversion.new)) + try: + if run(inversion.test): + return "still passes when inverted" + finally: + inversion.path.write_text(source) + return None + + +def main() -> int: + problems = 0 + for inversion in INVERSIONS: + problem = check(inversion) + problems += problem is not None + print(f"{problem or 'fails when inverted'}: {inversion.test}") + print(f"{len(INVERSIONS)} inversions, {len(INVERSIONS) - problems} confirmed") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/sdk/tests/mod/conftest.py b/sdk/tests/mod/conftest.py new file mode 100644 index 0000000..00970f6 --- /dev/null +++ b/sdk/tests/mod/conftest.py @@ -0,0 +1,31 @@ +import enum + +import pytest + +from dagger.client.base import Type + + +@pytest.fixture +def selections(): + """Reduce a binding to the fields it selects. + + A hand-written binding and a generated one never compare equal, even + when they send the same query. Their selections do. + """ + + def _value(value): + if isinstance(value, Type): + return _selections(value) + if isinstance(value, enum.Enum): + return value.name + if isinstance(value, list): + return [_value(v) for v in value] + return value + + def _selections(obj: Type): + return [ + (f.type_name, f.name, {k: _value(v) for k, v in f.args.items()}) + for f in obj._ctx.selections + ] + + return _selections diff --git a/sdk/tests/mod/golden/main.dang b/sdk/tests/mod/golden/main.dang index f2fa2f4..69d1f96 100644 --- a/sdk/tests/mod/golden/main.dang +++ b/sdk/tests/mod/golden/main.dang @@ -2,7 +2,6 @@ type Entrypoint implements ModuleEntrypoint { let moduleName: String! = "main" - let modulePath: String! = ".dagger/modules/main" let skippedDirs: [String!]! = [".venv", "__pycache__", "sdk"] let sourceFiles: [SourceFile!]! = [ SourceFile(path: ".python-version", digest: ""), @@ -24,43 +23,46 @@ type Entrypoint implements ModuleEntrypoint { fnName: String!, fnArgs: JSON!, ): JSON! { + # The module's declared local clients go with the call, never the + # workspace (see handover.dang). let request = JSON.encode({{ receiverType: receiverType, receiverValue: receiverValue, fnName: fnName, fnArgs: fnArgs, + clients: ClientHandover(workspace: workspace).clients.map { client => + {{name: client.name, source: client.source}} + }, }}) - let result = runtime(workspace) + let result = runtime .withExec(["python", "-m", "dagger.mod", "call", "--output", "/dagger/result.json"], stdin: request, experimentalPrivilegedNesting: true) .file("/dagger/result.json") .contents (result :: JSON!) } - let runtime(workspace: Workspace!): Container! { - let module = workspace.directory(if (modulePath == ".") { "/" } else { "/" + modulePath }) - if (module.exists("pyproject.toml") == false) { - raise "module \"" + moduleName + "\" was generated at \"" + modulePath + "\" and is not there; run `dagger generate` after moving it" - } else { - let changed = sourceFiles.filter { f => - if (f.digest == "") { - module.exists(f.path) - } else { - module.exists(f.path) == false or module.file(f.path).digest(excludeMetadata: true) != f.digest - } - }.map { f => f.path } - let added = module.glob("**/*.py").filter { p => - isSource(p) and sourceFiles.filter { f => f.path == p }.length == 0 - } - if ((changed + added).length > 0) { - raise "module \"" + moduleName + "\" changed since its entrypoint was generated (" + (changed + added).join(", ") + "); run `dagger generate`" + # The module's own source, not the workspace the engine hands over: that is + # the caller's, which holds the module only when the module sits in it. + let runtime: Container! { + let module = currentModule.source + let changed = sourceFiles.filter { f => + if (f.digest == "") { + module.exists(f.path) } else { - PythonModuleBuild( - contextDir: workspace.directory("/", include: [if (modulePath == ".") { "**" } else { modulePath + "/**" }], exclude: ["**/.venv", "**/__pycache__"]), - subPath: modulePath, - moduleName: moduleName, - ).installed + module.exists(f.path) == false or module.file(f.path).digest(excludeMetadata: true) != f.digest } + }.map { f => f.path } + let added = module.glob("**/*.py").filter { p => + isSource(p) and sourceFiles.filter { f => f.path == p }.length == 0 + } + if ((changed + added).length > 0) { + raise "module \"" + moduleName + "\" changed since its entrypoint was generated (" + (changed + added).join(", ") + "); run `dagger generate`" + } else { + PythonModuleBuild( + contextDir: directory.withDirectory(".", module, exclude: ["**/.venv", "**/__pycache__"]), + subPath: ".", + moduleName: moduleName, + ).installed } } diff --git a/sdk/tests/mod/test_api.py b/sdk/tests/mod/test_api.py new file mode 100644 index 0000000..900617c --- /dev/null +++ b/sdk/tests/mod/test_api.py @@ -0,0 +1,187 @@ +"""The hand-written API calls send what the generated bindings send.""" + +import pytest + +from dagger.client import gen +from dagger.client._session import SharedConnection +from dagger.client.gen import dag +from dagger.mod import _api + +Kind = gen.TypeDefKind +Policy = gen.FunctionCachePolicy + + +def string(api): + kind = "STRING_KIND" if api is _api else Kind.STRING_KIND + return api.type_def().with_kind(kind) + + +def func(api): + return api.function("fn", string(api)) + + +def json(api, value: str): + return value if api is _api else gen.JSON(value) + + +TYPE_DEF_CASES = { + "optional": lambda api: api.type_def().with_optional(True), + "kind": string, + "list_of": lambda api: api.type_def().with_list_of(string(api)), + "scalar": lambda api: api.type_def().with_scalar("S", description="doc"), + "scalar_no_doc": lambda api: api.type_def().with_scalar("S", description=None), + "enum": lambda api: api.type_def().with_enum("E", description=None), + "enum_member": lambda api: ( + api.type_def() + .with_enum("E", description="doc") + .with_enum_member("A", value="a", description=None, deprecated="old") + ), + "interface": lambda api: api.type_def().with_interface("I", description="doc"), + "object": lambda api: api.type_def().with_object( + "O", description=None, deprecated="old" + ), + "field": lambda api: ( + api.type_def() + .with_object("O") + .with_field("f", string(api), description="doc", deprecated=None) + ), + "function": lambda api: api.type_def().with_object("O").with_function(func(api)), + "constructor": lambda api: ( + api.type_def().with_object("O").with_constructor(func(api)) + ), +} + +FUNCTION_CASES = { + "description": lambda api: func(api).with_description("doc"), + "cache_never": lambda api: func(api).with_cache_policy( + "Never" if api is _api else Policy.Never + ), + "cache_session": lambda api: func(api).with_cache_policy( + "PerSession" if api is _api else Policy.PerSession + ), + "cache_ttl": lambda api: func(api).with_cache_policy( + "Default" if api is _api else Policy.Default, time_to_live="5m" + ), + "deprecated": lambda api: func(api).with_deprecated(reason="old"), + "markers": lambda api: ( + func(api).with_check().with_generator().with_up().with_agent() + ), + "arg": lambda api: func(api).with_arg( + "a", + string(api).with_optional(True), + description="doc", + default_value=json(api, '"x"'), + default_path="./src", + default_address="alpine:latest", + ignore=[".venv"], + deprecated="old", + ), + "arg_bare": lambda api: func(api).with_arg( + "a", + string(api), + description=None, + default_value=None, + default_path=None, + default_address=None, + ignore=None, + deprecated=None, + ), +} + +MODULE_CASES = { + "description": lambda api: api.module().with_description("doc"), + "object": lambda api: api.module().with_object(api.type_def().with_object("O")), + "interface": lambda api: api.module().with_interface( + api.type_def().with_interface("I") + ), + "enum": lambda api: api.module().with_enum(api.type_def().with_enum("E")), + "error": lambda api: api.error("boom").with_value("k", json(api, "1")), +} + +CASES = { + **{f"type_def.{k}": v for k, v in TYPE_DEF_CASES.items()}, + **{f"function.{k}": v for k, v in FUNCTION_CASES.items()}, + **{f"module.{k}": v for k, v in MODULE_CASES.items()}, +} + + +@pytest.mark.parametrize("build", CASES.values(), ids=CASES.keys()) +def test_selections_match_generated_bindings(selections, build): + assert selections(build(_api)) == selections(build(dag)) + + +class FakeSession: + """Answers any function call query, and keeps what it was asked.""" + + def __init__(self): + self.queries: list[str] = [] + + async def execute(self, query: str): + self.queries.append(query) + return { + "error": {"id": "error-id"}, + "currentFunctionCall": { + "parentName": "Main", + "name": "hello", + "parent": "{}", + "inputArgs": [{"name": "who", "value": '"you"'}], + "returnValue": None, + "returnError": None, + }, + } + + +@pytest.fixture +def session(monkeypatch: pytest.MonkeyPatch): + fake = FakeSession() + monkeypatch.setattr(SharedConnection, "session", fake) + return fake + + +async def _drive(api, session: FakeSession, value) -> list[str]: + call = api.current_function_call() + assert await call.parent_name() == "Main" + assert await call.name() == "hello" + assert await call.parent() == "{}" + await call.return_value(value) + await call.return_error(api.error("boom")) + queries, session.queries = session.queries, [] + return queries + + +@pytest.mark.anyio +async def test_function_call_queries_match_generated_bindings(session: FakeSession): + raw = await _drive(_api, session, '"hi"') + generated = await _drive(dag, session, gen.JSON('"hi"')) + + assert raw == generated + assert 'returnValue(value: "\\"hi\\"")' in raw[3] + assert 'returnError(error: "error-id")' in raw[5] + + +@pytest.mark.anyio +async def test_input_args_come_in_one_query(session: FakeSession): + args = await _api.current_function_call().input_args() + + assert args == [_api.ArgValue(name="who", value='"you"')] + assert session.queries == [ + "query {\n" + " currentFunctionCall {\n" + " inputArgs {\n" + " name\n" + " value\n" + " }\n" + " }\n" + "}" + ] + + +def test_enum_values_go_out_bare(): + query = ( + _api.type_def() + .with_kind("STRING_KIND") + ._ctx.select("TypeDef", "id", []) + .build() + ) + + assert "withKind(kind: STRING_KIND)" in query diff --git a/sdk/tests/mod/test_describe.py b/sdk/tests/mod/test_describe.py index 6fa92fa..8ef0572 100644 --- a/sdk/tests/mod/test_describe.py +++ b/sdk/tests/mod/test_describe.py @@ -6,15 +6,14 @@ import pytest from typing_extensions import Doc, Self -import dagger -from dagger import DefaultPath, Ignore, Name, dag +from dagger import DefaultPath, Ignore, Name +from dagger.client import gen +from dagger.client.gen import dag from dagger.mod import Module from dagger.mod._converter import to_typedef, typedef_from from dagger.mod._describe import TypeRef, describe_type from dagger.mod._module import _module_from -Kind = dagger.TypeDefKind - class Color(enum.Enum): """A color.""" @@ -45,7 +44,7 @@ class Helper: class Main: """The main object.""" - source: dagger.Directory + source: gen.Directory greeting: str = m.field(default="hello") count: Annotated[int, Doc("How many")] = m.field(default=1, name="howMany") extra: InitVar[str] = "" @@ -65,7 +64,7 @@ def lint(self) -> None: ... @m.function def helpers( - self, src: Annotated[dagger.Directory, DefaultPath("."), Ignore([".venv"])] + self, src: Annotated[gen.Directory, DefaultPath("."), Ignore([".venv"])] ) -> list[Helper]: ... @m.function @@ -78,7 +77,7 @@ def maybe(self) -> list[str] | None: ... def greeter(self, g: Greeter) -> Greeter: ... @m.function(deprecated="use shout") - def old(self, p: dagger.Platform) -> dagger.JSON: ... + def old(self, p: gen.Platform) -> gen.JSON: ... make_helper = m.function()(Helper) @@ -98,7 +97,7 @@ def test_objects_and_enums(mod: Module): assert [f.name for f in greeter.functions] == ["greet"] assert helper.description == "A helper." - assert helper.fields[0].type == TypeRef(Kind.ENUM_KIND, "Color", "A color.") + assert helper.fields[0].type == TypeRef("ENUM_KIND", "Color", "A color.") assert helper.constructor is None assert main.description == "The main object." @@ -119,8 +118,8 @@ def test_fields(mod: Module): main = mod.describe().objects[2] assert [(f.name, f.type.kind) for f in main.fields] == [ - ("greeting", Kind.STRING_KIND), - ("howMany", Kind.INTEGER_KIND), + ("greeting", "STRING_KIND"), + ("howMany", "INTEGER_KIND"), ] assert main.fields[1].description == "How many" @@ -130,9 +129,9 @@ def test_constructor(mod: Module): assert ctor is not None assert ctor.name == "" - assert ctor.returns == TypeRef(Kind.OBJECT_KIND, "Main") + assert ctor.returns == TypeRef("OBJECT_KIND", "Main") assert [a.name for a in ctor.args] == ["source", "greeting", "count", "extra"] - assert ctor.args[0].type == TypeRef(Kind.OBJECT_KIND, "Directory") + assert ctor.args[0].type == TypeRef("OBJECT_KIND", "Directory") assert ctor.args[1].default_value == '"hello"' assert ctor.args[2].description == "How many" assert ctor.args[3].default_value == '""' @@ -145,34 +144,34 @@ def test_function_metadata(mod: Module): assert shout.description == "Shout it" assert shout.cache == "never" assert shout.args[0].nullable is True - assert shout.args[0].type == TypeRef(Kind.STRING_KIND, optional=True) + assert shout.args[0].type == TypeRef("STRING_KIND", optional=True) assert shout.args[0].default_value == "null" assert shout.args[1].default_value == "1" assert functions["lint"].check is True - assert functions["lint"].returns == TypeRef(Kind.VOID_KIND, optional=True) + assert functions["lint"].returns == TypeRef("VOID_KIND", optional=True) src = functions["helpers"].args[0] assert src.default_path == "." assert src.ignore == (".venv",) assert functions["helpers"].returns == TypeRef( - Kind.LIST_KIND, elem=TypeRef(Kind.OBJECT_KIND, "Helper") + "LIST_KIND", elem=TypeRef("OBJECT_KIND", "Helper") ) assert functions["mob"].returns == TypeRef( - Kind.LIST_KIND, elem=TypeRef(Kind.OBJECT_KIND, "Main") + "LIST_KIND", elem=TypeRef("OBJECT_KIND", "Main") ) assert functions["maybe"].returns == TypeRef( - Kind.LIST_KIND, optional=True, elem=TypeRef(Kind.STRING_KIND) + "LIST_KIND", optional=True, elem=TypeRef("STRING_KIND") ) - assert functions["greeter"].returns == TypeRef(Kind.INTERFACE_KIND, "Greeter") + assert functions["greeter"].returns == TypeRef("INTERFACE_KIND", "Greeter") assert functions["container"].args[0].name == "from" old = functions["old"] assert old.deprecated == "use shout" - assert old.args[0].type.kind == Kind.SCALAR_KIND + assert old.args[0].type.kind == "SCALAR_KIND" assert old.args[0].type.name == "Platform" - assert old.returns.kind == Kind.SCALAR_KIND + assert old.returns.kind == "SCALAR_KIND" make_helper = functions["make_helper"] assert make_helper.description == "A helper." @@ -199,7 +198,7 @@ def test_unsupported_type(): describe_type(int | str) -def test_module_materialisation(): +def test_module_materialisation(selections): mod = Module("Foo") @mod.object_type @@ -213,7 +212,8 @@ def hello(self, who: str | None = None) -> str: """Say hello.""" return who or self.name - string = dag.type_def().with_kind(Kind.STRING_KIND) + kind = gen.TypeDefKind.STRING_KIND + string = dag.type_def().with_kind(kind) expected = dag.module().with_object( dag.type_def() .with_object("Foo", description="Foo doc.", deprecated=None) @@ -223,12 +223,9 @@ def hello(self, who: str | None = None) -> str: .with_description("Say hello.") .with_arg( "who", - dag.type_def() - .with_optional(True) - .with_kind(Kind.STRING_KIND) - .with_optional(True), + dag.type_def().with_optional(True).with_kind(kind).with_optional(True), description=None, - default_value=dagger.JSON("null"), + default_value=gen.JSON("null"), default_path=None, default_address=None, ignore=None, @@ -242,7 +239,7 @@ def hello(self, who: str | None = None) -> str: "name", string, description=None, - default_value=dagger.JSON('"foo"'), + default_value=gen.JSON('"foo"'), default_path=None, default_address=None, ignore=None, @@ -253,4 +250,4 @@ def hello(self, who: str | None = None) -> str: desc = mod.describe() assert desc.description is None - assert _module_from(desc) == expected + assert selections(_module_from(desc)) == selections(expected) diff --git a/sdk/tests/mod/test_dispatch.py b/sdk/tests/mod/test_dispatch.py index 8ea5f0a..8ab4c99 100644 --- a/sdk/tests/mod/test_dispatch.py +++ b/sdk/tests/mod/test_dispatch.py @@ -91,6 +91,10 @@ def module_dir(tmp_path: pathlib.Path) -> pathlib.Path: " @function\n" " def boom(self) -> str:\n" " raise RuntimeError('boom')\n" + " @function\n" + " def handed(self) -> str:\n" + " from dagger.client import _load\n" + " return repr(_load._handover and _load._handover.clients)\n" ) return tmp_path @@ -135,3 +139,51 @@ def test_command_failure_writes_nothing(module_dir: pathlib.Path): assert proc.returncode == 2 assert "boom" in proc.stderr assert not (module_dir / "out").exists() + + +def test_command_hands_the_clients_to_the_load(module_dir: pathlib.Path): + # The entrypoint sends the module's declared local clients with the call, + # because this process is not the module and resolves no path of the + # caller's itself. Never a workspace: a key for one is not read. + request = { + "receiverType": "Hello", + "receiverValue": "{}", + "fnName": "handed", + "fnArgs": "{}", + "clients": [{"name": "linter", "source": "bW9kdWxlU291cmNl"}], + "workspace": "d29ya3NwYWNl", + } + proc = _call(module_dir, request) + assert proc.returncode == 0, proc.stderr + got = json.loads((module_dir / "out" / "result.json").read_text()) + assert got == "{'linter': 'bW9kdWxlU291cmNl'}" + + +def test_command_without_clients_is_still_under_an_entrypoint( + module_dir: pathlib.Path, +): + # An entrypoint from before the handover sends none. The process is still + # one an entrypoint runs, so a local client fails rather than resolving + # in this container. + request = { + "receiverType": "Hello", + "receiverValue": "{}", + "fnName": "handed", + "fnArgs": "{}", + } + proc = _call(module_dir, request) + assert proc.returncode == 0, proc.stderr + assert json.loads((module_dir / "out" / "result.json").read_text()) == "None" + + +def test_command_refuses_malformed_clients(module_dir: pathlib.Path): + request = { + "receiverType": "Hello", + "receiverValue": "{}", + "fnName": "hi", + "fnArgs": '{"who": "you"}', + "clients": {"linter": "bW9kdWxlU291cmNl"}, + } + proc = _call(module_dir, request) + assert proc.returncode == 2 + assert "clients the entrypoint handed over" in proc.stderr diff --git a/sdk/tests/mod/test_entrypoint.py b/sdk/tests/mod/test_entrypoint.py index 10d3a58..949c592 100644 --- a/sdk/tests/mod/test_entrypoint.py +++ b/sdk/tests/mod/test_entrypoint.py @@ -10,8 +10,8 @@ import pytest from typing_extensions import Doc, Self -import dagger from dagger import DefaultPath, Ignore, Name +from dagger.client import gen from dagger.mod import Module from dagger.mod._entrypoint import ( _quote, @@ -55,7 +55,7 @@ class Helper: class Main: """The main object.""" - source: dagger.Directory + source: gen.Directory greeting: str = m.field(default="hello") count: Annotated[int, Doc("How many")] = m.field(default=1, name="howMany") @@ -73,7 +73,7 @@ def lint(self) -> None: ... @m.function def helpers( - self, src: Annotated[dagger.Directory, DefaultPath("."), Ignore([".venv"])] + self, src: Annotated[gen.Directory, DefaultPath("."), Ignore([".venv"])] ) -> list[Helper]: ... @m.function @@ -83,7 +83,7 @@ def mob(self, who: str | None = None) -> list[Self]: ... def greeter(self, g: Greeter) -> Greeter: ... @m.function(deprecated="use container") - def old(self, p: dagger.Platform) -> dagger.JSON: ... + def old(self, p: gen.Platform) -> gen.JSON: ... return m @@ -113,7 +113,7 @@ def test_types_golden(mod: Module): def test_main_golden(root: pathlib.Path): - rendered = render_main("main", ".dagger/modules/main", source_files(root)) + rendered = render_main("main", source_files(root)) _assert_golden("main.dang", rendered) @@ -129,7 +129,7 @@ def test_source_files(root: pathlib.Path): def test_absent_manifest_is_recorded_without_a_digest(root: pathlib.Path): - rendered = render_main("main", ".", source_files(root)) + rendered = render_main("main", source_files(root)) assert 'SourceFile(path: "uv.lock", digest: ""),' in rendered @@ -172,15 +172,9 @@ def fresh(self) -> str: ... render_types(mod.describe()) -@pytest.mark.parametrize("path", ["/abs", "../up", "a/../../b"]) -def test_path_must_stay_inside(path: str): - with pytest.raises(BadUsageError, match="relative"): - render_main("main", path, []) - - def test_write_entrypoint(mod: Module, root: pathlib.Path): out = root / "out" - write_entrypoint(mod.describe(), name="main", path=".", root=root, output=out) + write_entrypoint(mod.describe(), name="main", root=root, output=out) assert (out / "types.dang").read_text().startswith("# Code generated") assert 'SourceFile(path: "src/main/extra.py"' in (out / "main.dang").read_text() @@ -202,8 +196,6 @@ def test_command(tmp_path: pathlib.Path): "entrypoint", "--name", "hello", - "--path", - ".dagger/modules/hello", "--output", "out", ], diff --git a/sdk/tests/mod/test_future_annotations.py b/sdk/tests/mod/test_future_annotations.py index e908186..ce797b2 100644 --- a/sdk/tests/mod/test_future_annotations.py +++ b/sdk/tests/mod/test_future_annotations.py @@ -11,8 +11,8 @@ from typing_extensions import Doc -import dagger from dagger import DefaultPath, Deprecated, Ignore, Name +from dagger.client import gen from dagger.mod import Module @@ -25,7 +25,7 @@ class Foo: @mod.function def build( self, - src: Annotated[dagger.Directory, DefaultPath(".")], + src: Annotated[gen.Directory, DefaultPath(".")], ) -> str: return "ok" @@ -83,7 +83,7 @@ class Foo: @mod.function def build( self, - src: Annotated[dagger.Directory, Ignore(["*.tmp", ".git"])], + src: Annotated[gen.Directory, Ignore(["*.tmp", ".git"])], ) -> str: return "ok" diff --git a/sdk/tests/mod/test_registration.py b/sdk/tests/mod/test_registration.py index ea903ba..aceed7f 100644 --- a/sdk/tests/mod/test_registration.py +++ b/sdk/tests/mod/test_registration.py @@ -5,7 +5,8 @@ from typing_extensions import Doc, Self import dagger -from dagger import dag +from dagger.client import gen +from dagger.client.gen import dag from dagger.mod import Module from dagger.mod._converter import to_typedef from dagger.mod._exceptions import BadUsageError @@ -188,7 +189,7 @@ class Test: for param in fn.parameters.values(): assert param.name == "foo" assert param.doc == "a foo walks into a bar" - assert param.default_value == dagger.JSON('"bar"') + assert param.default_value == gen.JSON('"bar"') def test_external_alt_constructor_doc(): @@ -210,7 +211,7 @@ class Test: assert mod.get_object("Test").functions["external"].doc == "Factory constructor." -def test_void_return_type(): +def test_void_return_type(selections): mod = Module() @mod.object_type @@ -220,13 +221,13 @@ def void(self): ... func = mod.get_object("Test").functions["void"] assert func.return_type is None - assert to_typedef(func.return_type) == dag.type_def().with_optional(True).with_kind( - dagger.TypeDefKind.VOID_KIND + assert selections(to_typedef(func.return_type)) == selections( + dag.type_def().with_optional(True).with_kind(gen.TypeDefKind.VOID_KIND) ) @pytest.mark.anyio -async def test_self_return_type(): +async def test_self_return_type(selections): mod = Module() @mod.object_type @@ -245,5 +246,7 @@ def seq(self) -> list[Self]: assert iden.return_type is Test assert seq.return_type == list[Test] expected = dag.type_def().with_object("Test") - assert to_typedef(iden.return_type) == expected - assert to_typedef(seq.return_type) == dag.type_def().with_list_of(expected) + assert selections(to_typedef(iden.return_type)) == selections(expected) + assert selections(to_typedef(seq.return_type)) == selections( + dag.type_def().with_list_of(expected) + ) diff --git a/sdk/tests/mod/test_results.py b/sdk/tests/mod/test_results.py index b60b52c..c0bea04 100644 --- a/sdk/tests/mod/test_results.py +++ b/sdk/tests/mod/test_results.py @@ -7,7 +7,9 @@ import typing_extensions import dagger -from dagger import Doc, Name, dag +from dagger import Doc, Name +from dagger.client import gen +from dagger.client.gen import dag from dagger.mod import Module from dagger.mod._exceptions import RegistrationError @@ -23,7 +25,7 @@ async def test_unstructure_structure(): @mod.object_type class Bar: msg: Annotated[str, Doc("Echo message")] = mod.field(default="foobar") - ctr: Annotated[dagger.Container, Doc("A container")] = mod.field() + ctr: Annotated[gen.Container, Doc("A container")] = mod.field() @mod.function async def bar(self) -> str: diff --git a/sdk/tests/test_dagger_package.py b/sdk/tests/test_dagger_package.py new file mode 100644 index 0000000..7738680 --- /dev/null +++ b/sdk/tests/test_dagger_package.py @@ -0,0 +1,411 @@ +"""What the dagger package is, with and without the temporary global client.""" + +import importlib.util +import os +import pathlib +import subprocess +import sys +import textwrap + +import pytest +from graphql import build_schema + +import dagger +from codegen.packages import ( + client_package, + core_package, + global_package, + write_global, + write_package, +) +from dagger.client import Session, default_session + +SDL = """ + directive @sourceMap(module: String, filename: String) + on OBJECT | FIELD_DEFINITION | ENUM | ENUM_VALUE | INPUT_OBJECT + directive @expectedType(name: String!) on FIELD_DEFINITION | ARGUMENT_DEFINITION + + scalar Platform + type Directory { id: ID! @expectedType(name: "Directory") } + type Container { id: ID! @expectedType(name: "Container") } + type Binding { + name: String! + asLinter: Linter! @sourceMap(module: "linter") + } + type Linter @sourceMap(module: "linter") { + id: ID! @expectedType(name: "Linter") + lint(src: ID! @expectedType(name: "Directory")): String! + } + type Query { + directory: Directory! + container(platform: Platform): Container! + linter(source: ID! @expectedType(name: "Directory")): Linter! + @sourceMap(module: "linter") + } +""" + + +def test_this_environment_has_no_global_client(): + assert importlib.util.find_spec("dagger_global") is None + + +def test_dag_is_the_default_session(): + assert type(dagger.dag) is Session + assert dagger.dag is default_session() + + +def test_core_name_points_to_its_new_home(): + with pytest.raises(AttributeError) as info: + _ = dagger.Container + + assert str(info.value) == ( + "module 'dagger' has no attribute 'Container'. Core types moved to " + "dagger_clients.core: from dagger_clients.core import Container" + ) + + +def test_legacy_client_type_points_to_session(): + with pytest.raises(AttributeError) as info: + _ = dagger.Client + + assert str(info.value) == ( + "module 'dagger' has no attribute 'Client'. dagger.Connection now yields " + "a dagger.Session, and the API is on core() from dagger_clients.core." + ) + + +def test_other_missing_package_names_get_the_plain_message(): + plain = r"^module 'dagger' has no attribute 'nope'$" + with pytest.raises(AttributeError, match=plain): + _ = dagger.nope + + assert not hasattr(dagger, "Container") + + +def test_session_field_points_to_core(): + with pytest.raises(AttributeError) as info: + dagger.dag.container() + + assert str(info.value) == ( + "'Session' object has no attribute 'container'. The API is on the " + "clients now: core().container() for a core field " + "(from dagger_clients.core import core), or container() from " + "the client's package for a client. To keep dag.container() while " + "migrating, set global-client = true under [tool.dagger] and run " + "dagger generate." + ) + assert info.value.name == "container" + + +def test_session_field_points_to_a_client_too(): + # The session cannot tell a client from a core field without importing + # core, so one message has to be true of both. + with pytest.raises(AttributeError) as info: + dagger.dag.linter() + + assert "or linter() from the client's package for a client." in str(info.value) + + +def test_private_session_name_gets_the_plain_message(): + plain = r"^'Session' object has no attribute '_x'$" + with pytest.raises(AttributeError, match=plain): + _ = dagger.dag._x + + assert not hasattr(dagger.dag, "container") + + +def test_dir_lists_dag_with_the_sdk_names(): + names = dir(dagger) + + assert {"dag", "Session", "connection", "function"} <= set(names) + assert "Container" not in names + + +def test_star_import_without_the_global_client(): + namespace: dict = {} + exec("from dagger import *", namespace) + + assert namespace["dag"] is default_session() + assert {"Session", "connection", "function"} <= namespace.keys() + assert "Container" not in namespace + assert not {n for n in namespace if n.startswith("_")} - {"__builtins__"} + + +def _run(script: str, pythonpath: pathlib.Path | None = None) -> str: + env = dict(os.environ) + if pythonpath is not None: + env["PYTHONPATH"] = str(pythonpath) + proc = subprocess.run( + [sys.executable, "-c", textwrap.dedent(script)], + capture_output=True, + text=True, + check=False, + env=env, + ) + assert proc.returncode == 0, proc.stderr + return proc.stdout + + +NO_GENERATED_CODE = """ + import importlib.abc + import sys + + class Absent(importlib.abc.MetaPathFinder): + def find_spec(self, name, path=None, target=None): + if name == "dagger.client.gen": + raise RuntimeError("the legacy bindings were imported") + + sys.meta_path.insert(0, Absent()) + # Absent both to an import and to find_spec, whatever is installed. + for generated in ("dagger_clients", "dagger_gen", "dagger_global"): + sys.modules[generated] = None + + import dagger + + assert type(dagger.dag).__name__ == "Session", type(dagger.dag) + try: + dagger.Container + except AttributeError as e: + assert "dagger_clients.core" in str(e), e + else: + raise AssertionError("dagger.Container exists") + try: + dagger.dag.container() + except AttributeError as e: + assert "core().container()" in str(e), e + else: + raise AssertionError("dag.container exists") + print("ok") +""" + + +def test_package_imports_with_no_generated_code(): + assert _run(NO_GENERATED_CODE) == "ok\n" + + +@pytest.fixture +def generated(tmp_path: pathlib.Path) -> pathlib.Path: + """Core, the linter client and the global client, as an installed scope.""" + schema = build_schema(SDL) + write_package(tmp_path, "core", core_package(schema)) + write_package(tmp_path, *client_package(schema, "linter", "./linter")) + write_global(tmp_path, global_package([schema])) + return tmp_path + + +WITH_GLOBAL_CLIENT = """ + import dagger + import dagger_global + import dagger_clients.core as core + import dagger_clients.linter as linter + from dagger.client import Session, default_session + from dagger.client._core import Context + + assert type(dagger.dag) is dagger_global.Client, type(dagger.dag) + assert isinstance(dagger.dag, Session) + assert dagger.dag is default_session() + assert dagger.Container is core.Container + assert dagger.Linter is linter.Linter + assert dagger.Client is dagger_global.Client + + ctr = dagger.dag.container() + assert type(ctr) is core.Container, type(ctr) + assert ctr._ctx.conn is dagger.dag + + # Old and new calls mix: each takes what the other made. + old = dagger.dag.directory() + assert type(linter.linter(old)) is linter.Linter + new = core.core().directory() + assert new._ctx.conn is dagger.dag + assert type(dagger.dag.linter(new)) is linter.Linter + assert type(core.Binding(Context()).as_linter()) is linter.Linter + print("ok") +""" + + +def test_global_client_makes_dag_the_client(generated: pathlib.Path): + assert _run(WITH_GLOBAL_CLIENT, generated) == "ok\n" + + +GLOBAL_NAMES_LISTED = """ + import dagger + import dagger_clients.core as core + import dagger_global + + listed = dir(dagger) + assert {"dag", "Container", "Linter", "Client", "Session"} <= set(listed) + + namespace = {} + exec("from dagger import *", namespace) + assert namespace["dag"] is dagger_global.dag + assert namespace["Container"] is core.Container + assert namespace["Client"] is dagger_global.Client + assert {"Linter", "Session", "connection", "function"} <= namespace.keys() + print("ok") +""" + + +def test_global_client_names_are_listed_and_star_imported(generated: pathlib.Path): + assert _run(GLOBAL_NAMES_LISTED, generated) == "ok\n" + + +@pytest.mark.parametrize( + "first", + [ + "import dagger_clients.core", + "from dagger_clients.linter import linter", + "import dagger_global", + ], +) +def test_generated_code_imported_before_dagger(generated: pathlib.Path, first: str): + script = f""" + {first} + import dagger + import dagger_global + assert type(dagger.dag) is dagger_global.Client, type(dagger.dag) + print("ok") + """ + + assert _run(script, generated) == "ok\n" + + +CLIENT_BEFORE_DAG = """ + import dagger_clients.core as core + + root = core.core() + + import dagger + import dagger_global + assert root._ctx.conn is dagger_global.dag, type(root._ctx.conn) + assert dagger.dag is dagger_global.dag + print("ok") +""" + + +def test_client_called_before_dag_gets_the_global_session(generated: pathlib.Path): + assert _run(CLIENT_BEFORE_DAG, generated) == "ok\n" + + +BROKEN_GLOBAL_CLIENT = """ + import dagger + + try: + dagger.dag + except ModuleNotFoundError as e: + print(e.name) + else: + print(type(dagger.dag).__name__) +""" + + +def test_global_client_that_cannot_import_is_not_skipped(tmp_path: pathlib.Path): + # Otherwise dag quietly becomes a plain Session, and its message tells a + # user who has the flag to set it. + (tmp_path / "dagger_clients").mkdir() + (tmp_path / "dagger_global").mkdir() + (tmp_path / "dagger_global/__init__.py").write_text("import dagger_clients.gone\n") + + assert _run(BROKEN_GLOBAL_CLIENT, tmp_path) == "dagger_clients.gone\n" + + +IMPORT_WARNINGS = """ + import warnings + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + import dagger + + for w in caught: + if "dagger" in str(w.message): + print(w.category.__name__, w.message) +""" + + +def test_legacy_bindings_are_named_not_loaded(tmp_path: pathlib.Path): + legacy = tmp_path / "dagger_gen.py" + legacy.write_text("raise AssertionError('the legacy bindings were loaded')\n") + + assert _run(IMPORT_WARNINGS, tmp_path) == ( + f"UserWarning {legacy} is no longer loaded. If an earlier SDK generated " + "it, `dagger generate` removes it; if you wrote it, delete or rename it.\n" + ) + + +def test_no_legacy_bindings_no_warning(): + assert _run(IMPORT_WARNINGS) == "" + + +CONNECTION_WITH_GLOBAL_CLIENT = """ + import anyio + import dagger + import dagger_global + from dagger.client import default_session + from dagger.provisioning import _connection + + class Engine: + def get_shared_client_connection(self): + return default_session().connection + + async def setup_client(self, conn): + return conn + + class provision_engine: + def __init__(self, cfg): + pass + + async def __aenter__(self): + return Engine() + + async def __aexit__(self, *_): + pass + + _connection.provision_engine = provision_engine + + async def main(): + async with dagger.connection() as session: + assert session is dagger.dag + assert type(session) is dagger_global.Client + + anyio.run(main) + print("ok") +""" + + +def test_connection_yields_the_global_client(generated: pathlib.Path): + assert _run(CONNECTION_WITH_GLOBAL_CLIENT, generated) == "ok\n" + + +TYPED = """ + import dagger + from dagger_clients.core import Container + + ctr: Container = dagger.dag.container() + reveal_type(dagger.dag) +""" + + +@pytest.mark.slow +def test_mypy_types_dag_as_the_global_client(generated: pathlib.Path): + (generated / "typed.py").write_text(textwrap.dedent(TYPED)) + + proc = subprocess.run( + [ + sys.executable, + "-m", + "mypy", + "--cache-dir", + str(generated / ".mypy_cache"), + str(generated / "typed.py"), + ], + capture_output=True, + text=True, + check=False, + env={**os.environ, "MYPYPATH": str(generated)}, + ) + + # Notes and errors are not on one stream once mypy installs stubs. + out = proc.stdout + proc.stderr + assert 'Revealed type is "dagger_global.Client"' in out, out + # Nothing else about the user's file, and nothing about the SDK's init. + assert "typed.py:" not in out.replace("typed.py:6:13: note", ""), out + assert "__init__.py" not in out, out diff --git a/templates/default/pyproject.toml.tmpl b/templates/default/pyproject.toml.tmpl index 6a46215..adb9998 100644 --- a/templates/default/pyproject.toml.tmpl +++ b/templates/default/pyproject.toml.tmpl @@ -2,11 +2,15 @@ name = "{{ .ModuleName }}" version = "0.1.0" requires-python = ">=3.14" -dependencies = ["dagger-io"] +dependencies = ["dagger-io", "dagger-clients-core"] [build-system] requires = ["uv_build>=0.8.4,<0.12.0"] build-backend = "uv_build" +[tool.uv.workspace] +members = ["sdk", "clients/core"] + [tool.uv.sources] -dagger-io = { path = "sdk", editable = true } +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } diff --git a/templates/default/src/{{.ModulePackage}}/__init__.py.tmpl b/templates/default/src/{{.ModulePackage}}/__init__.py.tmpl index 172fc71..443bebc 100644 --- a/templates/default/src/{{.ModulePackage}}/__init__.py.tmpl +++ b/templates/default/src/{{.ModulePackage}}/__init__.py.tmpl @@ -1,16 +1,16 @@ -import dagger -from dagger import dag, function, object_type +from dagger import function, object_type +from dagger_clients.core import Container, Directory, Workspace, core @object_type class {{ .ModuleType }}: - source: dagger.Directory + source: Directory base_image_address: str @classmethod def create( cls, - ws: dagger.Workspace, + ws: Workspace, base_image_address: str = "alpine:3.24", ) -> {{ .ModuleType }}: return cls( @@ -29,10 +29,11 @@ class {{ .ModuleType }}: ) @function - def container(self) -> dagger.Container: + def container(self) -> Container: """A container with the workspace source, ready to build.""" return ( - dag.container() + core() + .container() .from_(self.base_image_address) .with_directory("/src", self.source) .with_workdir("/src") diff --git a/templates/empty/pyproject.toml.tmpl b/templates/empty/pyproject.toml.tmpl index 6a46215..adb9998 100644 --- a/templates/empty/pyproject.toml.tmpl +++ b/templates/empty/pyproject.toml.tmpl @@ -2,11 +2,15 @@ name = "{{ .ModuleName }}" version = "0.1.0" requires-python = ">=3.14" -dependencies = ["dagger-io"] +dependencies = ["dagger-io", "dagger-clients-core"] [build-system] requires = ["uv_build>=0.8.4,<0.12.0"] build-backend = "uv_build" +[tool.uv.workspace] +members = ["sdk", "clients/core"] + [tool.uv.sources] -dagger-io = { path = "sdk", editable = true } +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } diff --git a/templates/legacy/.gitattributes b/templates/legacy/.gitattributes index 8274184..263a3a8 100644 --- a/templates/legacy/.gitattributes +++ b/templates/legacy/.gitattributes @@ -1 +1,2 @@ /sdk/** linguist-generated +/clients/** linguist-generated diff --git a/templates/legacy/pyproject.toml.tmpl b/templates/legacy/pyproject.toml.tmpl index 6a46215..adb9998 100644 --- a/templates/legacy/pyproject.toml.tmpl +++ b/templates/legacy/pyproject.toml.tmpl @@ -2,11 +2,15 @@ name = "{{ .ModuleName }}" version = "0.1.0" requires-python = ">=3.14" -dependencies = ["dagger-io"] +dependencies = ["dagger-io", "dagger-clients-core"] [build-system] requires = ["uv_build>=0.8.4,<0.12.0"] build-backend = "uv_build" +[tool.uv.workspace] +members = ["sdk", "clients/core"] + [tool.uv.sources] -dagger-io = { path = "sdk", editable = true } +dagger-io = { workspace = true } +dagger-clients-core = { workspace = true } diff --git a/templates/legacy/src/{{.ModulePackage}}/main.py.tmpl b/templates/legacy/src/{{.ModulePackage}}/main.py.tmpl index ab9f04d..6350fd4 100644 --- a/templates/legacy/src/{{.ModulePackage}}/main.py.tmpl +++ b/templates/legacy/src/{{.ModulePackage}}/main.py.tmpl @@ -1,19 +1,20 @@ -import dagger -from dagger import dag, function, object_type +from dagger import function, object_type +from dagger_clients.core import Container, Directory, core @object_type class {{ .ModuleType }}: @function - def container_echo(self, string_arg: str) -> dagger.Container: + def container_echo(self, string_arg: str) -> Container: """Returns a container that echoes whatever string argument is provided""" - return dag.container().from_("alpine:latest").with_exec(["echo", string_arg]) + return core().container().from_("alpine:latest").with_exec(["echo", string_arg]) @function - async def grep_dir(self, directory_arg: dagger.Directory, pattern: str) -> str: + async def grep_dir(self, directory_arg: Directory, pattern: str) -> str: """Returns lines that match a pattern in the files of the provided Directory""" return await ( - dag.container() + core() + .container() .from_("alpine:latest") .with_mounted_directory("/mnt", directory_arg) .with_workdir("/mnt")