diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index dca60a9d43..1f28825a81 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -163,33 +163,6 @@ jobs: runner/dstack-shim-${{ matrix.goos }}-${{ matrix.goarch }} retention-days: 1 - gateway-build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up uv - uses: astral-sh/setup-uv@v5 - with: - python-version: 3.11 - - name: Build package - working-directory: gateway - run: | - echo "__version__ = \"${{ inputs.version }}\"" > src/dstack/gateway/version.py - # TODO: depend on a specific dstackai/dstack commit for staging builds? - if [[ "${{ inputs.staging }}" == "false" ]]; then - sed \ - -i.old \ - "s|@ https://github.com/dstackai/dstack/archive/refs/heads/master.tar.gz|== ${{ inputs.version }}|" \ - pyproject.toml - diff pyproject.toml pyproject.toml.old > /dev/null && echo "Could not set version" && exit 1 - fi - uv build - - uses: actions/upload-artifact@v4 - with: - name: dstack-gateway - path: gateway/dist/dstack_gateway-${{ inputs.version }}-py3-none-any.whl - retention-days: 1 - python-build: needs: [code-lint, frontend-build] runs-on: ubuntu-latest diff --git a/.github/workflows/upload-pre-pypi-artifacts.yml b/.github/workflows/upload-pre-pypi-artifacts.yml index 542f445e2e..91e94508bf 100644 --- a/.github/workflows/upload-pre-pypi-artifacts.yml +++ b/.github/workflows/upload-pre-pypi-artifacts.yml @@ -25,21 +25,6 @@ jobs: ignore-empty-workdir: true - name: Install AWS run: uv tool install awscli - - name: Download dstack-gateway - uses: actions/download-artifact@v4 - with: - name: dstack-gateway - path: gateway - - name: Upload dstack-gateway to S3 - working-directory: gateway - run: | - CHANNEL=release - if [ "${{ inputs.staging }}" = "true" ]; then - CHANNEL=stgn - fi - WHEEL=dstack_gateway-${{ inputs.version }}-py3-none-any.whl - aws s3 cp $WHEEL "s3://dstack-gateway-downloads/$CHANNEL/$WHEEL" - echo "${{ inputs.version }}" | aws s3 cp - "s3://dstack-gateway-downloads/$CHANNEL/latest-version" - name: Download dstack-runner uses: actions/download-artifact@v4 with: diff --git a/contributing/AUTOSCALING.md b/contributing/AUTOSCALING.md index eb5f6e1978..f977e4d60f 100644 --- a/contributing/AUTOSCALING.md +++ b/contributing/AUTOSCALING.md @@ -2,8 +2,8 @@ `dstack` features auto-scaling for services published via the gateway. The general flow is: -- STEP 1: `dstack-gateway` parses nginx `access.log` to collect per-second statistics about requests to the service and request times. -- STEP 2: `dstack-gateway` aggregates statistics over several predefined windows. +- STEP 1: The gateway parses nginx `access.log` to collect per-second statistics about requests to the service and request times. +- STEP 2: The gateway aggregates statistics over several predefined windows. - STEP 3: The server keeps gateway connections alive in the scheduled `process_gateways_connections` task and continuously collects stats from active gateways. This is separate from `GatewayPipeline`, which handles gateway provisioning and deletion. - STEP 4: When `RunPipeline` processes a service run, it loads the latest collected gateway stats for that service. - STEP 5: The autoscaler (configured via `dstack.yml`) computes the desired replica count for each replica group. diff --git a/contributing/PROXY.md b/contributing/PROXY.md index 6c738e999d..a8dc522cc8 100644 --- a/contributing/PROXY.md +++ b/contributing/PROXY.md @@ -137,7 +137,7 @@ Provisioning happens as follows: 1. Launch a non-GPU instance (usually the smallest) with all ports exposed. 2. Install Nginx, Certbot, and patch configs. 3. Create blue-green virtual environments. -4. Install the latest `dstack-gateway` package from the S3 bucket. `dstack-gateway` is a thin package that depends on the `dstack` package, which contains the actual gateway implementation. +4. Install the `dstack[gateway]` Python package. 5. Run the systemd service `dstack.gateway.service`. ### Update @@ -152,56 +152,18 @@ The update process looks like this: ## Gateway development -The gateway app needs to interact with Nginx and certbot, so running it locally can be challenging. One way to test your code is to upload your development branch to an existing gateway and run the gateway app from source. +The gateway app needs to interact with Nginx and certbot, so running it locally can be challenging. You can test your development version by deploying it to a live gateway. -1. Run `dstack server` with `DSTACK_SKIP_GATEWAY_UPDATE=1` environment variable. This will prevent `dstack` from updating and starting the standard gateway version on each server restart. +For example, push your version to GitHub and run `dstack server` with the `DSTACK_GATEWAY_PACKAGE_URL` environment variable, which will instruct `dstack` to deploy that version to all gateways. -1. Provision a gateway through `dstack`: - - ```shell - dstack apply -f my-gateway.dstack.yml - ``` - -1. Save the gateway key to a file: - - ```shell - sqlite3 ~/.dstack/server/data/sqlite.db "SELECT ssh_private_key FROM gateway_computes WHERE deleted = 0 AND ip_address = ''" > /tmp/gateway.key - chmod 600 /tmp/gateway.key - ``` - -1. Deliver your code to the gateway. For example, clone it from a remote repo: - - ```shell - ssh -i /tmp/gateway.key ubuntu@gateway.example "git clone https://github.com/dstackai/dstack.git ~/dstack-repo" - ``` - - Or push it from your machine: - - ```shell - ssh -i /tmp/gateway.key ubuntu@gateway.example "git init ~/dstack-repo" - git remote add gateway ubuntu@gateway.example:~/dstack-repo - GIT_SSH_COMMAND='ssh -i /tmp/gateway.key' git push gateway branch_name - ``` - -1. Connect to the gateway: - - ```shell - ssh -i /tmp/gateway.key ubuntu@gateway.example - ``` - -1. Prepare an environment with your development branch on the gateway: - - ```shell - cd ~/dstack-repo - git checkout branch_name - curl -LsSf https://astral.sh/uv/install.sh | sh - source ~/.local/bin/env - uv sync --extra gateway - ``` +```shell +DSTACK_GATEWAY_PACKAGE_URL=https://github.com/dstackai/dstack/archive/refs/heads/my_development_branch.zip dstack server +``` -1. Stop the gateway service and start your development version from source: +To access logs or troubleshoot, extract the gateway's private key from the database and connect to it via SSH. Logs are available in `journalctl`. - ```shell - sudo systemctl stop dstack.gateway.service - uv run uvicorn dstack._internal.proxy.gateway.main:app - ``` +```shell +sqlite3 ~/.dstack/server/data/sqlite.db "SELECT ssh_private_key FROM gateway_computes WHERE deleted = 0 AND ip_address = '$GATEWAY_IP_ADDR'" > /tmp/gateway.key +chmod 600 /tmp/gateway.key +ssh -i /tmp/gateway.key ubuntu@$GATEWAY_IP_ADDR sudo journalctl -u dstack.gateway.service +``` diff --git a/contributing/RUNNER-AND-SHIM.md b/contributing/RUNNER-AND-SHIM.md index 747a0745d2..bbb09ed4ff 100644 --- a/contributing/RUNNER-AND-SHIM.md +++ b/contributing/RUNNER-AND-SHIM.md @@ -56,6 +56,6 @@ All communication between the `dstack` server and `dstack-runner` happens via HT ## SSH tunnels -`dstack` expects a running SSH server right next to the `dstack-runner`. It provides a secure channel for communication with the runner API and forwarding any ports without listening for `0.0.0.0`. The `dstack-gateway` also uses this SSH server for forwarding requests from public endpoints. +`dstack` expects a running SSH server right next to the `dstack-runner`. It provides a secure channel for communication with the runner API and forwarding any ports without listening for `0.0.0.0`. The gateway also uses this SSH server for forwarding requests from public endpoints. `dstack-shim` must also be running next to the SSH server. The `dstack` server connects to this SSH server for interacting with both `dstack-shim` and `dstack-runner`. The CLI uses this SSH server as a jump host because the user wants to connect to the container. diff --git a/gateway/README.md b/gateway/README.md deleted file mode 100644 index 05d7678292..0000000000 --- a/gateway/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# `dstack-gateway` - -A thin package to deliver and install the gateway app. Expected to be merged with the `dstack` package in [#2251](https://github.com/dstackai/dstack/issues/2251). - -For details about gateways, see [contributing/PROXY.md](../contributing/PROXY.md). diff --git a/gateway/pyproject.toml b/gateway/pyproject.toml deleted file mode 100644 index 5cec27e095..0000000000 --- a/gateway/pyproject.toml +++ /dev/null @@ -1,23 +0,0 @@ -[build-system] -requires = ["setuptools>=61.0"] -build-backend = "setuptools.build_meta" - -[project] -name = "dstack-gateway" -authors = [ - { name = "dstack GmbH" }, -] -requires-python = ">=3.10" -dynamic = ["version"] -dependencies = [ - # release builds of dstack-gateway depend on a PyPI version of dstack instead - "dstack[gateway] @ https://github.com/dstackai/dstack/archive/refs/heads/master.tar.gz", -] - -[tool.setuptools.package-data] -"dstack.gateway" = [ - "resources/systemd/*", -] - -[tool.setuptools.dynamic] -version = {attr = "dstack.gateway.version.__version__"} diff --git a/gateway/src/dstack/gateway/__init__.py b/gateway/src/dstack/gateway/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/gateway/src/dstack/gateway/main.py b/gateway/src/dstack/gateway/main.py deleted file mode 100644 index 4d701eb257..0000000000 --- a/gateway/src/dstack/gateway/main.py +++ /dev/null @@ -1 +0,0 @@ -from dstack._internal.proxy.gateway.main import app as app diff --git a/gateway/src/dstack/gateway/systemd/__init__.py b/gateway/src/dstack/gateway/systemd/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/gateway/src/dstack/gateway/version.py b/gateway/src/dstack/gateway/version.py deleted file mode 100644 index 6c8e6b979c..0000000000 --- a/gateway/src/dstack/gateway/version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.0.0" diff --git a/gateway/src/tests/__init__.py b/gateway/src/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/mkdocs/docs/reference/env.md b/mkdocs/docs/reference/env.md index 3621e27f4e..9b1c9f17d2 100644 --- a/mkdocs/docs/reference/env.md +++ b/mkdocs/docs/reference/env.md @@ -173,6 +173,7 @@ slows down processing and may cause CPU spikes due to frequent SSH-connection es `https://dstack.example.com/{arch}/{version}/dstack-runner`. * `DSTACK_SHIM_DOWNLOAD_URL` – Overrides `dstack-shim` binary download URL. The URL can contain `{version}` and/or `{arch}` placeholders, see `DSTACK_RUNNER_DOWNLOAD_URL` for the details. + * `DSTACK_GATEWAY_PACKAGE_URL` – Overrides the URL the `dstack` package is installed from on gateway instances (used as `dstack[gateway] @ `). * `DSTACK_DEFAULT_CREDS_DISABLED` – Disables default credentials detection if set. Defaults to `None`. ## CLI diff --git a/src/dstack/_internal/core/backends/base/compute.py b/src/dstack/_internal/core/backends/base/compute.py index 6062acc3cd..0451ca2295 100644 --- a/src/dstack/_internal/core/backends/base/compute.py +++ b/src/dstack/_internal/core/backends/base/compute.py @@ -1218,25 +1218,23 @@ def get_latest_runner_build() -> Optional[str]: return None -def get_dstack_gateway_wheel(build: str) -> str: - channel = "release" if settings.DSTACK_RELEASE else "stgn" - base_url = f"https://dstack-gateway-downloads.s3.amazonaws.com/{channel}" - if build == "latest": - build = _fetch_version(f"{base_url}/latest-version") or "latest" - logger.debug("Found the latest gateway build: %s", build) - wheel = f"{base_url}/dstack_gateway-{build}-py3-none-any.whl" - return f"dstack-gateway @ {wheel}" +def get_dstack_gateway_package_and_target_version() -> tuple[str, str | None]: + if settings.DSTACK_GATEWAY_PACKAGE_URL: + return f"dstack[gateway] @ {settings.DSTACK_GATEWAY_PACKAGE_URL}", None + if settings.DSTACK_VERSION is not None: + return f"dstack[gateway]=={settings.DSTACK_VERSION}", settings.DSTACK_VERSION + package = "dstack[gateway] @ https://github.com/dstackai/dstack/archive/refs/heads/master.zip" + return package, None def get_dstack_gateway_commands() -> List[str]: - build = get_dstack_runner_version() or "latest" - gateway_package = get_dstack_gateway_wheel(build) + gateway_package, _ = get_dstack_gateway_package_and_target_version() return [ "mkdir -p /home/ubuntu/dstack", "python3 -m venv /home/ubuntu/dstack/blue", "python3 -m venv /home/ubuntu/dstack/green", f"/home/ubuntu/dstack/blue/bin/pip install '{gateway_package}'", - "sudo /home/ubuntu/dstack/blue/bin/python -m dstack.gateway.systemd install --run", + "sudo /home/ubuntu/dstack/blue/bin/python -m dstack._internal.proxy.gateway.systemd install --run", ] diff --git a/gateway/src/dstack/gateway/resources/systemd/dstack.gateway.service b/src/dstack/_internal/proxy/gateway/resources/systemd/dstack.gateway.service similarity index 100% rename from gateway/src/dstack/gateway/resources/systemd/dstack.gateway.service rename to src/dstack/_internal/proxy/gateway/resources/systemd/dstack.gateway.service diff --git a/gateway/src/dstack/gateway/resources/systemd/start.sh b/src/dstack/_internal/proxy/gateway/resources/systemd/start.sh old mode 100755 new mode 100644 similarity index 75% rename from gateway/src/dstack/gateway/resources/systemd/start.sh rename to src/dstack/_internal/proxy/gateway/resources/systemd/start.sh index 46afbe7733..932740acc5 --- a/gateway/src/dstack/gateway/resources/systemd/start.sh +++ b/src/dstack/_internal/proxy/gateway/resources/systemd/start.sh @@ -8,4 +8,4 @@ else version="blue" echo "$version" > "$root/version" fi -"$root/$version/bin/uvicorn" dstack.gateway.main:app +"$root/$version/bin/uvicorn" dstack._internal.proxy.gateway.main:app diff --git a/gateway/src/dstack/__init__.py b/src/dstack/_internal/proxy/gateway/systemd/__init__.py similarity index 100% rename from gateway/src/dstack/__init__.py rename to src/dstack/_internal/proxy/gateway/systemd/__init__.py diff --git a/gateway/src/dstack/gateway/systemd/__main__.py b/src/dstack/_internal/proxy/gateway/systemd/__main__.py similarity index 83% rename from gateway/src/dstack/gateway/systemd/__main__.py rename to src/dstack/_internal/proxy/gateway/systemd/__main__.py index 793bf876f7..2857eee978 100644 --- a/gateway/src/dstack/gateway/systemd/__main__.py +++ b/src/dstack/_internal/proxy/gateway/systemd/__main__.py @@ -30,14 +30,12 @@ def install_action(args): uid, gid = user.pw_uid, user.pw_gid print("Writing service file...") - service_file = importlib.resources.read_text( - "dstack.gateway.resources.systemd", service_path.name - ) + service_file = read_resource(service_path.name) service_path.write_text(service_file.format(working_dir=working_dir.as_posix())) for script_name in ["start.sh"]: print(f"Writing {script_name} script...") - script = importlib.resources.read_text("dstack.gateway.resources.systemd", script_name) + script = read_resource(script_name) script_path = working_dir / script_name script_path.write_text(script) os.chown(script_path, uid, gid) @@ -50,5 +48,13 @@ def install_action(args): assert subprocess.run(["systemctl", "enable", service_name] + args).returncode == 0 +def read_resource(file: str) -> str: + return ( + importlib.resources.files("dstack._internal.proxy.gateway") + .joinpath(f"resources/systemd/{file}") + .read_text() + ) + + if __name__ == "__main__": main() diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py index 339cfc2177..3ad0042d3b 100644 --- a/src/dstack/_internal/server/services/gateways/__init__.py +++ b/src/dstack/_internal/server/services/gateways/__init__.py @@ -16,8 +16,7 @@ import dstack._internal.utils.random_names as random_names from dstack._internal.core.backends.base.compute import ( - get_dstack_gateway_wheel, - get_dstack_runner_version, + get_dstack_gateway_package_and_target_version, ) from dstack._internal.core.backends.features import ( BACKENDS_WITH_GATEWAY_SUPPORT, @@ -782,11 +781,11 @@ async def init_gateways(session: AsyncSession): "Skipping gateway replicas update due to DSTACK_SKIP_GATEWAY_UPDATE env variable" ) else: - build = get_dstack_runner_version() or "latest" + gateway_package, target_version = get_dstack_gateway_package_and_target_version() for gateway_replica, res in await gather_map_async( gateway_replicas, - lambda r: _update_gateway_replica(r, build), + lambda r: _update_gateway_replica(r, gateway_package, target_version), return_exceptions=True, ): if isinstance(res, Exception): @@ -808,7 +807,11 @@ async def init_gateways(session: AsyncSession): ) -async def _update_gateway_replica(gateway_replica_model: GatewayReplicaModel, build: str) -> bool: +async def _update_gateway_replica( + gateway_replica_model: GatewayReplicaModel, + gateway_package: str, + target_version: str | None, +) -> bool: if gateway_replica_model.ip_address is None: logger.warning( "Gateway replica %s has no ip_address, cannot update", gateway_replica_model.id @@ -826,15 +829,13 @@ async def _update_gateway_replica(gateway_replica_model: GatewayReplicaModel, bu ) logger.debug("Updating gateway replica %s", connection.ip_address) - # Build package spec with extras and wheel URL - gateway_package = get_dstack_gateway_wheel(build) command = ( "/bin/sh -c " + shlex.quote(_GATEWAY_UPDATE_SCRIPT) + " sh " # $0 placeholder + shlex.quote(gateway_package) + " " - + shlex.quote(build) + + shlex.quote(target_version or "") ) stdout = await connection.tunnel.aexec(command) if "Update successfully completed" in stdout: @@ -854,20 +855,23 @@ async def _update_gateway_replica(gateway_replica_model: GatewayReplicaModel, bu else version=blue fi -current_build=$("$root/$version/bin/pip" show dstack-gateway | grep Version | awk '{print $2}') -if [ "$current_build" = "$build" ]; then - echo "The build $build is already installed. Skipping..." - exit 0 +if [ -n "$build" ]; then + current_build=$("$root/$version/bin/pip" show dstack | grep Version | awk '{print $2}') + if [ "$current_build" = "$build" ]; then + echo "The build $build is already installed. Skipping..." + exit 0 + fi fi if [ "$version" = blue ]; then version=green else version=blue fi +# dstack-gateway is a pre-0.21.4 package that may still be installed and require a conflicting dstack version "$root/$version/bin/pip" uninstall -y dstack-gateway dstack "$root/$version/bin/pip" cache remove dstack "$root/$version/bin/pip" install "$gateway_package" -sudo "$root/$version/bin/python" -m dstack.gateway.systemd install +sudo "$root/$version/bin/python" -m dstack._internal.proxy.gateway.systemd install echo "$version" > "$root/version" sudo systemctl daemon-reload sudo systemctl restart dstack.gateway diff --git a/src/dstack/_internal/settings.py b/src/dstack/_internal/settings.py index 7a7638c99c..7a37ba5cfb 100644 --- a/src/dstack/_internal/settings.py +++ b/src/dstack/_internal/settings.py @@ -17,6 +17,7 @@ DSTACK_SHIM_VERSION_URL = os.getenv("DSTACK_SHIM_VERSION_URL") DSTACK_SHIM_DOWNLOAD_URL = os.getenv("DSTACK_SHIM_DOWNLOAD_URL") DSTACK_USE_LATEST_FROM_BRANCH = os.getenv("DSTACK_USE_LATEST_FROM_BRANCH") is not None +DSTACK_GATEWAY_PACKAGE_URL = os.getenv("DSTACK_GATEWAY_PACKAGE_URL") DSTACK_DOCKER_BASE_IMAGE = os.getenv("DSTACK_DOCKER_BASE_IMAGE", "dstackai/base")