Fix remote container build fallback

cli/azd/docs/concurrency-model.md

@@ -185,6 +185,29 @@ race on `env` (one writing `KUBECONFIG=…`, the other reading it for an
 
 ---
 
+## `pkg/tools/docker.Cli`
+
+| Lock | Protects | Acquired by |
+|------|----------|-------------|
+| `engineMu sync.Mutex` | `containerEngine` and runtime selection | `ContainerEngine`, `selectContainerEngine`, `getContainerEngine` |
+
+**Contract**: Every read and write of `containerEngine` holds `engineMu`.
+`ContainerEngine` holds it across the cache check, environment/PATH detection,
+and publication. `detectContainerEngineLocked` requires the caller to hold it.
+`CheckInstalled` uses `selectContainerEngine` to select and snapshot under the
+lock, then validates that snapshot, including error names, without the lock.
+Each call repeats selection and readiness checks; failures are not cached.
+
+`Name`, `InstallUrl`, and container operations read through
+`getContainerEngine`, which snapshots under the lock without detection and
+defaults to Docker before selection. No engine lock is held during version
+checks, daemon checks, builds, or other container subprocesses.
+
+**Why it matters**: Parallel services and remote-build fallbacks share the
+singleton `docker.Cli`.
+
+---
+
 ## `pkg/project.containerAppTarget` and `pkg/project.aksTarget`
 
 These targets no longer carry package-level `envMu` / `aksEnvMu` mutexes

cli/azd/internal/grpcserver/container_service_test.go

@@ -7,11 +7,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/http"
 	osexec "os/exec"
 	"testing"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry"
+	"github.com/azure/azure-dev/cli/azd/internal"
 	"github.com/azure/azure-dev/cli/azd/pkg/azdext"
 	"github.com/azure/azure-dev/cli/azd/pkg/containerregistry"
 	"github.com/azure/azure-dev/cli/azd/pkg/environment"
@@ -378,6 +380,44 @@ func TestMapContainerPublishError_PreservesResponseError(t *testing.T) {
 	require.Same(t, err, mapContainerPublishError(err))
 }
 
+func TestMapContainerPublishError_LocalFallbackFailure(t *testing.T) {
+	t.Parallel()
+
+	buildErr := &azdexec.ExitError{Cmd: "docker", ExitCode: 1}
+	tests := []struct {
+		name string
+		err  error
+	}{
+		{"RuntimeUnavailable", errors.New("local container runtime unavailable")},
+		{"BuildFailed", buildErr},
+		{"PushFailed", &internal.ErrorWithSuggestion{Err: buildErr, Suggestion: "Check registry authentication."}},
+		{"Canceled", context.Canceled},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			responseErr := &azcore.ResponseError{
+				StatusCode: http.StatusForbidden,
+				ErrorCode:  "TasksOperationsNotAllowed",
+			}
+			remoteErr := &containerregistry.RemoteBuildUnavailableError{Err: responseErr}
+			err := fmt.Errorf("remote build failed: %w\n\nLocal fallback failed: %w", remoteErr, tt.err)
+			mapped := mapContainerToolError(mapContainerPublishError(err))
+			require.Same(t, err, mapped)
+			require.ErrorIs(t, mapped, responseErr)
+			require.ErrorIs(t, mapped, tt.err)
+
+			st, ok := status.FromError(mapHostError(mapped))
+			require.True(t, ok)
+			require.Equal(t, err.Error(), st.Message())
+			detail := requireServiceErrorDetail(t, st)
+			require.Equal(t, "TasksOperationsNotAllowed", detail.GetErrorCode())
+			require.Equal(t, int32(http.StatusForbidden), detail.GetStatusCode())
+			require.Empty(t, relayedExtensionErrorDetails(st))
+		})
+	}
+}
+
 func TestMapContainerToolError(t *testing.T) {
 	t.Parallel()
 

cli/azd/pkg/containerregistry/remote_build.go

@@ -126,6 +126,22 @@ func (r *RemoteBuildManager) UploadBuildSource(
 	return sourceUploadRes.SourceUploadDefinition, nil
 }
 
+// RemoteBuildUnavailableError indicates that ACR explicitly refused to schedule a build.
+// Callers may attempt a local build instead. Errors after submission do not use this type.
+type RemoteBuildUnavailableError struct {
+	Err error
+}
+
+// Error returns the original scheduling error.
+func (e *RemoteBuildUnavailableError) Error() string {
+	return e.Err.Error()
+}
+
+// Unwrap preserves the original Azure service error.
+func (e *RemoteBuildUnavailableError) Unwrap() error {
+	return e.Err
+}
+
 // RemoteBuildRunError represents a terminal failure reported by an Azure Container Registry remote build.
 type RemoteBuildRunError struct {
 	Status   armcontainerregistry.RunStatus
@@ -195,6 +211,10 @@ func (r *RemoteBuildManager) RunDockerBuildRequestWithLogs(
 
 	runPoller, err := regClient.BeginScheduleRun(ctx, resourceGroupName, registryName, buildRequest, nil)
 	if err != nil {
+		if responseErr, ok := errors.AsType[*azcore.ResponseError](err); ok &&
+			responseErr.ErrorCode == "TasksOperationsNotAllowed" {
+			return &RemoteBuildUnavailableError{Err: err}
+		}
 		return err
 	}
 

cli/azd/pkg/containerregistry/remote_build_test.go

@@ -227,33 +227,65 @@ func TestRunDockerBuildRequestWithLogs_CredentialError(t *testing.T) {
 func TestRunDockerBuildRequestWithLogs_ScheduleRunError(t *testing.T) {
 	t.Parallel()
 
-	mockCtx := mocks.NewMockContext(t.Context())
-	mockCtx.HttpClient.When(func(request *http.Request) bool {
-		return strings.Contains(request.URL.Path, "scheduleRun")
-	}).RespondFn(func(request *http.Request) (*http.Response, error) {
-		return mocks.CreateHttpResponseWithBody(
-			request, http.StatusBadRequest, map[string]any{
-				"error": map[string]string{
-					"code":    "BadRequest",
-					"message": "invalid build request",
-				},
-			},
-		)
-	})
-
-	mgr := NewRemoteBuildManager(
-		mockCtx.SubscriptionCredentialProvider,
-		armOptionsNoRetry(mockCtx.HttpClient),
-	)
-
-	err := mgr.RunDockerBuildRequestWithLogs(
-		t.Context(), testSubscriptionID, testResourceGroup, testRegistryName,
-		&armcontainerregistry.DockerBuildRequest{}, io.Discard,
-	)
-	require.Error(t, err)
-	require.Contains(t, err.Error(), "BadRequest")
-	var responseErr *azcore.ResponseError
-	require.ErrorAs(t, err, &responseErr)
+	tests := []struct {
+		name            string
+		code            string
+		status          int
+		afterSubmission bool
+		wantUnavailable bool
+	}{
+		{"TasksRefused", "TasksOperationsNotAllowed", http.StatusForbidden, false, true},
+		{"InvalidRequest", "BadRequest", http.StatusBadRequest, false, false},
+		{"AuthorizationFailed", "AuthorizationFailed", http.StatusForbidden, false, false},
+		{"ServerError", "InternalServerError", http.StatusInternalServerError, false, false},
+		{"RefusalAfterSubmission", "TasksOperationsNotAllowed", http.StatusForbidden, true, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			mockCtx := mocks.NewMockContext(t.Context())
+			errorResponse := func(request *http.Request) (*http.Response, error) {
+				return mocks.CreateHttpResponseWithBody(request, tt.status, map[string]any{
+					"error": map[string]string{
+						"code":    tt.code,
+						"message": "request refused",
+					},
+				})
+			}
+			mockCtx.HttpClient.When(func(request *http.Request) bool {
+				return strings.Contains(request.URL.Path, "scheduleRun")
+			}).RespondFn(func(request *http.Request) (*http.Response, error) {
+				if !tt.afterSubmission {
+					return errorResponse(request)
+				}
+				return mocks.CreateHttpResponseWithBody(request, http.StatusOK, armcontainerregistry.Run{
+					Properties: &armcontainerregistry.RunProperties{RunID: new("run-id")},
+				})
+			})
+			mockCtx.HttpClient.When(func(request *http.Request) bool {
+				return strings.Contains(request.URL.Path, "listLogSasUrl")
+			}).RespondFn(errorResponse)
+
+			mgr := NewRemoteBuildManager(
+				mockCtx.SubscriptionCredentialProvider,
+				armOptionsNoRetry(mockCtx.HttpClient),
+			)
+			err := mgr.RunDockerBuildRequestWithLogs(
+				t.Context(), testSubscriptionID, testResourceGroup, testRegistryName,
+				&armcontainerregistry.DockerBuildRequest{}, io.Discard,
+			)
+			require.Error(t, err)
+			responseErr, ok := errors.AsType[*azcore.ResponseError](err)
+			require.True(t, ok)
+			require.Equal(t, tt.code, responseErr.ErrorCode)
+			unavailableErr, ok := errors.AsType[*RemoteBuildUnavailableError](err)
+			require.Equal(t, tt.wantUnavailable, ok)
+			if ok {
+				require.Same(t, responseErr, unavailableErr.Unwrap())
+				require.Equal(t, responseErr.Error(), unavailableErr.Error())
+			}
+		})
+	}
 }
 
 func TestRunDockerBuildRequestWithLogs_TerminalStatus(t *testing.T) {

cli/azd/pkg/project/container_helper.go

@@ -404,6 +404,15 @@ func (ch *ContainerHelper) Build(
 		return &ServiceBuildResult{}, nil
 	}
 
+	return ch.buildLocalImage(ctx, serviceConfig, env, progress)
+}
+
+func (ch *ContainerHelper) buildLocalImage(
+	ctx context.Context,
+	serviceConfig *ServiceConfig,
+	env *environment.Environment,
+	progress *async.Progress[ServiceProgress],
+) (*ServiceBuildResult, error) {
 	dockerOptions := getDockerOptionsWithDefaults(serviceConfig.Docker)
 	resolveDockerPaths(serviceConfig, &dockerOptions)
 
@@ -601,6 +610,16 @@ func (ch *ContainerHelper) Package(
 		return &ServicePackageResult{}, nil
 	}
 
+	return ch.packageLocalImage(ctx, serviceConfig, serviceContext, env, progress)
+}
+
+func (ch *ContainerHelper) packageLocalImage(
+	ctx context.Context,
+	serviceConfig *ServiceConfig,
+	serviceContext *ServiceContext,
+	env *environment.Environment,
+	progress *async.Progress[ServiceProgress],
+) (*ServicePackageResult, error) {
 	var imageId string
 	var sourceImage string
 	var imageHash string
@@ -779,6 +798,10 @@ func (ch *ContainerHelper) Publish(
 		fields.ContainerRemoteBuildKey.Bool(serviceConfig.Docker.RemoteBuild),
 	)
 
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+
 	var remoteImage string
 
 	if err := validatePublishOptions(serviceConfig, options); err != nil {
@@ -807,20 +830,29 @@ func (ch *ContainerHelper) Publish(
 	} else if serviceConfig.Docker.RemoteBuild {
 		remoteImage, err = ch.runRemoteBuild(ctx, serviceConfig, targetResource, env, progress, imageOverride)
 		if err != nil {
-			// Check if a local container runtime (Docker/Podman) is available before falling back
-			if dockerErr := ch.docker.CheckInstalled(ctx); dockerErr != nil {
-				return nil, fmt.Errorf(
-					"remote build failed: %w\n\nLocal fallback unavailable: %w",
-					err, dockerErr)
+			remoteErr := err
+			if errors.Is(remoteErr, context.Canceled) || errors.Is(remoteErr, context.DeadlineExceeded) {
+				return nil, remoteErr
+			}
+			if ctxErr := ctx.Err(); ctxErr != nil {
+				return nil, fmt.Errorf("remote build failed: %w\n\nPublish canceled: %w", remoteErr, ctxErr)
+			}
+			if _, ok := errors.AsType[*containerregistry.RemoteBuildUnavailableError](remoteErr); !ok {
+				return nil, remoteErr
 			}
 
-			ch.console.MessageUxItem(ctx, &ux.WarningMessage{
-				Description: fmt.Sprintf(
-					"Remote build failed: %s\nFalling back to local Docker build.", err),
-				HidePrefix: false,
-			})
-			remoteImage, err = ch.publishLocalImage(
+			remoteImage, err = ch.publishLocalFallback(
 				ctx, serviceConfig, serviceContext, env, progress, imageOverride)
+			if err != nil {
+				err = fmt.Errorf("remote build failed: %w\n\nLocal fallback failed: %w", remoteErr, err)
+				if suggestion, ok := errors.AsType[*internal.ErrorWithSuggestion](err); ok {
+					// Rich CLI output renders suggestion.Err rather than its outer wrappers.
+					combined := *suggestion
+					combined.Err = err
+					return nil, &combined
+				}
+				return nil, err
+			}
 		}
 	} else if useDotnetPublishForDockerBuild(serviceConfig) {
 		remoteImage, err = ch.runDotnetPublish(ctx, serviceConfig, targetResource, env, progress)
@@ -848,7 +880,70 @@ func (ch *ContainerHelper) Publish(
 	}, nil
 }
 
-// publishLocalImage builds the image locally and pushes it to the remote registry, it returns the full remote image name.
+func (ch *ContainerHelper) publishLocalFallback(
+	ctx context.Context,
+	serviceConfig *ServiceConfig,
+	serviceContext *ServiceContext,
+	env *environment.Environment,
+	progress *async.Progress[ServiceProgress],
+	imageOverride *imageOverride,
+) (string, error) {
+	if err := ctx.Err(); err != nil {
+		return "", err
+	}
+	if err := ch.docker.CheckInstalled(ctx); err != nil {
+		return "", fmt.Errorf("local container runtime unavailable: %w", err)
+	}
+	if err := ctx.Err(); err != nil {
+		return "", err
+	}
+
+	var hasPackage bool
+	if serviceContext != nil {
+		artifact, found := serviceContext.Package.FindFirst(WithKind(ArtifactKindContainer))
+		hasPackage = found
+		if found && artifact.LocationKind == LocationKindRemote {
+			return "", errors.New("local fallback requires a local container package")
+		}
+	}
+
+	action := fmt.Sprintf("Building locally with %s.", ch.docker.Name())
+	if hasPackage {
+		action = fmt.Sprintf("Publishing the existing local image with %s.", ch.docker.Name())
+	}
+	ch.console.MessageUxItem(ctx, &ux.WarningMessage{
+		Description: fmt.Sprintf("ACR refused the build request with TasksOperationsNotAllowed. %s", action),
+	})
+
+	if !hasPackage {
+		// Remote mode skips local build/package. Keep fallback artifacts separate from completed lifecycle state.
+		serviceContext = NewServiceContext()
+		buildResult, err := ch.buildLocalImage(ctx, serviceConfig, env, progress)
+		if err != nil {
+			return "", fmt.Errorf("building local image: %w", err)
+		}
+		if err := serviceContext.Build.Add(buildResult.Artifacts...); err != nil {
+			return "", fmt.Errorf("adding local build artifacts: %w", err)
+		}
+		if err := ctx.Err(); err != nil {
+			return "", err
+		}
+		packageResult, err := ch.packageLocalImage(ctx, serviceConfig, serviceContext, env, progress)
+		if err != nil {
+			return "", fmt.Errorf("packaging local image: %w", err)
+		}
+		if err := serviceContext.Package.Add(packageResult.Artifacts...); err != nil {
+			return "", fmt.Errorf("adding local package artifacts: %w", err)
+		}
+	}
+
+	if err := ctx.Err(); err != nil {
+		return "", err
+	}
+	return ch.publishLocalImage(ctx, serviceConfig, serviceContext, env, progress, imageOverride)
+}
+
+// publishLocalImage publishes a prepared container package and returns the full remote image name.
 func (ch *ContainerHelper) publishLocalImage(
 	ctx context.Context,
 	serviceConfig *ServiceConfig,