Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 37 additions & 16 deletions crates/openshell-driver-docker/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,14 +40,15 @@ use openshell_core::progress::{
format_bytes, mark_progress_active, mark_progress_complete, mark_progress_detail,
};
use openshell_core::proto::compute::v1::{
CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse,
DeleteWorkspaceRequest, DeleteWorkspaceResponse, DriverCondition, DriverPlatformEvent,
DriverSandbox, DriverSandboxStatus, DriverSandboxTemplate, EnsureWorkspaceRequest,
EnsureWorkspaceResponse, GatewayListenerRequirement, GetCapabilitiesRequest,
GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest,
CpuResourceCapabilities, CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest,
DeleteSandboxResponse, DeleteWorkspaceRequest, DeleteWorkspaceResponse, DriverCondition,
DriverPlatformEvent, DriverSandbox, DriverSandboxStatus, DriverSandboxTemplate,
EnsureWorkspaceRequest, EnsureWorkspaceResponse, GatewayListenerRequirement,
GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest,
GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse,
GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest,
StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest,
GpuResourceCapabilities, GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse,
MemoryResourceCapabilities, ResourceCapabilities, StartSandboxRequest, StartSandboxResponse,
StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest,
ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent,
WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent,
compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector,
Expand Down Expand Up @@ -212,12 +213,17 @@ struct DockerDriverRuntimeConfig {
supervisor_bin: PathBuf,
guest_tls: Option<DockerGuestTlsPaths>,
daemon_version: String,
supports_gpu: bool,
allow_all_default_gpu: bool,
gpu: DockerGpuRuntimeCapabilities,
sandbox_pids_limit: i64,
enable_bind_mounts: bool,
}

#[derive(Debug, Clone, Copy)]
struct DockerGpuRuntimeCapabilities {
cdi_supported: bool,
wsl_all_gpu_fallback_enabled: bool,
}

#[derive(Debug, Clone, PartialEq, Eq)]
enum DockerGatewayRoute {
Bridge {
Expand Down Expand Up @@ -552,12 +558,16 @@ impl DockerComputeDriver {
let info = docker.info().await.map_err(|err| {
Error::execution(format!("failed to query Docker daemon info: {err}"))
})?;
let supports_gpu = info
let cdi_supported = info
.cdi_spec_dirs
.as_ref()
.is_some_and(|dirs| !dirs.is_empty());
let cdi_gpu_inventory = docker_cdi_gpu_inventory(&info);
let allow_all_default_gpu = docker_info_reports_wsl2(&info);
let wsl_all_gpu_fallback_enabled = docker_info_reports_wsl2(&info);
let gpu = DockerGpuRuntimeCapabilities {
cdi_supported,
wsl_all_gpu_fallback_enabled,
};
validate_sandbox_pids_limit(docker_config.sandbox_pids_limit)?;
let gateway_port = config.bind_address.port();
if gateway_port == 0 {
Expand Down Expand Up @@ -607,16 +617,15 @@ impl DockerComputeDriver {
supervisor_bin,
guest_tls,
daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()),
supports_gpu,
allow_all_default_gpu,
gpu,
sandbox_pids_limit: docker_config.sandbox_pids_limit,
enable_bind_mounts: docker_config.enable_bind_mounts,
},
events: broadcast::channel(WATCH_BUFFER).0,
pending: Arc::new(Mutex::new(HashMap::new())),
gpu_selector: Arc::new(CdiGpuDefaultSelector::new(
cdi_gpu_inventory,
allow_all_default_gpu,
gpu.wsl_all_gpu_fallback_enabled,
)),
lifecycle_event_fences: DockerLifecycleEventFences::default(),
};
Expand All @@ -635,6 +644,18 @@ impl DockerComputeDriver {
driver_version: self.config.daemon_version.clone(),
default_image: self.config.default_image.clone(),
gateway_manages_lifecycle: true,
resource_capabilities: Some(ResourceCapabilities {
cpu: Some(CpuResourceCapabilities {
limit_supported: true,
}),
memory: Some(MemoryResourceCapabilities {
limit_supported: true,
}),
gpu: Some(GpuResourceCapabilities {
default_selection_supported: self.config.gpu.cdi_supported,
count_selection_supported: self.config.gpu.cdi_supported,
}),
}),
}
}

Expand Down Expand Up @@ -666,7 +687,7 @@ impl DockerComputeDriver {
DockerSandboxDriverConfig::from_template(template).map_err(Status::invalid_argument)?;
validate_docker_driver_mounts(&driver_config.mounts, config.enable_bind_mounts)?;
let gpu_requirements = driver_gpu_requirements(spec.resource_requirements.as_ref());
Self::validate_gpu_request(gpu_requirements, config.supports_gpu, &driver_config)?;
Self::validate_gpu_request(gpu_requirements, config.gpu.cdi_supported, &driver_config)?;
Ok(ValidatedDockerSandbox {
template,
driver_config,
Expand Down Expand Up @@ -774,7 +795,7 @@ impl DockerComputeDriver {
.map_err(|err| internal_status("query Docker daemon info", err))?;
self.gpu_selector.refresh(
docker_cdi_gpu_inventory(&info),
self.config.allow_all_default_gpu,
self.config.gpu.wsl_all_gpu_fallback_enabled,
);
Ok(())
}
Expand Down
54 changes: 39 additions & 15 deletions crates/openshell-driver-docker/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,10 @@ fn runtime_config() -> DockerDriverRuntimeConfig {
key: PathBuf::from("/tmp/tls.key"),
}),
daemon_version: "28.0.0".to_string(),
supports_gpu: false,
allow_all_default_gpu: false,
gpu: DockerGpuRuntimeCapabilities {
cdi_supported: false,
wsl_all_gpu_fallback_enabled: false,
},
sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT,
enable_bind_mounts: false,
}
Expand Down Expand Up @@ -149,7 +151,7 @@ fn inspected_volume(driver: &str, options: HashMap<String, String>) -> bollard::
}

fn test_driver_with_config(config: DockerDriverRuntimeConfig) -> DockerComputeDriver {
let allow_all_default_gpu = config.allow_all_default_gpu;
let wsl_all_gpu_fallback_enabled = config.gpu.wsl_all_gpu_fallback_enabled;
DockerComputeDriver {
docker: Arc::new(
Docker::connect_with_http("http://127.0.0.1:2375", 1, bollard::API_DEFAULT_VERSION)
Expand All @@ -160,12 +162,34 @@ fn test_driver_with_config(config: DockerDriverRuntimeConfig) -> DockerComputeDr
pending: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
gpu_selector: Arc::new(CdiGpuDefaultSelector::new(
CdiGpuInventory::default(),
allow_all_default_gpu,
wsl_all_gpu_fallback_enabled,
)),
lifecycle_event_fences: DockerLifecycleEventFences::default(),
}
}

#[test]
fn capabilities_report_static_resource_support() {
let mut config = runtime_config();
let capabilities = test_driver_with_config(config.clone()).capabilities();
let resources = capabilities.resource_capabilities.unwrap();
assert!(resources.cpu.unwrap().limit_supported);
assert!(resources.memory.unwrap().limit_supported);
let gpu = resources.gpu.unwrap();
assert!(!gpu.default_selection_supported);
assert!(!gpu.count_selection_supported);

config.gpu.cdi_supported = true;
let gpu = test_driver_with_config(config)
.capabilities()
.resource_capabilities
.unwrap()
.gpu
.unwrap();
assert!(gpu.default_selection_supported);
assert!(gpu.count_selection_supported);
}

#[tokio::test]
async fn tracing_in_process_service_preserves_the_driver_rpc_server_boundary() {
use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider};
Expand Down Expand Up @@ -2118,7 +2142,7 @@ fn validate_sandbox_rejects_unknown_driver_config_fields() {
#[test]
fn validate_sandbox_accepts_gpu_count_request_shape() {
let mut config = runtime_config();
config.supports_gpu = true;
config.gpu.cdi_supported = true;
let mut sandbox = test_sandbox();
sandbox.spec.as_mut().unwrap().resource_requirements = Some(gpu_resources(Some(2)));

Expand All @@ -2129,7 +2153,7 @@ fn validate_sandbox_accepts_gpu_count_request_shape() {
#[test]
fn validate_sandbox_accepts_gpu_count_matching_cdi_devices() {
let mut config = runtime_config();
config.supports_gpu = true;
config.gpu.cdi_supported = true;
let mut sandbox = test_sandbox();
let spec = sandbox.spec.as_mut().unwrap();
spec.resource_requirements = Some(gpu_resources(Some(2)));
Expand All @@ -2145,7 +2169,7 @@ fn validate_sandbox_accepts_gpu_count_matching_cdi_devices() {
#[test]
fn validate_sandbox_accepts_single_cdi_device_without_gpu_count() {
let mut config = runtime_config();
config.supports_gpu = true;
config.gpu.cdi_supported = true;
let mut sandbox = test_sandbox();
let spec = sandbox.spec.as_mut().unwrap();
spec.resource_requirements = Some(gpu_resources(None));
Expand All @@ -2158,7 +2182,7 @@ fn validate_sandbox_accepts_single_cdi_device_without_gpu_count() {
#[test]
fn validate_sandbox_rejects_multiple_cdi_devices_without_gpu_count() {
let mut config = runtime_config();
config.supports_gpu = true;
config.gpu.cdi_supported = true;
let mut sandbox = test_sandbox();
let spec = sandbox.spec.as_mut().unwrap();
spec.resource_requirements = Some(gpu_resources(None));
Expand All @@ -2179,7 +2203,7 @@ fn validate_sandbox_rejects_multiple_cdi_devices_without_gpu_count() {
#[test]
fn validate_sandbox_rejects_cdi_devices_without_gpu_request() {
let mut config = runtime_config();
config.supports_gpu = true;
config.gpu.cdi_supported = true;
let mut sandbox = test_sandbox();
sandbox
.spec
Expand All @@ -2199,7 +2223,7 @@ fn validate_sandbox_rejects_cdi_devices_without_gpu_request() {
#[test]
fn validate_sandbox_rejects_gpu_count_mismatched_cdi_devices() {
let mut config = runtime_config();
config.supports_gpu = true;
config.gpu.cdi_supported = true;
let mut sandbox = test_sandbox();
let spec = sandbox.spec.as_mut().unwrap();
spec.resource_requirements = Some(gpu_resources(Some(2)));
Expand Down Expand Up @@ -2255,7 +2279,7 @@ fn validate_sandbox_auth_accepts_gateway_token() {
#[test]
fn build_container_create_body_maps_default_gpu_to_selected_cdi_device() {
let mut config = runtime_config();
config.supports_gpu = true;
config.gpu.cdi_supported = true;
let mut sandbox = test_sandbox();
sandbox.spec.as_mut().unwrap().resource_requirements = Some(gpu_resources(None));

Expand Down Expand Up @@ -2285,7 +2309,7 @@ fn build_container_create_body_maps_default_gpu_to_selected_cdi_device() {
#[test]
fn build_container_create_body_omits_devices_without_resolved_default_cdi_devices() {
let mut config = runtime_config();
config.supports_gpu = true;
config.gpu.cdi_supported = true;
let mut sandbox = test_sandbox();
sandbox.spec.as_mut().unwrap().resource_requirements = Some(gpu_resources(None));

Expand All @@ -2303,7 +2327,7 @@ fn build_container_create_body_omits_devices_without_resolved_default_cdi_device
#[test]
fn build_container_create_body_passes_explicit_cdi_device_id_through() {
let mut config = runtime_config();
config.supports_gpu = true;
config.gpu.cdi_supported = true;
let mut sandbox = test_sandbox();
let spec = sandbox.spec.as_mut().unwrap();
spec.resource_requirements = Some(gpu_resources(None));
Expand All @@ -2327,7 +2351,7 @@ fn build_container_create_body_passes_explicit_cdi_device_id_through() {
#[test]
fn build_container_create_body_rejects_gpu_count_mismatched_cdi_devices() {
let mut config = runtime_config();
config.supports_gpu = true;
config.gpu.cdi_supported = true;
let mut sandbox = test_sandbox();
let spec = sandbox.spec.as_mut().unwrap();
spec.resource_requirements = Some(gpu_resources(Some(2)));
Expand Down Expand Up @@ -2374,7 +2398,7 @@ fn build_container_create_body_rejects_empty_cdi_devices() {
#[test]
fn driver_default_gpu_selection_consumes_distinct_devices_for_creates() {
let mut config = runtime_config();
config.supports_gpu = true;
config.gpu.cdi_supported = true;
let driver = test_driver_with_config(config);
driver.gpu_selector.refresh(
CdiGpuInventory::new(["nvidia.com/gpu=0", "nvidia.com/gpu=1"]),
Expand Down
40 changes: 34 additions & 6 deletions crates/openshell-driver-kubernetes/src/driver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,13 @@ use openshell_core::progress::{
format_bytes, mark_progress_active, mark_progress_complete, mark_progress_detail,
};
use openshell_core::proto::compute::v1::{
DriverCondition as SandboxCondition, DriverPlatformEvent as PlatformEvent,
DriverSandbox as Sandbox, DriverSandboxSpec as SandboxSpec,
DriverSandboxStatus as SandboxStatus, DriverSandboxTemplate as SandboxTemplate,
GetCapabilitiesResponse, GpuResourceRequirements, WatchSandboxesDeletedEvent,
WatchSandboxesEvent, WatchSandboxesPlatformEvent, WatchSandboxesSandboxEvent,
watch_sandboxes_event,
CpuResourceCapabilities, DriverCondition as SandboxCondition,
DriverPlatformEvent as PlatformEvent, DriverSandbox as Sandbox,
DriverSandboxSpec as SandboxSpec, DriverSandboxStatus as SandboxStatus,
DriverSandboxTemplate as SandboxTemplate, GetCapabilitiesResponse, GpuResourceCapabilities,
GpuResourceRequirements, MemoryResourceCapabilities, ResourceCapabilities,
WatchSandboxesDeletedEvent, WatchSandboxesEvent, WatchSandboxesPlatformEvent,
WatchSandboxesSandboxEvent, watch_sandboxes_event,
};
use openshell_core::proto_struct::{struct_to_json_object, value_to_json};
use serde::Deserialize;
Expand Down Expand Up @@ -567,6 +568,18 @@ impl KubernetesComputeDriver {
driver_version: openshell_core::VERSION.to_string(),
default_image: self.config.default_image.clone(),
gateway_manages_lifecycle: false,
resource_capabilities: Some(ResourceCapabilities {
cpu: Some(CpuResourceCapabilities {
limit_supported: true,
}),
memory: Some(MemoryResourceCapabilities {
limit_supported: true,
}),
gpu: Some(GpuResourceCapabilities {
default_selection_supported: true,
count_selection_supported: true,
}),
}),
})
}

Expand Down Expand Up @@ -4648,6 +4661,21 @@ mod tests {
static ENV_LOCK: std::sync::LazyLock<std::sync::Mutex<()>> =
std::sync::LazyLock::new(|| std::sync::Mutex::new(()));

#[tokio::test]
async fn capabilities_report_static_resource_support() {
let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig::default());
let resources = driver
.capabilities()
.unwrap()
.resource_capabilities
.unwrap();
assert!(resources.cpu.unwrap().limit_supported);
assert!(resources.memory.unwrap().limit_supported);
let gpu = resources.gpu.unwrap();
assert!(gpu.default_selection_supported);
assert!(gpu.count_selection_supported);
}

#[tokio::test]
async fn tracing_create_sandbox_failure_exports_a_kubernetes_operation_span() {
use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider};
Expand Down
1 change: 1 addition & 0 deletions crates/openshell-driver-mxc/src/driver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,7 @@ impl MxcComputeBackend {
driver_version: DRIVER_VERSION.to_string(),
default_image: DEFAULT_IMAGE_SENTINEL.to_string(),
gateway_manages_lifecycle: false,
resource_capabilities: None,
}
}

Expand Down
32 changes: 30 additions & 2 deletions crates/openshell-driver-podman/src/driver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,9 @@ use openshell_core::proto::compute::v1::GatewayDefaultRouteInterfaceRequirement;
#[cfg(target_os = "macos")]
use openshell_core::proto::compute::v1::GatewayLoopbackInterfaceRequirement;
use openshell_core::proto::compute::v1::{
DriverSandbox, GatewayListenerRequirement, GetCapabilitiesResponse, GpuResourceRequirements,
gateway_listener_requirement::Selector,
CpuResourceCapabilities, DriverSandbox, GatewayListenerRequirement, GetCapabilitiesResponse,
GpuResourceCapabilities, GpuResourceRequirements, MemoryResourceCapabilities,
ResourceCapabilities, gateway_listener_requirement::Selector,
};
#[cfg(target_os = "linux")]
use std::net::{IpAddr, SocketAddr};
Expand Down Expand Up @@ -507,6 +508,18 @@ impl PodmanComputeDriver {
driver_version: openshell_core::VERSION.to_string(),
default_image: self.config.default_image.clone(),
gateway_manages_lifecycle: true,
resource_capabilities: Some(ResourceCapabilities {
cpu: Some(CpuResourceCapabilities {
limit_supported: true,
}),
memory: Some(MemoryResourceCapabilities {
limit_supported: true,
}),
gpu: Some(GpuResourceCapabilities {
default_selection_supported: true,
count_selection_supported: true,
}),
}),
})
}

Expand Down Expand Up @@ -1587,6 +1600,21 @@ mod tests {
use std::fs;
use std::path::{Path, PathBuf};

#[test]
fn capabilities_report_static_resource_support() {
let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig::default());
let resources = driver
.capabilities()
.unwrap()
.resource_capabilities
.unwrap();
assert!(resources.cpu.unwrap().limit_supported);
assert!(resources.memory.unwrap().limit_supported);
let gpu = resources.gpu.unwrap();
assert!(gpu.default_selection_supported);
assert!(gpu.count_selection_supported);
}

// ── socket resolution ───────────────────────────────────────────────
//
// These test resolve_socket_path directly with an injected detector, so
Expand Down
Loading
Loading