diff --git a/.agents/skills/add-app/SKILL.md b/.agents/skills/add-app/SKILL.md index cadfe2c7..009b03cc 100644 --- a/.agents/skills/add-app/SKILL.md +++ b/.agents/skills/add-app/SKILL.md @@ -29,6 +29,7 @@ Each image has a folder under `apps/` (or `engines/` for an engine image) with a 4. Write `catalog/applications/.yaml` from the closest neighbour. * Reference the package `apps..docker` with `dockerImage: {}`, and list the platforms the image was pinned for. * Bind the inference endpoint and published URLs with resolved values rather than literals, and keep any fixed login visible in `notes`. + * Declare `useCases` when the app is the master catalog's app for a use case (one app per use case), and `preferredAccelerators` (vendor names) when it runs best on one vendor's GPUs. Both are advisory metadata. * Leave out optional fields you cannot support. 5. Write the folder's `README.md` and `AGENTS.md` from the templates in `apps/AGENTS.md`. diff --git a/.agents/skills/add-model/SKILL.md b/.agents/skills/add-model/SKILL.md index 8d70e41a..020c74e9 100644 --- a/.agents/skills/add-model/SKILL.md +++ b/.agents/skills/add-model/SKILL.md @@ -25,7 +25,7 @@ Sizing is owned by the `model-sizing` skill. Catalog loading and validation live * Size each configuration with the `model-sizing` skill. * Leave out optional fields you cannot support. -4. Decide whether the model belongs in any hardware class's lists in `catalog/recommendations/`, for each use case, and place it by priority. Lists stay short and best first; add a memory minimum only where the model fits but leaves too little room for context. +4. Decide whether the model belongs in any hardware class's lists in `catalog/recommendations/`, for each use case, and place it by priority. An entry may set `contextTokensPerRequest`: coding entries carry the largest context per request that starts on an idle machine at the entry's minimum; the catalog test checks every entry fits. Lists stay short and best first; add a memory minimum only where the model fits but leaves too little room for context. 5. Validate the declaration and any list you changed with the catalog tests and the area gates. diff --git a/AGENTS.md b/AGENTS.md index 301d223f..25e7680c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -206,11 +206,12 @@ in the same change that adds, removes or renames a folder it lists. An **engine artifact** is one runnable build of it. - **hardware class**: a kind of execution hardware, nothing model-specific. **Hardware support** is a configuration's memory minimums and sizing constants for one class. -- **use case**: what an operator runs models for (coding and agents, general assistant, - translation, decision making); a fixed list, distinct from inference request kinds. +- **use case**: what an operator runs Dotcore for (chat assistant, coding, image generation); a + fixed list, distinct from inference request kinds. Each use case has at most one app; models + serve the use cases that have model lists. - **model recommendation**: a master-catalog entry naming a model profile to run on a hardware - class for a use case, in priority order, with optional memory minimums; advisory, never used for - admission or placement. + class for a use case, in priority order, with optional memory minimums and an optional context + per request (the profile still holds none); advisory, never used for admission or placement. - **maximum context**: the largest context one request can use with the model's weights; a configuration may lower it. No "effective" variant. - **total context** (`total-context`): the KV pool an engine holds, shared by active requests. diff --git a/backend/catalog/src/lib.rs b/backend/catalog/src/lib.rs index 3ea5813f..eb7f8738 100644 --- a/backend/catalog/src/lib.rs +++ b/backend/catalog/src/lib.rs @@ -2129,7 +2129,7 @@ artifacts: const NVIDIA_RECOMMENDATIONS: &str = r#"hardwareClassId: "nvidia" useCases: - - useCase: "USE_CASE_CODING_AND_AGENTS" + - useCase: "USE_CASE_CODING" entries: - modelId: "fixture-model" profileId: "balanced" @@ -2137,7 +2137,7 @@ useCases: minimumMemoryBytesPerDevice: "32000000000" - modelId: "fixture-model" profileId: "fast" - - useCase: "USE_CASE_TRANSLATION" + - useCase: "USE_CASE_CHAT_ASSISTANT" entries: - modelId: "fixture-model" profileId: "fast" @@ -2187,10 +2187,10 @@ useCases: [ ( "nvidia", - UseCase::CodingAndAgents, + UseCase::Coding, vec![("balanced", Some(32_000_000_000)), ("fast", None)] ), - ("nvidia", UseCase::Translation, vec![("fast", None)]), + ("nvidia", UseCase::ChatAssistant, vec![("fast", None)]), ] ); fs::remove_dir_all(root).expect("Catalog fixture cleanup"); @@ -2247,6 +2247,108 @@ useCases: fs::remove_dir_all(root).expect("Catalog fixture cleanup"); } + #[test] + fn master_catalog_applications_serve_distinct_use_cases() { + let root = temporary_catalog_root(); + create_empty_catalog_source(&root); + let mut chat = super::test_support::application(); + chat.use_cases = vec![UseCase::ChatAssistant as i32]; + let mut images = super::test_support::application(); + images.application_id = "fixture-images".to_owned(); + images.use_cases = vec![UseCase::ImageGeneration as i32]; + images.preferred_accelerators = vec!["nvidia".to_owned()]; + write_application(&root, "chat.yaml", &chat); + write_application(&root, "images.yaml", &images); + + let catalog = Catalog::load_with_policy( + &root, + OciArtifactPolicy::Development, + &ACCEPT_ENGINE_DECLARATIONS, + ) + .expect("applications serving distinct use cases load"); + assert_eq!( + catalog + .list_application_releases() + .iter() + .map(|application| ( + application.application_id.as_str(), + application.use_cases().collect::>(), + application.preferred_accelerators.clone(), + )) + .collect::>(), + [ + ( + "fixture-application", + vec![UseCase::ChatAssistant], + Vec::new() + ), + ( + "fixture-images", + vec![UseCase::ImageGeneration], + vec!["nvidia".to_owned()] + ), + ] + ); + + let mut second_chat = images; + second_chat.use_cases = vec![ + UseCase::ImageGeneration as i32, + UseCase::ChatAssistant as i32, + ]; + write_application(&root, "images.yaml", &second_chat); + let error = Catalog::load_with_policy( + &root, + OciArtifactPolicy::Development, + &ACCEPT_ENGINE_DECLARATIONS, + ) + .expect_err("a use case served by two applications is rejected"); + assert_eq!( + error.to_string(), + "Catalog diagnostics: applications/images.yaml: application \"fixture-images\" serves use case USE_CASE_CHAT_ASSISTANT, already served by application \"fixture-application\" in applications/chat.yaml" + ); + fs::remove_dir_all(root).expect("Catalog fixture cleanup"); + } + + #[test] + fn catalog_layers_cannot_declare_application_use_cases() { + let root = temporary_catalog_root(); + let master = root.join("master"); + let layer = root.join("layer"); + create_empty_catalog_source(&master); + create_empty_catalog_source(&layer); + let mut application = super::test_support::application(); + application.use_cases = vec![UseCase::Coding as i32]; + write_application(&layer, "coding.yaml", &application); + + let development = + Catalog::load_development_sources(&master, &layer, &ACCEPT_ENGINE_DECLARATIONS) + .expect_err("a development layer must not declare application use cases"); + let site = Catalog::load_sources_with_policy( + &master, + &layer, + OciArtifactPolicy::Development, + &ACCEPT_ENGINE_DECLARATIONS, + ) + .expect_err("a site layer must not declare application use cases"); + + assert_eq!( + development.to_string(), + "Catalog diagnostics: development catalog applications/coding.yaml: application use cases can only be declared by the master catalog" + ); + assert_eq!( + site.to_string(), + "Catalog diagnostics: site catalog applications/coding.yaml: application use cases can only be declared by the master catalog" + ); + + // A layer application that serves no use case still loads. + application.use_cases.clear(); + application.preferred_accelerators = vec!["nvidia".to_owned()]; + write_application(&layer, "coding.yaml", &application); + Catalog::load_development_sources(&master, &layer, &ACCEPT_ENGINE_DECLARATIONS) + .expect("a layer application without use cases loads"); + fs::remove_dir_all(root).expect("Catalog fixture cleanup"); + } + #[test] fn recommendations_on_wider_and_narrower_classes_than_the_supported_one_are_valid() { let root = recommendation_fixture(" - hardwareClassId: \"nvidia\"\n"); @@ -2265,7 +2367,7 @@ useCases: &root, class, &format!( - "hardwareClassId: \"{class}\"\nuseCases:\n - useCase: \"USE_CASE_GENERAL_ASSISTANT\"\n entries:\n - modelId: \"fixture-model\"\n profileId: \"balanced\"\n" + "hardwareClassId: \"{class}\"\nuseCases:\n - useCase: \"USE_CASE_CHAT_ASSISTANT\"\n entries:\n - modelId: \"fixture-model\"\n profileId: \"balanced\"\n" ), ); } @@ -2310,9 +2412,9 @@ useCases: lists.concat() ) }; - let assistant = |entries: &[(&str, &str)]| list("USE_CASE_GENERAL_ASSISTANT", entries); + let assistant = |entries: &[(&str, &str)]| list("USE_CASE_CHAT_ASSISTANT", entries); let balanced = [("fixture-model", "balanced")]; - let entry = "use case USE_CASE_GENERAL_ASSISTANT entry 1 (model \"fixture-model\" profile \"balanced\")"; + let entry = "use case USE_CASE_CHAT_ASSISTANT entry 1 (model \"fixture-model\" profile \"balanced\")"; let nvidia_support = " - hardwareClassId: \"nvidia\"\n"; let minimum_support = " - hardwareClassId: \"nvidia\"\n hardwareRequirements:\n minimumMemoryBytesPerDevice: \"16000000000\"\n"; @@ -2333,19 +2435,19 @@ useCases: "repeated use case", nvidia_support, vec![("nvidia", file("nvidia", &[assistant(&balanced), assistant(&[("fixture-model", "fast")])]))], - "recommendations/nvidia.yaml: model recommendations for hardware class \"nvidia\" list use case USE_CASE_GENERAL_ASSISTANT more than once".to_owned(), + "recommendations/nvidia.yaml: model recommendations for hardware class \"nvidia\" list use case USE_CASE_CHAT_ASSISTANT more than once".to_owned(), ), ( "unknown model", nvidia_support, vec![("nvidia", file("nvidia", &[assistant(&[("missing", "balanced")])]))], - "recommendations/nvidia.yaml: model recommendations for hardware class \"nvidia\" use case USE_CASE_GENERAL_ASSISTANT entry 1 (model \"missing\" profile \"balanced\") references an unknown model".to_owned(), + "recommendations/nvidia.yaml: model recommendations for hardware class \"nvidia\" use case USE_CASE_CHAT_ASSISTANT entry 1 (model \"missing\" profile \"balanced\") references an unknown model".to_owned(), ), ( "unknown profile", nvidia_support, vec![("nvidia", file("nvidia", &[assistant(&[("fixture-model", "missing")])]))], - "recommendations/nvidia.yaml: model recommendations for hardware class \"nvidia\" use case USE_CASE_GENERAL_ASSISTANT entry 1 (model \"fixture-model\" profile \"missing\") references an unknown profile".to_owned(), + "recommendations/nvidia.yaml: model recommendations for hardware class \"nvidia\" use case USE_CASE_CHAT_ASSISTANT entry 1 (model \"fixture-model\" profile \"missing\") references an unknown profile".to_owned(), ), ( "unrelated class", @@ -2353,12 +2455,6 @@ useCases: vec![("apple", file("apple", &[assistant(&balanced)]))], format!("recommendations/apple.yaml: model recommendations for hardware class \"apple\" {entry} runs a configuration that supports neither this hardware class, a class containing it, nor a class nested in it"), ), - ( - "decision making without decision requests", - nvidia_support, - vec![("nvidia", file("nvidia", &[list("USE_CASE_DECISION_MAKING", &balanced)]))], - "recommendations/nvidia.yaml: model recommendations for hardware class \"nvidia\" use case USE_CASE_DECISION_MAKING entry 1 (model \"fixture-model\" profile \"balanced\") runs a configuration that doesn't accept INFERENCE_REQUEST_KIND_DECISION requests".to_owned(), - ), ( "entry minimum below the configuration's", minimum_support, @@ -2371,11 +2467,35 @@ useCases: )], format!("recommendations/nvidia.yaml: model recommendations for hardware class \"nvidia\" {entry} sets minimum memory per device 8000000000 bytes, below its configuration's 16000000000 bytes"), ), + ( + "context per request above the maximum context", + nvidia_support, + vec![( + "nvidia", + file("nvidia", &[assistant(&balanced)]).replace( + "profileId: \"balanced\"\n", + "profileId: \"balanced\"\n contextTokensPerRequest: 262145\n", + ), + )], + format!("recommendations/nvidia.yaml: model recommendations for hardware class \"nvidia\" {entry} sets context per request 262145 tokens, above its configuration's maximum context of 262144 tokens"), + ), + ( + "context per request below the minimum context", + nvidia_support, + vec![( + "nvidia", + file("nvidia", &[assistant(&balanced)]).replace( + "profileId: \"balanced\"\n", + "profileId: \"balanced\"\n contextTokensPerRequest: 1023\n", + ), + )], + format!("recommendations/nvidia.yaml: model recommendations for hardware class \"nvidia\" {entry} sets context per request 1023 tokens, below the minimum context of 1024 tokens"), + ), ( "repeated entry", nvidia_support, vec![("nvidia", file("nvidia", &[assistant(&[balanced[0], balanced[0]])]))], - "recommendations/nvidia.yaml: model recommendations for hardware class \"nvidia\" use case USE_CASE_GENERAL_ASSISTANT entry 2 (model \"fixture-model\" profile \"balanced\") repeats an earlier entry".to_owned(), + "recommendations/nvidia.yaml: model recommendations for hardware class \"nvidia\" use case USE_CASE_CHAT_ASSISTANT entry 2 (model \"fixture-model\" profile \"balanced\") repeats an earlier entry".to_owned(), ), ( "class declared twice", @@ -2420,7 +2540,7 @@ useCases: } #[test] - fn chat_use_cases_require_a_configuration_accepting_chat_requests() { + fn coding_entries_require_a_configuration_accepting_chat_requests() { let root = recommendation_fixture(" - hardwareClassId: \"nvidia\"\n"); let path = root.join("models/fixture.yaml"); let model = fs::read_to_string(&path).expect("fixture model"); @@ -2435,7 +2555,7 @@ useCases: super::test_support::write_recommendations( &root, "nvidia", - "hardwareClassId: \"nvidia\"\nuseCases:\n - useCase: \"USE_CASE_DECISION_MAKING\"\n entries:\n - modelId: \"fixture-model\"\n profileId: \"balanced\"\n - useCase: \"USE_CASE_TRANSLATION\"\n entries:\n - modelId: \"fixture-model\"\n profileId: \"balanced\"\n", + "hardwareClassId: \"nvidia\"\nuseCases:\n - useCase: \"USE_CASE_CODING\"\n entries:\n - modelId: \"fixture-model\"\n profileId: \"balanced\"\n", ); let error = Catalog::load_with_policy( @@ -2443,12 +2563,12 @@ useCases: OciArtifactPolicy::Development, &ACCEPT_ENGINE_DECLARATIONS, ) - .expect_err("a chat use case on a decision-only configuration must reject") + .expect_err("a coding entry on a decision-only configuration must reject") .to_string(); assert_eq!( error, - "Catalog diagnostics: recommendations/nvidia.yaml: model recommendations for hardware class \"nvidia\" use case USE_CASE_TRANSLATION entry 1 (model \"fixture-model\" profile \"balanced\") runs a configuration that doesn't accept INFERENCE_REQUEST_KIND_CHAT requests" + "Catalog diagnostics: recommendations/nvidia.yaml: model recommendations for hardware class \"nvidia\" use case USE_CASE_CODING entry 1 (model \"fixture-model\" profile \"balanced\") runs a configuration that doesn't accept INFERENCE_REQUEST_KIND_CHAT requests" ); fs::remove_dir_all(root).expect("Catalog fixture cleanup"); } diff --git a/backend/catalog/src/service.rs b/backend/catalog/src/service.rs index 756e6dc8..56ac7c0a 100644 --- a/backend/catalog/src/service.rs +++ b/backend/catalog/src/service.rs @@ -297,6 +297,8 @@ mod tests { description: String::new(), notes: String::new(), version: "1".to_owned(), + use_cases: Vec::new(), + preferred_accelerators: Vec::new(), graph: Some(ApplicationGraph { services: vec![ApplicationService { service_name: "web".to_owned(), diff --git a/backend/catalog/src/source.rs b/backend/catalog/src/source.rs index 75a2ee65..58be4093 100644 --- a/backend/catalog/src/source.rs +++ b/backend/catalog/src/source.rs @@ -3,7 +3,8 @@ //! //! It walks the model, engine, hardware class, recommendation, application, and upstream-template //! directories and parses every file into a typed record, extracting each configuration's private -//! sizing layers along the way. Only the master catalog may declare model recommendations. +//! sizing layers along the way. Only the master catalog may declare model recommendations and the +//! use cases applications serve. //! A file that cannot be read or parsed is reported as a located diagnostic rather than //! aborting the whole load, so one bad file never hides the rest. @@ -57,7 +58,13 @@ pub(super) fn load_catalog_source( &mut records.recommendations, diagnostics, ); - load_applications(catalog_root, source, &mut records.applications, diagnostics); + load_applications( + catalog_root, + source, + role, + &mut records.applications, + diagnostics, + ); load_upstream_model_templates( catalog_root, source, @@ -294,14 +301,28 @@ fn load_recommendations( } } +/// Loads the applications of one catalog source. Only the master catalog may say which use cases +/// an application serves, since a layer's application would compete with the master's for its use +/// case, so a layer application that names one is reported instead. fn load_applications( catalog_root: &Path, source: &str, + role: CatalogSourceRole, applications: &mut Vec<(String, Application)>, diagnostics: &mut Vec, ) { for file in optional_catalog_files(catalog_root, source, APPLICATION_DIRECTORY, diagnostics) { - match yaml::from_file(&file.path) { + match yaml::from_file::(&file.path) { + Ok(application) + if role == CatalogSourceRole::Layer && !application.use_cases.is_empty() => + { + diagnostics.push(CatalogDiagnostic::new( + file.location, + anyhow::anyhow!( + "application use cases can only be declared by the master catalog" + ), + )); + } Ok(application) => applications.push((file.location, application)), Err(error) => diagnostics.push(CatalogDiagnostic::new( file.location, diff --git a/backend/catalog/src/validation/application.rs b/backend/catalog/src/validation/application.rs index 4a73f6cf..7756dde7 100644 --- a/backend/catalog/src/validation/application.rs +++ b/backend/catalog/src/validation/application.rs @@ -1,10 +1,16 @@ //! Application declaration graph, service, and binding validation. +//! +//! Besides its graph, an application may name the use cases it serves and the accelerator vendors +//! it prefers. Both are recommendation and display metadata: each use case is served by at most +//! one master-catalog application, and they never decide admission or placement. + +use std::collections::{BTreeMap, BTreeSet}; use anyhow::anyhow; -use dotcore_domain::valid_id; -use dotcore_proto::dotcore::catalog::v1::Application; +use dotcore_domain::{Accelerator, valid_id}; +use dotcore_proto::dotcore::catalog::v1::{Application, UseCase}; -use super::graph::validate_application_graph; +use super::{CatalogDiagnostic, graph::validate_application_graph}; use crate::OciArtifactPolicy; pub fn validate_application_with_policy( @@ -28,6 +34,8 @@ pub fn validate_application_with_policy( "application notes must be at most {MAX_NOTES_BYTES} bytes and contain no control characters other than line breaks" )); } + validate_use_cases(application, &mut errors); + validate_preferred_accelerators(application, &mut errors); match application.graph.as_ref() { Some(graph) => errors.extend(validate_application_graph(graph, oci_artifact_policy)), None => errors.push(anyhow!("application requires a service graph")), @@ -35,6 +43,64 @@ pub fn validate_application_with_policy( errors } +/// Reports every application that serves a use case an earlier application already serves, so +/// each use case leads to at most one application. +pub(super) fn report_shared_use_cases( + applications: &[(String, Application)], + diagnostics: &mut Vec, +) { + let mut served = BTreeMap::new(); + for (location, application) in applications { + for use_case in application.use_cases().collect::>() { + if use_case == UseCase::Unspecified { + continue; + } + match served.get(&use_case) { + Some((first_id, first_location)) => diagnostics.push(CatalogDiagnostic::new( + location, + anyhow!( + "application {:?} serves use case {}, already served by application {first_id:?} in {first_location}", + application.application_id, + use_case.as_str_name() + ), + )), + None => { + served.insert(use_case, (&application.application_id, location)); + } + } + } + } +} + +fn validate_use_cases(application: &Application, errors: &mut Vec) { + let id = &application.application_id; + let mut use_cases = BTreeSet::new(); + for use_case in &application.use_cases { + match UseCase::try_from(*use_case) { + Ok(UseCase::Unspecified) | Err(_) => { + errors.push(anyhow!("application {id:?} names an unspecified use case")) + } + Ok(use_case) if !use_cases.insert(use_case) => errors.push(anyhow!( + "application {id:?} serves use case {} more than once", + use_case.as_str_name() + )), + Ok(_) => {} + } + } +} + +fn validate_preferred_accelerators(application: &Application, errors: &mut Vec) { + let mut vendors = BTreeSet::new(); + if application.preferred_accelerators.iter().any(|name| { + Accelerator::from_catalog_name(name).is_none() || !vendors.insert(name.as_str()) + }) { + errors.push(anyhow!( + "application {:?} requires distinct preferred accelerators, each nvidia or apple", + application.application_id + )); + } +} + const MAX_NOTES_BYTES: usize = 1000; fn valid_notes(notes: &str) -> bool { @@ -64,7 +130,7 @@ mod tests { ApplicationServicePort, ApplicationServiceReadiness, BooleanParameterSchema, NixArtifact, NixDockerImage, NixSystemStorePath, OciPlatform, ParameterDefinition, StorageBinding, StorageKind, StorageScope, StringEnumParameterSchema, UnsignedIntegerParameterSchema, - application_configuration_binding, + UseCase, application_configuration_binding, application_service_artifact::Kind as ServiceArtifactKind, nix_artifact::Provides, parameter_definition::ParameterType, }; @@ -83,6 +149,8 @@ mod tests { description: String::new(), notes: String::new(), version: "0.10.1".to_owned(), + use_cases: Vec::new(), + preferred_accelerators: Vec::new(), graph: Some(ApplicationGraph { services: vec![ ApplicationService { @@ -217,6 +285,48 @@ mod tests { assert!(notes_error(&"é".repeat(501))); } + #[test] + fn application_use_cases_and_preferred_accelerators_are_validated() { + let errors = |use_cases: &[UseCase], accelerators: &[&str]| { + let mut application = application(); + application.use_cases = use_cases.iter().map(|use_case| *use_case as i32).collect(); + application.preferred_accelerators = accelerators + .iter() + .map(|accelerator| (*accelerator).to_owned()) + .collect(); + validate_application(&application) + .iter() + .map(ToString::to_string) + .collect::>() + }; + + assert_eq!( + errors( + &[UseCase::ChatAssistant, UseCase::Coding], + &["nvidia", "apple"] + ), + Vec::::new() + ); + assert_eq!(errors(&[], &[]), Vec::::new()); + assert_eq!( + errors(&[UseCase::Unspecified], &[]), + ["application \"open-webui\" names an unspecified use case"] + ); + assert_eq!( + errors(&[UseCase::Coding, UseCase::Coding], &[]), + ["application \"open-webui\" serves use case USE_CASE_CODING more than once"] + ); + for accelerators in [&["amd"][..], &["nvidia", "nvidia"], &[""]] { + assert_eq!( + errors(&[], accelerators), + [ + "application \"open-webui\" requires distinct preferred accelerators, each nvidia or apple" + ], + "{accelerators:?}" + ); + } + } + #[test] fn application_validation_requires_nonempty_unique_concrete_linux_platforms() { let cases = [ diff --git a/backend/catalog/src/validation/mod.rs b/backend/catalog/src/validation/mod.rs index 3631a63d..fd0a814a 100644 --- a/backend/catalog/src/validation/mod.rs +++ b/backend/catalog/src/validation/mod.rs @@ -183,6 +183,7 @@ pub(super) fn validate_catalog( &duplicate_recommendations, diagnostics, ); + application::report_shared_use_cases(&applications, diagnostics); let application_index = validate_declarations( applications, |application| application.application_id.clone(), diff --git a/backend/catalog/src/validation/recommendation.rs b/backend/catalog/src/validation/recommendation.rs index b48a0ba7..852e7cf3 100644 --- a/backend/catalog/src/validation/recommendation.rs +++ b/backend/catalog/src/validation/recommendation.rs @@ -2,7 +2,8 @@ //! they name. //! //! Each hardware class's file lists, per use case, models and profiles that must be able to run -//! on that class and accept the requests the use case sends. The classes that declare +//! on that class and accept the requests the use case sends, each with an optional context per +//! request within its configuration's context limits. The classes that declare //! recommendations must be disjoint or strictly nested, so the list a target uses is never //! ambiguous. Every problem is reported rather than only the first. @@ -14,7 +15,10 @@ use dotcore_proto::dotcore::catalog::v1::{ }; use super::{CatalogDiagnostic, model::non_nested_classes}; -use crate::selector::{SelectorRelation, selector_relation}; +use crate::{ + selector::{SelectorRelation, selector_relation}, + shared_parameters::{MINIMUM_CONTEXT_TOKENS, maximum_context_tokens}, +}; pub(super) fn validate_recommendations( recommendations: &HardwareClassRecommendations, @@ -61,7 +65,6 @@ pub(super) fn validate_recommendations( } validate_entry( entry, - use_case, hardware_class, models, hardware_classes, @@ -103,7 +106,6 @@ pub(super) fn report_overlapping_recommendation_classes( fn validate_entry( entry: &ModelRecommendation, - use_case: UseCase, hardware_class: &HardwareClass, models: &BTreeMap, hardware_classes: &BTreeMap, @@ -136,21 +138,29 @@ fn validate_entry( )); } - let required_kind = if use_case == UseCase::DecisionMaking { - InferenceRequestKind::Decision - } else { - InferenceRequestKind::Chat - }; if !configuration .request_kinds - .contains(&(required_kind as i32)) + .contains(&(InferenceRequestKind::Chat as i32)) { errors.push(anyhow::anyhow!( "{label} runs a configuration that doesn't accept {} requests", - required_kind.as_str_name() + InferenceRequestKind::Chat.as_str_name() )); } + if let Some(context_tokens) = entry.context_tokens_per_request { + let maximum = maximum_context_tokens(model, configuration); + if context_tokens < MINIMUM_CONTEXT_TOKENS { + errors.push(anyhow::anyhow!( + "{label} sets context per request {context_tokens} tokens, below the minimum context of {MINIMUM_CONTEXT_TOKENS} tokens" + )); + } else if context_tokens > u64::from(maximum) { + errors.push(anyhow::anyhow!( + "{label} sets context per request {context_tokens} tokens, above its configuration's maximum context of {maximum} tokens" + )); + } + } + if let Some(requirements) = entry.hardware_requirements.as_ref() && !related_support.is_empty() { diff --git a/backend/cli/src/hardware/recommend.rs b/backend/cli/src/hardware/recommend.rs index 85ab026f..51c89ed6 100644 --- a/backend/cli/src/hardware/recommend.rs +++ b/backend/cli/src/hardware/recommend.rs @@ -2,7 +2,9 @@ //! //! The machine is this one or a machine description file, treated as idle. For each use case it //! prints a table of recommended models, best first, with the execution target that runs each, -//! whether it fits there now, and the total context it starts with. +//! whether it fits there now, its context per request and the total context it starts with. Each +//! use case also names the catalog application that serves it, and says when no target runs it +//! or none holds the accelerators it prefers. use std::path::PathBuf; @@ -12,7 +14,7 @@ use dotcore::model::{IdleMachine, RecommendationFit, UseCaseRecommendations}; use dotcore_catalog::{Catalog, CatalogModelId}; use dotcore_domain::timing::Timing; use dotcore_node_hardware::MachineDescription; -use dotcore_proto::dotcore::catalog::v1::UseCase; +use dotcore_proto::dotcore::catalog::v1::{Application, UseCase}; use super::load_catalog; use crate::{RootPathArgs, terminal::Message}; @@ -34,25 +36,18 @@ pub(super) struct RecommendArgs { #[derive(Clone, Copy, Debug, ValueEnum)] enum UseCaseArg { Coding, - Assistant, - Translation, - Decision, + Chat, + Image, } impl UseCaseArg { - const ALL: [Self; 4] = [ - Self::Coding, - Self::Assistant, - Self::Translation, - Self::Decision, - ]; + const ALL: [Self; 3] = [Self::Coding, Self::Chat, Self::Image]; fn use_case(self) -> UseCase { match self { - Self::Coding => UseCase::CodingAndAgents, - Self::Assistant => UseCase::GeneralAssistant, - Self::Translation => UseCase::Translation, - Self::Decision => UseCase::DecisionMaking, + Self::Coding => UseCase::Coding, + Self::Chat => UseCase::ChatAssistant, + Self::Image => UseCase::ImageGeneration, } } } @@ -90,8 +85,8 @@ fn read_machine(file: &PathBuf) -> Result { .with_context(|| format!("invalid machine description {}", file.display())) } -/// Lays the recommendations out with the shared terminal layout: a heading and a table per use -/// case, or a line saying nothing is recommended. +/// Lays the recommendations out with the shared terminal layout: a heading per use case, then its +/// table of models and the application that serves it, or a line saying nothing is recommended. fn render( catalog: &Catalog, machine: &IdleMachine, @@ -103,12 +98,23 @@ fn render( text.push('\n'); } let heading = Message::Heading(use_case_label(list.use_case).to_owned()); - let body = if list.rows.is_empty() { - Message::Info("No recommendation for this machine".to_owned()) - } else { - table(catalog, machine, list) - }; - for message in [heading, body] { + let application = catalog + .list_application_releases() + .into_iter() + .find(|application| application.use_cases().any(|case| case == list.use_case)); + let mut messages = vec![heading]; + if !list.rows.is_empty() { + messages.push(table(catalog, machine, list)); + } + if let Some(application) = application { + messages.push(application_line(machine, &application)); + } + if messages.len() == 1 { + messages.push(Message::Info( + "No recommendation for this machine".to_owned(), + )); + } + for message in messages { for line in message.plain_lines() { text.push_str(&line); text.push('\n'); @@ -132,6 +138,7 @@ fn table(catalog: &Catalog, machine: &IdleMachine, list: &UseCaseRecommendations profile.clone(), target_name(machine, &target.placement.execution_target_id), fit_label(target.fit).to_owned(), + row.context_tokens_per_request.to_string(), target .starting_total_context_tokens .map_or("-".to_owned(), |tokens| tokens.to_string()), @@ -140,19 +147,46 @@ fn table(catalog: &Catalog, machine: &IdleMachine, list: &UseCaseRecommendations }) .collect(); Message::Table { - header: ["MODEL", "PROFILE", "TARGET", "STATUS", "CONTEXT TOKENS"] - .map(str::to_owned) - .to_vec(), + header: [ + "MODEL", + "PROFILE", + "TARGET", + "STATUS", + "CONTEXT PER REQUEST", + "TOTAL CONTEXT", + ] + .map(str::to_owned) + .to_vec(), rows, } } +/// Names the application that serves a use case, with a note when the machine can't run it or +/// holds none of the accelerators it prefers. +fn application_line(machine: &IdleMachine, application: &Application) -> Message { + let options = machine.recommend_application(application); + let note = if options.is_empty() { + Some("no target can run it".to_owned()) + } else if !options.iter().any(|option| option.matches_preference) { + Some(format!( + "no target matches its preferred accelerators: {}", + application.preferred_accelerators.join(", ") + )) + } else { + None + }; + let name = match note { + Some(note) => format!("{} ({note})", application.display_name), + None => application.display_name.clone(), + }; + Message::Info(format!("App: {name}")) +} + fn use_case_label(use_case: UseCase) -> &'static str { match use_case { - UseCase::CodingAndAgents => "Coding and agents", - UseCase::GeneralAssistant => "General assistant", - UseCase::Translation => "Translation", - UseCase::DecisionMaking => "Decision making", + UseCase::Coding => "Coding", + UseCase::ChatAssistant => "Chat assistant", + UseCase::ImageGeneration => "Image generation", UseCase::Unspecified => "Unspecified", } } diff --git a/backend/cli/tests/command_contract.rs b/backend/cli/tests/command_contract.rs index 2c75d5ce..dd1e94b9 100644 --- a/backend/cli/tests/command_contract.rs +++ b/backend/cli/tests/command_contract.rs @@ -6154,13 +6154,22 @@ fn hardware_recommend_prints_exact_tables_for_a_described_machine() { }; // The model that needs 24 GB per device is listed first on the roomy GPU and dropped on the tight one. + // Chat's app prefers Apple, so no target on this NVIDIA machine matches it; image generation's matches. assert_eq!( recommend(&roomy, &[]), - "Coding and agents\n MODEL PROFILE TARGET STATUS CONTEXT TOKENS\n Fixture Big Fixture Q4 Fixture GPU X fits now 174080\n Fixture Small Fixture Q4 Fixture GPU X fits now 314368\n\nGeneral assistant\nNo recommendation for this machine\n\nTranslation\nNo recommendation for this machine\n\nDecision making\nNo recommendation for this machine\n" + "Coding\n MODEL PROFILE TARGET STATUS CONTEXT PER REQUEST TOTAL CONTEXT\n Fixture Big Fixture Q4 Fixture GPU X fits now 32768 174080\n Fixture Small Fixture Q4 Fixture GPU X fits now 32768 314368\n\nChat assistant\n MODEL PROFILE TARGET STATUS CONTEXT PER REQUEST TOTAL CONTEXT\n Fixture Small Fixture Q4 Fixture GPU X fits now 32768 314368\nApp: Fixture Chat (no target matches its preferred accelerators: apple)\n\nImage generation\nApp: Fixture Art\n" ); assert_eq!( recommend(&tight, &["--use-case", "coding"]), - "Coding and agents\n MODEL PROFILE TARGET STATUS CONTEXT TOKENS\n Fixture Small Fixture Q4 Fixture GPU X fits now 59392\n" + "Coding\n MODEL PROFILE TARGET STATUS CONTEXT PER REQUEST TOTAL CONTEXT\n Fixture Small Fixture Q4 Fixture GPU X fits now 32768 59392\n" + ); + assert_eq!( + recommend(&tight, &["--use-case", "chat"]), + "Chat assistant\n MODEL PROFILE TARGET STATUS CONTEXT PER REQUEST TOTAL CONTEXT\n Fixture Small Fixture Q4 Fixture GPU X fits now 32768 59392\nApp: Fixture Chat (no target matches its preferred accelerators: apple)\n" + ); + assert_eq!( + recommend(&tight, &["--use-case", "image"]), + "Image generation\nApp: Fixture Art\n" ); } diff --git a/backend/cli/tests/support/fake_manager.rs b/backend/cli/tests/support/fake_manager.rs index 2b6d9e36..4426b5c7 100644 --- a/backend/cli/tests/support/fake_manager.rs +++ b/backend/cli/tests/support/fake_manager.rs @@ -47,10 +47,11 @@ use dotcore_proto::dotcore::{ model_deployment_snapshot, }, site::v1::{ - ConfiguredExecutionTarget, ExecutionTargetWorkloadKind, - ListConfiguredExecutionTargetsRequest, ListConfiguredExecutionTargetsResponse, - ListHardwareSnapshotsRequest, ListHardwareSnapshotsResponse, ListMachinesRequest, - ListMachinesResponse, + ConfiguredExecutionTarget, ExecutionTargetWorkloadKind, GetConsoleSettingsRequest, + GetConsoleSettingsResponse, ListConfiguredExecutionTargetsRequest, + ListConfiguredExecutionTargetsResponse, ListHardwareSnapshotsRequest, + ListHardwareSnapshotsResponse, ListMachinesRequest, ListMachinesResponse, + UpdateConsoleSettingsRequest, UpdateConsoleSettingsResponse, site_service_server::{SiteService, SiteServiceServer}, }, }; @@ -725,6 +726,13 @@ impl ApplicationDeploymentService for FakeApplications { ))) } + async fn list_application_targets( + &self, + _request: Request, + ) -> Result, Status> { + Err(not_faked()) + } + async fn update_application_deployment_metadata( &self, _request: Request, @@ -764,6 +772,20 @@ impl SiteService for FakeSite { ) -> Result, Status> { Err(not_faked()) } + + async fn get_console_settings( + &self, + _request: Request, + ) -> Result, Status> { + Err(not_faked()) + } + + async fn update_console_settings( + &self, + _request: Request, + ) -> Result, Status> { + Err(not_faked()) + } } /// Records one call and applies its script: a scripted failure is returned as the error, and a diff --git a/backend/core/src/application/controller/mod.rs b/backend/core/src/application/controller/mod.rs index eca41141..d1e84e84 100644 --- a/backend/core/src/application/controller/mod.rs +++ b/backend/core/src/application/controller/mod.rs @@ -50,6 +50,7 @@ use dotcore_proto::{ ApplicationReadinessState, ApplicationServiceLifecycleState, ApplicationServiceObservation, CreateApplicationDeploymentRequest, CurrentApplicationDeployment, ListApplicationDeploymentsResponse, + ListApplicationTargetsResponse, }, site::v1::{ExecutionTarget as SiteExecutionTarget, Site}, }, @@ -80,7 +81,8 @@ pub(crate) use node::{ }; pub(crate) use node_view::ApplicationNodeViewSection; use operations::{ - create_deployment, pause_deployment, remove_deployment, resume_deployment, update_metadata, + create_deployment, list_targets, pause_deployment, remove_deployment, resume_deployment, + update_metadata, }; use owner::ApplicationOwner; use projection::project; @@ -113,6 +115,13 @@ pub trait ApplicationLifecycleController: Send + Sync { deployment_id: String, ) -> impl Future> + Send; + /// Lists the application targets that can take a catalog application now, best first. The + /// read is advisory and never waits for a lifecycle operation in progress. + fn list_targets( + &self, + application_id: String, + ) -> impl Future> + Send; + fn create_deployment( &self, request: CreateApplicationDeploymentRequest, @@ -337,6 +346,15 @@ where } } + fn list_targets( + &self, + application_id: String, + ) -> impl Future> + Send + { + let owner = Arc::clone(&self.owner); + async move { list_targets(&owner, &application_id).await } + } + fn create_deployment( &self, request: CreateApplicationDeploymentRequest, diff --git a/backend/core/src/application/controller/operations.rs b/backend/core/src/application/controller/operations.rs index 2388c48d..2e698aed 100644 --- a/backend/core/src/application/controller/operations.rs +++ b/backend/core/src/application/controller/operations.rs @@ -2,20 +2,20 @@ use std::sync::Arc; -use dotcore_domain::{SiteNodeId, real_time}; +use dotcore_domain::{SiteNodeId, diagnostic::diagnostic_chain, real_time}; use dotcore_proto::dotcore::{ application::v1::{ ApplicationDeploymentDeclaration, ApplicationDeploymentDesiredState, ApplicationLifecycleState, CreateApplicationDeploymentRequest, - CurrentApplicationDeployment, ResolvedApplicationDeploymentSnapshot, - ResolvedApplicationServiceSelection, + CurrentApplicationDeployment, ListApplicationTargetsResponse, + ResolvedApplicationDeploymentSnapshot, ResolvedApplicationServiceSelection, }, catalog::v1::application_service_artifact::Kind as ServiceArtifactKind, deployment::v1::{ArtifactPreparationPolicy, ExactPlacement}, site::v1::{ExecutionTarget, Site}, }; use dotcore_site::node::target_inventory::CapabilityObservation; -use tracing::info; +use tracing::{debug, info, warn}; use super::{ ApplicationLifecycleError, ApplicationOwner, configuration, configuration_with_source, @@ -24,6 +24,7 @@ use super::{ }; use crate::application::{ published_port::{PUBLISHED_PORT_RANGE, allocate_published_port, deployment_start_offset}, + recommendation::recommend_application_targets, repository::{ ApplicationDeclarationRepository, ApplicationLifecycleGenerations, CurrentApplicationRepository, has_inference_route_projection, @@ -257,6 +258,51 @@ where result } +/// Lists the application targets that can take a catalog application now, best first, from the +/// current target observations and deployments. Unavailable observations leave nothing to list. +pub(super) async fn list_targets( + owner: &ApplicationOwner, + application_id: &str, +) -> Result +where + D: ApplicationDeclarationRepository, + S: CurrentApplicationRepository + ApplicationLifecycleGenerations, +{ + validate_id(application_id)?; + let application = resolve_application(owner, application_id)?; + let inventory = match owner.target_capabilities.observe().await { + Ok(inventory) => inventory, + Err(error) => { + warn!( + application_id, + error = %diagnostic_chain(&error), + "target observations are unavailable; application targets list nothing" + ); + return Ok(ListApplicationTargetsResponse::default()); + } + }; + let site = owner.application_site()?; + let deployments = owner.projection()?.deployments; + let manager = site + .nodes + .iter() + .find(|node| node.site_manager.is_some()) + .map(|node| node.node_id.as_str()); + let targets = recommend_application_targets( + &application, + &site.execution_targets, + manager, + &inventory, + &deployments, + ); + debug!( + application_id, + target_count = targets.len(), + "listed application targets" + ); + Ok(ListApplicationTargetsResponse { targets }) +} + fn find_desired( owner: &Arc>, deployment_id: &str, diff --git a/backend/core/src/application/controller/test_support.rs b/backend/core/src/application/controller/test_support.rs index e2c76ba4..8b25c580 100644 --- a/backend/core/src/application/controller/test_support.rs +++ b/backend/core/src/application/controller/test_support.rs @@ -1371,6 +1371,14 @@ where self.owner.get_deployment(deployment_id) } + fn list_targets( + &self, + application_id: String, + ) -> impl Future> + Send + { + self.owner.list_targets(application_id) + } + fn create_deployment( &self, request: CreateApplicationDeploymentRequest, @@ -3023,6 +3031,8 @@ pub(super) fn application() -> Application { description: String::new(), notes: String::new(), version: "1.0.0".to_owned(), + use_cases: Vec::new(), + preferred_accelerators: Vec::new(), graph: Some(ApplicationGraph { services: vec![ApplicationService { service_name: "web".to_owned(), diff --git a/backend/core/src/application/mod.rs b/backend/core/src/application/mod.rs index 6b5be8b2..d0a6bfd3 100644 --- a/backend/core/src/application/mod.rs +++ b/backend/core/src/application/mod.rs @@ -2,6 +2,7 @@ mod controller; mod published_port; +mod recommendation; pub mod repository; pub mod service; pub(crate) mod workloads; @@ -17,3 +18,4 @@ pub use controller::{ InferenceRouteResolutionError, InferenceRouteResolver, ParameterRejection, ResolvedInferenceRoute, RetainedTargetMismatch, TargetIneligibility, }; +pub(crate) use recommendation::recommend_application_targets; diff --git a/backend/core/src/application/recommendation.rs b/backend/core/src/application/recommendation.rs new file mode 100644 index 00000000..cc661b61 --- /dev/null +++ b/backend/core/src/application/recommendation.rs @@ -0,0 +1,565 @@ +//! Recommends where one catalog application can run now: the application targets that can take +//! it, best first. +//! +//! A target qualifies when its node reports it fresh, its runtime runs the platform the site +//! declares for it, and every service of the application approves that platform, as creating a +//! deployment there requires. Targets the application already runs or starts on come first, then +//! those whose accelerators the application prefers, then the site manager's. The answer is +//! advisory: it never admits, reserves, or places a deployment. + +use std::time::SystemTime; + +use dotcore_domain::ExecutionPlatform; +use dotcore_proto::dotcore::{ + application::v1::{ApplicationDeployment, ApplicationLifecycleState, ApplicationTargetOption}, + catalog::v1::Application, + deployment::v1::ExactPlacement, + site::v1::{ExecutionTarget, execution_target::RuntimeBinding}, +}; +use dotcore_site::node::target_inventory::{CapabilityObservation, TargetCapabilityInventory}; + +/// Returns the application targets among `targets` that can take `application` now, best first. +/// `site_manager_node_id` names the site manager's node, when it has one, and `deployments` are +/// the site's application deployments. A target matches the application's preference when it +/// holds accelerator devices of a vendor the application prefers; every target matches an +/// application with no preference. +pub(crate) fn recommend_application_targets( + application: &Application, + targets: &[ExecutionTarget], + site_manager_node_id: Option<&str>, + inventory: &TargetCapabilityInventory, + deployments: &[ApplicationDeployment], +) -> Vec { + let mut options = targets + .iter() + .filter(|target| can_take(application, target, inventory)) + .map(|target| { + let placement = ExactPlacement { + site_node_id: target.site_node_id.clone(), + execution_target_id: target.execution_target_id.clone(), + }; + ApplicationTargetOption { + matches_preference: matches_preference(application, target, inventory), + application_deployment_id: deployment_id(application, &placement, deployments), + target: Some(placement), + } + }) + .collect::>(); + options.sort_by_cached_key(|option| { + let placement = option.target.clone().unwrap_or_default(); + ( + option.application_deployment_id.is_none(), + !option.matches_preference, + Some(placement.site_node_id.as_str()) != site_manager_node_id, + placement.site_node_id, + placement.execution_target_id, + ) + }); + options +} + +/// Whether `target` is an application target that can take the application now: freshly +/// observed, on its declared platform, which every service of the application approves. +fn can_take( + application: &Application, + target: &ExecutionTarget, + inventory: &TargetCapabilityInventory, +) -> bool { + if !matches!( + target.runtime_binding, + Some(RuntimeBinding::ApplicationRuntime(_)) + ) { + return false; + } + let Some(capabilities) = inventory.for_target(&target.execution_target_id) else { + return false; + }; + let Some(observed) = fresh( + inventory, + &capabilities.platform, + capabilities.platform_observed_at, + ) else { + return false; + }; + let declared = target + .platform + .as_ref() + .and_then(|platform| ExecutionPlatform::try_from(platform).ok()); + if declared.as_ref() != Some(observed) { + return false; + } + let platform = observed.to_oci_platform(); + application.graph.as_ref().is_some_and(|graph| { + graph.services.iter().all(|service| { + service + .artifact + .as_ref() + .is_some_and(|artifact| artifact.supported_platforms.contains(&platform)) + }) + }) +} + +/// Whether `target` holds accelerator devices of a vendor the application prefers, or the +/// application prefers none. +fn matches_preference( + application: &Application, + target: &ExecutionTarget, + inventory: &TargetCapabilityInventory, +) -> bool { + if application.preferred_accelerators.is_empty() { + return true; + } + if target.accelerator_device_ids.is_empty() { + return false; + } + inventory + .for_target(&target.execution_target_id) + .and_then(|capabilities| { + fresh( + inventory, + &capabilities.accelerator, + capabilities.accelerator_observed_at, + ) + }) + .is_some_and(|vendor| { + application + .preferred_accelerators + .iter() + .any(|preferred| preferred == vendor.catalog_name()) + }) +} + +/// The first running or starting deployment of the application placed at `placement`. Paused, +/// stopped, failed, unavailable and stopping deployments don't count. +fn deployment_id( + application: &Application, + placement: &ExactPlacement, + deployments: &[ApplicationDeployment], +) -> Option { + deployments + .iter() + .find(|deployment| { + let same_application = deployment + .snapshot + .as_ref() + .is_some_and(|snapshot| snapshot.application_id == application.application_id); + let active = matches!( + deployment + .state + .as_ref() + .map(|state| state.lifecycle_state()), + Some( + ApplicationLifecycleState::Queued + | ApplicationLifecycleState::Preparing + | ApplicationLifecycleState::Starting + | ApplicationLifecycleState::Running + | ApplicationLifecycleState::Ready + ) + ); + deployment.placement.as_ref() == Some(placement) && same_application && active + }) + .map(|deployment| deployment.application_deployment_id.clone()) +} + +/// The observed value when it is available and was fresh when the inventory was read. +fn fresh<'a, T>( + inventory: &TargetCapabilityInventory, + observation: &'a CapabilityObservation, + observed_at: Option, +) -> Option<&'a T> { + match observation { + CapabilityObservation::Available(value) if inventory.is_fresh(observed_at) => Some(value), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use std::time::SystemTime; + + use dotcore_domain::timing::Timing; + use dotcore_node_hardware::MachineDescription; + use dotcore_proto::dotcore::{ + application::v1::{ + ApplicationDeployment, ApplicationDeploymentState, ApplicationLifecycleState, + ResolvedApplicationDeploymentSnapshot, + }, + catalog::v1::{ + Application, ApplicationGraph, ApplicationService, ApplicationServiceArtifact, + OciPlatform, + }, + deployment::v1::ExactPlacement, + site::v1::{ExecutionTarget, execution_target::RuntimeBinding}, + }; + use dotcore_site::{ + initial::inputs::described_execution_targets, + node::target_inventory::{ + NodeAcceleratorObservation, RuntimePlatformObservation, TargetCapabilityInventory, + TargetCapabilityReconciler, + }, + }; + + use super::recommend_application_targets; + + use ApplicationLifecycleState::{Failed, Paused, Ready, Starting}; + + /// Expected recommendations for one site of described machines. + struct Case { + name: &'static str, + site: Site, + application: Application, + deployments: Vec<(&'static str, &'static str, ApplicationLifecycleState)>, + /// Each expected target, best first: its node, whether it matches the application's + /// preference, and the application's deployment there. + expected: Vec<(&'static str, bool, Option<&'static str>)>, + } + + #[test] + fn application_targets_are_the_ones_that_can_take_the_application_best_first() { + let cases = [ + Case { + name: "the NVIDIA machine's Docker target holds its GPU, the Mac's holds none", + site: Site::of(&[("a-mac", MAC), ("z-gpu", NVIDIA)]), + application: application(&["nvidia"], &[LINUX_AMD64, LINUX_ARM64]), + deployments: Vec::new(), + expected: vec![("z-gpu", true, None), ("a-mac", false, None)], + }, + Case { + name: "a Mac alone never matches", + site: Site::of(&[("a-mac", MAC)]), + application: application(&["nvidia"], &[LINUX_AMD64, LINUX_ARM64]), + deployments: Vec::new(), + expected: vec![("a-mac", false, None)], + }, + Case { + name: "an NVIDIA machine's Docker target that holds no GPU doesn't match", + site: Site::of(&[("z-gpu", NVIDIA)]).without_application_devices(), + application: application(&["nvidia"], &[LINUX_AMD64, LINUX_ARM64]), + deployments: Vec::new(), + expected: vec![("z-gpu", false, None)], + }, + Case { + name: "a site without application targets offers none", + site: Site::of(&[("a-mac", MAC), ("z-gpu", NVIDIA)]).without_application_targets(), + application: application(&["nvidia"], &[LINUX_AMD64, LINUX_ARM64]), + deployments: Vec::new(), + expected: Vec::new(), + }, + Case { + name: "a target running or starting the application comes first, with its deployment", + site: Site::of(&[("a-mac", MAC), ("z-gpu", NVIDIA)]), + application: application(&["nvidia"], &[LINUX_AMD64, LINUX_ARM64]), + deployments: vec![ + ("a-mac", "app-on-mac", Starting), + ("a-mac", "app-on-mac-ready", Ready), + ], + expected: vec![("a-mac", false, Some("app-on-mac")), ("z-gpu", true, None)], + }, + Case { + name: "failed and paused deployments and other applications' don't count", + site: Site::of(&[("a-mac", MAC), ("z-gpu", NVIDIA)]), + application: application(&["nvidia"], &[LINUX_AMD64, LINUX_ARM64]), + deployments: vec![ + ("a-mac", "app-failed", Failed), + ("a-mac", "app-paused", Paused), + ("a-mac", OTHER_APPLICATION_DEPLOYMENT, Ready), + ], + expected: vec![("z-gpu", true, None), ("a-mac", false, None)], + }, + Case { + name: "a machine that stopped reporting is dropped, even running the application", + site: Site::of(&[("a-mac", MAC), ("z-gpu", NVIDIA)]).stale("z-gpu"), + application: application(&["nvidia"], &[LINUX_AMD64, LINUX_ARM64]), + deployments: vec![("z-gpu", "app-on-gpu", Ready)], + expected: vec![("a-mac", false, None)], + }, + Case { + name: "without a preference every target matches, the site manager's first", + site: Site::of(&[("a-mac", MAC), ("z-gpu", NVIDIA)]).managed_by("z-gpu"), + application: application(&[], &[LINUX_AMD64, LINUX_ARM64]), + deployments: Vec::new(), + expected: vec![("z-gpu", true, None), ("a-mac", true, None)], + }, + Case { + name: "without a preference or a manager, by node", + site: Site::of(&[("z-gpu", NVIDIA), ("a-mac", MAC)]), + application: application(&[], &[LINUX_AMD64, LINUX_ARM64]), + deployments: Vec::new(), + expected: vec![("a-mac", true, None), ("z-gpu", true, None)], + }, + Case { + name: "a platform one service doesn't approve is left out", + site: Site::of(&[("a-mac", MAC), ("z-gpu", NVIDIA)]), + application: application(&["nvidia"], &[LINUX_AMD64]), + deployments: Vec::new(), + expected: vec![("z-gpu", true, None)], + }, + ]; + + for case in cases { + let deployments = case + .deployments + .iter() + .map(|(node, id, state)| case.site.deployment(node, id, *state)) + .collect::>(); + let options = recommend_application_targets( + &case.application, + &case.site.targets, + case.site.manager.as_deref(), + &case.site.inventory, + &deployments, + ); + let actual = options + .into_iter() + .map(|option| { + let target = option.target.expect("a target"); + ( + target, + option.matches_preference, + option.application_deployment_id, + ) + }) + .collect::>(); + let expected = case + .expected + .iter() + .map(|(node, matches, deployment)| { + ( + case.site.application_placement(node), + *matches, + deployment.map(str::to_owned), + ) + }) + .collect::>(); + assert_eq!(actual, expected, "{}", case.name); + } + } + + /// A site of described machines, each its own node, observed now unless made stale. + struct Site { + machines: Vec<(&'static str, MachineDescription)>, + targets: Vec, + manager: Option, + stale: Vec<&'static str>, + inventory: TargetCapabilityInventory, + } + + impl Site { + /// Declares the targets site setup gives each named machine. + fn of(machines: &[(&'static str, &str)]) -> Self { + let machines = machines + .iter() + .map(|(node, yaml)| { + let description = + MachineDescription::from_yaml(node, yaml).expect("described machine"); + (*node, description) + }) + .collect::>(); + let targets = machines + .iter() + .flat_map(|(node, description)| { + described_execution_targets(node, description).expect("described targets") + }) + .collect(); + Self { + machines, + targets, + manager: None, + stale: Vec::new(), + inventory: TargetCapabilityInventory::empty_now(), + } + .observed() + } + + fn without_application_devices(mut self) -> Self { + for target in self + .targets + .iter_mut() + .filter(|target| is_application(target)) + { + target.accelerator_device_ids.clear(); + } + self.observed() + } + + fn without_application_targets(mut self) -> Self { + self.targets.retain(|target| !is_application(target)); + self.observed() + } + + fn stale(mut self, node: &'static str) -> Self { + self.stale.push(node); + self.observed() + } + + fn managed_by(mut self, node: &str) -> Self { + self.manager = Some(node.to_owned()); + self + } + + /// Observes every target's platform and every node's accelerators, long ago on stale + /// nodes. + fn observed(mut self) -> Self { + let now = SystemTime::now(); + let max_age = Timing::test().node_control.capability_max_age; + let reported_at = |node: &str| { + if self.stale.contains(&node) { + now - 2 * max_age + } else { + now + } + }; + let platforms = self + .targets + .iter() + .map(|target| { + RuntimePlatformObservation::available_at( + target.execution_target_id.clone(), + target + .platform + .as_ref() + .expect("target platform") + .try_into() + .expect("supported platform"), + reported_at(&target.site_node_id), + ) + }) + .collect::>(); + let accelerators = self + .machines + .iter() + .map(|(node, description)| { + let snapshot = description + .render(&[], reported_at(node)) + .expect("rendered snapshot"); + NodeAcceleratorObservation::from_hardware_snapshot( + *node, + &snapshot, + self.stale.contains(node), + ) + }) + .collect::>(); + self.inventory = TargetCapabilityReconciler::reconcile( + &self.targets, + platforms, + accelerators, + max_age, + now, + ); + self + } + + /// The placement of the node's one application target. + fn application_placement(&self, node: &str) -> ExactPlacement { + let mut targets = self + .targets + .iter() + .filter(|target| target.site_node_id == node && is_application(target)); + let target = targets.next().expect("an application target"); + assert!(targets.next().is_none(), "one application target on {node}"); + ExactPlacement { + site_node_id: node.to_owned(), + execution_target_id: target.execution_target_id.clone(), + } + } + + /// A deployment on the node's application target, of the fixture application unless + /// its id says otherwise. + fn deployment( + &self, + node: &str, + id: &str, + lifecycle_state: ApplicationLifecycleState, + ) -> ApplicationDeployment { + let application_id = if id == OTHER_APPLICATION_DEPLOYMENT { + "other-application" + } else { + APPLICATION_ID + }; + ApplicationDeployment { + application_deployment_id: id.to_owned(), + placement: Some(self.application_placement(node)), + snapshot: Some(ResolvedApplicationDeploymentSnapshot { + application_id: application_id.to_owned(), + ..Default::default() + }), + state: Some(ApplicationDeploymentState { + lifecycle_state: lifecycle_state as i32, + ..Default::default() + }), + ..Default::default() + } + } + } + + fn is_application(target: &ExecutionTarget) -> bool { + matches!( + target.runtime_binding, + Some(RuntimeBinding::ApplicationRuntime(_)) + ) + } + + /// The fixture application with two services approving `platforms`, preferring + /// `accelerators`. + fn application(accelerators: &[&str], platforms: &[(&str, &str)]) -> Application { + let platforms = platforms + .iter() + .map(|(operating_system, architecture)| OciPlatform { + operating_system: (*operating_system).to_owned(), + architecture: (*architecture).to_owned(), + variant: None, + }) + .collect::>(); + let service = |name: &str| ApplicationService { + service_name: name.to_owned(), + artifact: Some(ApplicationServiceArtifact { + kind: None, + supported_platforms: platforms.clone(), + }), + ..Default::default() + }; + Application { + application_id: APPLICATION_ID.to_owned(), + preferred_accelerators: accelerators + .iter() + .map(|accelerator| (*accelerator).to_owned()) + .collect(), + graph: Some(ApplicationGraph { + services: vec![service("web"), service("worker")], + ..Default::default() + }), + ..Default::default() + } + } + + const APPLICATION_ID: &str = "fixture-application"; + + /// The id of a deployment of another application. + const OTHER_APPLICATION_DEPLOYMENT: &str = "other-application-deployment"; + + const LINUX_AMD64: (&str, &str) = ("linux", "amd64"); + + const LINUX_ARM64: (&str, &str) = ("linux", "arm64"); + + /// A dedicated 32 GiB NVIDIA GPU beside 64 GiB of host memory. + const NVIDIA: &str = r#" +platform: linux/amd64 +cpuCount: 16 +hostMemory: { totalBytes: 68719476736, idleUsedBytes: 4294967296 } +gpus: + - id: GPU-fixture + vendor: nvidia + model: Fixture GPU + driverVersion: "580.173.02" + memory: { architecture: dedicated, totalBytes: 34359738368, reservedBytes: 536870912 } +"#; + + /// A 32 GiB Mac, 6 GiB in use at idle. + const MAC: &str = r#" +platform: darwin/arm64 +cpuCount: 10 +hostMemory: { totalBytes: 34359738368, idleUsedBytes: 6442450944 } +gpus: + - { id: apple-gpu-0, vendor: apple, model: Fixture Apple GPU, memory: { architecture: unified } } +"#; +} diff --git a/backend/core/src/application/repository.rs b/backend/core/src/application/repository.rs index 09818443..3b0748a2 100644 --- a/backend/core/src/application/repository.rs +++ b/backend/core/src/application/repository.rs @@ -602,6 +602,8 @@ fn validate_resolved_snapshot( notes: snapshot.notes.clone(), version: snapshot.version.clone(), graph: snapshot.graph.clone(), + use_cases: Vec::new(), + preferred_accelerators: Vec::new(), }; if !validate_application_with_policy(&application, OciArtifactPolicy::Development).is_empty() || snapshot.placement.as_ref().is_none_or(|placement| { @@ -1462,6 +1464,8 @@ mod tests { description: String::new(), notes: String::new(), version: "1.0.0".to_owned(), + use_cases: Vec::new(), + preferred_accelerators: Vec::new(), graph: Some(ApplicationGraph { services: vec![ApplicationService { service_name: "service".to_owned(), diff --git a/backend/core/src/application/service.rs b/backend/core/src/application/service.rs index 377428e4..dd6aa17f 100644 --- a/backend/core/src/application/service.rs +++ b/backend/core/src/application/service.rs @@ -10,7 +10,8 @@ use dotcore_proto::dotcore::application::v1::{ CreateApplicationDeploymentRequest, CreateApplicationDeploymentResponse, FollowDeploymentLogsRequest, FollowDeploymentLogsResponse, GetApplicationDeploymentRequest, GetApplicationDeploymentResponse, ListApplicationDeploymentsRequest, - ListApplicationDeploymentsResponse, PauseDeploymentRequest, PauseDeploymentResponse, + ListApplicationDeploymentsResponse, ListApplicationTargetsRequest, + ListApplicationTargetsResponse, PauseDeploymentRequest, PauseDeploymentResponse, ReadDeploymentLogsRequest, ReadDeploymentLogsResponse, RemoveApplicationDeploymentRequest, RemoveApplicationDeploymentResponse, ResumeDeploymentRequest, ResumeDeploymentResponse, UpdateApplicationDeploymentMetadataRequest, UpdateApplicationDeploymentMetadataResponse, @@ -86,6 +87,17 @@ where .map_err(|error| error.to_tonic_status()) } + async fn list_application_targets( + &self, + request: Request, + ) -> Result, Status> { + self.controller + .list_targets(request.into_inner().application_id) + .await + .map(Response::new) + .map_err(|error| error.to_tonic_status()) + } + async fn get_application_deployment( &self, request: Request, @@ -536,8 +548,8 @@ mod tests { use dotcore_proto::{ dotcore::application::v1::{ ApplicationDeployment, CreateApplicationDeploymentRequest, DeploymentLogResumePosition, - ListApplicationDeploymentsResponse, ReadDeploymentLogsRequest, - UpdateApplicationDeploymentMetadataRequest, + ListApplicationDeploymentsResponse, ListApplicationTargetsResponse, + ReadDeploymentLogsRequest, UpdateApplicationDeploymentMetadataRequest, }, google::protobuf::Timestamp, }; @@ -1062,6 +1074,15 @@ mod tests { ready(Ok(ApplicationDeployment::default())) } + fn list_targets( + &self, + _application_id: String, + ) -> impl std::future::Future< + Output = Result, + > + Send { + ready(Ok(ListApplicationTargetsResponse::default())) + } + fn create_deployment( &self, _request: CreateApplicationDeploymentRequest, diff --git a/backend/core/src/composition/isolated.rs b/backend/core/src/composition/isolated.rs index 120ee2c2..3b298346 100644 --- a/backend/core/src/composition/isolated.rs +++ b/backend/core/src/composition/isolated.rs @@ -16,8 +16,8 @@ use dotcore_proto::dotcore::{ FollowDeploymentLogsRequest as FollowApplicationLogsRequest, GetApplicationDeploymentRequest, GetApplicationDeploymentResponse, ListApplicationDeploymentsRequest, ListApplicationDeploymentsResponse, - PauseDeploymentRequest, PauseDeploymentResponse, - ReadDeploymentLogsRequest as ReadApplicationLogsRequest, + ListApplicationTargetsRequest, ListApplicationTargetsResponse, PauseDeploymentRequest, + PauseDeploymentResponse, ReadDeploymentLogsRequest as ReadApplicationLogsRequest, ReadDeploymentLogsResponse as ReadApplicationLogsResponse, RemoveApplicationDeploymentRequest, RemoveApplicationDeploymentResponse, ResumeDeploymentRequest, ResumeDeploymentResponse, @@ -276,6 +276,13 @@ impl ApplicationDeploymentServiceContract for IsolatedApplicationService { self.service.list_application_deployments(request).await } + async fn list_application_targets( + &self, + request: Request, + ) -> Result, Status> { + self.service.list_application_targets(request).await + } + async fn get_application_deployment( &self, request: Request, diff --git a/backend/core/src/composition/mod.rs b/backend/core/src/composition/mod.rs index 0a955a0e..52dc8d96 100644 --- a/backend/core/src/composition/mod.rs +++ b/backend/core/src/composition/mod.rs @@ -374,6 +374,7 @@ where log_admission, )?; let site_service = Arc::new(SiteService::new( + Arc::new(inputs.site_repository.clone()), Arc::new(inputs.site_repository.clone()), bound_node.node_id.clone(), manager_node_registry.clone(), diff --git a/backend/core/src/model/compatibility.rs b/backend/core/src/model/compatibility.rs index 8dd0122b..90d92d25 100644 --- a/backend/core/src/model/compatibility.rs +++ b/backend/core/src/model/compatibility.rs @@ -311,25 +311,13 @@ pub(crate) fn node_workloads( ) -> NodeWorkloads { let mut workloads = NodeWorkloads::new(); for deployment in deployments { - let Some(status) = deployment.state.as_ref().map(|state| state.status()) else { + let Some(phase) = workload_phase(deployment) else { continue; }; let memory = deployment .accelerator_memory .as_ref() .and_then(|memory| memory.result.as_ref()); - let phase = match status { - ModelDeploymentStatus::Queued - | ModelDeploymentStatus::Preparing - | ModelDeploymentStatus::Downloading - | ModelDeploymentStatus::Loading - | ModelDeploymentStatus::WarmingUp => WorkloadPhase::Starting, - // Its estimate counts until a reading measures it, rather than offering memory the - // next reading will show in use. - ModelDeploymentStatus::Ready if memory_pending(memory) => WorkloadPhase::Starting, - ModelDeploymentStatus::Ready => WorkloadPhase::Ready, - _ => WorkloadPhase::Other, - }; let local = match deployment .snapshot .as_ref() @@ -391,6 +379,30 @@ pub(crate) fn node_workloads( workloads } +/// The phase a model deployment's workload is in, or none without a lifecycle state. Deployments +/// queued, preparing, downloading, loading or warming up are starting, and so are ready ones whose +/// memory no reading has measured yet; other ready ones are ready, and the rest are in another +/// phase. +pub(crate) fn workload_phase(deployment: &ModelDeployment) -> Option { + let status = deployment.state.as_ref()?.status(); + let memory = deployment + .accelerator_memory + .as_ref() + .and_then(|memory| memory.result.as_ref()); + Some(match status { + ModelDeploymentStatus::Queued + | ModelDeploymentStatus::Preparing + | ModelDeploymentStatus::Downloading + | ModelDeploymentStatus::Loading + | ModelDeploymentStatus::WarmingUp => WorkloadPhase::Starting, + // Its estimate counts until a reading measures it, rather than offering memory the next + // reading will show in use. + ModelDeploymentStatus::Ready if memory_pending(memory) => WorkloadPhase::Starting, + ModelDeploymentStatus::Ready => WorkloadPhase::Ready, + _ => WorkloadPhase::Other, + }) +} + /// Whether a deployment's accelerator memory has no reading yet only because none was taken since /// its process started. fn memory_pending(memory: Option<&accelerator_memory_observation::Result>) -> bool { diff --git a/backend/core/src/model/controller/mod.rs b/backend/core/src/model/controller/mod.rs index 523200b4..a86d5884 100644 --- a/backend/core/src/model/controller/mod.rs +++ b/backend/core/src/model/controller/mod.rs @@ -212,6 +212,7 @@ pub(crate) trait DesiredDeploymentRepository: ModelRealizationInput { fn model_recommendations( &self, _use_cases: &[dotcore_proto::dotcore::catalog::v1::UseCase], + _can_start_now: bool, _inventory: Option<&TargetCapabilityInventory>, _deployments: &[ModelDeployment], ) -> Result { @@ -328,11 +329,13 @@ pub trait ModelLifecycleController: Send + Sync { } /// Ranks the catalog's recommended model profiles for each of `use_cases` on the site's - /// configured model execution targets, marking the profiles already deployed. Advisory: it - /// never changes deployments, admission or placement. + /// configured model execution targets, naming the deployments already running them, and + /// keeping only what can start now when `can_start_now` is set. Advisory: it never changes + /// deployments, admission or placement. fn list_model_recommendations( &self, _use_cases: Vec, + _can_start_now: bool, ) -> impl Future> + Send { std::future::ready(Err(ModelLifecycleError::configuration( @@ -484,10 +487,11 @@ impl DesiredDeploymentRepository for crate::model::repository::ModelDeclarationR fn model_recommendations( &self, use_cases: &[dotcore_proto::dotcore::catalog::v1::UseCase], + can_start_now: bool, inventory: Option<&TargetCapabilityInventory>, deployments: &[ModelDeployment], ) -> Result { - self.model_recommendations(use_cases, inventory, deployments) + self.model_recommendations(use_cases, can_start_now, inventory, deployments) } fn load_upstream_model( diff --git a/backend/core/src/model/controller/operations.rs b/backend/core/src/model/controller/operations.rs index 510fd5b7..1ca599c8 100644 --- a/backend/core/src/model/controller/operations.rs +++ b/backend/core/src/model/controller/operations.rs @@ -168,6 +168,7 @@ where fn list_model_recommendations( &self, use_cases: Vec, + can_start_now: bool, ) -> impl std::future::Future< Output = Result< dotcore_proto::dotcore::model::v1::ListModelRecommendationsResponse, @@ -179,12 +180,13 @@ where let span = debug_span!( "model_recommendations", operation = "model_recommendations", + can_start_now, outcome = field::Empty, error_kind = field::Empty, ); let operation_span = span.clone(); async move { - let result = execute_model_recommendations(&inner, &use_cases).await; + let result = execute_model_recommendations(&inner, &use_cases, can_start_now).await; record_operation_result(&operation_span, &result); result } @@ -561,6 +563,7 @@ where async fn execute_model_recommendations( inner: &ManagerInferenceControllerInner, use_cases: &[dotcore_proto::dotcore::catalog::v1::UseCase], + can_start_now: bool, ) -> Result where D: super::DesiredDeploymentRepository, @@ -582,7 +585,7 @@ where let deployments = inner.public_projection()?.deployments; inner .desired - .model_recommendations(use_cases, inventory.as_ref(), &deployments) + .model_recommendations(use_cases, can_start_now, inventory.as_ref(), &deployments) .map_err(ModelLifecycleError::from_model_repository) } diff --git a/backend/core/src/model/recommendation/machine.rs b/backend/core/src/model/recommendation/machine.rs index 71c84f5f..61423309 100644 --- a/backend/core/src/model/recommendation/machine.rs +++ b/backend/core/src/model/recommendation/machine.rs @@ -9,7 +9,8 @@ use dotcore_catalog::{Catalog, SupportedHardwareClass}; use dotcore_domain::{real_time, timing::NodeControlTiming}; use dotcore_node_hardware::{MachineDescription, MachineError}; use dotcore_proto::dotcore::{ - catalog::v1::{HardwareClass, UseCase}, + application::v1::ApplicationTargetOption, + catalog::v1::{Application, HardwareClass, UseCase}, site::v1::{ExecutionTarget, execution_target::RuntimeBinding}, }; use dotcore_site::{ @@ -23,6 +24,7 @@ use tracing::debug; use super::{UseCaseRecommendations, recommend_models}; use crate::{ + application::recommend_application_targets, hardware::selection::select_hardware_class, model::compatibility::{StaleTargets, check_targets, node_workloads}, }; @@ -139,9 +141,27 @@ impl IdleMachine { Some(&self.inventory), &[], use_cases, + false, ) } + /// Returns the machine's application targets that can run `application`, best first, with + /// nothing deployed on them. + pub fn recommend_application(&self, application: &Application) -> Vec { + let targets = self + .targets + .iter() + .filter(|target| { + matches!( + target.runtime_binding, + Some(RuntimeBinding::ApplicationRuntime(_)) + ) + }) + .cloned() + .collect::>(); + recommend_application_targets(application, &targets, None, &self.inventory, &[]) + } + /// The targets that run models, which recommendations and hardware classes apply to. fn model_targets(&self) -> Vec { self.targets @@ -192,10 +212,7 @@ mod tests { assert_eq!( rows( &mac, - &mac.recommend( - &catalog, - &[UseCase::CodingAndAgents, UseCase::DecisionMaking] - ) + &mac.recommend(&catalog, &[UseCase::Coding, UseCase::ChatAssistant]) ), [ vec![ @@ -203,7 +220,10 @@ mod tests { "mid/q4 on Fixture Apple GPU: FitsNow from 454656", "small/q4 on Fixture Apple GPU: FitsNow from 494592", ], - vec!["decider/q8 on Fixture Apple GPU: FitsNow from 504832"], + vec![ + "mid/q4 on Fixture Apple GPU: FitsNow from 454656", + "small/q4 on Fixture Apple GPU: FitsNow from 494592", + ], ] ); } @@ -220,7 +240,7 @@ mod tests { assert_eq!( rows( &gpu, - &gpu.recommend(&catalog, &[UseCase::CodingAndAgents, UseCase::Translation]) + &gpu.recommend(&catalog, &[UseCase::Coding, UseCase::ChatAssistant]) ), [ vec![ @@ -264,7 +284,7 @@ mod tests { .iter() .flat_map(|row| { row.targets.iter().map(move |target| { - assert!(!target.deployed); + assert_eq!(target.model_deployment_id, None); let name = machine .targets() .iter() diff --git a/backend/core/src/model/recommendation/mod.rs b/backend/core/src/model/recommendation/mod.rs index 19339f00..5cd71b11 100644 --- a/backend/core/src/model/recommendation/mod.rs +++ b/backend/core/src/model/recommendation/mod.rs @@ -3,9 +3,14 @@ //! //! Each target reads the list of the most specific hardware class that has one for the use case, //! and each entry is judged with the same profile resolution and memory fit as the compatibility -//! read, the entry's own minimums added to its configuration's. Entries that can't run on a target -//! or exceed its total memory are left out there, and the rest merge across targets by their best -//! declared position. The answer is advisory: it never admits, reserves, or places a deployment. +//! read, the entry's own minimums added to its configuration's. An entry is sized for one request +//! using its whole total context, at the entry's context per request or else the profile's +//! default. Entries that can't run on a target or exceed its total memory are left out there, and +//! the rest merge across targets by their best declared position and context per request. Each +//! target names the model's running or starting deployment of the same profile there, whatever its +//! context. Asked for what can start now, it keeps only reachable targets that are deployed or fit +//! free memory, deployed rows first. The answer is advisory: it never admits, reserves, or places a +//! deployment. use std::{cmp::Ordering, collections::BTreeMap}; @@ -16,7 +21,10 @@ use dotcore_proto::dotcore::{ UseCase, }, deployment::v1::{ExactPlacement, MemoryFit}, - model::v1::{ModelCompatibilityStatus, ModelDeployment, model_deployment_snapshot}, + model::v1::{ + ModelCompatibilityStatus, ModelDeployment, SizingScenarioUnsupportedReason, + model_deployment_snapshot, + }, site::v1::ExecutionTarget, }; use dotcore_site::node::target_inventory::TargetCapabilityInventory; @@ -27,16 +35,21 @@ mod machine; pub use machine::{IdleMachine, IdleMachineError}; use crate::{ - hardware::selection::{ - HardwareClassSelectionError, SelectedTargetCapabilities, select_hardware_class, + hardware::{ + capacity::WorkloadPhase, + selection::{ + HardwareClassSelectionError, SelectedTargetCapabilities, select_hardware_class, + }, }, model::{ compatibility::{ CheckedTarget, ProfileEvaluation, ProfileRequest, StaleTargets, check_targets, - evaluate_profile, node_workloads, + evaluate_profile, node_workloads, workload_phase, }, engine::EngineIntegrationRegistry, - sizing_scenario::{ProfileSizing, size_profile}, + sizing_scenario::{ + ProfileSizing, SizingScenario, default_context_per_request, size_profile, + }, }, }; @@ -55,7 +68,12 @@ pub struct Recommendation { pub profile_id: String, - /// Targets whose list has the profile and that can run it, by node then target. + /// Context per request in tokens the profile is sized with: the entry's own, else the + /// profile's default. + pub context_tokens_per_request: u64, + + /// Targets whose list has the profile and that can run it, by node then target; deployed + /// targets come first when only what can start now is asked for. pub targets: Vec, } @@ -66,8 +84,9 @@ pub struct TargetRecommendation { pub fit: RecommendationFit, - /// Whether a model deployment of the same model and profile is placed on the target. - pub deployed: bool, + /// The running or starting model deployment of the same model and profile on the target, + /// whatever its context per request. + pub model_deployment_id: Option, /// Total context in tokens a deployment of the profile starts from there; absent without a /// largest fit or a profile default. @@ -95,14 +114,16 @@ pub enum RecommendationFit { /// `targets`, which must be configured model execution targets. `inventory` is the current target /// observation, or none when it couldn't be read, which leaves nothing to recommend; a target that /// stopped reporting is judged on its last observations, its rows marked unreachable, until it -/// leaves the site. `deployments` are the site's model deployments, which hold memory and mark the -/// profiles already deployed. +/// leaves the site. `deployments` are the site's model deployments, which hold memory and name the +/// profiles already deployed. With `can_start_now`, each list keeps only what can start now, as +/// `startable` describes. pub(crate) fn recommend_models( catalog: &Catalog, targets: &[ExecutionTarget], inventory: Option<&TargetCapabilityInventory>, deployments: &[ModelDeployment], use_cases: &[UseCase], + can_start_now: bool, ) -> Vec { let workloads = node_workloads(deployments, inventory); let checked = check_targets( @@ -120,20 +141,23 @@ pub(crate) fn recommend_models( }; use_cases .iter() - .map(|&use_case| UseCaseRecommendations { - use_case, - rows: recommender.rank(use_case, &checked, deployments), + .map(|&use_case| { + let rows = recommender.rank(use_case, &checked, deployments); + UseCaseRecommendations { + use_case, + rows: if can_start_now { startable(rows) } else { rows }, + } }) .collect() } /// What one call reads for every use case: the catalog, its declarations indexed by id, and each -/// profile's sizing once it has been needed. +/// profile's sizing at each context per request once it has been needed. struct Recommender<'a> { catalog: &'a Catalog, declarations: &'a Declarations, registry: EngineIntegrationRegistry, - sizings: BTreeMap<(String, String), ProfileSizing>, + sizings: BTreeMap, } impl Recommender<'_> { @@ -153,7 +177,7 @@ impl Recommender<'_> { hardware_requirements: None, }) .collect::>(); - let mut rows = BTreeMap::<(String, String), RankedRow>::new(); + let mut rows = BTreeMap::::new(); for checked in checked { let Ok(selected) = &checked.check else { continue; @@ -174,8 +198,11 @@ impl Recommender<'_> { let Some((model, profile)) = declarations.profile(entry) else { continue; }; + let Some(scenario) = entry_scenario(self.catalog, model, profile, entry) else { + continue; + }; let Some((fit, starting_total_context_tokens)) = - self.judge(model, profile, entry, checked.target, selected) + self.judge(model, profile, entry, scenario, checked.target, selected) else { continue; }; @@ -189,7 +216,7 @@ impl Recommender<'_> { execution_target_id: checked.target.execution_target_id.clone(), }; let row = rows - .entry((entry.model_id.clone(), entry.profile_id.clone())) + .entry(row_key(entry, scenario)) .or_insert_with(|| RankedRow { position, intelligence: model @@ -200,7 +227,7 @@ impl Recommender<'_> { }); row.position = row.position.min(position); row.targets.push(TargetRecommendation { - deployed: deployed(deployments, &placement, entry), + model_deployment_id: deployment_id(deployments, &placement, entry), placement, fit, starting_total_context_tokens, @@ -210,21 +237,22 @@ impl Recommender<'_> { ranked(rows) } - /// Judges one entry on one target with the compatibility read's evaluation, its minimums - /// added to its configuration's: its fit and starting total context there, or none when it is - /// left out there. + /// Judges one entry sized for `scenario` on one target with the compatibility read's + /// evaluation, its minimums added to its configuration's: its fit and starting total context + /// there, or none when it is left out there. fn judge( &mut self, model: &Model, profile: &ModelProfile, entry: &ModelRecommendation, + scenario: SizingScenario, target: &ExecutionTarget, selected: &SelectedTargetCapabilities, ) -> Option<(RecommendationFit, Option)> { let sizing = self .sizings - .entry((entry.model_id.clone(), entry.profile_id.clone())) - .or_insert_with(|| size_profile(self.catalog, model, profile, None)); + .entry(row_key(entry, scenario)) + .or_insert_with(|| size_profile(self.catalog, model, profile, Some(&scenario))); let evaluation = evaluate_profile( self.catalog, &self.registry, @@ -241,6 +269,9 @@ impl Recommender<'_> { } } +/// A row's model id, profile id and context per request in tokens; also keys each sizing. +type RowKey = (String, String, u64); + /// One row while targets are merged: its best declared position, its model's intelligence score, /// and the targets that can run it. struct RankedRow { @@ -300,9 +331,35 @@ impl Declarations { } } +/// The sizing scenario an entry is judged with: one request using the whole total context, both +/// the entry's context per request or else the profile's default. Returns none when the profile's +/// defaults don't resolve. +fn entry_scenario( + catalog: &Catalog, + model: &Model, + profile: &ModelProfile, + entry: &ModelRecommendation, +) -> Option { + let context_tokens = entry + .context_tokens_per_request + .or_else(|| default_context_per_request(catalog, model, profile))?; + // Catalog validation keeps an entry's context within the configuration's maximum context. + let context_tokens = u32::try_from(context_tokens).ok()?; + SizingScenario::new(context_tokens, Some(context_tokens)).ok() +} + +/// The row an entry sized for `scenario` merges into. +fn row_key(entry: &ModelRecommendation, scenario: SizingScenario) -> RowKey { + ( + entry.model_id.clone(), + entry.profile_id.clone(), + scenario.total_context_tokens(), + ) +} + /// The fit of an evaluated entry on a target and the total context a deployment starts from -/// there, or none when the entry is left out there: its profile can't run there, or its memory -/// fit, with the entry's minimums, exceeds the target's total. +/// there, or none when the entry is left out there: its profile can't run there or serve its +/// context there, or its memory fit, with the entry's minimums, exceeds the target's total. fn target_fit(evaluation: &ProfileEvaluation) -> Option<(RecommendationFit, Option)> { let running = match evaluation { ProfileEvaluation::Runs(running) => running, @@ -312,6 +369,9 @@ fn target_fit(evaluation: &ProfileEvaluation) -> Option<(RecommendationFit, Opti } ProfileEvaluation::Fails(_) => return None, }; + if running.fit.unsupported_reason != SizingScenarioUnsupportedReason::Unspecified { + return None; + } let fit = match (running.fit.memory_fit, running.fit.free_memory_fit) { (MemoryFit::Exceeds, _) => return None, (MemoryFit::Fits, MemoryFit::Fits) => RecommendationFit::FitsNow, @@ -326,30 +386,39 @@ fn target_fit(evaluation: &ProfileEvaluation) -> Option<(RecommendationFit, Opti )) } -/// Whether a model deployment of the entry's model and profile is placed at `placement`. -fn deployed( +/// The first running or starting model deployment of the entry's model and profile placed at +/// `placement`, whatever its context per request. It shares the memory accounting's lifecycle +/// phases, so failed, paused, unavailable and unloading deployments don't count. +fn deployment_id( deployments: &[ModelDeployment], placement: &ExactPlacement, entry: &ModelRecommendation, -) -> bool { - deployments.iter().any(|deployment| { - let placed = deployment - .intent - .as_ref() - .and_then(|intent| intent.placement.as_ref()) - == Some(placement); - let same_profile = matches!( - deployment.snapshot.as_ref().and_then(|snapshot| snapshot.source.as_ref()), - Some(model_deployment_snapshot::Source::Local(local)) - if local.model_id == entry.model_id && local.profile_id == entry.profile_id - ); - placed && same_profile - }) +) -> Option { + deployments + .iter() + .find(|deployment| { + let placed = deployment + .intent + .as_ref() + .and_then(|intent| intent.placement.as_ref()) + == Some(placement); + let same_profile = matches!( + deployment.snapshot.as_ref().and_then(|snapshot| snapshot.source.as_ref()), + Some(model_deployment_snapshot::Source::Local(local)) + if local.model_id == entry.model_id && local.profile_id == entry.profile_id + ); + let active = matches!( + workload_phase(deployment), + Some(WorkloadPhase::Starting | WorkloadPhase::Ready) + ); + placed && same_profile && active + }) + .map(|deployment| deployment.model_deployment_id.clone()) } -/// Orders merged rows by best declared position, then higher intelligence score, then model and -/// profile id. -fn ranked(rows: BTreeMap<(String, String), RankedRow>) -> Vec { +/// Orders merged rows by best declared position, then higher intelligence score, then model id, +/// profile id and context per request. +fn ranked(rows: BTreeMap) -> Vec { let mut rows = rows.into_iter().collect::>(); rows.sort_by(|(left_id, left), (right_id, right)| { left.position @@ -363,14 +432,50 @@ fn ranked(rows: BTreeMap<(String, String), RankedRow>) -> Vec { .then_with(|| left_id.cmp(right_id)) }); rows.into_iter() - .map(|((model_id, profile_id), row)| Recommendation { - model_id, - profile_id, - targets: row.targets, - }) + .map( + |((model_id, profile_id, context_tokens_per_request), row)| Recommendation { + model_id, + profile_id, + context_tokens_per_request, + targets: row.targets, + }, + ) .collect() } +/// Keeps what can start now from ranked rows: unreachable targets go first, deployed or not; a row +/// keeps its deployed targets and those that fit free memory now, and goes when none remain. Rows +/// with a deployed target come first, each group in rank order; within a row deployed targets +/// come first, then by node and target. +fn startable(rows: Vec) -> Vec { + let mut rows = rows + .into_iter() + .filter_map(|mut row| { + row.targets.retain(|target| { + target.fit != RecommendationFit::Unreachable + && (target.model_deployment_id.is_some() + || target.fit == RecommendationFit::FitsNow) + }); + row.targets.sort_by_key(|target| { + ( + target.model_deployment_id.is_none(), + target.fit != RecommendationFit::FitsNow, + target.placement.site_node_id.clone(), + target.placement.execution_target_id.clone(), + ) + }); + (!row.targets.is_empty()).then_some(row) + }) + .collect::>(); + // A stable sort keeps the rank order within each group. + rows.sort_by_key(|row| { + !row.targets + .iter() + .any(|target| target.model_deployment_id.is_some()) + }); + rows +} + /// Logs why a target has no list for a use case: no class with one matches it, or, against /// catalog validation, several match with none more specific. fn log_no_list(use_case: UseCase, target: &ExecutionTarget, error: &HardwareClassSelectionError) { diff --git a/backend/core/src/model/recommendation/tests.rs b/backend/core/src/model/recommendation/tests.rs index 7366e95f..6de2287b 100644 --- a/backend/core/src/model/recommendation/tests.rs +++ b/backend/core/src/model/recommendation/tests.rs @@ -24,7 +24,7 @@ use dotcore_site::node::target_inventory::{ use super::{RecommendationFit, UseCaseRecommendations, recommend_models}; use RecommendationFit::{FitsNow, NeedsMemoryFreed, Unknown, Unreachable}; -use UseCase::{CodingAndAgents, DecisionMaking, GeneralAssistant, Translation}; +use UseCase::{ChatAssistant, Coding, ImageGeneration}; /// Expected recommendations for one site of declared machines under one occupancy and use case. struct Case { @@ -43,6 +43,12 @@ enum Occupancy { /// estimate. Running(&'static str, &'static str), + /// One ready deployment of the model and profile on the site's first machine, using 30 GB. + Filling(&'static str, &'static str), + + /// One failed deployment of the model and profile on the site's first machine, using nothing. + Failed(&'static str, &'static str), + /// One deployment of unknown size still downloading on the site's first machine. StartingUnsized, @@ -54,182 +60,180 @@ enum Occupancy { StaleRunning(&'static str, &'static str), } -/// One expected row: `model/profile`, then each target with its fit, whether the profile is -/// deployed there, and the total context a deployment starts from. +/// One expected row: `model/profile`, its context per request, then each target with its fit, +/// the model deployment of the profile running or starting there, and the total context a +/// deployment starts from. type Row = ( &'static str, - Vec<(&'static str, RecommendationFit, bool, u64)>, + u64, + Vec<(&'static str, RecommendationFit, Option<&'static str>, u64)>, ); #[test] fn declared_machines_get_exact_ranked_recommendations_per_use_case_and_occupancy() { let cases = [ // 8 GiB GPU: the GPU-model entry is unsupported and the entries above its memory or their - // minimum are dropped; translation and decision making fall back to the vendor class. + // minimum are dropped; the chat assistant falls back to the vendor class. Case { machines: &["s8"], occupancy: Occupancy::Idle, - use_case: CodingAndAgents, - rows: vec![("small/q4", vec![("s8", FitsNow, false, 59_392)])], + use_case: Coding, + rows: vec![("small/q4", 32_768, vec![("s8", FitsNow, None, 59_392)])], }, Case { machines: &["s8"], occupancy: Occupancy::Idle, - use_case: Translation, - rows: vec![("small/q4", vec![("s8", FitsNow, false, 59_392)])], - }, - Case { - machines: &["s8"], - occupancy: Occupancy::Idle, - use_case: DecisionMaking, - rows: vec![("decider/q8", vec![("s8", FitsNow, false, 69_632)])], + use_case: ChatAssistant, + rows: vec![("small/q4", 32_768, vec![("s8", FitsNow, None, 59_392)])], }, // 32 GiB GPU of the model the GPU-model class names: its only configuration comes first. Case { machines: &["x32"], occupancy: Occupancy::Idle, - use_case: CodingAndAgents, + use_case: Coding, rows: vec![ - ("x-only/x", vec![("x32", FitsNow, false, 293_888)]), - ("big/q4", vec![("x32", FitsNow, false, 174_080)]), - ("mid/q4", vec![("x32", FitsNow, false, 274_432)]), - ("small/q4", vec![("x32", FitsNow, false, 314_368)]), + ("x-only/x", 32_768, vec![("x32", FitsNow, None, 293_888)]), + ("big/q4", 32_768, vec![("x32", FitsNow, None, 174_080)]), + ("mid/q4", 32_768, vec![("x32", FitsNow, None, 274_432)]), + ("small/q4", 32_768, vec![("x32", FitsNow, None, 314_368)]), ], }, Case { machines: &["x32"], occupancy: Occupancy::Idle, - use_case: Translation, + use_case: ChatAssistant, rows: vec![ - ("mid/q4", vec![("x32", FitsNow, false, 274_432)]), - ("small/q4", vec![("x32", FitsNow, false, 314_368)]), + ("mid/q4", 32_768, vec![("x32", FitsNow, None, 274_432)]), + ("small/q4", 32_768, vec![("x32", FitsNow, None, 314_368)]), ], }, - Case { - machines: &["x32"], - occupancy: Occupancy::Idle, - use_case: DecisionMaking, - rows: vec![("decider/q8", vec![("x32", FitsNow, false, 324_608)])], - }, // The running model is deployed and a second one needs its memory freed; it starts from // the profile default without room for a largest fit. Case { machines: &["x32"], occupancy: Occupancy::Running("big", "q4"), - use_case: CodingAndAgents, + use_case: Coding, rows: vec![ - ("x-only/x", vec![("x32", FitsNow, false, 101_376)]), - ("big/q4", vec![("x32", NeedsMemoryFreed, true, 32_768)]), - ("mid/q4", vec![("x32", FitsNow, false, 81_920)]), - ("small/q4", vec![("x32", FitsNow, false, 121_856)]), + ("x-only/x", 32_768, vec![("x32", FitsNow, None, 101_376)]), + ( + "big/q4", + 32_768, + vec![("x32", NeedsMemoryFreed, Some("big-q4"), 32_768)], + ), + ("mid/q4", 32_768, vec![("x32", FitsNow, None, 81_920)]), + ("small/q4", 32_768, vec![("x32", FitsNow, None, 121_856)]), ], }, Case { machines: &["x32"], occupancy: Occupancy::Running("big", "q4"), - use_case: Translation, + use_case: ChatAssistant, rows: vec![ - ("mid/q4", vec![("x32", FitsNow, false, 81_920)]), - ("small/q4", vec![("x32", FitsNow, false, 121_856)]), + ("mid/q4", 32_768, vec![("x32", FitsNow, None, 81_920)]), + ("small/q4", 32_768, vec![("x32", FitsNow, None, 121_856)]), ], }, // Macs: entry minimums on system memory separate 16, 32 and 64 GiB. Case { machines: &["mac16"], occupancy: Occupancy::Idle, - use_case: CodingAndAgents, - rows: vec![("small/q4", vec![("mac16", FitsNow, false, 87_040)])], + use_case: Coding, + rows: vec![("small/q4", 32_768, vec![("mac16", FitsNow, None, 87_040)])], }, Case { machines: &["mac16"], occupancy: Occupancy::Idle, - use_case: Translation, - rows: vec![("small/q4", vec![("mac16", FitsNow, false, 87_040)])], - }, - Case { - machines: &["mac16"], - occupancy: Occupancy::Idle, - use_case: DecisionMaking, - rows: vec![("decider/q8", vec![("mac16", FitsNow, false, 97_280)])], + use_case: ChatAssistant, + rows: vec![("small/q4", 32_768, vec![("mac16", FitsNow, None, 87_040)])], }, Case { machines: &["mac32"], occupancy: Occupancy::Idle, - use_case: CodingAndAgents, + use_case: Coding, rows: vec![ - ("mid/q4", vec![("mac32", FitsNow, false, 176_128)]), - ("small/q4", vec![("mac32", FitsNow, false, 216_064)]), + ("mid/q4", 32_768, vec![("mac32", FitsNow, None, 176_128)]), + ("small/q4", 32_768, vec![("mac32", FitsNow, None, 216_064)]), ], }, Case { machines: &["mac32"], occupancy: Occupancy::Idle, - use_case: Translation, + use_case: ChatAssistant, rows: vec![ - ("mid/q4", vec![("mac32", FitsNow, false, 176_128)]), - ("small/q4", vec![("mac32", FitsNow, false, 216_064)]), + ("mid/q4", 32_768, vec![("mac32", FitsNow, None, 176_128)]), + ("small/q4", 32_768, vec![("mac32", FitsNow, None, 216_064)]), ], }, Case { machines: &["mac64"], occupancy: Occupancy::Idle, - use_case: CodingAndAgents, + use_case: Coding, rows: vec![ - ("big/q4", vec![("mac64", FitsNow, false, 355_328)]), - ("mid/q4", vec![("mac64", FitsNow, false, 454_656)]), - ("small/q4", vec![("mac64", FitsNow, false, 494_592)]), + ("big/q4", 32_768, vec![("mac64", FitsNow, None, 355_328)]), + ("mid/q4", 32_768, vec![("mac64", FitsNow, None, 454_656)]), + ("small/q4", 32_768, vec![("mac64", FitsNow, None, 494_592)]), ], }, Case { machines: &["mac64"], occupancy: Occupancy::Running("mid", "q4"), - use_case: CodingAndAgents, + use_case: Coding, rows: vec![ - ("big/q4", vec![("mac64", FitsNow, false, 262_144)]), - ("mid/q4", vec![("mac64", FitsNow, true, 362_496)]), - ("small/q4", vec![("mac64", FitsNow, false, 402_432)]), + ("big/q4", 32_768, vec![("mac64", FitsNow, None, 262_144)]), + ( + "mid/q4", + 32_768, + vec![("mac64", FitsNow, Some("mid-q4"), 362_496)], + ), + ("small/q4", 32_768, vec![("mac64", FitsNow, None, 402_432)]), ], }, - Case { - machines: &["mac64"], - occupancy: Occupancy::Idle, - use_case: DecisionMaking, - rows: vec![("decider/q8", vec![("mac64", FitsNow, false, 504_832)])], - }, - // Unified 128 GiB: its own coding list, the vendor class's translation list. + // Unified 128 GiB: its own coding list, the vendor class's chat assistant list. Case { machines: &["uni128"], occupancy: Occupancy::Idle, - use_case: CodingAndAgents, + use_case: Coding, rows: vec![ - ("big/q4", vec![("uni128", FitsNow, false, 1_020_928)]), - ("mid/q4", vec![("uni128", FitsNow, false, 1_120_256)]), + ("big/q4", 32_768, vec![("uni128", FitsNow, None, 1_020_928)]), + ("mid/q4", 32_768, vec![("uni128", FitsNow, None, 1_120_256)]), ], }, Case { machines: &["uni128"], occupancy: Occupancy::Idle, - use_case: Translation, + use_case: ChatAssistant, rows: vec![ - ("mid/q4", vec![("uni128", FitsNow, false, 1_120_256)]), - ("small/q4", vec![("uni128", FitsNow, false, 1_160_192)]), + ("mid/q4", 32_768, vec![("uni128", FitsNow, None, 1_120_256)]), + ( + "small/q4", + 32_768, + vec![("uni128", FitsNow, None, 1_160_192)], + ), ], }, + // A deployment of unknown size still starting leaves free memory unknown, while the + // total memory still decides what is kept. Case { machines: &["uni128"], - occupancy: Occupancy::Idle, - use_case: DecisionMaking, - rows: vec![("decider/q8", vec![("uni128", FitsNow, false, 1_170_432)])], + occupancy: Occupancy::StartingUnsized, + use_case: Coding, + rows: vec![ + ("big/q4", 32_768, vec![("uni128", Unknown, None, 32_768)]), + ("mid/q4", 32_768, vec![("uni128", Unknown, None, 32_768)]), + ], }, - // A deployment of unknown size still starting leaves free memory unknown, while the - // total memory still decides what is kept. + // A starting deployment names its profile's row. Case { machines: &["uni128"], occupancy: Occupancy::StartingUnsized, - use_case: CodingAndAgents, + use_case: ChatAssistant, rows: vec![ - ("big/q4", vec![("uni128", Unknown, false, 32_768)]), - ("mid/q4", vec![("uni128", Unknown, false, 32_768)]), + ("mid/q4", 32_768, vec![("uni128", Unknown, None, 32_768)]), + ( + "small/q4", + 32_768, + vec![("uni128", Unknown, Some("small-q4"), 32_768)], + ), ], }, // Two machines: rows merge by their best position in either list, then the higher @@ -237,28 +241,31 @@ fn declared_machines_get_exact_ranked_recommendations_per_use_case_and_occupancy Case { machines: &["mac64", "x32"], occupancy: Occupancy::Idle, - use_case: CodingAndAgents, + use_case: Coding, rows: vec![ - ("x-only/x", vec![("x32", FitsNow, false, 293_888)]), + ("x-only/x", 32_768, vec![("x32", FitsNow, None, 293_888)]), ( "big/q4", + 32_768, vec![ - ("mac64", FitsNow, false, 355_328), - ("x32", FitsNow, false, 174_080), + ("mac64", FitsNow, None, 355_328), + ("x32", FitsNow, None, 174_080), ], ), ( "mid/q4", + 32_768, vec![ - ("mac64", FitsNow, false, 454_656), - ("x32", FitsNow, false, 274_432), + ("mac64", FitsNow, None, 454_656), + ("x32", FitsNow, None, 274_432), ], ), ( "small/q4", + 32_768, vec![ - ("mac64", FitsNow, false, 494_592), - ("x32", FitsNow, false, 314_368), + ("mac64", FitsNow, None, 494_592), + ("x32", FitsNow, None, 314_368), ], ), ], @@ -266,29 +273,31 @@ fn declared_machines_get_exact_ranked_recommendations_per_use_case_and_occupancy Case { machines: &["mac64", "x32"], occupancy: Occupancy::Idle, - use_case: Translation, + use_case: ChatAssistant, rows: vec![ ( "mid/q4", + 32_768, vec![ - ("mac64", FitsNow, false, 454_656), - ("x32", FitsNow, false, 274_432), + ("mac64", FitsNow, None, 454_656), + ("x32", FitsNow, None, 274_432), ], ), ( "small/q4", + 32_768, vec![ - ("mac64", FitsNow, false, 494_592), - ("x32", FitsNow, false, 314_368), + ("mac64", FitsNow, None, 494_592), + ("x32", FitsNow, None, 314_368), ], ), ], }, - // No class has a list for the general assistant. + // Image generation has no model lists. Case { machines: &["mac64", "x32"], occupancy: Occupancy::Idle, - use_case: GeneralAssistant, + use_case: ImageGeneration, rows: vec![], }, // A machine that stopped reporting keeps the rows its last reported hardware holds, each @@ -296,12 +305,12 @@ fn declared_machines_get_exact_ranked_recommendations_per_use_case_and_occupancy Case { machines: &["x32"], occupancy: Occupancy::Stale, - use_case: CodingAndAgents, + use_case: Coding, rows: vec![ - ("x-only/x", vec![("x32", Unreachable, false, 32_768)]), - ("big/q4", vec![("x32", Unreachable, false, 32_768)]), - ("mid/q4", vec![("x32", Unreachable, false, 32_768)]), - ("small/q4", vec![("x32", Unreachable, false, 32_768)]), + ("x-only/x", 32_768, vec![("x32", Unreachable, None, 32_768)]), + ("big/q4", 32_768, vec![("x32", Unreachable, None, 32_768)]), + ("mid/q4", 32_768, vec![("x32", Unreachable, None, 32_768)]), + ("small/q4", 32_768, vec![("x32", Unreachable, None, 32_768)]), ], }, // What runs on an unreachable machine is deployed there, and never drops an entry its @@ -309,40 +318,47 @@ fn declared_machines_get_exact_ranked_recommendations_per_use_case_and_occupancy Case { machines: &["x32"], occupancy: Occupancy::StaleRunning("big", "q4"), - use_case: CodingAndAgents, + use_case: Coding, rows: vec![ - ("x-only/x", vec![("x32", Unreachable, false, 32_768)]), - ("big/q4", vec![("x32", Unreachable, true, 32_768)]), - ("mid/q4", vec![("x32", Unreachable, false, 32_768)]), - ("small/q4", vec![("x32", Unreachable, false, 32_768)]), + ("x-only/x", 32_768, vec![("x32", Unreachable, None, 32_768)]), + ( + "big/q4", + 32_768, + vec![("x32", Unreachable, Some("big-q4"), 32_768)], + ), + ("mid/q4", 32_768, vec![("x32", Unreachable, None, 32_768)]), + ("small/q4", 32_768, vec![("x32", Unreachable, None, 32_768)]), ], }, // An unreachable machine beside a reachable one: each target keeps its own fit in a row. Case { machines: &["mac64", "x32"], occupancy: Occupancy::Stale, - use_case: CodingAndAgents, + use_case: Coding, rows: vec![ - ("x-only/x", vec![("x32", FitsNow, false, 293_888)]), + ("x-only/x", 32_768, vec![("x32", FitsNow, None, 293_888)]), ( "big/q4", + 32_768, vec![ - ("mac64", Unreachable, false, 32_768), - ("x32", FitsNow, false, 174_080), + ("mac64", Unreachable, None, 32_768), + ("x32", FitsNow, None, 174_080), ], ), ( "mid/q4", + 32_768, vec![ - ("mac64", Unreachable, false, 32_768), - ("x32", FitsNow, false, 274_432), + ("mac64", Unreachable, None, 32_768), + ("x32", FitsNow, None, 274_432), ], ), ( "small/q4", + 32_768, vec![ - ("mac64", Unreachable, false, 32_768), - ("x32", FitsNow, false, 314_368), + ("mac64", Unreachable, None, 32_768), + ("x32", FitsNow, None, 314_368), ], ), ], @@ -365,6 +381,7 @@ fn declared_machines_get_exact_ranked_recommendations_per_use_case_and_occupancy Some(&inventory), &deployments, &[case.use_case], + false, ); assert_eq!( @@ -389,21 +406,30 @@ fn every_use_case_asked_gets_its_own_list_in_the_order_asked() { &targets, Some(&inventory), &deployments, - &[DecisionMaking, GeneralAssistant, CodingAndAgents], + &[ImageGeneration, ChatAssistant, Coding], + false, ); assert_eq!( recommendations.iter().map(rows).collect::>(), [ - owned(vec![("decider/q8", vec![("s8", FitsNow, false, 69_632)])]), Vec::new(), - owned(vec![("small/q4", vec![("s8", FitsNow, false, 59_392)])]), + owned(vec![( + "small/q4", + 32_768, + vec![("s8", FitsNow, None, 59_392)] + )]), + owned(vec![( + "small/q4", + 32_768, + vec![("s8", FitsNow, None, 59_392)] + )]), ] ); // Without an inventory every target is unknown and nothing is recommended. assert_eq!( - recommend_models(&catalog, &targets, None, &deployments, &[CodingAndAgents],) + recommend_models(&catalog, &targets, None, &deployments, &[Coding], false) .iter() .map(rows) .collect::>(), @@ -411,6 +437,286 @@ fn every_use_case_asked_gets_its_own_list_in_the_order_asked() { ); } +#[test] +fn entries_are_sized_with_their_context_per_request() { + // The dedicated coding entry asks for 131072 tokens per request, the dedicated chat entry + // leaves it to the profile's default, and the unified coding entry asks for 32768. + let catalog = catalog_with_lists(&[ + ( + "nvidia-dedicated", + r#" +hardwareClassId: "nvidia-dedicated" +useCases: + - useCase: "USE_CASE_CODING" + entries: + - { modelId: "mid", profileId: "q4", contextTokensPerRequest: 131072 } + - useCase: "USE_CASE_CHAT_ASSISTANT" + entries: + - { modelId: "mid", profileId: "q4" } +"#, + ), + ( + "nvidia-unified", + r#" +hardwareClassId: "nvidia-unified" +useCases: + - useCase: "USE_CASE_CODING" + entries: + - { modelId: "mid", profileId: "q4", contextTokensPerRequest: 32768 } +"#, + ), + ]); + let read = |machines, occupancy| { + let (targets, inventory, deployments) = site(machines, occupancy); + recommend_models( + &catalog, + &targets, + Some(&inventory), + &deployments, + &[Coding, ChatAssistant], + false, + ) + .iter() + .map(rows) + .collect::>() + }; + + // Idle 32 GiB GPU: both contexts fit, and the starting total context, bounded by free memory + // alone, is the same. + assert_eq!( + read(&["x32"], Occupancy::Idle), + [ + owned(vec![( + "mid/q4", + 131_072, + vec![("x32", FitsNow, None, 274_432)] + )]), + owned(vec![( + "mid/q4", + 32_768, + vec![("x32", FitsNow, None, 274_432)] + )]), + ] + ); + + // Beside a running model, 131072 tokens per request need memory freed where 32768 fit, and + // the two coding contexts stay separate rows. + assert_eq!( + read(&["x32", "uni128"], Occupancy::Running("big", "q4")), + [ + owned(vec![ + ("mid/q4", 32_768, vec![("uni128", FitsNow, None, 1_120_256)]), + ( + "mid/q4", + 131_072, + vec![("x32", NeedsMemoryFreed, None, 32_768)] + ), + ]), + owned(vec![( + "mid/q4", + 32_768, + vec![("x32", FitsNow, None, 81_920)] + )]), + ] + ); +} + +#[test] +fn can_start_now_keeps_deployed_and_fitting_targets_with_deployed_rows_first() { + let cases = [ + // Idle 32 GiB GPU: every row fits now. + Case { + machines: &["x32"], + occupancy: Occupancy::Idle, + use_case: Coding, + rows: vec![ + ("x-only/x", 32_768, vec![("x32", FitsNow, None, 293_888)]), + ("big/q4", 32_768, vec![("x32", FitsNow, None, 174_080)]), + ("mid/q4", 32_768, vec![("x32", FitsNow, None, 274_432)]), + ("small/q4", 32_768, vec![("x32", FitsNow, None, 314_368)]), + ], + }, + // The running model's row comes first, its deployed target before the fitting one; other + // rows keep their rank and order their targets by machine. + Case { + machines: &["x32", "mac64"], + occupancy: Occupancy::Running("mid", "q4"), + use_case: Coding, + rows: vec![ + ( + "mid/q4", + 32_768, + vec![ + ("x32", FitsNow, Some("mid-q4"), 181_248), + ("mac64", FitsNow, None, 454_656), + ], + ), + ("x-only/x", 32_768, vec![("x32", FitsNow, None, 201_728)]), + ( + "big/q4", + 32_768, + vec![ + ("mac64", FitsNow, None, 355_328), + ("x32", FitsNow, None, 81_920), + ], + ), + ( + "small/q4", + 32_768, + vec![ + ("mac64", FitsNow, None, 494_592), + ("x32", FitsNow, None, 221_184), + ], + ), + ], + }, + // A machine filled by another model starts nothing new, but its own row stays. + Case { + machines: &["x32"], + occupancy: Occupancy::Filling("x-only", "x"), + use_case: ChatAssistant, + rows: vec![], + }, + Case { + machines: &["x32"], + occupancy: Occupancy::Filling("x-only", "x"), + use_case: Coding, + rows: vec![( + "x-only/x", + 32_768, + vec![("x32", NeedsMemoryFreed, Some("x-only-x"), 32_768)], + )], + }, + // An unreachable machine's targets are dropped, even with a deployment there. + Case { + machines: &["mac64", "x32"], + occupancy: Occupancy::StaleRunning("big", "q4"), + use_case: Coding, + rows: vec![ + ("x-only/x", 32_768, vec![("x32", FitsNow, None, 293_888)]), + ("big/q4", 32_768, vec![("x32", FitsNow, None, 174_080)]), + ("mid/q4", 32_768, vec![("x32", FitsNow, None, 274_432)]), + ("small/q4", 32_768, vec![("x32", FitsNow, None, 314_368)]), + ], + }, + // A failed deployment names no row and holds no memory. + Case { + machines: &["x32"], + occupancy: Occupancy::Failed("x-only", "x"), + use_case: Coding, + rows: vec![ + ("x-only/x", 32_768, vec![("x32", FitsNow, None, 293_888)]), + ("big/q4", 32_768, vec![("x32", FitsNow, None, 174_080)]), + ("mid/q4", 32_768, vec![("x32", FitsNow, None, 274_432)]), + ("small/q4", 32_768, vec![("x32", FitsNow, None, 314_368)]), + ], + }, + // Unknown free memory keeps only the starting deployment's row. + Case { + machines: &["uni128"], + occupancy: Occupancy::StartingUnsized, + use_case: ChatAssistant, + rows: vec![( + "small/q4", + 32_768, + vec![("uni128", Unknown, Some("small-q4"), 32_768)], + )], + }, + ]; + + let catalog = catalog(); + for case in cases { + let label = format!( + "{} {:?} {}", + case.machines.join("+"), + case.occupancy, + case.use_case.as_str_name() + ); + let (targets, inventory, deployments) = site(case.machines, case.occupancy); + + let recommendations = recommend_models( + &catalog, + &targets, + Some(&inventory), + &deployments, + &[case.use_case], + true, + ); + + assert_eq!( + recommendations.iter().map(rows).collect::>(), + [owned(case.rows)], + "{label}" + ); + } +} + +#[test] +fn a_running_deployment_serves_every_context_per_request_of_its_profile() { + // Coding asks for 262144 tokens per request and the chat assistant for the default 32768. + let catalog = catalog_with_lists(&[( + "nvidia-dedicated", + r#" +hardwareClassId: "nvidia-dedicated" +useCases: + - useCase: "USE_CASE_CODING" + entries: + - { modelId: "mid", profileId: "q4", contextTokensPerRequest: 262144 } + - useCase: "USE_CASE_CHAT_ASSISTANT" + entries: + - { modelId: "mid", profileId: "q4" } +"#, + )]); + let read = |occupancy, can_start_now| { + let (targets, inventory, deployments) = site(&["x32"], occupancy); + recommend_models( + &catalog, + &targets, + Some(&inventory), + &deployments, + &[Coding, ChatAssistant], + can_start_now, + ) + .iter() + .map(rows) + .collect::>() + }; + + // Idle, 262144 tokens fit now. + assert_eq!( + read(Occupancy::Idle, true), + [ + owned(vec![( + "mid/q4", + 262_144, + vec![("x32", FitsNow, None, 274_432)] + )]), + owned(vec![( + "mid/q4", + 32_768, + vec![("x32", FitsNow, None, 274_432)] + )]), + ] + ); + + // Beside the same profile running at 32768 tokens, 262144 more would need memory freed, but + // the running deployment is offered for coding too, with or without the option. + let running = [ + owned(vec![( + "mid/q4", + 262_144, + vec![("x32", NeedsMemoryFreed, Some("mid-q4"), 32_768)], + )]), + owned(vec![( + "mid/q4", + 32_768, + vec![("x32", FitsNow, Some("mid-q4"), 181_248)], + )]), + ]; + assert_eq!(read(Occupancy::Running("mid", "q4"), true), running); + assert_eq!(read(Occupancy::Running("mid", "q4"), false), running); +} + /// The rows of one use case's recommendations, in the form `Row` describes. fn rows(recommendations: &UseCaseRecommendations) -> Vec { recommendations @@ -424,12 +730,16 @@ fn rows(recommendations: &UseCaseRecommendations) -> Vec { ( target.placement.execution_target_id.clone(), target.fit, - target.deployed, + target.model_deployment_id.clone(), target.starting_total_context_tokens, ) }) .collect(); - (format!("{}/{}", row.model_id, row.profile_id), targets) + ( + format!("{}/{}", row.model_id, row.profile_id), + row.context_tokens_per_request, + targets, + ) }) .collect() } @@ -437,19 +747,28 @@ fn rows(recommendations: &UseCaseRecommendations) -> Vec { /// Expected rows in the form `rows` reads them. fn owned(rows: Vec) -> Vec { rows.into_iter() - .map(|(name, targets)| { + .map(|(name, context_tokens_per_request, targets)| { let targets = targets .into_iter() - .map(|(target, fit, deployed, starting)| { - (target.to_owned(), fit, deployed, Some(starting)) + .map(|(target, fit, deployment, starting)| { + ( + target.to_owned(), + fit, + deployment.map(str::to_owned), + Some(starting), + ) }) .collect(); - (name.to_owned(), targets) + (name.to_owned(), context_tokens_per_request, targets) }) .collect() } -type OwnedRow = (String, Vec<(String, RecommendationFit, bool, Option)>); +type OwnedRow = ( + String, + u64, + Vec<(String, RecommendationFit, Option, Option)>, +); /// One target per machine, named after it, with the inventory the machines report under /// `occupancy` and the site's deployments. @@ -477,8 +796,14 @@ fn site( let architecture = described[0].accelerator_facts()[0].memory_architecture(); let (deployments, workloads) = match occupancy { Occupancy::Idle | Occupancy::Stale => (Vec::new(), Vec::new()), - Occupancy::Running(model, profile) | Occupancy::StaleRunning(model, profile) => { - let bytes = default_estimate(model); + Occupancy::Running(model, profile) + | Occupancy::Filling(model, profile) + | Occupancy::StaleRunning(model, profile) => { + let bytes = if matches!(occupancy, Occupancy::Filling(..)) { + 30_000_000_000 + } else { + default_estimate(model) + }; ( vec![deployment( first, @@ -493,6 +818,16 @@ fn site( }], ) } + Occupancy::Failed(model, profile) => ( + vec![deployment( + first, + architecture, + (model, profile), + ModelDeploymentStatus::Failed, + Some(default_estimate(model)), + )], + Vec::new(), + ), Occupancy::StartingUnsized => ( vec![deployment( first, @@ -712,7 +1047,6 @@ struct FixtureModel { weight_bytes: u64, intelligence: u32, classes: &'static [&'static str], - request_kind: &'static str, profile: &'static str, } @@ -722,7 +1056,6 @@ const MODELS: &[FixtureModel] = &[ weight_bytes: 4_000_000_000, intelligence: 45, classes: &["nvidia-x"], - request_kind: "INFERENCE_REQUEST_KIND_CHAT", profile: "x", }, FixtureModel { @@ -730,7 +1063,6 @@ const MODELS: &[FixtureModel] = &[ weight_bytes: 16_000_000_000, intelligence: 40, classes: CHAT_CLASSES, - request_kind: "INFERENCE_REQUEST_KIND_CHAT", profile: "q4", }, FixtureModel { @@ -738,7 +1070,6 @@ const MODELS: &[FixtureModel] = &[ weight_bytes: 6_000_000_000, intelligence: 25, classes: CHAT_CLASSES, - request_kind: "INFERENCE_REQUEST_KIND_CHAT", profile: "q4", }, FixtureModel { @@ -746,17 +1077,8 @@ const MODELS: &[FixtureModel] = &[ weight_bytes: 2_000_000_000, intelligence: 10, classes: CHAT_CLASSES, - request_kind: "INFERENCE_REQUEST_KIND_CHAT", profile: "q4", }, - FixtureModel { - id: "decider", - weight_bytes: 1_000_000_000, - intelligence: 5, - classes: CHAT_CLASSES, - request_kind: "INFERENCE_REQUEST_KIND_DECISION", - profile: "q8", - }, ]; const CHAT_CLASSES: &[&str] = &["nvidia-dedicated", "nvidia-unified", "apple"]; @@ -765,6 +1087,17 @@ const CHAT_CLASSES: &[&str] = &["nvidia-dedicated", "nvidia-unified", "apple"]; /// one llama.cpp engine for linux/amd64, linux/arm64 and darwin/arm64, the fixture models, and /// their recommendation lists. pub(super) fn catalog() -> Catalog { + catalog_with_lists(&[ + ("nvidia", NVIDIA_LISTS), + ("nvidia-dedicated", DEDICATED_LISTS), + ("nvidia-unified", UNIFIED_LISTS), + ("apple", APPLE_LISTS), + ]) +} + +/// Writes and loads the made-up catalog with `lists`, each a hardware class id and its +/// recommendation file. +fn catalog_with_lists(lists: &[(&str, &str)]) -> Catalog { let root = tempfile::tempdir().expect("catalog root"); let mut files = vec![ ("hardware-classes/nvidia.yaml", class("nvidia", "")), @@ -792,14 +1125,16 @@ pub(super) fn catalog() -> Catalog { .to_owned(), ), ("engines/llama.yaml", ENGINE.to_owned()), - ("recommendations/nvidia.yaml", NVIDIA_LISTS.to_owned()), - ( - "recommendations/nvidia-dedicated.yaml", - DEDICATED_LISTS.to_owned(), - ), - ("recommendations/nvidia-unified.yaml", UNIFIED_LISTS.to_owned()), - ("recommendations/apple.yaml", APPLE_LISTS.to_owned()), ]; + let list_files = lists + .iter() + .map(|(class, body)| (format!("recommendations/{class}.yaml"), (*body).to_owned())) + .collect::>(); + files.extend( + list_files + .iter() + .map(|(path, body)| (path.as_str(), body.clone())), + ); let models = MODELS .iter() .map(|model| (format!("models/{}.yaml", model.id), model_yaml(model))) @@ -837,7 +1172,6 @@ fn model_yaml(model: &FixtureModel) -> String { weight_bytes: base, intelligence, classes, - request_kind, profile, } = model; let support = classes @@ -890,7 +1224,7 @@ configurations: engineKind: "ENGINE_KIND_LLAMA_CPP" llamaCpp: {{ fixedSettings: {{ gpuLayers: 0 }} }} executionCapabilities: {{ supportedParallelShapes: ["MODEL_PARALLEL_SHAPE_ONE"] }} - requestKinds: ["{request_kind}"] + requestKinds: ["INFERENCE_REQUEST_KIND_CHAT"] inputCapabilities: {{ supportedModalities: ["MODEL_INPUT_MODALITY_TEXT"] }} profiles: - profileId: "{profile}" @@ -942,20 +1276,17 @@ artifacts: const NVIDIA_LISTS: &str = r#" hardwareClassId: "nvidia" useCases: - - useCase: "USE_CASE_TRANSLATION" + - useCase: "USE_CASE_CHAT_ASSISTANT" entries: - { modelId: "mid", profileId: "q4" } - { modelId: "small", profileId: "q4" } - - useCase: "USE_CASE_DECISION_MAKING" - entries: - - { modelId: "decider", profileId: "q8" } "#; /// Dedicated GPUs' coding list: the GPU-model entry first, then minimums on the GPU's memory. const DEDICATED_LISTS: &str = r#" hardwareClassId: "nvidia-dedicated" useCases: - - useCase: "USE_CASE_CODING_AND_AGENTS" + - useCase: "USE_CASE_CODING" entries: - { modelId: "x-only", profileId: "x" } - { modelId: "big", profileId: "q4", hardwareRequirements: { minimumMemoryBytesPerDevice: "24000000000" } } @@ -966,7 +1297,7 @@ useCases: const UNIFIED_LISTS: &str = r#" hardwareClassId: "nvidia-unified" useCases: - - useCase: "USE_CASE_CODING_AND_AGENTS" + - useCase: "USE_CASE_CODING" entries: - { modelId: "big", profileId: "q4" } - { modelId: "mid", profileId: "q4" } @@ -976,16 +1307,13 @@ useCases: const APPLE_LISTS: &str = r#" hardwareClassId: "apple" useCases: - - useCase: "USE_CASE_CODING_AND_AGENTS" + - useCase: "USE_CASE_CODING" entries: - { modelId: "big", profileId: "q4", hardwareRequirements: { minimumSystemMemoryBytes: "48000000000" } } - { modelId: "mid", profileId: "q4", hardwareRequirements: { minimumSystemMemoryBytes: "24000000000" } } - { modelId: "small", profileId: "q4" } - - useCase: "USE_CASE_TRANSLATION" + - useCase: "USE_CASE_CHAT_ASSISTANT" entries: - { modelId: "mid", profileId: "q4", hardwareRequirements: { minimumSystemMemoryBytes: "24000000000" } } - { modelId: "small", profileId: "q4" } - - useCase: "USE_CASE_DECISION_MAKING" - entries: - - { modelId: "decider", profileId: "q8" } "#; diff --git a/backend/core/src/model/repository.rs b/backend/core/src/model/repository.rs index ab7767fa..72f5169e 100644 --- a/backend/core/src/model/repository.rs +++ b/backend/core/src/model/repository.rs @@ -290,11 +290,12 @@ impl ModelDeclarationRepository { /// Ranks the catalog's recommended model profiles for each of `use_cases` on the site's /// configured model execution targets against `inventory` and the site's `deployments`, - /// without reading or mutating the retained target capabilities. A missing inventory - /// recommends nothing. + /// without reading or mutating the retained target capabilities, keeping only what can start + /// now when `can_start_now` is set. A missing inventory recommends nothing. pub(crate) fn model_recommendations( &self, use_cases: &[UseCase], + can_start_now: bool, inventory: Option<&TargetCapabilityInventory>, deployments: &[ModelDeployment], ) -> Result { @@ -305,8 +306,14 @@ impl ModelDeclarationRepository { .into_iter() .filter(is_model_target) .collect::>(); - let recommendations = - recommend_models(&self.catalog, &targets, inventory, deployments, use_cases); + let recommendations = recommend_models( + &self.catalog, + &targets, + inventory, + deployments, + use_cases, + can_start_now, + ); Ok(recommendations_response(recommendations)) } @@ -697,6 +704,7 @@ fn recommendations_response( .map(|row| RecommendedModelProfile { model_id: row.model_id, profile_id: row.profile_id, + context_tokens_per_request: row.context_tokens_per_request, targets: row .targets .into_iter() @@ -713,7 +721,7 @@ fn recommendations_response( } } .into(), - deployed: target.deployed, + model_deployment_id: target.model_deployment_id, starting_total_context_tokens: target.starting_total_context_tokens, }) .collect(), diff --git a/backend/core/src/model/service.rs b/backend/core/src/model/service.rs index fcd807a2..07de6e1b 100644 --- a/backend/core/src/model/service.rs +++ b/backend/core/src/model/service.rs @@ -352,9 +352,9 @@ where &self, request: Request, ) -> Result, Status> { - let use_cases = parse_model_recommendations_request(request.into_inner())?; + let (use_cases, can_start_now) = parse_model_recommendations_request(request.into_inner())?; self.controller - .list_model_recommendations(use_cases) + .list_model_recommendations(use_cases, can_start_now) .await .map(Response::new) .map_err(Self::translate_controller_error) @@ -840,13 +840,14 @@ fn parse_metadata_patch( }) } -/// Validates the use cases of a recommendations read: each must be a known, specified use case. -/// Duplicates collapse to their first occurrence, and none asks for every use case. +/// Validates a recommendations read: each use case must be a known, specified one. Duplicates +/// collapse to their first occurrence, and none asks for every use case. Returns the use cases and +/// whether only what can start now is asked for. fn parse_model_recommendations_request( request: ListModelRecommendationsRequest, -) -> Result, Status> { +) -> Result<(Vec, bool), Status> { if request.use_cases.is_empty() { - return Ok(ALL_USE_CASES.to_vec()); + return Ok((ALL_USE_CASES.to_vec(), request.can_start_now)); } let mut use_cases = Vec::new(); for value in request.use_cases { @@ -860,15 +861,14 @@ fn parse_model_recommendations_request( use_cases.push(use_case); } } - Ok(use_cases) + Ok((use_cases, request.can_start_now)) } /// Every use case a recommendations read ranks when the request names none. -const ALL_USE_CASES: [UseCase; 4] = [ - UseCase::CodingAndAgents, - UseCase::GeneralAssistant, - UseCase::Translation, - UseCase::DecisionMaking, +const ALL_USE_CASES: [UseCase; 3] = [ + UseCase::Coding, + UseCase::ChatAssistant, + UseCase::ImageGeneration, ]; /// Validated compatibility read selection: the model, the target and the sizing scenario. @@ -1345,32 +1345,50 @@ mod tests { #[test] fn recommendation_requests_default_to_all_use_cases_and_reject_unspecified_ones() { - let parse = |use_cases: &[i32]| { + let parse = |use_cases: &[i32], can_start_now: bool| { super::parse_model_recommendations_request(ListModelRecommendationsRequest { use_cases: use_cases.to_vec(), + can_start_now, }) }; assert_eq!( - parse(&[]).expect("all use cases"), - [ - UseCase::CodingAndAgents, - UseCase::GeneralAssistant, - UseCase::Translation, - UseCase::DecisionMaking, - ] + parse(&[], false).expect("all use cases"), + ( + vec![ + UseCase::Coding, + UseCase::ChatAssistant, + UseCase::ImageGeneration, + ], + false + ) ); assert_eq!( - parse(&[ - UseCase::Translation as i32, - UseCase::CodingAndAgents as i32, - UseCase::Translation as i32, - ]) + parse(&[], true).expect("all use cases that can start now"), + ( + vec![ + UseCase::Coding, + UseCase::ChatAssistant, + UseCase::ImageGeneration, + ], + true + ) + ); + assert_eq!( + parse( + &[ + UseCase::ImageGeneration as i32, + UseCase::Coding as i32, + UseCase::ImageGeneration as i32, + ], + true + ) .expect("named use cases"), - [UseCase::Translation, UseCase::CodingAndAgents] + (vec![UseCase::ImageGeneration, UseCase::Coding], true) ); for invalid in [UseCase::Unspecified as i32, 99] { - let status = parse(&[UseCase::Translation as i32, invalid]).expect_err("invalid"); + let status = + parse(&[UseCase::ImageGeneration as i32, invalid], false).expect_err("invalid"); assert_eq!(status.code(), tonic::Code::InvalidArgument); assert_eq!( status.message(), diff --git a/backend/core/src/model/sizing_scenario/mod.rs b/backend/core/src/model/sizing_scenario/mod.rs index 8969b702..701bc330 100644 --- a/backend/core/src/model/sizing_scenario/mod.rs +++ b/backend/core/src/model/sizing_scenario/mod.rs @@ -262,6 +262,22 @@ pub(crate) fn size_profile( } } +/// Context per request a profile's defaults resolve to: the engine's default total context, capped +/// at the configuration's maximum context. Returns none when the defaults don't resolve or hold no +/// total context. +pub(crate) fn default_context_per_request( + catalog: &Catalog, + model: &Model, + profile: &ModelProfile, +) -> Option { + let (defaults, declarations) = profile_defaults(catalog, model, profile)?; + let values = with_filled_context_per_request( + &declarations, + defaults.engine_parameter_values.unwrap_or_default(), + ); + unsigned_parameter_value(&values, CONTEXT_PER_REQUEST_PARAMETER_ID) +} + /// Judges the memory fit of a profile that runs with its defaults, resolved as `snapshot`, on the /// target capabilities `selected_target` the compatibility read selected, and the largest total /// context that fits there. Both fits compare one estimate, of the profile's sizing scenario on the diff --git a/backend/core/src/resource_watch/hub.rs b/backend/core/src/resource_watch/hub.rs index de320d2d..d4f4692a 100644 --- a/backend/core/src/resource_watch/hub.rs +++ b/backend/core/src/resource_watch/hub.rs @@ -49,6 +49,8 @@ pub(crate) enum ResourceKind { ExecutionTargets, NodeMemory, + + ConsoleSettings, } impl ResourceKind { @@ -62,6 +64,7 @@ impl ResourceKind { Self::Routes => "routes", Self::ExecutionTargets => "execution targets", Self::NodeMemory => "node memory", + Self::ConsoleSettings => "console settings", } } } @@ -104,6 +107,8 @@ pub(crate) struct ResourceReads { pub(crate) execution_targets: Arc, pub(crate) node_memory: Arc, + + pub(crate) console_settings: Arc, } impl ResourceReads { @@ -116,6 +121,7 @@ impl ResourceReads { ResourceKind::Routes => &self.routes, ResourceKind::ExecutionTargets => &self.execution_targets, ResourceKind::NodeMemory => &self.node_memory, + ResourceKind::ConsoleSettings => &self.console_settings, }) } } @@ -587,6 +593,7 @@ mod tests { ResourceKind::ApplicationDeployments, ResourceKind::ExecutionTargets, ResourceKind::NodeMemory, + ResourceKind::ConsoleSettings, ] { assert_eq!(harness.reads(kind), 0, "{} are not watched", kind.name()); } @@ -936,8 +943,8 @@ pub(super) mod test_support { inference::v1::ListRoutesResponse, model::v1::ListModelDeploymentsResponse, site::v1::{ - ListConfiguredExecutionTargetsResponse, ListHardwareSnapshotsResponse, - ListMachinesResponse, + ConsoleSettings, GetConsoleSettingsResponse, ListConfiguredExecutionTargetsResponse, + ListHardwareSnapshotsResponse, ListMachinesResponse, }, watch::v1::keyed_resource::Payload, }; @@ -955,7 +962,7 @@ pub(super) mod test_support { /// Real-time bound for progress the test waits on; never reached when the hub works. const PROGRESS_DEADLINE: Duration = Duration::from_secs(10); - pub(crate) const ALL_KINDS: [ResourceKind; 7] = [ + pub(crate) const ALL_KINDS: [ResourceKind; 8] = [ ResourceKind::Machines, ResourceKind::HardwareSnapshots, ResourceKind::ModelDeployments, @@ -963,6 +970,7 @@ pub(super) mod test_support { ResourceKind::Routes, ResourceKind::ExecutionTargets, ResourceKind::NodeMemory, + ResourceKind::ConsoleSettings, ]; /// A hub over fake owners and a virtual clock. @@ -1016,6 +1024,7 @@ pub(super) mod test_support { routes: read(ResourceKind::Routes), execution_targets: read(ResourceKind::ExecutionTargets), node_memory: read(ResourceKind::NodeMemory), + console_settings: read(ResourceKind::ConsoleSettings), }; let hub = ResourceWatchHub::new("a".repeat(32), reads, Arc::new(clock.clone()), config); Self { @@ -1146,6 +1155,12 @@ pub(super) mod test_support { ResourceKind::NodeMemory => Payload::NodeMemory(ListNodeMemoryResponse { nodes: vec![Default::default(); version], }), + // Console settings hold one flag, so their only versions are 0 and 1. + ResourceKind::ConsoleSettings => Payload::ConsoleSettings(GetConsoleSettingsResponse { + settings: Some(ConsoleSettings { + get_started_hidden: version == 1, + }), + }), } } @@ -1158,6 +1173,12 @@ pub(super) mod test_support { Payload::Routes(response) => response.routes.len(), Payload::ExecutionTargets(response) => response.execution_targets.len(), Payload::NodeMemory(response) => response.nodes.len(), + Payload::ConsoleSettings(response) => usize::from( + response + .settings + .as_ref() + .is_some_and(|settings| settings.get_started_hidden), + ), } } diff --git a/backend/core/src/resource_watch/mod.rs b/backend/core/src/resource_watch/mod.rs index 1779b618..2e892914 100644 --- a/backend/core/src/resource_watch/mod.rs +++ b/backend/core/src/resource_watch/mod.rs @@ -253,6 +253,9 @@ fn subscriptions(request: WatchResourcesRequest) -> Result ResourceKind::NodeMemory, + Some(resource_subscription::Kind::ConsoleSettings(_)) => { + ResourceKind::ConsoleSettings + } None => { return Err(Status::invalid_argument( "Resource watch subscription kind is required", @@ -538,6 +541,9 @@ mod tests { ResourceKind::NodeMemory => { resource_subscription::Kind::NodeMemory(Default::default()) } + ResourceKind::ConsoleSettings => { + resource_subscription::Kind::ConsoleSettings(Default::default()) + } }), }; assert_eq!( diff --git a/backend/core/src/resource_watch/reads.rs b/backend/core/src/resource_watch/reads.rs index cd2ad3e8..b4991a81 100644 --- a/backend/core/src/resource_watch/reads.rs +++ b/backend/core/src/resource_watch/reads.rs @@ -20,8 +20,9 @@ use crate::{ node_memory::NodeMemoryRead, route_manager::RouteManager, }; -/// The reads of every resource kind from their owners: the site inventory, the inference and -/// apps controllers, the route manager, and the capacity component's node memory. +/// The reads of every resource kind from their owners: the site inventory and console settings, +/// the inference and apps controllers, the route manager, and the capacity component's node +/// memory. pub(crate) fn owner_reads( site: Arc, models: M, @@ -42,9 +43,12 @@ where let site = Arc::clone(&site); move || site.hardware_snapshots().map(Payload::HardwareSnapshots) })), - execution_targets: Arc::new(BlockingRead::new(move || { - site.configured_execution_targets() - .map(Payload::ExecutionTargets) + execution_targets: Arc::new(BlockingRead::new({ + let site = Arc::clone(&site); + move || { + site.configured_execution_targets() + .map(Payload::ExecutionTargets) + } })), model_deployments: Arc::new(ModelDeploymentsRead(models)), application_deployments: Arc::new(ApplicationDeploymentsRead(applications)), @@ -52,6 +56,9 @@ where node_memory: Arc::new(BlockingRead::new(move || { node_memory.list().map(Payload::NodeMemory) })), + console_settings: Arc::new(BlockingRead::new(move || { + site.console_settings().map(Payload::ConsoleSettings) + })), } } diff --git a/backend/core/tests/backend/backend_integration.rs b/backend/core/tests/backend/backend_integration.rs index 7c9bf21a..e7cba202 100644 --- a/backend/core/tests/backend/backend_integration.rs +++ b/backend/core/tests/backend/backend_integration.rs @@ -23,10 +23,10 @@ use dotcore_proto::dotcore::{ application::v1::{ ApplicationDeployment, ApplicationDeploymentDeclaration, ApplicationDeploymentDesiredState, ApplicationDeploymentState, ApplicationLifecycleState, ApplicationReadinessState, - CreateApplicationDeploymentRequest, CurrentApplicationDeployment, - GetApplicationDeploymentRequest, ListApplicationDeploymentsRequest, PauseDeploymentRequest, - RemoveApplicationDeploymentRequest, ResumeDeploymentRequest, - UpdateApplicationDeploymentMetadataRequest, + ApplicationTargetOption, CreateApplicationDeploymentRequest, CurrentApplicationDeployment, + GetApplicationDeploymentRequest, ListApplicationDeploymentsRequest, + ListApplicationTargetsRequest, PauseDeploymentRequest, RemoveApplicationDeploymentRequest, + ResumeDeploymentRequest, UpdateApplicationDeploymentMetadataRequest, application_deployment_service_server::ApplicationDeploymentService as ApplicationServiceContract, }, catalog::v1::{ @@ -1097,12 +1097,12 @@ async fn application_paused_snapshot_survives_catalog_removal_and_remove_never_s fs::create_dir_all(root.join("catalog/applications")).expect("Application catalog directory"); fs::write( root.join("catalog/applications/open-webui.yaml"), - support::with_oci_service_images( + site_catalog_application(&support::with_oci_service_images( &fs::read_to_string( support::workspace_root().join("catalog/applications/open-webui.yaml"), ) .expect("local application catalog release"), - ), + )), ) .expect("local application catalog release"); let empty_master = root.join("empty-master-catalog"); @@ -2071,7 +2071,7 @@ fn configure_route_name_application(root: &Path) { let source = fs::read_to_string(support::workspace_root().join("catalog/applications/open-webui.yaml")) .expect("route-name application source"); - let source = support::with_oci_service_images(&source); + let source = site_catalog_application(&support::with_oci_service_images(&source)); let source = source .replace("applicationId: \"open-webui\"", "applicationId: \"route-test\"") .replace("displayName: \"Open WebUI\"", "displayName: \"Route Test\"") @@ -2085,6 +2085,16 @@ fn configure_route_name_application(root: &Path) { .expect("route-name application fixture"); } +/// A master-catalog application declaration as a site catalog may declare it: without the use +/// cases it serves, which only the master catalog names. +fn site_catalog_application(source: &str) -> String { + source + .lines() + .filter(|line| !line.starts_with("useCases:")) + .map(|line| format!("{line}\n")) + .collect() +} + struct ApplicationHealthServer { port: u16, task: tokio::task::JoinHandle<()>, @@ -5043,7 +5053,7 @@ async fn profile_compatibility_reads_the_simulated_site_through_the_model_servic } /// Master catalog copy whose only recommendations are the synthetic ones the recommendation read -/// test expects: coding lists the fixture profile on the fixture class, and decision making lists +/// test expects: coding lists the fixture profile on the fixture class, and the other use cases list /// nothing. fn recommendation_master_catalog(root: &Path) -> PathBuf { let master = root.join("cache/recommendation-master-catalog"); @@ -5055,7 +5065,7 @@ fn recommendation_master_catalog(root: &Path) -> PathBuf { "fixture-nvidia", r#"hardwareClassId: "fixture-nvidia" useCases: - - useCase: "USE_CASE_CODING_AND_AGENTS" + - useCase: "USE_CASE_CODING" entries: - modelId: "fixture-model" profileId: "fixture-profile" @@ -5077,10 +5087,11 @@ async fn model_recommendations_rank_the_simulated_site_through_the_model_service let mut paths = support::configuration(&root); paths.master_catalog_root = Some(recommendation_master_catalog(&root)); let service = model_service_for(&paths).await.expect("model service"); - let read = |use_cases: Vec| { + let read = |use_cases: Vec, can_start_now: bool| { ModelServiceContract::list_model_recommendations( &service, Request::new(ListModelRecommendationsRequest { + can_start_now, use_cases: use_cases .into_iter() .map(|use_case| use_case as i32) @@ -5092,7 +5103,7 @@ async fn model_recommendations_rank_the_simulated_site_through_the_model_service // The manager's own node reports its first observation asynchronously after composition. let deadline = Instant::now() + Duration::from_secs(10); let response = loop { - let response = read(vec![UseCase::CodingAndAgents]) + let response = read(vec![UseCase::Coding], false) .await .expect("recommendation read") .into_inner(); @@ -5126,39 +5137,48 @@ async fn model_recommendations_rank_the_simulated_site_through_the_model_service .starting_total_context_tokens .map(u64::from) .expect("a starting total context"); - assert_eq!( - response.use_cases, - [UseCaseModelRecommendations { - use_case: UseCase::CodingAndAgents as i32, - rows: vec![RecommendedModelProfile { - model_id: "fixture-model".to_owned(), - profile_id: "fixture-profile".to_owned(), - targets: vec![RecommendedProfileTarget { - target: Some(support::exact_placement("fixture-node", "fixture-target")), - fit: ModelRecommendationFit::FitsNow as i32, - deployed: false, - starting_total_context_tokens: Some(starting_total_context_tokens), - }], + let expected = [UseCaseModelRecommendations { + use_case: UseCase::Coding as i32, + rows: vec![RecommendedModelProfile { + model_id: "fixture-model".to_owned(), + profile_id: "fixture-profile".to_owned(), + // The entry sets none, so the row reads the profile's default. + context_tokens_per_request: 32_768, + targets: vec![RecommendedProfileTarget { + target: Some(support::exact_placement("fixture-node", "fixture-target")), + fit: ModelRecommendationFit::FitsNow as i32, + model_deployment_id: None, + starting_total_context_tokens: Some(starting_total_context_tokens), }], - }] - ); + }], + }]; + assert_eq!(response.use_cases, expected); - // No use case asks for all four, each ranked even when nothing is recommended. - let all = read(Vec::new()).await.expect("all use cases").into_inner(); + // The idle site's target fits now, so asking for what can start now keeps it. + let startable = read(vec![UseCase::Coding], true) + .await + .expect("can start now read") + .into_inner(); + assert_eq!(startable.use_cases, expected); + + // No use case asks for all three, each ranked even when nothing is recommended. + let all = read(Vec::new(), false) + .await + .expect("all use cases") + .into_inner(); assert_eq!( all.use_cases .iter() .map(|ranked| (ranked.use_case, ranked.rows.len())) .collect::>(), [ - (UseCase::CodingAndAgents as i32, 1), - (UseCase::GeneralAssistant as i32, 0), - (UseCase::Translation as i32, 0), - (UseCase::DecisionMaking as i32, 0), + (UseCase::Coding as i32, 1), + (UseCase::ChatAssistant as i32, 0), + (UseCase::ImageGeneration as i32, 0), ] ); - let unspecified = read(vec![UseCase::CodingAndAgents, UseCase::Unspecified]) + let unspecified = read(vec![UseCase::Coding, UseCase::Unspecified], false) .await .expect_err("unspecified use case"); assert_eq!(unspecified.code(), tonic::Code::InvalidArgument); @@ -5168,6 +5188,57 @@ async fn model_recommendations_rank_the_simulated_site_through_the_model_service ); } +#[tokio::test] +async fn application_targets_list_the_simulated_site_through_the_application_service() { + let root = support::temporary_fast_fixture(); + support::declare_fixture_machine(&root, &["rtx-5090"]); + let service = application_service_for(&support::configuration(&root)) + .await + .expect("application service"); + let read = |application_id: &str| { + ApplicationServiceContract::list_application_targets( + &service, + Request::new(ListApplicationTargetsRequest { + application_id: application_id.to_owned(), + }), + ) + }; + + // The manager's own node reports its first observation asynchronously after composition. + let deadline = Instant::now() + Duration::from_secs(10); + let targets = loop { + let targets = read("comfyui") + .await + .expect("application target read") + .into_inner() + .targets; + if !targets.is_empty() || Instant::now() >= deadline { + break targets; + } + tokio::time::sleep(Duration::from_millis(10)).await; + }; + + // ComfyUI prefers NVIDIA, but the simulated application target holds no GPU of the machine. + assert_eq!( + targets, + [ApplicationTargetOption { + target: Some(support::exact_placement( + "fixture-node", + "fixture-application-target" + )), + matches_preference: false, + application_deployment_id: None, + }] + ); + + let unknown = read("missing-application") + .await + .expect_err("unknown application"); + assert_eq!(unknown.code(), tonic::Code::NotFound); + let invalid = read("Invalid").await.expect_err("invalid application id"); + assert_eq!(invalid.code(), tonic::Code::InvalidArgument); +} + #[tokio::test] async fn rejected_local_configurations_return_their_reason_through_the_model_service() { let root = support::temporary_fast_fixture(); diff --git a/backend/core/tests/backend/catalog_contracts.rs b/backend/core/tests/backend/catalog_contracts.rs index 1d8e1e38..d290d3c0 100644 --- a/backend/core/tests/backend/catalog_contracts.rs +++ b/backend/core/tests/backend/catalog_contracts.rs @@ -433,7 +433,7 @@ fn repository_recommendation_entries_fit_idle_machines_of_their_class() { assert_eq!(failures, Vec::::new()); } -/// ninfer leads coding and agents on each card of the GB202 chip it is built for, and is never +/// ninfer leads coding on each card of the GB202 chip it is built for, and is never /// offered on another dedicated GPU, even one with more memory. #[test] fn repository_ninfer_leads_coding_on_gb202_cards_only() { @@ -443,7 +443,7 @@ fn repository_ninfer_leads_coding_on_gb202_cards_only() { let coding = |machine: &dotcore_node_hardware::MachineDescription| { IdleMachine::new(machine, Timing::test().node_control) .expect("idle simulated machine") - .recommend(&catalog, &[UseCase::CodingAndAgents]) + .recommend(&catalog, &[UseCase::Coding]) .into_iter() .flat_map(|recommendations| recommendations.rows) .map(|row| (row.model_id, row.profile_id)) @@ -472,9 +472,9 @@ fn repository_ninfer_leads_coding_on_gb202_cards_only() { assert_eq!( coding(&large_4090), [ - ("qwen3-8-27b".to_owned(), "q4-xl".to_owned()), - ("qwen3-5-9b".to_owned(), "q4-xl".to_owned()), - ("qwen3-5-4b".to_owned(), "q4-xl".to_owned()), + ("qwen3-8-27b".to_owned(), "q3-xl-q8-kv".to_owned()), + ("gemma-4-12b-it-qat".to_owned(), "gpu".to_owned()), + ("qwen3-5-4b".to_owned(), "q3-xl".to_owned()), ] ); } @@ -487,7 +487,7 @@ fn recommendation_entry_checks_fail_on_a_minimum_set_too_low() { format!( r#"hardwareClassId: "nvidia-dedicated" useCases: - - useCase: "USE_CASE_CODING_AND_AGENTS" + - useCase: "USE_CASE_CODING" entries: - {{ modelId: "big", profileId: "q4", hardwareRequirements: {{ minimumMemoryBytesPerDevice: "{minimum}" }} }} - {{ modelId: "small", profileId: "q4" }} @@ -520,7 +520,7 @@ useCases: assert_eq!( recommendation_entry_failures(&catalog_with("8000000000")), [ - "recommendations/nvidia-dedicated.yaml, USE_CASE_CODING_AND_AGENTS, big/q4: \ + "recommendations/nvidia-dedicated.yaml, USE_CASE_CODING, big/q4: \ not recommended on idle simulated hardware rtx-4080, rtx-4090, rtx-5090, rtx-pro-6000 \ resized to its minimum of 8000000000 bytes" ] diff --git a/backend/core/tests/backend/route_exposure_integrity.rs b/backend/core/tests/backend/route_exposure_integrity.rs index 63e4216d..666f95e8 100644 --- a/backend/core/tests/backend/route_exposure_integrity.rs +++ b/backend/core/tests/backend/route_exposure_integrity.rs @@ -2,7 +2,7 @@ use std::fs; -use dotcore::composition::{SelectedModelService, model_service_for}; +use dotcore::composition::SelectedModelService; use dotcore_proto::dotcore::{ inference::v1::{ CreateRouteRequest, FallbackGroup, FallbackGroupSelection, InferenceRoute, @@ -25,7 +25,7 @@ use crate::support; #[tokio::test] async fn direct_load_persists_one_deployment_and_its_direct_route() { let root = support::temporary_fast_fixture(); - let service = model_service_for(&support::configuration(&root)) + let service = support::model_service_for(&support::configuration(&root)) .await .expect("Direct model service"); let deployment = ModelServiceContract::create_model_deployment( @@ -119,7 +119,7 @@ async fn direct_load_and_unload_return_with_the_route_snapshot_already_current() #[tokio::test] async fn hidden_load_persists_one_deployment_without_a_route() { let root = support::temporary_fast_fixture(); - let service = model_service_for(&support::configuration(&root)) + let service = support::model_service_for(&support::configuration(&root)) .await .expect("Hidden model service"); let deployment = ModelServiceContract::create_model_deployment( @@ -148,7 +148,7 @@ async fn hidden_load_persists_one_deployment_without_a_route() { #[tokio::test] async fn local_deployments_are_independent_and_a_conflicting_route_is_atomic() { let root = support::temporary_fast_fixture(); - let service = model_service_for(&support::configuration(&root)) + let service = support::model_service_for(&support::configuration(&root)) .await .expect("Model service"); let first = create_local_deployment(&service, "public-model") @@ -225,7 +225,7 @@ async fn unload_deletes_sole_member_routes_and_keeps_unrelated_routes() { unrelated.clone(), ]; write_site(&root, &site); - let service = model_service_for(&configuration) + let service = support::model_service_for(&configuration) .await .expect("Model service"); @@ -290,7 +290,7 @@ async fn shared_route_blocks_unload_by_name_without_mutation() { }, ]; write_site(&root, &site); - let service = model_service_for(&configuration) + let service = support::model_service_for(&configuration) .await .expect("Model service"); @@ -535,7 +535,7 @@ async fn only_route( } async fn load_hidden_then_drop(root: &std::path::Path) -> String { - let service = model_service_for(&support::configuration(root)) + let service = support::model_service_for(&support::configuration(root)) .await .expect("Model service"); let deployment_id = ModelServiceContract::create_model_deployment( diff --git a/backend/core/tests/transport/server.rs b/backend/core/tests/transport/server.rs index fda9cb2e..4a06760a 100644 --- a/backend/core/tests/transport/server.rs +++ b/backend/core/tests/transport/server.rs @@ -57,17 +57,18 @@ use dotcore_proto::{ RuntimeObservation, accelerator_observation, host_observation, runtime_observation, }, site::v1::{ - CurrentHardwareSnapshotAvailability, HardwareRefreshState, - ListHardwareSnapshotsResponse, ListMachinesResponse, Machine, MachineHardwareSnapshot, - NodeObservationFreshness, NodeObservationState, Site, SiteNode, - current_hardware_snapshot_availability, hardware_refresh_state, + ConsoleSettings, CurrentHardwareSnapshotAvailability, GetConsoleSettingsResponse, + HardwareRefreshState, ListHardwareSnapshotsResponse, ListMachinesResponse, Machine, + MachineHardwareSnapshot, NodeObservationFreshness, NodeObservationState, Site, + SiteNode, UpdateConsoleSettingsRequest, current_hardware_snapshot_availability, + hardware_refresh_state, site_service_client::SiteServiceClient, }, watch::v1::{ KeyedResource, ResourceSubscription, WatchResourcesRequest, WatchResourcesResponse, keyed_resource::Payload, resource_subscription, watch_resources_response, }, }, - google::protobuf::{Empty, Timestamp}, + google::protobuf::{Empty, FieldMask, Timestamp}, }; use dotcore_runtime_backends::simulated::{ SimulatedRuntimeCallCounts, SimulatedRuntimeTestSupport, @@ -1437,6 +1438,10 @@ async fn manager_backend_server_streams_resource_watches() { "routes", resource_subscription::Kind::Routes(Default::default()), ), + subscription( + "console", + resource_subscription::Kind::ConsoleSettings(Default::default()), + ), ], }; let mut watch = GrpcWebStream::open( @@ -1465,7 +1470,7 @@ async fn manager_backend_server_streams_resource_watches() { // The first round is read on open, before any tick. assert_eq!(first_tick, 0); first_round.sort_by(|left, right| left.key.cmp(&right.key)); - let [models_round, routes_round] = first_round.as_slice() else { + let [console_round, models_round, routes_round] = first_round.as_slice() else { panic!("one payload per key in the first round: {first_round:?}"); }; assert_eq!(models_round.key, "models"); @@ -1485,6 +1490,15 @@ async fn manager_backend_server_streams_resource_watches() { payload: Some(Payload::Routes(Default::default())), } ); + assert_eq!( + console_round, + &KeyedResource { + key: "console".to_owned(), + payload: Some(Payload::ConsoleSettings(GetConsoleSettingsResponse { + settings: Some(ConsoleSettings::default()), + })), + } + ); // A pause reaches the watch as an update of the changed kind alone. models @@ -1514,6 +1528,46 @@ async fn manager_backend_server_streams_resource_watches() { "update tick {update} follows {first_tick}" ); + // A console settings update reaches the watch as the accepted settings. + let hidden = ConsoleSettings { + get_started_hidden: true, + }; + let accepted = SiteServiceClient::connect(format!("http://{address}")) + .await + .expect("site API connection") + .update_console_settings(UpdateConsoleSettingsRequest { + settings: Some(hidden), + update_mask: Some(FieldMask { + paths: vec!["get_started_hidden".to_owned()], + }), + }) + .await + .expect("update console settings") + .into_inner(); + assert_eq!(accepted.settings, Some(hidden)); + let console_update = loop { + let response = next_resource_event(&mut watch, &clock).await; + let Some(watch_resources_response::Event::Resource(resource)) = response.event else { + panic!("a console settings update follows: {response:?}"); + }; + assert_ne!(resource.key, "routes", "routes are unchanged"); + if resource.key == "console" { + assert_eq!( + resource.payload, + Some(Payload::ConsoleSettings(GetConsoleSettingsResponse { + settings: Some(hidden), + })) + ); + break response.tick_sequence; + } + }; + // A tick reads every watched kind at once, so the console read of the pause's tick may + // already see the update and publish it under that same tick. + assert!( + console_update >= update, + "console update tick {console_update} is not before {update}" + ); + // An idle stream heartbeats once fifteen seconds pass without an event. clock.advance(Duration::from_secs(15)); let heartbeat = loop { @@ -1523,7 +1577,7 @@ async fn manager_backend_server_streams_resource_watches() { } }; assert_eq!(heartbeat.manager_session_id, session_id); - assert!(heartbeat.tick_sequence >= update); + assert!(heartbeat.tick_sequence >= console_update); drop(watch); shutdown.send(()).expect("server shutdown"); diff --git a/backend/fixtures/recommendation-catalog/applications/fixture-art.yaml b/backend/fixtures/recommendation-catalog/applications/fixture-art.yaml new file mode 100644 index 00000000..89314987 --- /dev/null +++ b/backend/fixtures/recommendation-catalog/applications/fixture-art.yaml @@ -0,0 +1,26 @@ +applicationId: "fixture-art" +displayName: "Fixture Art" +description: "Image generation fixture that prefers NVIDIA." +version: "1.0.0" +useCases: ["USE_CASE_IMAGE_GENERATION"] +preferredAccelerators: ["nvidia"] +graph: + services: + - serviceName: "web" + artifact: + nix: + flake: "self" + package: "apps.fixture-art.docker" + dockerImage: {} + supportedPlatforms: + - operatingSystem: "linux" + architecture: "amd64" + - operatingSystem: "linux" + architecture: "arm64" + ports: + - portName: "http" + portNumber: 8080 + primaryEndpoint: + serviceName: "web" + portName: "http" + protocol: "APPLICATION_ENDPOINT_PROTOCOL_HTTP" diff --git a/backend/fixtures/recommendation-catalog/applications/fixture-chat.yaml b/backend/fixtures/recommendation-catalog/applications/fixture-chat.yaml new file mode 100644 index 00000000..c911de24 --- /dev/null +++ b/backend/fixtures/recommendation-catalog/applications/fixture-chat.yaml @@ -0,0 +1,26 @@ +applicationId: "fixture-chat" +displayName: "Fixture Chat" +description: "Chat fixture that prefers Apple." +version: "1.0.0" +useCases: ["USE_CASE_CHAT_ASSISTANT"] +preferredAccelerators: ["apple"] +graph: + services: + - serviceName: "web" + artifact: + nix: + flake: "self" + package: "apps.fixture-chat.docker" + dockerImage: {} + supportedPlatforms: + - operatingSystem: "linux" + architecture: "amd64" + - operatingSystem: "linux" + architecture: "arm64" + ports: + - portName: "http" + portNumber: 8080 + primaryEndpoint: + serviceName: "web" + portName: "http" + protocol: "APPLICATION_ENDPOINT_PROTOCOL_HTTP" diff --git a/backend/fixtures/recommendation-catalog/recommendations/nvidia-dedicated.yaml b/backend/fixtures/recommendation-catalog/recommendations/nvidia-dedicated.yaml index 480a4898..6688d5a6 100644 --- a/backend/fixtures/recommendation-catalog/recommendations/nvidia-dedicated.yaml +++ b/backend/fixtures/recommendation-catalog/recommendations/nvidia-dedicated.yaml @@ -1,6 +1,9 @@ hardwareClassId: "nvidia-dedicated" useCases: - - useCase: "USE_CASE_CODING_AND_AGENTS" + - useCase: "USE_CASE_CODING" entries: - { modelId: "big", profileId: "q4", hardwareRequirements: { minimumMemoryBytesPerDevice: "24000000000" } } - { modelId: "small", profileId: "q4" } + - useCase: "USE_CASE_CHAT_ASSISTANT" + entries: + - { modelId: "small", profileId: "q4" } diff --git a/backend/proto/tests/application_contract.rs b/backend/proto/tests/application_contract.rs index 8a60aeeb..9f822077 100644 --- a/backend/proto/tests/application_contract.rs +++ b/backend/proto/tests/application_contract.rs @@ -402,6 +402,11 @@ fn application_descriptors_define_portable_graph_and_owner_separated_records() { "UpdateApplicationDeploymentMetadataRequest", "UpdateApplicationDeploymentMetadataResponse", ), + ( + "ListApplicationTargets", + "ListApplicationTargetsRequest", + "ListApplicationTargetsResponse", + ), ], ); // The application contracts share types through owner-neutral packages, never the model @@ -432,6 +437,7 @@ fn application_descriptors_define_portable_graph_and_owner_separated_records() { "inference_routes", "hardware_observation", "model_deployment_identity_allocator", + "console_settings", ] ); let deployment = descriptor_file(&descriptor, "dotcore/application/v1/deployment.proto"); diff --git a/backend/proto/tests/model_contract.rs b/backend/proto/tests/model_contract.rs index e7d72a5c..84908345 100644 --- a/backend/proto/tests/model_contract.rs +++ b/backend/proto/tests/model_contract.rs @@ -1385,14 +1385,26 @@ fn model_descriptors_define_typed_configuration_and_resolved_plan_contracts() { ); assert_eq!( field_layout(service, "ListModelRecommendationsRequest"), - [("use_cases", 1, ".dotcore.catalog.v1.UseCase")] + [ + ("use_cases", 1, ".dotcore.catalog.v1.UseCase"), + ("can_start_now", 2, "TYPE_BOOL"), + ] + ); + assert_eq!( + field_layout(service, "RecommendedModelProfile"), + [ + ("model_id", 1, "TYPE_STRING"), + ("profile_id", 2, "TYPE_STRING"), + ("context_tokens_per_request", 4, "TYPE_UINT64"), + ("targets", 3, ".dotcore.model.v1.RecommendedProfileTarget"), + ] ); assert_eq!( field_layout(service, "RecommendedProfileTarget"), [ ("target", 1, ".dotcore.deployment.v1.ExactPlacement"), ("fit", 2, ".dotcore.model.v1.ModelRecommendationFit"), - ("deployed", 3, "TYPE_BOOL"), + ("model_deployment_id", 3, "TYPE_STRING"), ("starting_total_context_tokens", 4, "TYPE_UINT64"), ] ); diff --git a/backend/proto/tests/site_contract.rs b/backend/proto/tests/site_contract.rs index 5671f938..23b6a257 100644 --- a/backend/proto/tests/site_contract.rs +++ b/backend/proto/tests/site_contract.rs @@ -560,6 +560,8 @@ fn complete_application() -> Application { display_name: "Application One".to_owned(), description: String::new(), notes: "First line.\nSecond line.".to_owned(), + use_cases: Vec::new(), + preferred_accelerators: Vec::new(), version: "1.0.0".to_owned(), graph: Some(ApplicationGraph { services: vec![ @@ -1202,6 +1204,16 @@ fn generated_descriptors_preserve_site_inventory_contracts() { ".dotcore.site.v1.ListHardwareSnapshotsRequest", ".dotcore.site.v1.ListHardwareSnapshotsResponse", ), + ( + "GetConsoleSettings", + ".dotcore.site.v1.GetConsoleSettingsRequest", + ".dotcore.site.v1.GetConsoleSettingsResponse", + ), + ( + "UpdateConsoleSettings", + ".dotcore.site.v1.UpdateConsoleSettingsRequest", + ".dotcore.site.v1.UpdateConsoleSettingsResponse", + ), ] )); } diff --git a/backend/site/src/console_settings.rs b/backend/site/src/console_settings.rs new file mode 100644 index 00000000..31a0cce1 --- /dev/null +++ b/backend/site/src/console_settings.rs @@ -0,0 +1,107 @@ +//! Site-wide console settings: operator choices about the console kept with the desired site. +//! +//! The site manager keeps one set of console settings for the whole site, so every browser that +//! opens the console sees the same choices. Updates name the fields they write and the last writer +//! wins. The settings drive no controller and never reach a node. + +use std::collections::BTreeSet; + +use dotcore_proto::{dotcore::site::v1::ConsoleSettings, google::protobuf::FieldMask}; + +use crate::repository::{ScopedSiteMutationError, SiteRepository, SiteRepositoryError}; + +const GET_STARTED_HIDDEN_PATH: &str = "get_started_hidden"; + +/// Reads and writes the site-wide console settings. +pub trait ConsoleSettingsStore: Send + Sync { + /// Reads the current settings, with defaults for values never written. + fn console_settings(&self) -> Result; + + /// Applies one patch and returns the complete settings it produced. + fn update_console_settings( + &self, + patch: &ConsoleSettingsPatch, + ) -> Result; +} + +impl ConsoleSettingsStore for SiteRepository { + fn console_settings(&self) -> Result { + Ok(self.snapshot()?.console_settings.unwrap_or_default()) + } + + fn update_console_settings( + &self, + patch: &ConsoleSettingsPatch, + ) -> Result { + self.mutate_console_settings(|settings| { + patch.apply(settings); + Ok::<_, std::convert::Infallible>(*settings) + }) + .map_err(|error| match error { + ScopedSiteMutationError::Repository(error) => error, + }) + } +} + +/// A validated update of the console settings fields an update mask selects. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConsoleSettingsPatch { + get_started_hidden: Option, +} + +impl ConsoleSettingsPatch { + /// Parses the masked fields of `settings`; every mask path must name a writable field once. + pub fn parse( + settings: Option<&ConsoleSettings>, + update_mask: Option<&FieldMask>, + ) -> Result { + let settings = settings.ok_or(ConsoleSettingsPatchError::MissingSettings)?; + let update_mask = update_mask.ok_or(ConsoleSettingsPatchError::MissingUpdateMask)?; + if update_mask.paths.is_empty() { + return Err(ConsoleSettingsPatchError::EmptyUpdateMask); + } + + let mut paths = BTreeSet::new(); + let mut patch = Self { + get_started_hidden: None, + }; + for path in &update_mask.paths { + if !paths.insert(path.as_str()) { + return Err(ConsoleSettingsPatchError::DuplicatePath(path.clone())); + } + match path.as_str() { + GET_STARTED_HIDDEN_PATH => { + patch.get_started_hidden = Some(settings.get_started_hidden) + } + _ => return Err(ConsoleSettingsPatchError::UnwritablePath(path.clone())), + } + } + Ok(patch) + } + + /// Writes the masked fields into `settings`, leaving the others unchanged. + pub fn apply(&self, settings: &mut ConsoleSettings) { + if let Some(hidden) = self.get_started_hidden { + settings.get_started_hidden = hidden; + } + } +} + +/// Rejects a console settings update before it changes the site. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum ConsoleSettingsPatchError { + #[error("console settings are required")] + MissingSettings, + + #[error("console settings update_mask is required")] + MissingUpdateMask, + + #[error("console settings update_mask must not be empty")] + EmptyUpdateMask, + + #[error("console settings update_mask repeats path {0:?}")] + DuplicatePath(String), + + #[error("console settings update_mask path {0:?} is not writable")] + UnwritablePath(String), +} diff --git a/backend/site/src/initial/inputs.rs b/backend/site/src/initial/inputs.rs index f89dd048..9abc9b66 100644 --- a/backend/site/src/initial/inputs.rs +++ b/backend/site/src/initial/inputs.rs @@ -99,6 +99,7 @@ impl InitialSiteInputs { application_deployments: Vec::new(), inference_routes: Vec::new(), hardware_observation: None, + console_settings: None, model_deployment_identity_allocator: Some(ModelDeploymentIdentityAllocator { namespace: crate::store::model_deployment_identity_namespace(identities.site_id()), next_sequence: 1, diff --git a/backend/site/src/lib.rs b/backend/site/src/lib.rs index 5ded23c4..1b9fb137 100644 --- a/backend/site/src/lib.rs +++ b/backend/site/src/lib.rs @@ -11,6 +11,7 @@ compile_error!("Dotcore supports only Linux and macOS targets"); pub mod atomic; +pub mod console_settings; pub mod initial; pub mod manager_trust; pub mod membership; diff --git a/backend/site/src/repository/mod.rs b/backend/site/src/repository/mod.rs index afc07289..3cf76e51 100644 --- a/backend/site/src/repository/mod.rs +++ b/backend/site/src/repository/mod.rs @@ -17,7 +17,7 @@ use dotcore_proto::{ ModelDeploymentDeclaration, ModelDeploymentDesiredState, ModelDeploymentLifecycleTransition, }, - site::v1::Site, + site::v1::{ConsoleSettings, Site}, }, yaml, }; @@ -178,7 +178,7 @@ impl SiteRepository { &self, operation: impl FnOnce(&mut ModelSiteTransaction<'_>) -> Result, ) -> Result> { - self.mutate_deployment_metadata(|candidate| { + self.mutate_without_reconciliation(|candidate| { let mut transaction = ModelSiteTransaction { deployments: &mut candidate.model_deployments, }; @@ -194,11 +194,28 @@ impl SiteRepository { &self, operation: impl FnOnce(&mut Vec) -> Result, ) -> Result> { - self.mutate_deployment_metadata(|candidate| { + self.mutate_without_reconciliation(|candidate| { operation(&mut candidate.application_deployments) }) } + /// Persists a site-wide console settings mutation without triggering reconciliation. + /// + /// Console settings are operator choices about the console only: no controller reads them + /// and they never enter a node's desired view, so their update wakes nothing. + pub fn mutate_console_settings( + &self, + operation: impl FnOnce(&mut ConsoleSettings) -> Result, + ) -> Result> { + self.mutate_without_reconciliation(|candidate| { + operation( + candidate + .console_settings + .get_or_insert_with(Default::default), + ) + }) + } + /// Accepts one owner-prepared deployment and its optional Direct-exposure route as one /// authoritative site update. pub fn accept_model_deployment_exposure( @@ -336,7 +353,9 @@ impl SiteRepository { }) } - fn mutate_deployment_metadata( + /// Accepts and writes a mutation of sections no controller or node view projects, without + /// running change observers or waking listeners. + fn mutate_without_reconciliation( &self, operation: impl FnOnce(&mut Site) -> Result, ) -> Result> { diff --git a/backend/site/src/repository/tests.rs b/backend/site/src/repository/tests.rs index 31550e3f..66ae9dab 100644 --- a/backend/site/src/repository/tests.rs +++ b/backend/site/src/repository/tests.rs @@ -23,18 +23,21 @@ use dotcore_proto::dotcore::{ model_deployment_snapshot, }, site::v1::{ - ApplicationRuntimeBinding, ExecutionTarget, ModelRuntimeBinding, + ApplicationRuntimeBinding, ConsoleSettings, ExecutionTarget, ModelRuntimeBinding, SimulatedApplicationRuntimeAdapter, SimulatedRuntimeAdapter, SimulatedRuntimeScenario, Site, SiteManagerRole, SiteNode, application_runtime_binding, execution_target, model_runtime_binding, }, }; use dotcore_proto::yaml; +use futures_util::FutureExt; use super::{SiteFileWriter, SiteRepository, SiteRepositoryError, SiteWriteFault}; use crate::{ + console_settings::ConsoleSettingsStore, persistence::SiteStorageError, view::contract::{MAX_APPLICATION_ASSIGNMENTS, MAX_MODEL_ASSIGNMENTS, MAX_NODE_VIEW_BYTES}, + view::fencing::NodeViewProjection, }; #[tokio::test] @@ -775,6 +778,64 @@ async fn metadata_mutations_are_owner_scoped_and_do_not_notify_reconciliation() fs::remove_dir_all(directory).expect("fixture cleanup"); } +#[test] +fn console_settings_persist_alone_without_waking_controllers_or_node_views() { + let directory = test_directory(); + let path = directory.join("site.yaml"); + let mut original = node_site(); + original.model_deployments = vec![ + model_deployment("deployment-one", "model-one"), + node_deployment("deployment-two", 0), + ]; + fs::write( + &path, + yaml::to_string(&original).expect("desired site YAML"), + ) + .expect("desired site fixture"); + let site = SiteRepository::load(&path, crate::test_support::site_declaration_rules()) + .expect("desired site"); + let listener = site.register_change_listener(); + let observed = counted_notifications(&site); + let views_before = node_views(&original); + assert_eq!( + ConsoleSettingsStore::console_settings(&site).expect("default console settings"), + ConsoleSettings::default() + ); + + let accepted = site + .mutate_console_settings(|settings| { + settings.get_started_hidden = true; + Ok::<_, std::convert::Infallible>(*settings) + }) + .expect("console settings mutation"); + + assert_eq!( + accepted, + ConsoleSettings { + get_started_hidden: true, + } + ); + let mut expected = original.clone(); + expected.console_settings = Some(ConsoleSettings { + get_started_hidden: true, + }); + assert_eq!(site.snapshot().expect("site after mutation"), expected); + let reloaded = SiteRepository::load(&path, crate::test_support::site_declaration_rules()) + .expect("reloaded desired site"); + assert_eq!(reloaded.snapshot().expect("reloaded site"), expected); + assert_eq!(node_views(&expected), views_before); + assert_eq!( + observed.load(Ordering::SeqCst), + 0, + "no change observer runs" + ); + assert!( + listener.notified().now_or_never().is_none(), + "console settings must not wake a controller" + ); + fs::remove_dir_all(directory).expect("fixture cleanup"); +} + #[test] fn failed_and_uncertain_writes_including_metadata_leave_the_site_dirty() { let directory = test_directory(); @@ -1434,6 +1495,14 @@ fn counted_notifications(site: &SiteRepository) -> Arc { notifications } +/// Every node's projected desired view of `site`, in node order. +fn node_views(site: &Site) -> Vec { + site.nodes + .iter() + .map(|node| NodeViewProjection::for_site_node(site, &node.node_id).expect("node view")) + .collect() +} + fn simulated_runtime() -> SimulatedRuntimeAdapter { SimulatedRuntimeAdapter { default_scenario: Some(SimulatedRuntimeScenario::default()), diff --git a/backend/site/src/service.rs b/backend/site/src/service.rs index 9ae05359..6841fa50 100644 --- a/backend/site/src/service.rs +++ b/backend/site/src/service.rs @@ -1,7 +1,9 @@ -//! Serves the public read-only gRPC API for the site: its machines, execution targets, and current hardware. +//! Serves the public gRPC API for the site: its machines, execution targets, current hardware and +//! site-wide console settings. //! //! It answers inventory and hardware reads from current site configuration and the latest -//! site-manager-owned node observation. Requests perform no collection or mutation. +//! site-manager-owned node observation, without collecting anything. The console settings are +//! the only site configuration it writes. use std::sync::Arc; @@ -15,11 +17,13 @@ use dotcore_proto::{ site::private::v1::node_hardware_observation, site::v1::{ ConfiguredExecutionTarget, CurrentHardwareSnapshotAvailability, - ExecutionTargetAvailability, ExecutionTargetWorkloadKind, HardwareRefreshState, + ExecutionTargetAvailability, ExecutionTargetWorkloadKind, GetConsoleSettingsRequest, + GetConsoleSettingsResponse, HardwareRefreshState, ListConfiguredExecutionTargetsRequest, ListConfiguredExecutionTargetsResponse, ListHardwareSnapshotsRequest, ListHardwareSnapshotsResponse, ListMachinesRequest, ListMachinesResponse, Machine, MachineHardwareSnapshot, NodeObservationFreshness, NodeObservationState, RuntimeTargetAvailability, Site, SiteNode, + UpdateConsoleSettingsRequest, UpdateConsoleSettingsResponse, current_hardware_snapshot_availability, execution_target, hardware_refresh_state, runtime_target_availability, site_service_server::SiteService as SiteServiceContract, }, @@ -27,8 +31,9 @@ use dotcore_proto::{ google::protobuf::Empty, }; use tonic::{Request, Response, Status}; -use tracing::{Instrument, debug_span, error, field}; +use tracing::{Instrument, debug_span, error, field, info}; +use crate::console_settings::{ConsoleSettingsPatch, ConsoleSettingsStore}; use crate::node::registry::{NodeConnectionState, NodeObservationView, NodeRegistry}; use crate::repository::{SiteRepository, SiteRepositoryError}; @@ -59,10 +64,13 @@ impl SiteConfigurationSource for Site { } } -/// Public site inventory service over the observations every SiteNode reports to the manager. +/// Public site service: inventory over the observations every SiteNode reports to the manager, +/// and the site-wide console settings. pub struct SiteService { site: Arc, + console_settings: Arc, + /// SiteNode hosting the manager process, projected with the site-manager marker. manager_node_id: SiteNodeId, @@ -70,19 +78,29 @@ pub struct SiteService { } impl SiteService { - /// Creates a cache-only public service over the manager's registry of node observations. + /// Creates the public service over the manager's registry of node observations; inventory + /// reads answer from caches only. pub fn new( site: Arc, + console_settings: Arc, manager_node_id: SiteNodeId, nodes: NodeRegistry, ) -> Self { Self { site, + console_settings, manager_node_id, nodes, } } + /// Answers the site-wide console settings, with defaults for values never written. + pub fn console_settings(&self) -> Result { + Ok(GetConsoleSettingsResponse { + settings: Some(self.console_settings.console_settings()?), + }) + } + /// Answers every configured machine, in machine-identifier order. pub fn machines(&self) -> Result { let site = self.site.snapshot()?; @@ -277,6 +295,18 @@ fn site_status(error: SiteRepositoryError) -> Status { Status::internal("Site configuration snapshot failed") } +/// Maps a failed console settings write. A storage failure leaves the accepted settings in memory, +/// so the caller learns the outcome is uncertain rather than failed. +fn console_settings_update_status(error: SiteRepositoryError) -> Status { + error!(error = %error, "Console settings update failed"); + match error { + SiteRepositoryError::Storage(_) => Status::unavailable( + "Console settings were accepted but saving them is uncertain. Retry or read them back.", + ), + _ => Status::internal("Console settings update failed"), + } +} + /// Configured nodes in node-identifier order. fn sorted_nodes(site: &Site) -> Vec<&SiteNode> { let mut nodes = site.nodes.iter().collect::>(); @@ -476,6 +506,36 @@ impl SiteServiceContract for SiteService { span.record("outcome", if result.is_ok() { "success" } else { "failed" }); result } + + async fn get_console_settings( + &self, + _request: Request, + ) -> Result, Status> { + self.console_settings() + .map(Response::new) + .map_err(site_status) + } + + async fn update_console_settings( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let patch = + ConsoleSettingsPatch::parse(request.settings.as_ref(), request.update_mask.as_ref()) + .map_err(|error| Status::invalid_argument(error.to_string()))?; + let settings = self + .console_settings + .update_console_settings(&patch) + .map_err(console_settings_update_status)?; + info!( + get_started_hidden = settings.get_started_hidden, + "Console settings updated" + ); + Ok(Response::new(UpdateConsoleSettingsResponse { + settings: Some(settings), + })) + } } #[cfg(test)] @@ -497,8 +557,8 @@ mod tests { NodeObservationAcceptance, NodeObservationSnapshot, node_hardware_observation, }, site::v1::{ - AcceleratorResolutionStatus, ApplicationRuntimeBinding, ExecutionTarget, - HardwareObservationConfiguration, HardwareRefreshState, + AcceleratorResolutionStatus, ApplicationRuntimeBinding, ConsoleSettings, + ExecutionTarget, HardwareObservationConfiguration, HardwareRefreshState, ListConfiguredExecutionTargetsRequest, ListHardwareSnapshotsRequest, ListMachinesRequest, ModelRuntimeBinding, SimulatedApplicationRuntimeAdapter, SimulatedRuntimeAdapter, Site, SiteManagerRole, SiteNode, @@ -506,7 +566,7 @@ mod tests { model_runtime_binding, }, }, - google::protobuf::{Empty, Timestamp}, + google::protobuf::{Empty, FieldMask, Timestamp}, }; use tokio::time::{Duration, advance}; @@ -529,6 +589,114 @@ mod tests { } } + /// Console settings held in memory, as the site repository keeps them. + #[derive(Default)] + struct MemoryConsoleSettings(Mutex); + + impl ConsoleSettingsStore for MemoryConsoleSettings { + fn console_settings(&self) -> Result { + Ok(*self.0.lock().expect("console settings lock")) + } + + fn update_console_settings( + &self, + patch: &ConsoleSettingsPatch, + ) -> Result { + let mut settings = self.0.lock().expect("console settings lock"); + patch.apply(&mut settings); + Ok(*settings) + } + } + + #[tokio::test] + async fn console_settings_read_their_defaults_and_apply_a_masked_update() { + let store = Arc::new(MemoryConsoleSettings::default()); + let (service, _) = service_with_console_settings(store.clone()); + + let initial = SiteServiceContract::get_console_settings( + &service, + Request::new(GetConsoleSettingsRequest {}), + ) + .await + .expect("console settings read") + .into_inner(); + assert_eq!(initial.settings, Some(ConsoleSettings::default())); + + let accepted = SiteServiceContract::update_console_settings( + &service, + Request::new(UpdateConsoleSettingsRequest { + settings: Some(ConsoleSettings { + get_started_hidden: true, + }), + update_mask: Some(FieldMask { + paths: vec!["get_started_hidden".to_owned()], + }), + }), + ) + .await + .expect("console settings update") + .into_inner(); + let hidden = ConsoleSettings { + get_started_hidden: true, + }; + assert_eq!(accepted.settings, Some(hidden)); + let read = SiteServiceContract::get_console_settings( + &service, + Request::new(GetConsoleSettingsRequest {}), + ) + .await + .expect("console settings read after update") + .into_inner(); + assert_eq!(read.settings, Some(hidden)); + } + + #[tokio::test] + async fn console_settings_updates_without_a_valid_mask_are_rejected_unchanged() { + let hidden = ConsoleSettings { + get_started_hidden: true, + }; + let mask = |paths: &[&str]| { + Some(FieldMask { + paths: paths.iter().map(|path| (*path).to_owned()).collect(), + }) + }; + let requests = [ + (None, mask(&["get_started_hidden"])), + (Some(ConsoleSettings::default()), None), + (Some(ConsoleSettings::default()), mask(&[])), + ( + Some(ConsoleSettings::default()), + mask(&["get_started_hidden", "get_started_hidden"]), + ), + (Some(ConsoleSettings::default()), mask(&["unknown_setting"])), + ]; + for (settings, update_mask) in requests { + let store = Arc::new(MemoryConsoleSettings(Mutex::new(hidden))); + let (service, _) = service_with_console_settings(store.clone()); + + let error = SiteServiceContract::update_console_settings( + &service, + Request::new(UpdateConsoleSettingsRequest { + settings, + update_mask: update_mask.clone(), + }), + ) + .await + .expect_err("invalid console settings update"); + + assert_eq!( + error.code(), + tonic::Code::InvalidArgument, + "{update_mask:?}" + ); + assert_eq!( + store.console_settings().expect("console settings"), + hidden, + "{update_mask:?}" + ); + } + } + #[tokio::test] async fn list_machines_is_sorted_and_availability_is_node_keyed() { let mut site = node_site(); @@ -568,7 +736,12 @@ mod tests { let site = Arc::new(MutableSite(Mutex::new(manager_site()))); let source: Arc = site.clone(); let registry = NodeRegistry::new(Arc::clone(&source)); - let service = SiteService::new(source, SiteNodeId::for_test("node-a"), registry.clone()); + let service = SiteService::new( + source, + console_settings(), + SiteNodeId::for_test("node-a"), + registry.clone(), + ); let reported = manager_site(); registry .accept(manager_report( @@ -763,6 +936,7 @@ mod tests { let registry = NodeRegistry::with_coordinator(Arc::new(site.clone()), coordinator.clone()); let service = SiteService::new( Arc::new(site.clone()), + console_settings(), SiteNodeId::for_test("node-a"), registry.clone(), ); @@ -926,6 +1100,7 @@ mod tests { let registry = NodeRegistry::with_coordinator(Arc::new(site.clone()), coordinator.clone()); let service = SiteService::new( Arc::new(site.clone()), + console_settings(), SiteNodeId::for_test("manager-node"), registry.clone(), ); @@ -1243,12 +1418,33 @@ mod tests { service_for(site, "node-a") } + fn service_with_console_settings( + store: Arc, + ) -> (SiteService, NodeRegistry) { + let site: Arc = Arc::new(node_site()); + let registry = NodeRegistry::new(Arc::clone(&site)); + ( + SiteService::new( + site, + store, + SiteNodeId::for_test("manager-node"), + registry.clone(), + ), + registry, + ) + } + + fn console_settings() -> Arc { + Arc::new(MemoryConsoleSettings::default()) + } + fn service_for(site: Site, manager_node_id: &str) -> (SiteService, NodeRegistry) { let site: Arc = Arc::new(site); let registry = NodeRegistry::new(Arc::clone(&site)); ( SiteService::new( site, + console_settings(), SiteNodeId::for_test(manager_node_id), registry.clone(), ), diff --git a/catalog/applications/comfyui.yaml b/catalog/applications/comfyui.yaml index be1cf7ec..4ae19778 100644 --- a/catalog/applications/comfyui.yaml +++ b/catalog/applications/comfyui.yaml @@ -10,6 +10,8 @@ notes: |- The app has no login. Anyone who can reach its published port can run workflows that load any custom node of the deployment, which run as root with the container's network and GPU access. The only access control is the address the node publishes applications on, plus the network in front of it, so publish ComfyUI on a trusted network only. version: "0.42.5" +useCases: ["USE_CASE_IMAGE_GENERATION"] +preferredAccelerators: ["nvidia"] graph: parameters: - advanced: true diff --git a/catalog/applications/open-webui.yaml b/catalog/applications/open-webui.yaml index fc316abc..ce1ad575 100644 --- a/catalog/applications/open-webui.yaml +++ b/catalog/applications/open-webui.yaml @@ -4,6 +4,7 @@ applicationId: "open-webui" displayName: "Open WebUI" description: "Self-hosted chat interface for the models on this site." version: "0.11.4" +useCases: ["USE_CASE_CHAT_ASSISTANT"] graph: services: - serviceName: "web" diff --git a/catalog/models/qwen3-8-27b.yaml b/catalog/models/qwen3-8-27b.yaml index 45bc446f..f599d738 100644 --- a/catalog/models/qwen3-8-27b.yaml +++ b/catalog/models/qwen3-8-27b.yaml @@ -526,6 +526,20 @@ profiles: values: - parameterId: "prompt-cache-memory-mib" unsignedInteger: "2048" + - profileId: "q3-xl-q8-kv" + displayName: "Unsloth Q3_K_XL with 8-bit KV cache" + description: "Uses the smaller Q3 weights and stores the context cache at 8 bits, which fits more context in the same memory." + configurationId: "llama-cpp" + defaultModelArtifactBindingId: "q3-xl" + engineParameterDefaults: + values: + - parameterId: "prompt-cache-memory-mib" + unsignedInteger: "2048" + optionDefaults: + - optionId: "kv-cache" + values: + - parameterId: "precision" + string: "q8_0" - profileId: "q4-xl" displayName: "Unsloth Q4_K_XL" configurationId: "llama-cpp" diff --git a/catalog/recommendations/apple.yaml b/catalog/recommendations/apple.yaml index 06688bf8..150f70bd 100644 --- a/catalog/recommendations/apple.yaml +++ b/catalog/recommendations/apple.yaml @@ -1,55 +1,43 @@ # Model recommendations for Apple Silicon Macs, best first. Minimums on system memory separate Mac -# sizes: 16, 24, 32, 64 and 128 GB. Memory bandwidth limits generation, so the general assistant -# prefers a mixture-of-experts model on Macs large enough for it. +# sizes: 24, 32 and 64 GB, with the last entries for 16 GB Macs and smaller. Coding entries carry +# the largest context per request that starts on an idle Mac of their size. hardwareClassId: "apple" useCases: - - useCase: "USE_CASE_CODING_AND_AGENTS" + - useCase: "USE_CASE_CODING" entries: - modelId: "qwen3-8-27b" profileId: "q5-xl" + contextTokensPerRequest: 245760 hardwareRequirements: minimumSystemMemoryBytes: "64000000000" - - modelId: "qwen3-8-27b" - profileId: "q4-xl" + - modelId: "gemma-4-26b-a4b-it" + profileId: "q3-xl" + contextTokensPerRequest: 196608 hardwareRequirements: minimumSystemMemoryBytes: "32000000000" - - modelId: "qwen3-5-9b" - profileId: "q4-xl" + - modelId: "gemma-4-12b-it-qat" + profileId: "gpu" + contextTokensPerRequest: 262144 hardwareRequirements: - minimumSystemMemoryBytes: "16000000000" + minimumSystemMemoryBytes: "24000000000" - modelId: "qwen3-5-4b" - profileId: "q4-xl" - - useCase: "USE_CASE_GENERAL_ASSISTANT" + profileId: "q5-xl" + contextTokensPerRequest: 139264 + - useCase: "USE_CASE_CHAT_ASSISTANT" entries: - - modelId: "qwen3-6-35b-a3b" - profileId: "q4-xl" + - modelId: "qwen3-8-27b" + profileId: "q5-xl" + contextTokensPerRequest: 32768 hardwareRequirements: minimumSystemMemoryBytes: "64000000000" - modelId: "qwen3-8-27b" - profileId: "q4-xl" - hardwareRequirements: - minimumSystemMemoryBytes: "32000000000" - - modelId: "gemma-4-12b-it-qat" - profileId: "gpu" - hardwareRequirements: - minimumSystemMemoryBytes: "16000000000" - - modelId: "gemma-4-e4b-it-qat" - profileId: "gpu" - - useCase: "USE_CASE_TRANSLATION" - entries: - - modelId: "gemma-4-26b-a4b-it" - profileId: "q4-xl" + profileId: "q3-xl" + contextTokensPerRequest: 32768 hardwareRequirements: minimumSystemMemoryBytes: "32000000000" - modelId: "gemma-4-12b-it-qat" profileId: "gpu" - hardwareRequirements: - minimumSystemMemoryBytes: "16000000000" + contextTokensPerRequest: 32768 - modelId: "gemma-4-e4b-it-qat" profileId: "gpu" - - useCase: "USE_CASE_DECISION_MAKING" - entries: - - modelId: "clef-flash" - profileId: "q8-0" - - modelId: "clef-flash" - profileId: "q4-k-m" + contextTokensPerRequest: 32768 diff --git a/catalog/recommendations/nvidia-dedicated.yaml b/catalog/recommendations/nvidia-dedicated.yaml index f51e1841..66445f14 100644 --- a/catalog/recommendations/nvidia-dedicated.yaml +++ b/catalog/recommendations/nvidia-dedicated.yaml @@ -1,45 +1,47 @@ # Model recommendations for NVIDIA GPUs with their own memory, best first. Minimums per GPU separate -# card sizes: 8, 16, 24 and 32 GB. Entries a GPU can't run, such as ninfer off an RTX 5090 or RTX PRO 6000, are -# skipped for it. +# card sizes: 12, 16 and 24 GB, with the last entry for 8 GB cards. Entries a GPU can't run, such as +# ninfer off an RTX 5090 or RTX PRO 6000, are skipped for it. Coding entries carry the largest +# context per request that starts on an idle card of their size. hardwareClassId: "nvidia-dedicated" useCases: - - useCase: "USE_CASE_CODING_AND_AGENTS" + - useCase: "USE_CASE_CODING" entries: - modelId: "qwen3-8-27b" profileId: "ninfer-nvfp4" + contextTokensPerRequest: 229376 - modelId: "qwen3-8-27b" - profileId: "q4-xl" + profileId: "q3-xl-q8-kv" + contextTokensPerRequest: 155648 hardwareRequirements: minimumMemoryBytesPerDevice: "24000000000" - - modelId: "qwen3-5-9b" - profileId: "q4-xl" + - modelId: "gemma-4-12b-it-qat" + profileId: "gpu" + contextTokensPerRequest: 163840 hardwareRequirements: minimumMemoryBytesPerDevice: "12000000000" - modelId: "qwen3-5-4b" - profileId: "q4-xl" - - useCase: "USE_CASE_GENERAL_ASSISTANT" + profileId: "q3-xl" + contextTokensPerRequest: 131072 + - useCase: "USE_CASE_CHAT_ASSISTANT" entries: - modelId: "qwen3-8-27b" profileId: "ninfer-nvfp4" + contextTokensPerRequest: 32768 - modelId: "qwen3-8-27b" profileId: "q4-xl" + contextTokensPerRequest: 32768 hardwareRequirements: minimumMemoryBytesPerDevice: "24000000000" - - modelId: "gemma-4-12b-it-qat" - profileId: "gpu" + - modelId: "muse-glimmer-30b" + profileId: "q3-xl" + contextTokensPerRequest: 32768 hardwareRequirements: - minimumMemoryBytesPerDevice: "12000000000" - - modelId: "qwen3-5-4b" - profileId: "q4-xl" - - useCase: "USE_CASE_TRANSLATION" - entries: - - modelId: "gemma-4-26b-a4b-it" - profileId: "q4-xl" - hardwareRequirements: - minimumMemoryBytesPerDevice: "24000000000" + minimumMemoryBytesPerDevice: "16000000000" - modelId: "gemma-4-12b-it-qat" profileId: "gpu" + contextTokensPerRequest: 32768 hardwareRequirements: minimumMemoryBytesPerDevice: "12000000000" - modelId: "qwen3-5-4b" - profileId: "q4-xl" + profileId: "q5-xl" + contextTokensPerRequest: 32768 diff --git a/catalog/recommendations/nvidia-unified.yaml b/catalog/recommendations/nvidia-unified.yaml index edeaa9ca..adeeabda 100644 --- a/catalog/recommendations/nvidia-unified.yaml +++ b/catalog/recommendations/nvidia-unified.yaml @@ -1,29 +1,27 @@ # Model recommendations for NVIDIA GPUs sharing the host's memory, such as the GB10, best first. -# Memory bandwidth limits generation there, so the general assistant prefers mixture-of-experts -# models, which read fewer weights per token. Decision making uses the nvidia list. +# Later entries are fallbacks for smaller machines. Coding entries carry the largest context per +# request that starts on an idle GB10. hardwareClassId: "nvidia-unified" useCases: - - useCase: "USE_CASE_CODING_AND_AGENTS" + - useCase: "USE_CASE_CODING" entries: - modelId: "qwen3-8-27b" profileId: "nvfp4-text" + contextTokensPerRequest: 262144 - modelId: "qwen3-6-35b-a3b" profileId: "nvfp4" + contextTokensPerRequest: 262144 - modelId: "qwen3-5-9b" profileId: "q4-xl" - - useCase: "USE_CASE_GENERAL_ASSISTANT" + contextTokensPerRequest: 262144 + - useCase: "USE_CASE_CHAT_ASSISTANT" entries: - - modelId: "ornith-1-5-35b-a3b" - profileId: "nvfp4-multimodal" - modelId: "qwen3-8-27b" profileId: "nvfp4-multimodal" + contextTokensPerRequest: 32768 + - modelId: "ornith-1-5-35b-a3b" + profileId: "nvfp4-multimodal" + contextTokensPerRequest: 32768 - modelId: "gemma-4-12b-it-qat" profileId: "gpu" - - useCase: "USE_CASE_TRANSLATION" - entries: - - modelId: "gemma-4-31b-it" - profileId: "nvfp4" - - modelId: "gemma-4-26b-a4b-it" - profileId: "nvfp4" - - modelId: "gemma-4-12b-it-qat" - profileId: "gpu" + contextTokensPerRequest: 32768 diff --git a/catalog/recommendations/nvidia.yaml b/catalog/recommendations/nvidia.yaml deleted file mode 100644 index 8a358c23..00000000 --- a/catalog/recommendations/nvidia.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# Model recommendations shared by every NVIDIA target. Dedicated and unified targets use their own -# class's list for a use case first and fall back to this one. -hardwareClassId: "nvidia" -useCases: - - useCase: "USE_CASE_DECISION_MAKING" - entries: - - modelId: "clef-flash" - profileId: "q8-0" - - modelId: "clef-flash" - profileId: "q4-k-m" diff --git a/docs/specs/access.md b/docs/specs/access.md index 69998fbb..00baeaef 100644 --- a/docs/specs/access.md +++ b/docs/specs/access.md @@ -111,6 +111,9 @@ flowchart TB - Users, machine identities, roles, permissions, sessions, API keys, identity provider configuration and access policy, all owned by the identity and access manager. +- Per-user preferences, such as how one person's console looks, wait for this manager. Until then, + console settings are site-wide, not per user, and the site manager owns them + ([sites](sites.md#site-manager)). ### Secrets diff --git a/docs/specs/apps.md b/docs/specs/apps.md index 3441174a..5b9e00a8 100644 --- a/docs/specs/apps.md +++ b/docs/specs/apps.md @@ -137,6 +137,9 @@ What an application deployment keeps from creation: - Own app configuration resolution and endpoint projection, including the site inference base URL and optional route links. - Own typed app parameter resolution. +- Answer an advisory read of the application targets that can run an app now, best first: targets + with the app already running or starting, then targets matching its preferred accelerators. It + never changes admission or placement; a deployment still names its exact target. - Keep the target's GPU set in the snapshot at creation and pass that whole set to every unit at each launch. - Before compiling any launch, recheck the kept GPU set against the target's current configured diff --git a/docs/specs/catalog.md b/docs/specs/catalog.md index e9aea477..9647e4fa 100644 --- a/docs/specs/catalog.md +++ b/docs/specs/catalog.md @@ -161,8 +161,10 @@ minimums. - Only the master catalog declares recommendations. Site and development catalogs can't add or replace them. -- Validation keeps each entry runnable on its class and serving its use case's request kind, and - keeps the classes that declare lists disjoint or nested. +- An entry may set its own context per request, within the model's minimum and maximum context. + Recommendations size every entry with it. +- Validation keeps each entry runnable on its class and accepting chat requests, and keeps the + classes that declare lists disjoint or nested. - A site turns them into advisory recommendations per use case ([models](models.md#advisory-reads)). @@ -170,6 +172,9 @@ minimums. - Carries the app's portable artifact, configuration, storage, secret, dependency, readiness and endpoint requirements, and its typed deployment parameters, shown as basic or advanced. +- May name the use cases it serves (at most one master-catalog app per use case) and the + accelerator vendors it prefers. Both are recommendation and display metadata, with no admission, + placement or fit authority. Site and development catalogs can't claim a use case. - Engine releases and apps declare the storage they need. Deployments keep that declaration ([runtime](runtime.md)). @@ -193,7 +198,7 @@ The artifact manager also owns artifact bytes ([storage](storage.md)). - Catalog declarations are immutable. Deployments keep what they resolved ([models](models.md#deployment-snapshot)). -- Scores, licences and recommendations never affect admission, placement or fit. +- Scores, licences, app use cases, preferred accelerators and recommendations never affect admission, placement or fit. ## Future direction diff --git a/docs/specs/console.md b/docs/specs/console.md index 6a5de957..428876a6 100644 --- a/docs/specs/console.md +++ b/docs/specs/console.md @@ -9,7 +9,7 @@ Code: `frontend/` (its `AGENTS.md` holds the code map). - Manage and observe the site, its machines, deployments and routes, through the management API. - See live updates as the site changes. -- Start with a guided quick start that recommends models for your use cases and hardware. +- Start with a guided Get started page that proposes, for each use case, a model, an app or both that can start now on your hardware. - Check fit and preview memory before deploying, from the server's capacity answers. - Explore metric history across machines, devices, deployments and routes. - Try inference routes: conversations with chat routes, and typed questions to decision routes @@ -62,12 +62,17 @@ flowchart LR - Draw what the server answers: owner reads, capacity verdicts, previews, recommendations and metrics. +- Hide Get started for the whole site once a ready step shows or the operator dismisses it. The + profile menu still opens the page without changing the choice. The choice is a site-wide console + setting, read from and updated through the site manager ([sites](sites.md#site-manager)). - Explain the impact of actions, confirm destructive ones, and show actionable failures ([README](README.md#principles)). #### Boundaries -- Owns no product state. +- Owns no product state. Site-wide console settings belong to the site manager, not the console. +- Browser storage holds only per-browser preferences and drafts, never a setting the whole site + shares. - Holds no fit, headroom, sizing or starting-value rule ([machines](machines.md)). Each figure has one source, so two screens never disagree. - Never shows representative or fixture data as infrastructure-backed. diff --git a/docs/specs/models.md b/docs/specs/models.md index 4919f96e..6dfd06ed 100644 --- a/docs/specs/models.md +++ b/docs/specs/models.md @@ -13,8 +13,8 @@ recommendations), `backend/core/src/telemetry/` (workload resource records). execution target. - Use profiles as presets, and override typed options, engine parameters and expert launch arguments when you create a deployment. -- Get model recommendations for your use cases across the site's targets, through a guided quick - start. +- Get model recommendations for your use cases across the site's targets, through the Get started + page. - See which configured targets can run each profile, and narrow the catalog to profiles whose fit allows them there. - See advisory memory estimates for a configuration when measured evidence exists, and its fit on @@ -167,6 +167,9 @@ flowchart LR - Recommendations: each target takes its most specific hardware class's list per use case. Entries are kept or dropped by the capacity component's verdicts, entry minimums included, and the result is one ranking per use case across the site. +- A caller may ask for "can start now": rows keep only targets that already run the model or fit + free memory now, and rows with a running or starting deployment come first. Each row's target + names that deployment, and each row carries its context per request. - A target that stopped reporting is judged on its last observations for recommendations, marked unreachable with free memory unknown, until its node leaves the site. This is the only read that accepts stale observations. diff --git a/docs/specs/sites.md b/docs/specs/sites.md index b48dce6a..fc545a4d 100644 --- a/docs/specs/sites.md +++ b/docs/specs/sites.md @@ -180,6 +180,9 @@ What tells one server process which declared node it is. - Own site identity, membership, topology and runtime configuration, and the reconciled inventory, health and maintenance view. +- Own the site-wide console settings: operator choices about how the console behaves for everyone + on the site, such as whether Get started is hidden. They are desired configuration of the site, + kept with it and never mixed with observations or history. - Hold authoritative site intent and owner-controlled management in the process bound to the site manager node. - Request node facts and supported administrative actions from node agents. @@ -204,6 +207,8 @@ What tells one server process which declared node it is. - Provide: node views, view republication and acknowledgements. - Provide: the reconciled inventory, read by the management API, owner admission and the capacity component, instead of node-local transport. +- Receive and provide: the console settings, read by the console and updated by it one setting at a + time. Updating them never wakes controllers or changes any node's view. ### Node agent diff --git a/frontend/src/api/application-client.ts b/frontend/src/api/application-client.ts index 8c90fc09..aa191a50 100644 --- a/frontend/src/api/application-client.ts +++ b/frontend/src/api/application-client.ts @@ -34,6 +34,7 @@ export type ApplicationCatalogClient = Pick, "list type GeneratedApplicationDeploymentServiceClient = Pick< Client, | "listApplicationDeployments" + | "listApplicationTargets" | "getApplicationDeployment" | "createApplicationDeployment" | "removeApplicationDeployment" @@ -71,6 +72,7 @@ export function createApplicationClient(baseUrl = applicationServerUrl): Applica catalog: createClient(CatalogService, transport), application: { listApplicationDeployments: application.listApplicationDeployments, + listApplicationTargets: application.listApplicationTargets, getApplicationDeployment: application.getApplicationDeployment, createApplicationDeployment: async (request) => validateCreateApplicationResponse( diff --git a/frontend/src/api/model-client.ts b/frontend/src/api/model-client.ts index 70efd2a4..17ef73b4 100644 --- a/frontend/src/api/model-client.ts +++ b/frontend/src/api/model-client.ts @@ -40,6 +40,7 @@ export type CatalogClient = Pick< type GeneratedModelDeploymentServiceClient = Pick< Client, | "listModelDeployments" + | "listModelRecommendations" | "createModelDeployment" | "updateModelDeployment" | "removeModelDeployment" @@ -92,6 +93,7 @@ export function createModelClient(baseUrl = modelServerUrl): ModelClient { catalog: createClient(CatalogService, transport), model: { listModelDeployments: model.listModelDeployments, + listModelRecommendations: model.listModelRecommendations, previewLocalModelDeployment: (local) => model.previewLocalModelDeployment( create(PreviewLocalModelDeploymentRequestSchema, { local }), diff --git a/frontend/src/api/site-client.test.ts b/frontend/src/api/site-client.test.ts index 9765b8e4..c89c93dd 100644 --- a/frontend/src/api/site-client.test.ts +++ b/frontend/src/api/site-client.test.ts @@ -5,24 +5,31 @@ import { ListConfiguredExecutionTargetsResponseSchema } from "@/gen/dotcore/site import type { SiteClient } from "./site-client"; -const { createClient, createGrpcWebTransport, generatedListConfiguredExecutionTargets } = - vi.hoisted(() => { - const generatedListConfiguredExecutionTargets = vi.fn(); - const createGrpcWebTransport = vi.fn(() => ({}) as never); - const createClient = vi.fn((...args: unknown[]) => { - void args; - return { - listConfiguredExecutionTargets: generatedListConfiguredExecutionTargets, - } as never; - }); - +const { + createClient, + createGrpcWebTransport, + generatedListConfiguredExecutionTargets, + generatedUpdateConsoleSettings, +} = vi.hoisted(() => { + const generatedListConfiguredExecutionTargets = vi.fn(); + const generatedUpdateConsoleSettings = vi.fn(); + const createGrpcWebTransport = vi.fn(() => ({}) as never); + const createClient = vi.fn((...args: unknown[]) => { + void args; return { - createClient, - createGrpcWebTransport, - generatedListConfiguredExecutionTargets, - }; + listConfiguredExecutionTargets: generatedListConfiguredExecutionTargets, + updateConsoleSettings: generatedUpdateConsoleSettings, + } as never; }); + return { + createClient, + createGrpcWebTransport, + generatedListConfiguredExecutionTargets, + generatedUpdateConsoleSettings, + }; +}); + vi.mock("@connectrpc/connect-web", () => ({ createGrpcWebTransport })); vi.mock("@connectrpc/connect", async (importOriginal) => ({ ...(await importOriginal()), @@ -68,7 +75,9 @@ describe("Site client endpoint resolution", () => { }); it("exposes only public site management methods to browser callers", async () => { - expectTypeOf().toEqualTypeOf<"listConfiguredExecutionTargets">(); + expectTypeOf().toEqualTypeOf< + "listConfiguredExecutionTargets" | "updateConsoleSettings" + >(); const { createSiteClient } = await import("./site-client"); const siteClient = createSiteClient(); @@ -82,4 +91,11 @@ describe("Site client endpoint resolution", () => { await siteClient.listConfiguredExecutionTargets({}); expect(generatedListConfiguredExecutionTargets).toHaveBeenCalledWith({}, undefined); }); + + it("forwards console settings updates unchanged", async () => { + const { createSiteClient } = await import("./site-client"); + const request = { settings: { getStartedHidden: true } }; + await createSiteClient().updateConsoleSettings(request); + expect(generatedUpdateConsoleSettings).toHaveBeenCalledWith(request, undefined); + }); }); diff --git a/frontend/src/api/site-client.ts b/frontend/src/api/site-client.ts index 9f8f6f73..b1979ae3 100644 --- a/frontend/src/api/site-client.ts +++ b/frontend/src/api/site-client.ts @@ -4,7 +4,10 @@ import { createConsoleTransport } from "@/api/console-transport"; import { resolveManagementOrigin } from "@/api/management-origin"; import { SiteService } from "@/gen/dotcore/site/v1/service_pb"; -export type SiteClient = Pick, "listConfiguredExecutionTargets">; +export type SiteClient = Pick< + Client, + "listConfiguredExecutionTargets" | "updateConsoleSettings" +>; const siteServerUrl = resolveManagementOrigin(import.meta.env.VITE_DOTCORE_MANAGEMENT_ORIGIN, { allowTestFallback: import.meta.env.MODE === "test", @@ -17,5 +20,7 @@ export function createSiteClient(baseUrl = siteServerUrl): SiteClient { return { listConfiguredExecutionTargets: (request, options) => generatedClient.listConfiguredExecutionTargets(request, options), + updateConsoleSettings: (request, options) => + generatedClient.updateConsoleSettings(request, options), }; } diff --git a/frontend/src/app/console-layout.test.tsx b/frontend/src/app/console-layout.test.tsx index 69bd5fd5..6c94432e 100644 --- a/frontend/src/app/console-layout.test.tsx +++ b/frontend/src/app/console-layout.test.tsx @@ -9,6 +9,7 @@ import { CONSOLE_PREFERENCES_STORAGE_KEY, writeSidebarCollapsed } from "@/lib/co import { productionTiming, setTiming, timing } from "@/lib/timing"; import { PanelLink } from "@/panels/panel-navigation"; import { usePanelPendingNavigation } from "@/panels/panel-navigation-pending"; +import { FakeConsoleSettings } from "@/test/fake-console-settings"; import { renderWithClients } from "@/test/render-with-clients"; import { setupUser } from "@/test/user"; import { ThemeProvider } from "@/theme/theme-provider"; @@ -124,6 +125,7 @@ function renderFeaturePanelLayout(initialEntries: string[]) { > , + { resourceWatchClient: new FakeConsoleSettings().resourceWatchClient() }, ); return { router, ...view }; } @@ -157,6 +159,7 @@ function renderLayout( > , + { resourceWatchClient: new FakeConsoleSettings().resourceWatchClient() }, ); return { router, storage, ...view }; @@ -268,7 +271,7 @@ describe("ConsoleLayout", () => { await user.tab(); expect(document.activeElement).toHaveAccessibleName("Open profile menu"); - for (const path of ["/overview", "/machines"]) { + for (const path of ["/get-started", "/overview", "/machines"]) { await user.tab(); expect(document.activeElement).toHaveAttribute("href", path); } @@ -317,6 +320,7 @@ describe("ConsoleLayout", () => { > , + { resourceWatchClient: new FakeConsoleSettings().resourceWatchClient() }, ); const footerToggle = screen.getByRole("button", { name: "Collapse" }); @@ -353,6 +357,7 @@ describe("ConsoleLayout", () => { > , + { resourceWatchClient: new FakeConsoleSettings().resourceWatchClient() }, ); // Nothing stored yet: open (expanded), the collapsed field's proto3 zero value. diff --git a/frontend/src/app/console-layout.tsx b/frontend/src/app/console-layout.tsx index 73404ad9..0dd48974 100644 --- a/frontend/src/app/console-layout.tsx +++ b/frontend/src/app/console-layout.tsx @@ -13,6 +13,7 @@ import { SidebarInset, SidebarProvider } from "@/components/ui/sidebar"; import { type ChatMode, ChatOpenerContext } from "@/features/chat/chat-opener"; import { ChatPanel } from "@/features/chat/chat-panel"; import { ChatSessionProvider } from "@/features/chat/chat-session-provider"; +import { GetStartedHiddenProvider } from "@/features/get-started/get-started-hidden"; import { ModelsPollingProvider } from "@/features/models/polling/models-polling-provider"; import { WatchPauseProvider } from "@/hooks/watch-pause-provider"; import type { StorageLike } from "@/lib/browser-storage"; @@ -36,11 +37,13 @@ export function ConsoleLayout({ eventTarget, storage }: ConsoleLayoutProps) { - - - - - + + + + + + + diff --git a/frontend/src/app/console-sidebar-groups.test.tsx b/frontend/src/app/console-sidebar-groups.test.tsx index e08b8a1d..9924036d 100644 --- a/frontend/src/app/console-sidebar-groups.test.tsx +++ b/frontend/src/app/console-sidebar-groups.test.tsx @@ -13,6 +13,7 @@ import { import { SidebarProvider } from "@/components/ui/sidebar"; import { CONSOLE_PREFERENCES_STORAGE_KEY, writeSidebarCollapsed } from "@/lib/console-preferences"; import { productionTiming, setTiming } from "@/lib/timing"; +import { StaticGetStartedHidden } from "@/test/static-get-started-hidden"; import { setupUser } from "@/test/user"; afterEach(() => { @@ -314,7 +315,9 @@ function renderGroups(initialEntry = "/models", storage = new FakeStorage()) { path: "*", element: ( - + + + ), }, diff --git a/frontend/src/app/console-sidebar.test.tsx b/frontend/src/app/console-sidebar.test.tsx index 9286a830..aeeaa5fa 100644 --- a/frontend/src/app/console-sidebar.test.tsx +++ b/frontend/src/app/console-sidebar.test.tsx @@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { ConsoleLayout } from "@/app/console-layout"; import { productionTiming, setTiming } from "@/lib/timing"; import { usePanelPendingNavigation } from "@/panels/panel-navigation-pending"; +import { FakeConsoleSettings } from "@/test/fake-console-settings"; import { useFakeTimersWithWaits } from "@/test/fake-timers"; import { renderWithClients } from "@/test/render-with-clients"; import { setupUser } from "@/test/user"; @@ -67,6 +68,7 @@ function renderNavigation(initialEntry = "/models", pending = false) { > , + { resourceWatchClient: new FakeConsoleSettings().resourceWatchClient() }, ); return router; @@ -99,9 +101,10 @@ async function openMobileNavigation(user: UserEvent) { } describe("ConsoleSidebar primary destinations", () => { - it("groups the Models and Catalog links under Models, followed by Routes", () => { + it("groups the Models and Catalog links under Models, followed by Routes", async () => { setViewportWidth(1280); renderNavigation(); + await screen.findByRole("link", { name: "Get started" }); const group = screen.getByRole("button", { name: "Models" }); const groupRow = group.closest('[data-slot="sidebar-menu-item"]')!; @@ -123,7 +126,7 @@ describe("ConsoleSidebar primary destinations", () => { within(groupRow.parentElement as HTMLElement) .getAllByRole("link") .map((link) => link.getAttribute("href")), - ).toEqual(["/overview", "/machines", "/models", "/models/catalog", "/routes"]); + ).toEqual(["/get-started", "/overview", "/machines", "/models", "/models/catalog", "/routes"]); }); it("groups the Apps and Catalog links under Apps, open on the Apps pages", () => { diff --git a/frontend/src/app/console-sidebar.tsx b/frontend/src/app/console-sidebar.tsx index be77b110..dfe99cda 100644 --- a/frontend/src/app/console-sidebar.tsx +++ b/frontend/src/app/console-sidebar.tsx @@ -9,6 +9,7 @@ import { AppsSectionIcon, CatalogSectionIcon, ChatSectionIcon, + GetStartedSectionIcon, MachinesSectionIcon, ModelsSectionIcon, OverviewSectionIcon, @@ -30,6 +31,7 @@ import { SidebarSeparator, useSidebar, } from "@/components/ui/sidebar"; +import { useGetStartedHidden } from "@/features/get-started/use-get-started-hidden"; import { timing } from "@/lib/timing"; import { usePanelNavigationPending } from "@/panels/panel-navigation-pending"; @@ -49,7 +51,14 @@ interface NavItem { children?: NavChildItem[]; } +const getStartedNavItem: NavItem = { + icon: GetStartedSectionIcon, + label: "Get started", + path: "/get-started", +}; + const primaryNavItems: NavItem[] = [ + getStartedNavItem, { icon: OverviewSectionIcon, label: "Overview", path: "/overview" }, { icon: MachinesSectionIcon, label: "Machines", path: "/machines" }, { @@ -76,12 +85,13 @@ const footerNavItem: NavItem = { icon: SettingsSectionIcon, label: "Settings" }; /** * Icon-collapsible left navigation with direct destinations and optional disclosure groups. * Settings has no route yet and renders disabled. Primary items default to the console - * destinations; supplied items use the same linked, disabled and collapsible row treatments. + * destinations, with Get started until dismissed; supplied items use the same linked, disabled and + * collapsible row treatments. */ export function ConsoleSidebar({ chatCurrent = false, chatOpen = false, - items = primaryNavItems, + items, onChatToggle, }: { /** Whether the Chat page is the current page, which marks the Chat item current. */ @@ -92,6 +102,11 @@ export function ConsoleSidebar({ onChatToggle?: () => void; } = {}) { const { pathname } = useLocation(); + const { hidden: getStartedHidden } = useGetStartedHidden(); + // Until the site's settings load the item stays out: a site that hid it never sees it flash. + const navItems = + items ?? + primaryNavItems.filter((item) => !(getStartedHidden !== false && item === getStartedNavItem)); const { state } = useSidebar(); const { hovering, onPointerEnter, onPointerLeave, suppressHover } = useHoverPeek(); const { focusWithin, onBlur, onFocus, onKeyDown, onPointerDown, suppressFocus } = @@ -126,7 +141,7 @@ export function ConsoleSidebar({ - {items.map((item) => ( + {navItems.map((item) => ( ))} diff --git a/frontend/src/app/get-started-entry-points.test.tsx b/frontend/src/app/get-started-entry-points.test.tsx new file mode 100644 index 00000000..504c56e6 --- /dev/null +++ b/frontend/src/app/get-started-entry-points.test.tsx @@ -0,0 +1,137 @@ +import { create } from "@bufbuild/protobuf"; +import { act, screen, within } from "@testing-library/react"; +import { createMemoryRouter, RouterProvider } from "react-router-dom"; +import { describe, expect, it } from "vitest"; + +import { routes } from "@/app/routes"; +import { + ListConfiguredExecutionTargetsResponseSchema, + UpdateConsoleSettingsRequestSchema, +} from "@/gen/dotcore/site/v1/service_pb"; +import { FakeConsoleSettings } from "@/test/fake-console-settings"; +import { renderWithClients } from "@/test/render-with-clients"; +import { setupUser } from "@/test/user"; +import { ThemeProvider } from "@/theme/theme-provider"; + +const hiddenRequest = (hidden: boolean) => + create(UpdateConsoleSettingsRequestSchema, { + settings: { getStartedHidden: hidden }, + updateMask: { paths: ["get_started_hidden"] }, + }); + +function renderConsole(initialEntry: string, settings: FakeConsoleSettings) { + const router = createMemoryRouter(routes, { initialEntries: [initialEntry] }); + const view = renderWithClients( + + + , + { + resourceWatchClient: settings.resourceWatchClient(), + siteClient: { + listConfiguredExecutionTargets: () => + Promise.resolve(create(ListConfiguredExecutionTargetsResponseSchema)), + updateConsoleSettings: settings.updateConsoleSettings, + }, + }, + ); + + return { router, ...view }; +} + +function sidebarLink(name: string) { + return screen.queryByRole("link", { name }); +} + +async function openGetStartedFromProfileMenu(user: ReturnType) { + await user.click(screen.getByRole("button", { name: "Open profile menu" })); + await user.click(within(screen.getByRole("menu")).getByRole("menuitem", { name: "Get started" })); +} + +describe("Get started entry points", () => { + it("shows the sidebar item first while the site shows it and opens the page", async () => { + const user = setupUser(); + const settings = new FakeConsoleSettings(false); + const { router } = renderConsole("/overview", settings); + + await screen.findByRole("link", { name: "Get started" }); + const links = screen.getAllByRole("link"); + expect(links[0]).toHaveTextContent("Get started"); + expect(links[0]).toHaveAttribute("href", "/get-started"); + + await user.click(links[0]); + + expect(router.state.location.pathname).toBe("/get-started"); + expect(await screen.findByRole("heading", { name: "Get started" })).toBeVisible(); + expect(settings.updates).toEqual([]); + }); + + it("leaves the item out of a site that hides it, with no write", async () => { + const settings = new FakeConsoleSettings(true); + renderConsole("/overview", settings); + + expect(await screen.findByRole("link", { name: "Overview" })).toBeVisible(); + await act(() => settings.watch.tick()); + + expect(sidebarLink("Get started")).not.toBeInTheDocument(); + expect(settings.updates).toEqual([]); + }); + + it("hides the item for the whole site on Dismiss and goes to the overview", async () => { + const user = setupUser(); + const settings = new FakeConsoleSettings(false); + const first = renderConsole("/get-started", settings); + await screen.findByRole("link", { name: "Get started" }); + + await user.click(await screen.findByRole("button", { name: "Dismiss" })); + + expect(settings.updates).toEqual([hiddenRequest(true)]); + expect(first.router.state.location.pathname).toBe("/overview"); + expect(sidebarLink("Get started")).not.toBeInTheDocument(); + + // The watch confirms it on its next tick, with no flicker back. + await act(() => settings.watch.tick()); + expect(sidebarLink("Get started")).not.toBeInTheDocument(); + + // Another browser tab loading the console agrees. + first.unmount(); + renderConsole("/overview", settings); + expect(await screen.findByRole("link", { name: "Overview" })).toBeVisible(); + expect(sidebarLink("Get started")).not.toBeInTheDocument(); + }); + + it("opens the page from the profile menu without changing the hidden setting", async () => { + const user = setupUser(); + const settings = new FakeConsoleSettings(true); + const { router } = renderConsole("/overview", settings); + await screen.findByRole("link", { name: "Overview" }); + expect(sidebarLink("Get started")).not.toBeInTheDocument(); + + await openGetStartedFromProfileMenu(user); + + expect(await screen.findByRole("heading", { name: "Get started" })).toBeVisible(); + expect(router.state.location.pathname).toBe("/get-started"); + expect(settings.updates).toEqual([]); + expect(sidebarLink("Get started")).not.toBeInTheDocument(); + }); + + it("stays on the page with the reason when Dismiss cannot be saved", async () => { + const user = setupUser(); + const settings = new FakeConsoleSettings(false); + const { router } = renderConsole("/get-started", settings); + await screen.findByRole("link", { name: "Get started" }); + settings.failing = true; + + await user.click(await screen.findByRole("button", { name: "Dismiss" })); + + expect( + await screen.findByText(/Saving the Get started setting outcome is uncertain/), + ).toBeVisible(); + expect(router.state.location.pathname).toBe("/get-started"); + expect(sidebarLink("Get started")).toBeVisible(); + expect(settings.updates).toEqual([hiddenRequest(true)]); + }); +}); diff --git a/frontend/src/app/profile-menu.test.tsx b/frontend/src/app/profile-menu.test.tsx index 328be719..d85eee66 100644 --- a/frontend/src/app/profile-menu.test.tsx +++ b/frontend/src/app/profile-menu.test.tsx @@ -6,6 +6,7 @@ import { describe, expect, it } from "vitest"; import type { DiagnosticLogOperations } from "@/api/diagnostic-log-client"; import { ProfileMenu } from "@/app/profile-menu"; import { renderWithClients } from "@/test/render-with-clients"; +import { StaticGetStartedHidden } from "@/test/static-get-started-hidden"; import { setupUser } from "@/test/user"; import { ThemeProvider } from "@/theme/theme-provider"; @@ -27,7 +28,9 @@ function renderProfileMenu(diagnostics?: DiagnosticLogOperations) { eventTarget={new EventTarget()} storage={new FakeStorage()} > - + + + , { diagnosticLogOperations: diagnostics }, @@ -79,6 +82,7 @@ describe("ProfileMenu", () => { ...within(menu).getAllByRole("menuitemcheckbox"), ]; expect(items.map((item) => item.textContent)).toEqual([ + "Get started", "Preferences", "Site logs", "Dark mode", @@ -106,9 +110,9 @@ describe("ProfileMenu", () => { trigger.focus(); await user.keyboard("{Enter}"); await waitFor(() => - expect(screen.getByRole("menuitem", { name: "Preferences" })).toHaveFocus(), + expect(screen.getByRole("menuitem", { name: "Get started" })).toHaveFocus(), ); - await user.keyboard("{ArrowDown}"); + await user.keyboard("{ArrowDown}{ArrowDown}"); await waitFor(() => expect(screen.getByRole("menuitem", { name: "Site logs" })).toHaveFocus(), ); diff --git a/frontend/src/app/profile-menu.tsx b/frontend/src/app/profile-menu.tsx index 572d536b..0e4dc275 100644 --- a/frontend/src/app/profile-menu.tsx +++ b/frontend/src/app/profile-menu.tsx @@ -10,7 +10,7 @@ import { DropdownMenuItem, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; -import { PreferencesSectionIcon } from "@/components/ui/section-icons"; +import { GetStartedSectionIcon, PreferencesSectionIcon } from "@/components/ui/section-icons"; import { DiagnosticLogViewer } from "@/features/diagnostic-logs/diagnostic-log-viewer"; import { ColorScheme } from "@/gen/dotcore/console/v1/preferences_pb"; import { useTheme } from "@/theme/theme-context"; @@ -63,6 +63,10 @@ export function ProfileMenu() { className="w-[min(12rem,calc(100vw-16px))] border border-border" sideOffset={8} > + void navigate("/get-started")}> + void navigate("/preferences")}>